diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
index 6e15d64154960..9efb9c8b498aa 100644
--- a/.buildkite/pipelines/intake.yml
+++ b/.buildkite/pipelines/intake.yml
@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.3", "8.17.1", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
index abd11068e7a65..b1e5a7bf933c9 100644
--- a/.buildkite/pipelines/periodic-packaging.yml
+++ b/.buildkite/pipelines/periodic-packaging.yml
@@ -287,8 +287,8 @@ steps:
         env:
           BWC_VERSION: 8.15.5
 
-      - label: "{{matrix.image}} / 8.16.2 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.2
+      - label: "{{matrix.image}} / 8.16.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.3
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -301,7 +301,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.16.2
+          BWC_VERSION: 8.16.3
 
       - label: "{{matrix.image}} / 8.17.1 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.1
diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml
index c5846a763f5e8..ea0d7b13b55b4 100644
--- a/.buildkite/pipelines/periodic-platform-support.yml
+++ b/.buildkite/pipelines/periodic-platform-support.yml
@@ -63,6 +63,7 @@ steps:
             image:
               - almalinux-8-aarch64
               - ubuntu-2004-aarch64
+              - ubuntu-2404-aarch64
             GRADLE_TASK:
               - checkPart1
               - checkPart2
diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
index f2d169cd2b30d..4c593bae62d7a 100644
--- a/.buildkite/pipelines/periodic.yml
+++ b/.buildkite/pipelines/periodic.yml
@@ -306,8 +306,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.16.2 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.2#bwcTest
+      - label: 8.16.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.3#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -316,7 +316,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.16.2
+          BWC_VERSION: 8.16.3
         retry:
           automatic:
             - exit_status: "-1"
@@ -448,7 +448,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.3", "8.17.1", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -490,7 +490,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.3", "8.17.1", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
diff --git a/.buildkite/scripts/get-latest-test-mutes.sh b/.buildkite/scripts/get-latest-test-mutes.sh
index 5721e29f1b773..1dafcebec24b1 100755
--- a/.buildkite/scripts/get-latest-test-mutes.sh
+++ b/.buildkite/scripts/get-latest-test-mutes.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-if [[ ! "${BUILDKITE_PULL_REQUEST:-}" || "${BUILDKITE_AGENT_META_DATA_PROVIDER:-}" == "k8s" ]]; then
+if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false"  || "${BUILDKITE_AGENT_META_DATA_PROVIDER:-}" == "k8s" ]]; then
   exit 0
 fi
 
diff --git a/.ci/bwcVersions b/.ci/bwcVersions
index 3cb983373138f..cf12ee8c15419 100644
--- a/.ci/bwcVersions
+++ b/.ci/bwcVersions
@@ -15,7 +15,7 @@ BWC_VERSION:
   - "8.13.4"
   - "8.14.3"
   - "8.15.5"
-  - "8.16.2"
+  - "8.16.3"
   - "8.17.1"
   - "8.18.0"
   - "9.0.0"
diff --git a/.ci/init.gradle b/.ci/init.gradle
index 3e1f23804cf98..15d63f8ca7d20 100644
--- a/.ci/init.gradle
+++ b/.ci/init.gradle
@@ -1,95 +1,3 @@
-import com.bettercloud.vault.VaultConfig
-import com.bettercloud.vault.Vault
-
-initscript {
-  repositories {
-    mavenCentral()
-  }
-  dependencies {
-    classpath 'com.bettercloud:vault-java-driver:4.1.0'
-  }
-}
-
-boolean USE_ARTIFACTORY = false
-
-if (System.getenv('VAULT_ADDR') == null) {
-  // When trying to reproduce errors outside of CI, it can be useful to allow this to just return rather than blowing up
-  if (System.getenv('CI') == null) {
-    return
-  }
-
-  throw new GradleException("You must set the VAULT_ADDR environment variable to use this init script.")
-}
-
-if (System.getenv('VAULT_ROLE_ID') == null && System.getenv('VAULT_SECRET_ID') == null && System.getenv('VAULT_TOKEN') == null) {
-    // When trying to reproduce errors outside of CI, it can be useful to allow this to just return rather than blowing up
-  if (System.getenv('CI') == null) {
-    return
-  }
-
-  throw new GradleException("You must set either the VAULT_ROLE_ID and VAULT_SECRET_ID environment variables, " +
-    "or the VAULT_TOKEN environment variable to use this init script.")
-}
-
-final String vaultPathPrefix = System.getenv('VAULT_ADDR') ==~ /.+vault-ci.+\.dev.*/ ? "secret/ci/elastic-elasticsearch/migrated" : "secret/elasticsearch-ci"
-
-final String vaultToken = System.getenv('VAULT_TOKEN') ?: new Vault(
-  new VaultConfig()
-    .address(System.env.VAULT_ADDR)
-    .engineVersion(1)
-    .build()
-)
-  .withRetries(5, 1000)
-  .auth()
-  .loginByAppRole("approle", System.env.VAULT_ROLE_ID, System.env.VAULT_SECRET_ID)
-  .getAuthClientToken()
-
-final Vault vault = new Vault(
-  new VaultConfig()
-    .address(System.env.VAULT_ADDR)
-    .engineVersion(1)
-    .token(vaultToken)
-    .build()
-)
-  .withRetries(5, 1000)
-
-
-if (USE_ARTIFACTORY) {
-  final Map<String, String> artifactoryCredentials = vault.logical()
-    .read("${vaultPathPrefix}/artifactory.elstc.co")
-    .getData()
-  logger.info("Using elastic artifactory repos")
-  Closure configCache = {
-    return {
-      name "artifactory-gradle-release"
-      url "https://artifactory.elstc.co/artifactory/gradle-release"
-      credentials {
-        username artifactoryCredentials.get("username")
-        password artifactoryCredentials.get("token")
-      }
-    }
-  }
-  settingsEvaluated { settings ->
-    settings.pluginManagement {
-      repositories {
-        maven configCache()
-      }
-    }
-  }
-  projectsLoaded {
-    allprojects {
-      buildscript {
-        repositories {
-          maven configCache()
-        }
-      }
-      repositories {
-        maven configCache()
-      }
-    }
-  }
-}
-
 gradle.settingsEvaluated { settings ->
   settings.pluginManager.withPlugin("com.gradle.develocity") {
     settings.develocity {
@@ -98,14 +6,10 @@ gradle.settingsEvaluated { settings ->
   }
 }
 
-
 final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
 final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))
 
 if (buildCacheUrl) {
-  final Map<String, String> buildCacheCredentials = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ? [:] : vault.logical()
-    .read("${vaultPathPrefix}/gradle-build-cache")
-    .getData()
   gradle.settingsEvaluated { settings ->
     settings.buildCache {
       local {
@@ -116,11 +20,10 @@ if (buildCacheUrl) {
         url = buildCacheUrl
         push = buildCachePush
         credentials {
-          username = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ?: buildCacheCredentials.get("username")
-          password = System.getenv("GRADLE_BUILD_CACHE_PASSWORD") ?: buildCacheCredentials.get("password")
+          username = System.getenv("GRADLE_BUILD_CACHE_USERNAME")
+          password = System.getenv("GRADLE_BUILD_CACHE_PASSWORD")
         }
       }
     }
   }
 }
-
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
index e05c0774c9819..68c6ad5601546 100644
--- a/.ci/snapshotBwcVersions
+++ b/.ci/snapshotBwcVersions
@@ -1,5 +1,5 @@
 BWC_VERSION:
-  - "8.16.2"
+  - "8.16.3"
   - "8.17.1"
   - "8.18.0"
   - "9.0.0"
diff --git a/.gitignore b/.gitignore
index d1af97cbaea3b..8b2da4dc0832a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -69,3 +69,6 @@ testfixtures_shared/
 # Generated
 checkstyle_ide.xml
 x-pack/plugin/esql/src/main/generated-src/generated/
+
+# JEnv
+.java-version
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java
index 9aab4a3e3210f..d3259b9604717 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -71,12 +72,11 @@ public class EvalBenchmark {
         BigArrays.NON_RECYCLING_INSTANCE
     );
 
+    private static final FoldContext FOLD_CONTEXT = FoldContext.small();
+
     private static final int BLOCK_LENGTH = 8 * 1024;
 
-    static final DriverContext driverContext = new DriverContext(
-        BigArrays.NON_RECYCLING_INSTANCE,
-        BlockFactory.getInstance(new NoopCircuitBreaker("noop"), BigArrays.NON_RECYCLING_INSTANCE)
-    );
+    static final DriverContext driverContext = new DriverContext(BigArrays.NON_RECYCLING_INSTANCE, blockFactory);
 
     static {
         // Smoke test all the expected values and force loading subclasses more like prod
@@ -114,11 +114,12 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
         return switch (operation) {
             case "abs" -> {
                 FieldAttribute longField = longField();
-                yield EvalMapper.toEvaluator(new Abs(Source.EMPTY, longField), layout(longField)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Abs(Source.EMPTY, longField), layout(longField)).get(driverContext);
             }
             case "add" -> {
                 FieldAttribute longField = longField();
                 yield EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
                     new Add(Source.EMPTY, longField, new Literal(Source.EMPTY, 1L, DataType.LONG)),
                     layout(longField)
                 ).get(driverContext);
@@ -126,6 +127,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
             case "add_double" -> {
                 FieldAttribute doubleField = doubleField();
                 yield EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
                     new Add(Source.EMPTY, doubleField, new Literal(Source.EMPTY, 1D, DataType.DOUBLE)),
                     layout(doubleField)
                 ).get(driverContext);
@@ -140,7 +142,8 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                     lhs = new Add(Source.EMPTY, lhs, new Literal(Source.EMPTY, 1L, DataType.LONG));
                     rhs = new Add(Source.EMPTY, rhs, new Literal(Source.EMPTY, 1L, DataType.LONG));
                 }
-                yield EvalMapper.toEvaluator(new Case(Source.EMPTY, condition, List.of(lhs, rhs)), layout(f1, f2)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Case(Source.EMPTY, condition, List.of(lhs, rhs)), layout(f1, f2))
+                    .get(driverContext);
             }
             case "date_trunc" -> {
                 FieldAttribute timestamp = new FieldAttribute(
@@ -149,6 +152,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                     new EsField("timestamp", DataType.DATETIME, Map.of(), true)
                 );
                 yield EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
                     new DateTrunc(Source.EMPTY, new Literal(Source.EMPTY, Duration.ofHours(24), DataType.TIME_DURATION), timestamp),
                     layout(timestamp)
                 ).get(driverContext);
@@ -156,6 +160,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
             case "equal_to_const" -> {
                 FieldAttribute longField = longField();
                 yield EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
                     new Equals(Source.EMPTY, longField, new Literal(Source.EMPTY, 100_000L, DataType.LONG)),
                     layout(longField)
                 ).get(driverContext);
@@ -163,21 +168,21 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
             case "long_equal_to_long" -> {
                 FieldAttribute lhs = longField();
                 FieldAttribute rhs = longField();
-                yield EvalMapper.toEvaluator(new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext);
             }
             case "long_equal_to_int" -> {
                 FieldAttribute lhs = longField();
                 FieldAttribute rhs = intField();
-                yield EvalMapper.toEvaluator(new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext);
             }
             case "mv_min", "mv_min_ascending" -> {
                 FieldAttribute longField = longField();
-                yield EvalMapper.toEvaluator(new MvMin(Source.EMPTY, longField), layout(longField)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, new MvMin(Source.EMPTY, longField), layout(longField)).get(driverContext);
             }
             case "rlike" -> {
                 FieldAttribute keywordField = keywordField();
                 RLike rlike = new RLike(Source.EMPTY, keywordField, new RLikePattern(".ar"));
-                yield EvalMapper.toEvaluator(rlike, layout(keywordField)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, rlike, layout(keywordField)).get(driverContext);
             }
             default -> throw new UnsupportedOperationException();
         };
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java
index b294fe97c7e7c..fdb09594a1cda 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java
@@ -83,6 +83,8 @@ public class VectorScorerBenchmark {
 
     RandomVectorScorer luceneDotScorerQuery;
     RandomVectorScorer nativeDotScorerQuery;
+    RandomVectorScorer luceneSqrScorerQuery;
+    RandomVectorScorer nativeSqrScorerQuery;
 
     @Setup
     public void setup() throws IOException {
@@ -130,6 +132,8 @@ public void setup() throws IOException {
         }
         luceneDotScorerQuery = luceneScorer(values, VectorSimilarityFunction.DOT_PRODUCT, queryVec);
         nativeDotScorerQuery = factory.getInt7SQVectorScorer(VectorSimilarityFunction.DOT_PRODUCT, values, queryVec).get();
+        luceneSqrScorerQuery = luceneScorer(values, VectorSimilarityFunction.EUCLIDEAN, queryVec);
+        nativeSqrScorerQuery = factory.getInt7SQVectorScorer(VectorSimilarityFunction.EUCLIDEAN, values, queryVec).get();
 
         // sanity
         var f1 = dotProductLucene();
@@ -157,6 +161,12 @@ public void setup() throws IOException {
         if (q1 != q2) {
             throw new AssertionError("query: lucene[" + q1 + "] != " + "native[" + q2 + "]");
         }
+
+        var sqr1 = squareDistanceLuceneQuery();
+        var sqr2 = squareDistanceNativeQuery();
+        if (sqr1 != sqr2) {
+            throw new AssertionError("query: lucene[" + q1 + "] != " + "native[" + q2 + "]");
+        }
     }
 
     @TearDown
@@ -217,6 +227,16 @@ public float squareDistanceScalar() {
         return 1 / (1f + adjustedDistance);
     }
 
+    @Benchmark
+    public float squareDistanceLuceneQuery() throws IOException {
+        return luceneSqrScorerQuery.score(1);
+    }
+
+    @Benchmark
+    public float squareDistanceNativeQuery() throws IOException {
+        return nativeSqrScorerQuery.score(1);
+    }
+
     QuantizedByteVectorValues vectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
         var sq = new ScalarQuantizer(0.1f, 0.9f, (byte) 7);
         var slice = in.slice("values", 0, in.length());
diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GitInfoPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GitInfoPlugin.java
index 28b90714508bd..6f64edfc8c0dc 100644
--- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GitInfoPlugin.java
+++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GitInfoPlugin.java
@@ -10,47 +10,45 @@
 package org.elasticsearch.gradle.internal.conventions;
 
 import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
+import org.elasticsearch.gradle.internal.conventions.info.GitInfoValueSource;
 import org.elasticsearch.gradle.internal.conventions.util.Util;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
-import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 
-import javax.inject.Inject;
 import java.io.File;
 
-class GitInfoPlugin implements Plugin<Project> {
+import javax.inject.Inject;
 
-    private ProviderFactory factory;
-    private ObjectFactory objectFactory;
+public abstract class GitInfoPlugin implements Plugin<Project> {
 
+    private ProviderFactory factory;
     private Provider<String> revision;
-    private Property<GitInfo> gitInfo;
 
     @Inject
-    GitInfoPlugin(ProviderFactory factory, ObjectFactory objectFactory) {
+    public GitInfoPlugin(ProviderFactory factory) {
         this.factory = factory;
-        this.objectFactory = objectFactory;
     }
 
     @Override
     public void apply(Project project) {
-        File rootDir = Util.locateElasticsearchWorkspace(project.getGradle());
-        gitInfo = objectFactory.property(GitInfo.class).value(factory.provider(() ->
-            GitInfo.gitInfo(rootDir)
-        ));
-        gitInfo.disallowChanges();
-        gitInfo.finalizeValueOnRead();
-
-        revision = gitInfo.map(info -> info.getRevision() == null ? info.getRevision() : "main");
+        File rootDir = getGitRootDir(project);
+        getGitInfo().convention(factory.of(GitInfoValueSource.class, spec -> { spec.getParameters().getPath().set(rootDir); }));
+        revision = getGitInfo().map(info -> info.getRevision() == null ? info.getRevision() : "main");
     }
 
-    public Property<GitInfo> getGitInfo() {
-        return gitInfo;
+    private static File getGitRootDir(Project project) {
+        File rootDir = project.getRootDir();
+        if (new File(rootDir, ".git").exists()) {
+            return rootDir;
+        }
+        return Util.locateElasticsearchWorkspace(project.getGradle());
     }
 
+    public abstract Property<GitInfo> getGitInfo();
+
     public Provider<String> getRevision() {
         return revision;
     }
diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java
index ba170d083c886..63514ae671bf3 100644
--- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java
+++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java
@@ -15,9 +15,10 @@
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 
-import javax.inject.Inject;
 import java.util.Map;
 
+import javax.inject.Inject;
+
 public class LicensingPlugin implements Plugin<Project> {
     static final String ELASTIC_LICENSE_URL_PREFIX = "https://raw.githubusercontent.com/elastic/elasticsearch/";
     static final String ELASTIC_LICENSE_URL_POSTFIX = "/licenses/ELASTIC-LICENSE-2.0.txt";
@@ -33,24 +34,33 @@ public LicensingPlugin(ProviderFactory providerFactory) {
     @Override
     public void apply(Project project) {
         Provider<String> revision = project.getRootProject().getPlugins().apply(GitInfoPlugin.class).getRevision();
-        Provider<String> licenseCommitProvider = providerFactory.provider(() ->
-             isSnapshotVersion(project) ? revision.get() : "v" + project.getVersion()
+        Provider<String> licenseCommitProvider = providerFactory.provider(
+            () -> isSnapshotVersion(project) ? revision.get() : "v" + project.getVersion()
         );
 
-        Provider<String> elasticLicenseURL = licenseCommitProvider.map(licenseCommit -> ELASTIC_LICENSE_URL_PREFIX +
-                licenseCommit + ELASTIC_LICENSE_URL_POSTFIX);
-        Provider<String> agplLicenseURL = licenseCommitProvider.map(licenseCommit -> ELASTIC_LICENSE_URL_PREFIX +
-            licenseCommit + AGPL_ELASTIC_LICENSE_URL_POSTFIX);
+        Provider<String> elasticLicenseURL = licenseCommitProvider.map(
+            licenseCommit -> ELASTIC_LICENSE_URL_PREFIX + licenseCommit + ELASTIC_LICENSE_URL_POSTFIX
+        );
+        Provider<String> agplLicenseURL = licenseCommitProvider.map(
+            licenseCommit -> ELASTIC_LICENSE_URL_PREFIX + licenseCommit + AGPL_ELASTIC_LICENSE_URL_POSTFIX
+        );
         // But stick the Elastic license url in project.ext so we can get it if we need to switch to it
         project.getExtensions().getExtraProperties().set("elasticLicenseUrl", elasticLicenseURL);
 
-        MapProperty<String, String> licensesProperty = project.getObjects().mapProperty(String.class, String.class).convention(
-                providerFactory.provider(() -> Map.of(
-                        "Server Side Public License, v 1", "https://www.mongodb.com/licensing/server-side-public-license",
-                        "Elastic License 2.0", elasticLicenseURL.get(),
-                        "GNU Affero General Public License Version 3", agplLicenseURL.get())
+        MapProperty<String, Provider<String>> licensesProperty = project.getObjects()
+            .mapProperty(String.class, (Class<Provider<String>>) (Class<?>) Provider.class)
+            .convention(
+                providerFactory.provider(
+                    () -> Map.of(
+                        "Server Side Public License, v 1",
+                        providerFactory.provider(() -> "https://www.mongodb.com/licensing/server-side-public-license"),
+                        "Elastic License 2.0",
+                        elasticLicenseURL,
+                        "GNU Affero General Public License Version 3",
+                        agplLicenseURL
+                    )
                 )
-        );
+            );
 
         // Default to the SSPL+Elastic dual license
         project.getExtensions().getExtraProperties().set("projectLicenses", licensesProperty);
diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java
index c3124812e5089..22b0ab1918024 100644
--- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java
+++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java
@@ -28,6 +28,7 @@
 import org.gradle.api.plugins.JavaLibraryPlugin;
 import org.gradle.api.plugins.JavaPlugin;
 import org.gradle.api.provider.MapProperty;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.publish.PublishingExtension;
 import org.gradle.api.publish.maven.MavenPublication;
@@ -42,6 +43,7 @@
 import java.io.File;
 import java.util.Map;
 import java.util.concurrent.Callable;
+
 import javax.inject.Inject;
 
 public class PublishPlugin implements Plugin<Project> {
@@ -81,7 +83,7 @@ private void configurePublications(Project project) {
             }
         });
         @SuppressWarnings("unchecked")
-        var projectLicenses = (MapProperty<String, String>) project.getExtensions().getExtraProperties().get("projectLicenses");
+        var projectLicenses = (MapProperty<String, Provider<String>>) project.getExtensions().getExtraProperties().get("projectLicenses");
         publication.getPom().withXml(xml -> {
             var node = xml.asNode();
             node.appendNode("inceptionYear", "2009");
@@ -89,7 +91,7 @@ private void configurePublications(Project project) {
             projectLicenses.get().entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> {
                 Node license = licensesNode.appendNode("license");
                 license.appendNode("name", entry.getKey());
-                license.appendNode("url", entry.getValue());
+                license.appendNode("url", entry.getValue().get());
                 license.appendNode("distribution", "repo");
             });
             var developer = node.appendNode("developers").appendNode("developer");
@@ -194,7 +196,6 @@ static void configureSourcesJar(Project project) {
         });
     }
 
-
     /**
      * Format the generated pom files to be in a sort of reproducible order.
      */
diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfo.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfo.java
index dbd3b3f9c48ad..e6a41093205cc 100644
--- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfo.java
+++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfo.java
@@ -22,6 +22,7 @@
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Objects;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -190,4 +191,15 @@ public String urlFromOrigin() {
         }
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (o == null || getClass() != o.getClass()) return false;
+        GitInfo gitInfo = (GitInfo) o;
+        return Objects.equals(revision, gitInfo.revision) && Objects.equals(origin, gitInfo.origin);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(revision, origin);
+    }
 }
diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfoValueSource.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfoValueSource.java
new file mode 100644
index 0000000000000..c422c2eb74e39
--- /dev/null
+++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfoValueSource.java
@@ -0,0 +1,22 @@
+package org.elasticsearch.gradle.internal.conventions.info;
+
+import org.gradle.api.provider.Property;
+import org.gradle.api.provider.ValueSource;
+import org.gradle.api.provider.ValueSourceParameters;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.File;
+
+public abstract class GitInfoValueSource implements ValueSource<GitInfo, GitInfoValueSource.Parameters> {
+
+    @Nullable
+    @Override
+    public GitInfo obtain() {
+        File path = getParameters().getPath().get();
+        return GitInfo.gitInfo(path);
+    }
+
+    public interface Parameters extends ValueSourceParameters {
+        Property<File> getPath();
+    }
+}
diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/util/Util.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/util/Util.java
index a21606156b093..16e8af5832c56 100644
--- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/util/Util.java
+++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/util/Util.java
@@ -14,18 +14,19 @@
 import org.gradle.api.Project;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.initialization.IncludedBuild;
+import org.gradle.api.internal.GradleInternal;
 import org.gradle.api.invocation.Gradle;
 import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.api.tasks.util.PatternFilterable;
 
-import javax.annotation.Nullable;
 import java.io.File;
-import java.util.Collection;
 import java.util.Optional;
 import java.util.function.Supplier;
 
+import javax.annotation.Nullable;
+
 public class Util {
 
     public static boolean getBooleanProperty(String property, boolean defaultValue) {
@@ -120,6 +121,14 @@ public static SourceSetContainer getJavaSourceSets(Project project) {
         return project.getExtensions().getByType(JavaPluginExtension.class).getSourceSets();
     }
 
+    public static File getRootFolder(Gradle gradle) {
+        Gradle parent = gradle.getParent();
+        if (parent == null) {
+            return gradle.getRootProject().getRootDir();
+        }
+        return getRootFolder(parent);
+    }
+
     public static File locateElasticsearchWorkspace(Gradle gradle) {
         if (gradle.getRootProject().getName().startsWith("build-tools")) {
             File buildToolsParent = gradle.getRootProject().getRootDir().getParentFile();
diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties
index 22286c90de3d1..e712035eabc7b 100644
--- a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties
+++ b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=89d4e70e4e84e2d2dfbb63e4daa53e21b25017cc70c37e4eea31ee51fb15098a
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-all.zip
+distributionSha256Sum=7ebdac923867a3cec0098302416d1e3c6c0c729fc4e2e05c10637a8af33a76c5
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy
index 01effd52dafef..f7c05894d3e95 100644
--- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy
+++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy
@@ -24,7 +24,7 @@ abstract class AbstractGitAwareGradleFuncTest extends AbstractGradleFuncTest {
 
     def setup() {
         remoteGitRepo = new File(setupGitRemote(), '.git')
-        "git clone ${remoteGitRepo.absolutePath} cloned".execute(Collections.emptyList(), testProjectDir.root).waitFor()
+        execute("git clone ${remoteGitRepo.absolutePath} cloned", testProjectDir.root)
         buildFile = new File(testProjectDir.root, 'cloned/build.gradle')
         settingsFile = new File(testProjectDir.root, 'cloned/settings.gradle')
     }
diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy
index 65f124e5f88e8..cc551057cd600 100644
--- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy
+++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy
@@ -45,7 +45,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         file("build/distributions/hello-world-1.0-javadoc.jar").exists()
         file("build/distributions/hello-world-1.0-sources.jar").exists()
         file("build/distributions/hello-world-1.0.pom").exists()
-        assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """
+        assertXmlEquals(
+            file("build/distributions/hello-world-1.0.pom").text, """
             <project xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
   <!-- This module was also published with a richer model, Gradle metadata,  -->
   <!-- which should be used instead. Do not delete the following line which  -->
@@ -130,7 +131,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         file("build/distributions/hello-world-1.0-javadoc.jar").exists()
         file("build/distributions/hello-world-1.0-sources.jar").exists()
         file("build/distributions/hello-world-1.0.pom").exists()
-        assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """
+        assertXmlEquals(
+            file("build/distributions/hello-world-1.0.pom").text, """
             <project xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
               <modelVersion>4.0.0</modelVersion>
               <groupId>org.acme</groupId>
@@ -219,7 +221,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         file("build/distributions/hello-world-1.0-javadoc.jar").exists()
         file("build/distributions/hello-world-1.0-sources.jar").exists()
         file("build/distributions/hello-world-1.0.pom").exists()
-        assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """
+        assertXmlEquals(
+            file("build/distributions/hello-world-1.0.pom").text, """
             <project xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
               <modelVersion>4.0.0</modelVersion>
               <groupId>org.acme</groupId>
@@ -282,7 +285,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
 
             esplugin {
                 name = 'hello-world-plugin'
-                classname 'org.acme.HelloWorldPlugin'
+                classname = 'org.acme.HelloWorldPlugin'
                 description = "shadowed es plugin"
             }
 
@@ -312,7 +315,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         file("build/distributions/hello-world-plugin-1.0-javadoc.jar").exists()
         file("build/distributions/hello-world-plugin-1.0-sources.jar").exists()
         file("build/distributions/hello-world-plugin-1.0.pom").exists()
-        assertXmlEquals(file("build/distributions/hello-world-plugin-1.0.pom").text, """
+        assertXmlEquals(
+            file("build/distributions/hello-world-plugin-1.0.pom").text, """
             <project xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
               <!-- This module was also published with a richer model, Gradle metadata,  -->
               <!-- which should be used instead. Do not delete the following line which  -->
@@ -371,7 +375,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
 
             esplugin {
                 name = 'hello-world-plugin'
-                classname 'org.acme.HelloWorldPlugin'
+                classname = 'org.acme.HelloWorldPlugin'
                 description = "custom project description"
             }
 
@@ -389,7 +393,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         then:
         result.task(":generatePom").outcome == TaskOutcome.SUCCESS
         file("build/distributions/hello-world-plugin-2.0.pom").exists()
-        assertXmlEquals(file("build/distributions/hello-world-plugin-2.0.pom").text, """
+        assertXmlEquals(
+            file("build/distributions/hello-world-plugin-2.0.pom").text, """
             <project xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
               <!-- This module was also published with a richer model, Gradle metadata,  -->
               <!-- which should be used instead. Do not delete the following line which  -->
@@ -439,7 +444,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         // scm info only added for internal builds
         internalBuild()
         buildFile << """
-            buildParams.setGitOrigin("https://some-repo.com/repo.git")
+            buildParams.setGitOrigin(project.providers.provider(() -> "https://some-repo.com/repo.git"))
             apply plugin:'elasticsearch.java'
             apply plugin:'elasticsearch.publish'
 
@@ -447,7 +452,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
             group = 'org.acme'
             description = "just a test project"
 
-            ext.projectLicenses.set(['The Apache Software License, Version 2.0': 'http://www.apache.org/licenses/LICENSE-2.0'])
+            ext.projectLicenses.set(['The Apache Software License, Version 2.0': project.providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')])
         """
 
         when:
@@ -456,7 +461,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         then:
         result.task(":generatePom").outcome == TaskOutcome.SUCCESS
         file("build/distributions/hello-world-1.0.pom").exists()
-        assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """
+        assertXmlEquals(
+            file("build/distributions/hello-world-1.0.pom").text, """
             <project xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
           <!-- This module was also published with a richer model, Gradle metadata,  -->
           <!-- which should be used instead. Do not delete the following line which  -->
@@ -493,15 +499,15 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
 
     private boolean assertXmlEquals(String toTest, String expected) {
         def diff = DiffBuilder.compare(Input.fromString(expected))
-                .ignoreWhitespace()
-                .ignoreComments()
-                .normalizeWhitespace()
-                .withTest(Input.fromString(toTest))
-                .build()
+            .ignoreWhitespace()
+            .ignoreComments()
+            .normalizeWhitespace()
+            .withTest(Input.fromString(toTest))
+            .build()
         diff.differences.each { difference ->
             println difference
         }
-        if(diff.differences.size() > 0) {
+        if (diff.differences.size() > 0) {
             println """ given:
 $toTest
 """
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle
index 847eda7a355c0..797dc8bd0641b 100644
--- a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle
+++ b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle
@@ -98,8 +98,8 @@ develocity {
           link 'Source', "${prBaseUrl}/tree/${System.getenv('BUILDKITE_COMMIT')}"
           link 'Pull Request', "https://github.com/${repository}/pull/${prId}"
         } else {
-          value 'Git Commit ID', gitRevision
-          link 'Source', "https://github.com/${repository}/tree/${gitRevision}"
+          value 'Git Commit ID', gitRevision.get()
+          link 'Source', "https://github.com/${repository}/tree/${gitRevision.get()}"
         }
 
         buildFinished { result ->
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
index 895cca2af7967..90a4f74b5e9f4 100644
--- a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
+++ b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
@@ -10,6 +10,8 @@
 import org.elasticsearch.gradle.util.Pair
 import org.elasticsearch.gradle.util.GradleUtils
 import org.elasticsearch.gradle.internal.test.TestUtil
+import org.elasticsearch.gradle.internal.idea.EnablePreviewFeaturesTask
+import org.elasticsearch.gradle.internal.idea.IdeaXmlUtil
 import org.jetbrains.gradle.ext.JUnit
 
 import java.nio.file.Files
@@ -24,12 +26,17 @@ allprojects {
   }
 }
 
+interface Injected {
+  @Inject FileSystemOperations getFs()
+}
+
 // Applying this stuff, particularly the idea-ext plugin, has a cost so avoid it unless we're running in the IDE
 if (providers.systemProperty('idea.active').getOrNull() == 'true') {
   project.apply(plugin: org.jetbrains.gradle.ext.IdeaExtPlugin)
 
   def elasticsearchProject = locateElasticsearchWorkspace(gradle)
 
+  def rootFolder = project.rootDir
   tasks.register('configureIdeCheckstyle') {
     group = 'ide'
     description = 'Generated a suitable checkstyle config for IDEs'
@@ -39,10 +46,10 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     String checkstyleConfig = "${resources}/checkstyle.xml"
     String checkstyleSuppressions = "${resources}/checkstyle_suppressions.xml"
     String checkstyleIdeFragment = "${resources}/checkstyle_ide_fragment.xml"
-    String checkstyleIdeConfig = "${rootDir}/checkstyle_ide.xml"
+    String checkstyleIdeConfig = "${rootFolder}/checkstyle_ide.xml"
 
     String checkstylePluginConfigTemplate = "${resources}/checkstyle-idea.xml"
-    String checkstylePluginConfig = "${rootDir}/.idea/checkstyle-idea.xml"
+    String checkstylePluginConfig = "${rootFolder}/.idea/checkstyle-idea.xml"
 
     inputs.files(
       file(checkstyleConfig),
@@ -53,31 +60,33 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
       file(checkstyleIdeConfig),
       file(checkstylePluginConfig)
     )
+    def injected = project.objects.newInstance(Injected)
 
+    def projectFolder = project.layout.projectDirectory.asFile
     doLast {
       // Configure the IntelliJ Checkstyle plugin by copying a standard file. We don't simply commit
       // the result to version control, because the plugin has a habit of modifying the file and
       // replacing the `$PROJECT_DIR$` placeholders, which developers must then revert.
-      project.copy {
+      injected.fs.copy {
         from(checkstylePluginConfigTemplate)
-        into("${rootDir}/.idea")
+        into("${rootFolder}/.idea")
         expand(jarLocation: buildConventionsJar, configLocation: checkstyleIdeConfig)
       }
 
       // Create an IDE-specific checkstyle config by first copying the standard config
       Files.copy(
-        Paths.get(file(checkstyleConfig).getPath()),
-        Paths.get(file(checkstyleIdeConfig).getPath()),
+        Paths.get(new File(checkstyleConfig).getPath()),
+        Paths.get(new File(checkstyleIdeConfig).getPath()),
         StandardCopyOption.REPLACE_EXISTING
       )
 
       // There are some rules that we only want to enable in an IDE. These
       // are extracted to a separate file, and merged into the IDE-specific
       // Checkstyle config.
-      Node xmlFragment = parseXml(checkstyleIdeFragment)
+      Node xmlFragment = IdeaXmlUtil.parseXml(checkstyleIdeFragment)
 
       // Edit the copy so that IntelliJ can copy with it
-      modifyXml(checkstyleIdeConfig, { xml ->
+      IdeaXmlUtil.modifyXml(checkstyleIdeConfig, { xml ->
         // Add all the nodes from the fragment file
         Node treeWalker = xml.module.find { it.'@name' == 'TreeWalker' }
         xmlFragment.module.each { treeWalker.append(it) }
@@ -103,7 +112,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     description = 'Configures the appropriate JVM for Gradle'
 
     doLast {
-      modifyXml('.idea/gradle.xml') { xml ->
+      IdeaXmlUtil.modifyXml('.idea/gradle.xml') { xml ->
         def gradleSettings = xml.component.find { it.'@name' == 'GradleSettings' }.option[0].GradleProjectSettings
         // Remove configured JVM option to force IntelliJ to use the project JDK for Gradle
         gradleSettings.option.findAll { it.'@name' == 'gradleJvm' }.each { it.parent().remove(it) }
@@ -127,7 +136,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     description = 'Enable per-module *.iml files'
 
     doLast {
-      modifyXml('.idea/misc.xml') {xml ->
+      IdeaXmlUtil.modifyXml('.idea/misc.xml') {xml ->
         def externalStorageConfig = xml.component.find { it.'@name' == 'ExternalStorageConfigurationManager' }
         if (externalStorageConfig) {
           xml.remove(externalStorageConfig)
@@ -137,19 +146,10 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
   }
 
   // modifies the idea module config to enable preview features on ':libs:native' module
-  tasks.register("enablePreviewFeatures") {
+  tasks.register("enablePreviewFeatures", EnablePreviewFeaturesTask) {
     group = 'ide'
     description = 'Enables preview features on native library module'
     dependsOn tasks.named("enableExternalConfiguration")
-
-    ext {
-      enablePreview = { moduleFile, languageLevel ->
-        modifyXml(moduleFile) { xml ->
-          xml.component.find { it.'@name' == 'NewModuleRootManager' }?.'@LANGUAGE_LEVEL' = languageLevel
-        }
-      }
-    }
-
     doLast {
       enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.main.iml', 'JDK_21_PREVIEW')
       enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.test.iml', 'JDK_21_PREVIEW')
@@ -159,7 +159,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
   tasks.register('buildDependencyArtifacts') {
     group = 'ide'
     description = 'Builds artifacts needed as dependency for IDE modules'
-    dependsOn([':plugins:repository-hdfs:hadoop-client-api:shadowJar',
+    dependsOn([':plugins:repository-hdfs:hadoop-client-api:jar',
       ':x-pack:plugin:esql:compute:ann:jar',
       ':x-pack:plugin:esql:compute:gen:jar',
       ':server:generateModulesList',
@@ -270,42 +270,6 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
   }
 }
 
-/**
- * Parses a given XML file, applies a set of changes, and writes those changes back to the original file.
- *
- * @param path Path to existing XML file
- * @param action Action to perform on parsed XML document
- * @param preface optional front matter to add after the XML declaration
- * but before the XML document, e.g. a doctype or comment
- */
-void modifyXml(Object path, Action<? super Node> action, String preface = null) {
-  if (project.file(path).exists()) {
-    Node xml = parseXml(path)
-    action.execute(xml)
-
-    File xmlFile = project.file(path)
-    xmlFile.withPrintWriter { writer ->
-      def printer = new XmlNodePrinter(writer)
-      printer.namespaceAware = true
-      printer.preserveWhitespace = true
-      writer.write("<?xml version=\"1.0\"?>\n")
-
-      if (preface != null) {
-        writer.write(preface)
-      }
-      printer.print(xml)
-    }
-  }
-}
-
-Node parseXml(Object path) {
-  File xmlFile = project.file(path)
-  XmlParser xmlParser = new XmlParser(false, true, true)
-  xmlParser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false)
-  Node xml = xmlParser.parse(xmlFile)
-  return xml
-}
-
 Pair<File, IncludedBuild> locateElasticsearchWorkspace(Gradle gradle) {
   if (gradle.parent == null) {
     // See if any of these included builds is the Elasticsearch gradle
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java
index d54eb798ce783..6dfb337a22ac4 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java
@@ -22,7 +22,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:1b51ff6dba78c98d3e02b0cd64a8ce3238c7a40408d21e3af12a329d44db6f23",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:eef54b3a414aa53b98f0f8df2633aed83c3ba6230722769282925442968f0364",
         "-wolfi",
         "apk"
     ),
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java
index e62c26c7fbc01..a4477d049460c 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java
@@ -27,15 +27,20 @@
 import org.gradle.api.plugins.JavaLibraryPlugin;
 import org.gradle.api.plugins.JavaPlugin;
 import org.gradle.api.provider.Property;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.api.tasks.bundling.Jar;
 import org.gradle.api.tasks.javadoc.Javadoc;
 import org.gradle.external.javadoc.CoreJavadocOptions;
+import org.gradle.jvm.toolchain.JavaLanguageVersion;
+import org.gradle.jvm.toolchain.JavaToolchainService;
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
 
 import java.io.File;
 import java.util.Map;
 
+import javax.inject.Inject;
+
 import static org.elasticsearch.gradle.internal.conventions.util.Util.toStringable;
 import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;
 
@@ -44,6 +49,14 @@
  * common configuration for production code.
  */
 public class ElasticsearchJavaPlugin implements Plugin<Project> {
+
+    private final JavaToolchainService javaToolchains;
+
+    @Inject
+    ElasticsearchJavaPlugin(JavaToolchainService javaToolchains) {
+        this.javaToolchains = javaToolchains;
+    }
+
     @Override
     public void apply(Project project) {
         project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class);
@@ -55,7 +68,7 @@ public void apply(Project project) {
         // configureConfigurations(project);
         configureJars(project, buildParams.get());
         configureJarManifest(project, buildParams.get());
-        configureJavadoc(project);
+        configureJavadoc(project, buildParams.get());
         testCompileOnlyDeps(project);
     }
 
@@ -108,12 +121,12 @@ public void execute(Task task) {
     }
 
     private static void configureJarManifest(Project project, BuildParameterExtension buildParams) {
-        String gitOrigin = buildParams.getGitOrigin();
-        String gitRevision = buildParams.getGitRevision();
+        Provider<String> gitOrigin = buildParams.getGitOrigin();
+        Provider<String> gitRevision = buildParams.getGitRevision();
 
         project.getPlugins().withType(InfoBrokerPlugin.class).whenPluginAdded(manifestPlugin -> {
-            manifestPlugin.add("Module-Origin", toStringable(() -> gitOrigin));
-            manifestPlugin.add("Change", toStringable(() -> gitRevision));
+            manifestPlugin.add("Module-Origin", toStringable(() -> gitOrigin.get()));
+            manifestPlugin.add("Change", toStringable(() -> gitRevision.get()));
             manifestPlugin.add("X-Compile-Elasticsearch-Version", toStringable(VersionProperties::getElasticsearch));
             manifestPlugin.add("X-Compile-Lucene-Version", toStringable(VersionProperties::getLucene));
             manifestPlugin.add(
@@ -128,7 +141,7 @@ private static void configureJarManifest(Project project, BuildParameterExtensio
         project.getPluginManager().apply("nebula.info-jar");
     }
 
-    private static void configureJavadoc(Project project) {
+    private void configureJavadoc(Project project, BuildParameterExtension buildParams) {
         project.getTasks().withType(Javadoc.class).configureEach(javadoc -> {
             /*
              * Generate docs using html5 to suppress a warning from `javadoc`
@@ -136,6 +149,10 @@ private static void configureJavadoc(Project project) {
              */
             CoreJavadocOptions javadocOptions = (CoreJavadocOptions) javadoc.getOptions();
             javadocOptions.addBooleanOption("html5", true);
+
+            javadoc.getJavadocTool().set(javaToolchains.javadocToolFor(spec -> {
+                spec.getLanguageVersion().set(JavaLanguageVersion.of(buildParams.getMinimumRuntimeVersion().getMajorVersion()));
+            }));
         });
 
         TaskProvider<Javadoc> javadoc = project.getTasks().withType(Javadoc.class).named("javadoc");
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java
index f7e2f3d0d6c30..c2547b72e21fa 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java
@@ -134,14 +134,14 @@ private void registerEmptyDirectoryTasks(Project project) {
         });
 
         File pluginsDir = new File(project.getBuildDir(), "plugins-hack/plugins");
-        project.getExtensions().add("pluginsDir", pluginsDir);
+        project.getExtensions().getExtraProperties().set("pluginsDir", pluginsDir);
         project.getTasks().register("createPluginsDir", EmptyDirTask.class, t -> {
             t.setDir(pluginsDir);
             t.setDirMode(0755);
         });
 
         File jvmOptionsDir = new File(project.getBuildDir(), "jvm-options-hack/jvm.options.d");
-        project.getExtensions().add("jvmOptionsDir", jvmOptionsDir);
+        project.getExtensions().getExtraProperties().set("jvmOptionsDir", jvmOptionsDir);
         project.getTasks().register("createJvmOptionsDir", EmptyDirTask.class, t -> {
             t.setDir(jvmOptionsDir);
             t.setDirMode(0750);
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java
index da26cb66122ad..c38ea5b4f0850 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java
@@ -17,12 +17,12 @@
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
+import org.gradle.api.file.FileSystemOperations;
 import org.gradle.api.file.ProjectLayout;
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.plugins.JvmToolchainsPlugin;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
-import org.gradle.api.tasks.Copy;
 import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.jvm.toolchain.JavaToolchainService;
@@ -54,11 +54,17 @@ public class InternalDistributionBwcSetupPlugin implements Plugin<Project> {
     private final ObjectFactory objectFactory;
     private ProviderFactory providerFactory;
     private JavaToolchainService toolChainService;
+    private FileSystemOperations fileSystemOperations;
 
     @Inject
-    public InternalDistributionBwcSetupPlugin(ObjectFactory objectFactory, ProviderFactory providerFactory) {
+    public InternalDistributionBwcSetupPlugin(
+        ObjectFactory objectFactory,
+        ProviderFactory providerFactory,
+        FileSystemOperations fileSystemOperations
+    ) {
         this.objectFactory = objectFactory;
         this.providerFactory = providerFactory;
+        this.fileSystemOperations = fileSystemOperations;
     }
 
     @Override
@@ -76,7 +82,8 @@ public void apply(Project project) {
                 providerFactory,
                 objectFactory,
                 toolChainService,
-                isCi
+                isCi,
+                fileSystemOperations
             );
         });
     }
@@ -88,7 +95,8 @@ private static void configureBwcProject(
         ProviderFactory providerFactory,
         ObjectFactory objectFactory,
         JavaToolchainService toolChainService,
-        Boolean isCi
+        Boolean isCi,
+        FileSystemOperations fileSystemOperations
     ) {
         ProjectLayout layout = project.getLayout();
         Provider<BwcVersions.UnreleasedVersionInfo> versionInfoProvider = providerFactory.provider(() -> versionInfo);
@@ -120,11 +128,20 @@ private static void configureBwcProject(
         List<DistributionProject> distributionProjects = resolveArchiveProjects(checkoutDir.get(), bwcVersion.get());
 
         // Setup gradle user home directory
-        project.getTasks().register("setupGradleUserHome", Copy.class, copy -> {
-            copy.into(project.getGradle().getGradleUserHomeDir().getAbsolutePath() + "-" + project.getName());
-            copy.from(project.getGradle().getGradleUserHomeDir().getAbsolutePath(), copySpec -> {
-                copySpec.include("gradle.properties");
-                copySpec.include("init.d/*");
+        // We don't use a normal `Copy` task here as snapshotting the entire gradle user home is very expensive. This task is cheap, so
+        // up-to-date checking doesn't buy us much
+        project.getTasks().register("setupGradleUserHome", task -> {
+            File gradleUserHome = project.getGradle().getGradleUserHomeDir();
+            String projectName = project.getName();
+            task.doLast(t -> {
+                fileSystemOperations.copy(copy -> {
+                    String absoluteGradleUserHomePath = gradleUserHome.getAbsolutePath();
+                    copy.into(absoluteGradleUserHomePath + "-" + projectName);
+                    copy.from(absoluteGradleUserHomePath, copySpec -> {
+                        copySpec.include("gradle.properties");
+                        copySpec.include("init.d/*");
+                    });
+                });
             });
         });
 
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java
index 7c488e6e73fee..5402e0a04fe8f 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java
@@ -86,14 +86,14 @@ public void apply(Project project) {
             configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.TEST_SOURCE_SET_NAME), 21);
             for (int javaVersion : mainVersions) {
                 String mainSourceSetName = SourceSet.MAIN_SOURCE_SET_NAME + javaVersion;
-                SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion);
+                SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion, true);
                 configureSourceSetInJar(project, mainSourceSet, javaVersion);
                 addJar(project, mainSourceSet, javaVersion);
                 mainSourceSets.add(mainSourceSetName);
                 testSourceSets.add(mainSourceSetName);
 
                 String testSourceSetName = SourceSet.TEST_SOURCE_SET_NAME + javaVersion;
-                SourceSet testSourceSet = addSourceSet(project, javaExtension, testSourceSetName, testSourceSets, javaVersion);
+                SourceSet testSourceSet = addSourceSet(project, javaExtension, testSourceSetName, testSourceSets, javaVersion, false);
                 testSourceSets.add(testSourceSetName);
                 createTestTask(project, buildParams, testSourceSet, javaVersion, mainSourceSets);
             }
@@ -121,7 +121,8 @@ private SourceSet addSourceSet(
         JavaPluginExtension javaExtension,
         String sourceSetName,
         List<String> parentSourceSets,
-        int javaVersion
+        int javaVersion,
+        boolean isMainSourceSet
     ) {
         SourceSet sourceSet = javaExtension.getSourceSets().maybeCreate(sourceSetName);
         for (String parentSourceSetName : parentSourceSets) {
@@ -135,6 +136,13 @@ private SourceSet addSourceSet(
             CompileOptions compileOptions = compileTask.getOptions();
             compileOptions.getRelease().set(javaVersion);
         });
+        if (isMainSourceSet) {
+            project.getTasks().create(sourceSet.getJavadocTaskName(), Javadoc.class, javadocTask -> {
+                javadocTask.getJavadocTool().set(javaToolchains.javadocToolFor(spec -> {
+                    spec.getLanguageVersion().set(JavaLanguageVersion.of(javaVersion));
+                }));
+            });
+        }
         configurePreviewFeatures(project, sourceSet, javaVersion);
 
         // Since we configure MRJAR sourcesets to allow preview apis, class signatures for those
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java
index 28776f03d17e8..c44b1212e6489 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java
@@ -14,14 +14,14 @@
 import java.util.List;
 
 public class InternalElasticsearchDistributionTypes {
-    public static ElasticsearchDistributionType DEB = new DebElasticsearchDistributionType();
-    public static ElasticsearchDistributionType RPM = new RpmElasticsearchDistributionType();
-    public static ElasticsearchDistributionType DOCKER = new DockerElasticsearchDistributionType();
-    public static ElasticsearchDistributionType DOCKER_IRONBANK = new DockerIronBankElasticsearchDistributionType();
-    public static ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType();
-    public static ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType DEB = new DebElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType RPM = new RpmElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType DOCKER = new DockerElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType DOCKER_IRONBANK = new DockerIronBankElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType();
 
-    public static List<ElasticsearchDistributionType> ALL_INTERNAL = List.of(
+    public static final List<ElasticsearchDistributionType> ALL_INTERNAL = List.of(
         DEB,
         RPM,
         DOCKER,
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/idea/EnablePreviewFeaturesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/idea/EnablePreviewFeaturesTask.java
new file mode 100644
index 0000000000000..f8c8b5127827f
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/idea/EnablePreviewFeaturesTask.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.idea;
+
+import groovy.util.Node;
+import groovy.util.NodeList;
+
+import org.gradle.api.DefaultTask;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+
+import javax.xml.parsers.ParserConfigurationException;
+
+public class EnablePreviewFeaturesTask extends DefaultTask {
+
+    public void enablePreview(String moduleFile, String languageLevel) throws IOException, ParserConfigurationException, SAXException {
+        IdeaXmlUtil.modifyXml(moduleFile, xml -> {
+            // Find the 'component' node
+            NodeList nodes = (NodeList) xml.depthFirst();
+            Node componentNode = null;
+            for (Object node : nodes) {
+                Node currentNode = (Node) node;
+                if ("component".equals(currentNode.name()) && "NewModuleRootManager".equals(currentNode.attribute("name"))) {
+                    componentNode = currentNode;
+                    break;
+                }
+            }
+
+            // Add the attribute to the 'component' node
+            if (componentNode != null) {
+                componentNode.attributes().put("LANGUAGE_LEVEL", languageLevel);
+            }
+        });
+    }
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/idea/IdeaXmlUtil.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/idea/IdeaXmlUtil.java
new file mode 100644
index 0000000000000..b7cc2862a0af1
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/idea/IdeaXmlUtil.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.idea;
+
+import groovy.util.Node;
+import groovy.util.XmlParser;
+import groovy.xml.XmlNodePrinter;
+
+import org.gradle.api.Action;
+import org.xml.sax.SAXException;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import javax.xml.parsers.ParserConfigurationException;
+
+public class IdeaXmlUtil {
+
+    static Node parseXml(String xmlPath) throws IOException, SAXException, ParserConfigurationException {
+        File xmlFile = new File(xmlPath);
+        XmlParser xmlParser = new XmlParser(false, true, true);
+        xmlParser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+        Node xml = xmlParser.parse(xmlFile);
+        return xml;
+    }
+
+    /**
+     * Parses a given XML file, applies a set of changes, and writes those changes back to the original file.
+     *
+     * @param path Path to existing XML file
+     * @param action Action to perform on parsed XML document
+     * but before the XML document, e.g. a doctype or comment
+     */
+    static void modifyXml(String xmlPath, Action<? super Node> action) throws IOException, ParserConfigurationException, SAXException {
+        modifyXml(xmlPath, action, null);
+    }
+
+    /**
+     * Parses a given XML file, applies a set of changes, and writes those changes back to the original file.
+     *
+     * @param path Path to existing XML file
+     * @param action Action to perform on parsed XML document
+     * @param preface optional front matter to add after the XML declaration
+     * but before the XML document, e.g. a doctype or comment
+     */
+    static void modifyXml(String xmlPath, Action<? super Node> action, String preface) throws IOException, ParserConfigurationException,
+        SAXException {
+        File xmlFile = new File(xmlPath);
+        if (xmlFile.exists()) {
+            Node xml = parseXml(xmlPath);
+            action.execute(xml);
+
+            try (PrintWriter writer = new PrintWriter(xmlFile)) {
+                var printer = new XmlNodePrinter(writer);
+                printer.setNamespaceAware(true);
+                printer.setPreserveWhitespace(true);
+                writer.write("<?xml version=\"1.0\"?>\n");
+                if (preface != null) {
+                    writer.write(preface);
+                }
+                printer.print(xml);
+            }
+        }
+    }
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java
index e80dc6ef1b44c..ef9055b3728d3 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java
@@ -46,9 +46,9 @@ public interface BuildParameterExtension {
 
     Provider<String> getRuntimeJavaDetails();
 
-    String getGitRevision();
+    Provider<String> getGitRevision();
 
-    String getGitOrigin();
+    Provider<String> getGitOrigin();
 
     ZonedDateTime getBuildDate();
 
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/DefaultBuildParameterExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/DefaultBuildParameterExtension.java
index faac406d974c6..283c02428e4e6 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/DefaultBuildParameterExtension.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/DefaultBuildParameterExtension.java
@@ -36,7 +36,7 @@ public abstract class DefaultBuildParameterExtension implements BuildParameterEx
     private final Provider<JavaVersion> runtimeJavaVersion;
     private final Provider<? extends Action<JavaToolchainSpec>> javaToolChainSpec;
     private final Provider<String> runtimeJavaDetails;
-    private final String gitRevision;
+    private final Provider<String> gitRevision;
 
     private transient AtomicReference<ZonedDateTime> buildDate = new AtomicReference<>();
     private final String testSeed;
@@ -46,7 +46,7 @@ public abstract class DefaultBuildParameterExtension implements BuildParameterEx
 
     // not final for testing
     private Provider<BwcVersions> bwcVersions;
-    private String gitOrigin;
+    private Provider<String> gitOrigin;
 
     public DefaultBuildParameterExtension(
         ProviderFactory providers,
@@ -59,8 +59,8 @@ public DefaultBuildParameterExtension(
         JavaVersion minimumCompilerVersion,
         JavaVersion minimumRuntimeVersion,
         JavaVersion gradleJavaVersion,
-        String gitRevision,
-        String gitOrigin,
+        Provider<String> gitRevision,
+        Provider<String> gitOrigin,
         String testSeed,
         boolean isCi,
         int defaultParallel,
@@ -155,12 +155,12 @@ public Provider<String> getRuntimeJavaDetails() {
     }
 
     @Override
-    public String getGitRevision() {
+    public Provider<String> getGitRevision() {
         return gitRevision;
     }
 
     @Override
-    public String getGitOrigin() {
+    public Provider<String> getGitOrigin() {
         return gitOrigin;
     }
 
@@ -239,7 +239,7 @@ public void setBwcVersions(Provider<BwcVersions> bwcVersions) {
     }
 
     // for testing; not part of public api
-    public void setGitOrigin(String gitOrigin) {
+    public void setGitOrigin(Provider<String> gitOrigin) {
         this.gitOrigin = gitOrigin;
     }
 }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java
index 86f59aa0ab41e..675f1198b2a7d 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java
@@ -14,6 +14,7 @@
 import org.apache.commons.io.IOUtils;
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.internal.BwcVersions;
+import org.elasticsearch.gradle.internal.conventions.GitInfoPlugin;
 import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
 import org.elasticsearch.gradle.internal.conventions.info.ParallelDetector;
 import org.elasticsearch.gradle.internal.conventions.util.Util;
@@ -96,6 +97,8 @@ public void apply(Project project) {
         }
         this.project = project;
         project.getPlugins().apply(JvmToolchainsPlugin.class);
+        Provider<GitInfo> gitInfo = project.getPlugins().apply(GitInfoPlugin.class).getGitInfo();
+
         toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
         GradleVersion minimumGradleVersion = GradleVersion.version(getResourceContents("/minimumGradleVersion"));
         if (GradleVersion.current().compareTo(minimumGradleVersion) < 0) {
@@ -111,8 +114,6 @@ public void apply(Project project) {
             ? explicitRuntimeJavaHome
             : resolveJavaHomeFromToolChainService(VersionProperties.getBundledJdkMajorVersion());
 
-        GitInfo gitInfo = GitInfo.gitInfo(project.getRootDir());
-
         Provider<JvmInstallationMetadata> runtimeJdkMetaData = actualRuntimeJavaHome.map(
             runtimeJavaHome -> metadataDetector.getMetadata(getJavaInstallation(runtimeJavaHome))
         );
@@ -143,8 +144,8 @@ public void apply(Project project) {
                 minimumCompilerVersion,
                 minimumRuntimeVersion,
                 Jvm.current().getJavaVersion(),
-                gitInfo.getRevision(),
-                gitInfo.getOrigin(),
+                gitInfo.map(g -> g.getRevision()),
+                gitInfo.map(g -> g.getOrigin()),
                 getTestSeed(),
                 System.getenv("JENKINS_URL") != null || System.getenv("BUILDKITE_BUILD_URL") != null || System.getProperty("isCI") != null,
                 ParallelDetector.findDefaultParallel(project),
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java
index dbbe35905d208..dc8ea4424ba89 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java
@@ -42,7 +42,8 @@ public TaskProvider<? extends Task> createTask(Project project) {
         File checkstyleDir = new File(project.getBuildDir(), "checkstyle");
         File checkstyleSuppressions = new File(checkstyleDir, "checkstyle_suppressions.xml");
         File checkstyleConf = new File(checkstyleDir, "checkstyle.xml");
-        TaskProvider<Task> copyCheckstyleConf = project.getTasks().register("copyCheckstyleConf");
+        TaskProvider<CopyCheckStyleConfTask> copyCheckstyleConf = project.getTasks()
+            .register("copyCheckstyleConf", CopyCheckStyleConfTask.class);
         // configure inputs and outputs so up to date works properly
         copyCheckstyleConf.configure(t -> t.getOutputs().files(checkstyleSuppressions, checkstyleConf));
         if ("jar".equals(checkstyleConfUrl.getProtocol())) {
diff --git a/server/src/main/java/org/elasticsearch/inference/EmptySettingsConfiguration.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CopyCheckStyleConfTask.java
similarity index 61%
rename from server/src/main/java/org/elasticsearch/inference/EmptySettingsConfiguration.java
rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CopyCheckStyleConfTask.java
index 8a3f96750f2ea..9e0f9c24bcef8 100644
--- a/server/src/main/java/org/elasticsearch/inference/EmptySettingsConfiguration.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CopyCheckStyleConfTask.java
@@ -7,13 +7,15 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.inference;
+package org.elasticsearch.gradle.internal.precommit;
 
-import java.util.Collections;
-import java.util.Map;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.file.FileSystemOperations;
 
-public class EmptySettingsConfiguration {
-    public static Map<String, SettingsConfiguration> get() {
-        return Collections.emptyMap();
-    }
+import javax.inject.Inject;
+
+public abstract class CopyCheckStyleConfTask extends DefaultTask {
+
+    @Inject
+    public abstract FileSystemOperations getFs();
 }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java
index e45a1d3dd25b1..7046a22204efa 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java
@@ -16,12 +16,14 @@
 import org.gradle.api.Task;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.component.ModuleComponentIdentifier;
+import org.gradle.api.file.FileCollection;
 import org.gradle.api.tasks.TaskProvider;
 
 import java.io.File;
 import java.nio.file.Path;
 
 import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createFileCollectionFromNonTransitiveArtifactsView;
+import static org.elasticsearch.gradle.internal.util.DependenciesUtils.thirdPartyDependenciesView;
 import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;
 
 public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
@@ -47,7 +49,6 @@ public TaskProvider<? extends Task> createTask(Project project) {
                 project.getDependencies().add(JDK_JAR_HELL_CONFIG_NAME, elasticsearchCoreProject);
             }
         }
-
         TaskProvider<ExportElasticsearchBuildResourcesTask> resourcesTask = project.getTasks()
             .register("thirdPartyAuditResources", ExportElasticsearchBuildResourcesTask.class);
         Path resourcesDir = project.getBuildDir().toPath().resolve("third-party-audit-config");
@@ -59,9 +60,11 @@ public TaskProvider<? extends Task> createTask(Project project) {
         // usually only one task is created. but this construct makes our integTests easier to setup
         project.getTasks().withType(ThirdPartyAuditTask.class).configureEach(t -> {
             Configuration runtimeConfiguration = project.getConfigurations().getByName("runtimeClasspath");
+            FileCollection runtimeThirdParty = thirdPartyDependenciesView(runtimeConfiguration);
             Configuration compileOnly = project.getConfigurations()
                 .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
-            t.setClasspath(runtimeConfiguration.plus(compileOnly));
+            FileCollection compileOnlyThirdParty = thirdPartyDependenciesView(compileOnly);
+            t.getThirdPartyClasspath().from(runtimeThirdParty, compileOnlyThirdParty);
             t.getJarsToScan()
                 .from(
                     createFileCollectionFromNonTransitiveArtifactsView(
@@ -78,7 +81,7 @@ public TaskProvider<? extends Task> createTask(Project project) {
             t.getJavaHome().set(buildParams.flatMap(params -> params.getRuntimeJavaHome()).map(File::getPath));
             t.setSignatureFile(resourcesDir.resolve("forbidden/third-party-audit.txt").toFile());
             t.getJdkJarHellClasspath().from(jdkJarHellConfig);
-            t.getForbiddenAPIsClasspath().from(project.getConfigurations().getByName("forbiddenApisCliJar").plus(compileOnly));
+            t.getForbiddenAPIsClasspath().from(project.getConfigurations().getByName("forbiddenApisCliJar").plus(compileOnlyThirdParty));
         });
         return audit;
     }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java
index 442797775de2f..59ba9bae0a57d 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java
@@ -17,7 +17,6 @@
 import org.gradle.api.JavaVersion;
 import org.gradle.api.file.ArchiveOperations;
 import org.gradle.api.file.ConfigurableFileCollection;
-import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileSystemOperations;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.file.ProjectLayout;
@@ -96,8 +95,6 @@ public abstract class ThirdPartyAuditTask extends DefaultTask {
 
     private final ProjectLayout projectLayout;
 
-    private FileCollection classpath;
-
     @Inject
     public ThirdPartyAuditTask(
         ArchiveOperations archiveOperations,
@@ -198,9 +195,7 @@ public Set<String> getMissingClassExcludes() {
     public abstract Property<JavaVersion> getRuntimeJavaVersion();
 
     @Classpath
-    public FileCollection getClasspath() {
-        return classpath;
-    }
+    public abstract ConfigurableFileCollection getThirdPartyClasspath();
 
     @TaskAction
     public void runThirdPartyAudit() throws IOException {
@@ -345,7 +340,7 @@ private String runForbiddenAPIsCli() throws IOException {
             if (javaHome.isPresent()) {
                 spec.setExecutable(javaHome.get() + "/bin/java");
             }
-            spec.classpath(getForbiddenAPIsClasspath(), classpath);
+            spec.classpath(getForbiddenAPIsClasspath(), getThirdPartyClasspath());
             // Enable explicitly for each release as appropriate. Just JDK 20/21/22/23 for now, and just the vector module.
             if (isJavaVersion(VERSION_20) || isJavaVersion(VERSION_21) || isJavaVersion(VERSION_22) || isJavaVersion(VERSION_23)) {
                 spec.jvmArgs("--add-modules", "jdk.incubator.vector");
@@ -383,7 +378,7 @@ private boolean isJavaVersion(JavaVersion version) {
     private Set<String> runJdkJarHellCheck() throws IOException {
         ByteArrayOutputStream standardOut = new ByteArrayOutputStream();
         ExecResult execResult = execOperations.javaexec(spec -> {
-            spec.classpath(getJdkJarHellClasspath(), classpath);
+            spec.classpath(getJdkJarHellClasspath(), getThirdPartyClasspath());
             spec.getMainClass().set(JDK_JAR_HELL_MAIN_CLASS);
             spec.args(getJarExpandDir());
             spec.setIgnoreExitValue(true);
@@ -402,8 +397,4 @@ private Set<String> runJdkJarHellCheck() throws IOException {
         return new TreeSet<>(Arrays.asList(jdkJarHellCheckList.split("\\r?\\n")));
     }
 
-    public void setClasspath(FileCollection classpath) {
-        this.classpath = classpath;
-    }
-
 }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java
index 211718c151ba9..08e3c92307d72 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java
@@ -31,6 +31,7 @@
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.dsl.DependencyHandler;
 import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
+import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.specs.Specs;
@@ -88,8 +89,8 @@ public void apply(Project project) {
         Map<String, TaskProvider<?>> versionTasks = versionTasks(project, "destructiveDistroUpgradeTest", buildParams.getBwcVersions());
         TaskProvider<Task> destructiveDistroTest = project.getTasks().register("destructiveDistroTest");
 
-        Configuration examplePlugin = configureExamplePlugin(project);
-
+        Configuration examplePluginConfiguration = configureExamplePlugin(project);
+        FileCollection examplePluginFileCollection = examplePluginConfiguration;
         List<TaskProvider<Test>> windowsTestTasks = new ArrayList<>();
         Map<ElasticsearchDistributionType, List<TaskProvider<Test>>> linuxTestTasks = new HashMap<>();
 
@@ -102,9 +103,9 @@ public void apply(Project project) {
                     t2 -> distribution.isDocker() == false || dockerSupport.get().getDockerAvailability().isAvailable()
                 );
                 addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath);
-                addDistributionSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePlugin.getSingleFile().toString());
+                addDistributionSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePluginFileCollection.getSingleFile().toString());
                 t.exclude("**/PackageUpgradeTests.class");
-            }, distribution, examplePlugin.getDependencies());
+            }, distribution, examplePluginConfiguration.getDependencies());
 
             if (distribution.getPlatform() == Platform.WINDOWS) {
                 windowsTestTasks.add(destructiveTask);
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ErrorReportingTestListener.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ErrorReportingTestListener.java
index f0b28f5381b98..80e64ad813230 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ErrorReportingTestListener.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ErrorReportingTestListener.java
@@ -172,6 +172,11 @@ public Destination getDestination() {
                             return Destination.StdErr;
                         }
 
+                        @Override
+                        public long getLogTime() {
+                            return result.getEndTime();
+                        }
+
                         @Override
                         public String getMessage() {
                             return message;
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunTaskExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunTaskExtension.java
index f08d3dadc1bb0..8e4519b55e15b 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunTaskExtension.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunTaskExtension.java
@@ -30,7 +30,7 @@ public class TestRerunTaskExtension {
     /**
      * The name of the extension added to each test task.
      */
-    public static String NAME = "rerun";
+    public static final String NAME = "rerun";
 
     private final Property<Integer> maxReruns;
 
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java
index 8fbe40aa82399..bb26bfd16721d 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java
@@ -54,16 +54,16 @@ public String url(String os, String arch, String extension) {
         }
     }
 
-    record EarlyAccessJdkBuild(JavaLanguageVersion languageVersion, String version, String buildNumber) implements JdkBuild {
-
+    record EarlyAccessJdkBuild(JavaLanguageVersion languageVersion) implements JdkBuild {
         @Override
         public String url(String os, String arch, String extension) {
+            String buildNumber = resolveBuildNumber(languageVersion.asInt());
             return "https://download.java.net/java/early_access/jdk"
-                + version
+                + languageVersion.asInt()
                 + "/"
-                + version
+                + buildNumber
                 + "/GPL/openjdk-"
-                + version
+                + languageVersion.asInt()
                 + "-ea+"
                 + buildNumber
                 + "_"
@@ -73,6 +73,29 @@ public String url(String os, String arch, String extension) {
                 + "_bin."
                 + extension;
         }
+
+        private static String resolveBuildNumber(int version) {
+            String buildNumber = System.getProperty("runtime.java." + version + ".build");
+            if (buildNumber != null) {
+                System.out.println("buildNumber = " + buildNumber);
+                return buildNumber;
+            }
+            buildNumber = System.getProperty("runtime.java.build");
+            if (buildNumber != null) {
+                System.out.println("buildNumber2 = " + buildNumber);
+                return buildNumber;
+            }
+
+            switch (version) {
+                case 24:
+                    // latest explicitly found build number for 24
+                    return "29";
+                case 25:
+                    return "3";
+                default:
+                    throw new IllegalArgumentException("Unsupported version " + version);
+            }
+        }
     }
 
     private static final Pattern VERSION_PATTERN = Pattern.compile(
@@ -88,8 +111,8 @@ public String url(String os, String arch, String extension) {
     // package private so it can be replaced by tests
     List<JdkBuild> builds = List.of(
         getBundledJdkBuild(),
-        // 23 early access
-        new EarlyAccessJdkBuild(JavaLanguageVersion.of(23), "23", "24")
+        new EarlyAccessJdkBuild(JavaLanguageVersion.of(24)),
+        new EarlyAccessJdkBuild(JavaLanguageVersion.of(25))
     );
 
     private JdkBuild getBundledJdkBuild() {
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java
index 9080f62f19937..5d7386e2c2150 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java
@@ -9,12 +9,16 @@
 
 package org.elasticsearch.gradle.internal.util;
 
+import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin;
+
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.ResolvableDependencies;
 import org.gradle.api.artifacts.component.ComponentIdentifier;
+import org.gradle.api.artifacts.component.ProjectComponentIdentifier;
 import org.gradle.api.artifacts.result.ResolvedComponentResult;
 import org.gradle.api.artifacts.result.ResolvedDependencyResult;
 import org.gradle.api.file.FileCollection;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.specs.AndSpec;
 import org.gradle.api.specs.Spec;
 
@@ -29,7 +33,7 @@ public static FileCollection createFileCollectionFromNonTransitiveArtifactsView(
     ) {
         ResolvableDependencies incoming = configuration.getIncoming();
         return incoming.artifactView(viewConfiguration -> {
-            Set<ComponentIdentifier> firstLevelDependencyComponents = incoming.getResolutionResult()
+            Provider<Set<ComponentIdentifier>> firstLevelDependencyComponents = incoming.getResolutionResult()
                 .getRootComponent()
                 .map(
                     rootComponent -> rootComponent.getDependencies()
@@ -39,12 +43,36 @@ public static FileCollection createFileCollectionFromNonTransitiveArtifactsView(
                         .filter(dependency -> dependency.getSelected() instanceof ResolvedComponentResult)
                         .map(dependency -> dependency.getSelected().getId())
                         .collect(Collectors.toSet())
-                )
-                .get();
+                );
             viewConfiguration.componentFilter(
-                new AndSpec<>(identifier -> firstLevelDependencyComponents.contains(identifier), componentFilter)
+                new AndSpec<>(identifier -> firstLevelDependencyComponents.get().contains(identifier), componentFilter)
             );
         }).getFiles();
     }
 
+    /**
+     * This method gives us an artifact view of a configuration that filters out all
+     * project dependencies that are not shadowed jars.
+     * Basically a thirdparty only view of the dependency tree.
+     */
+    public static FileCollection thirdPartyDependenciesView(Configuration configuration) {
+        ResolvableDependencies incoming = configuration.getIncoming();
+        return incoming.artifactView(v -> {
+            // resolve componentIdentifier for all shadowed project dependencies
+            Provider<Set<ComponentIdentifier>> shadowedDependencies = incoming.getResolutionResult()
+                .getRootComponent()
+                .map(
+                    root -> root.getDependencies()
+                        .stream()
+                        .filter(dep -> dep instanceof ResolvedDependencyResult)
+                        .map(dep -> (ResolvedDependencyResult) dep)
+                        .filter(dep -> dep.getResolvedVariant().getDisplayName() == ShadowBasePlugin.COMPONENT_NAME)
+                        .filter(dep -> dep.getSelected() instanceof ResolvedComponentResult)
+                        .map(dep -> dep.getSelected().getId())
+                        .collect(Collectors.toSet())
+                );
+            // filter out project dependencies if they are not a shadowed dependency
+            v.componentFilter(i -> (i instanceof ProjectComponentIdentifier == false || shadowedDependencies.get().contains(i)));
+        }).getFiles();
+    }
 }
diff --git a/build-tools-internal/src/main/resources/changelog-schema.json b/build-tools-internal/src/main/resources/changelog-schema.json
index 451701d74d690..9692af7adc5e6 100644
--- a/build-tools-internal/src/main/resources/changelog-schema.json
+++ b/build-tools-internal/src/main/resources/changelog-schema.json
@@ -279,6 +279,7 @@
     "compatibilityChangeArea": {
       "type": "string",
       "enum": [
+        "Aggregations",
         "Analysis",
         "Authorization",
         "Cluster and node setting",
@@ -295,6 +296,7 @@
         "Painless",
         "REST API",
         "Rollup",
+        "Search",
         "System requirement",
         "Transform"
       ]
diff --git a/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt b/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt
index a9da7995c2b36..53480a4a27b0b 100644
--- a/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt
+++ b/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt
@@ -155,10 +155,8 @@ org.elasticsearch.cluster.ClusterState#compatibilityVersions()
 
 @defaultMessage ClusterFeatures#nodeFeatures is for internal use only. Use FeatureService#clusterHasFeature to determine if a feature is present on the cluster.
 org.elasticsearch.cluster.ClusterFeatures#nodeFeatures()
-@defaultMessage ClusterFeatures#allNodeFeatures is for internal use only. Use FeatureService#clusterHasFeature to determine if a feature is present on the cluster.
-org.elasticsearch.cluster.ClusterFeatures#allNodeFeatures()
 @defaultMessage ClusterFeatures#clusterHasFeature is for internal use only. Use FeatureService#clusterHasFeature to determine if a feature is present on the cluster.
-org.elasticsearch.cluster.ClusterFeatures#clusterHasFeature(org.elasticsearch.features.NodeFeature)
+org.elasticsearch.cluster.ClusterFeatures#clusterHasFeature(org.elasticsearch.cluster.node.DiscoveryNodes, org.elasticsearch.features.NodeFeature)
 
 @defaultMessage Do not construct this records outside the source files they are declared in
 org.elasticsearch.cluster.SnapshotsInProgress$ShardSnapshotStatus#<init>(java.lang.String, org.elasticsearch.cluster.SnapshotsInProgress$ShardState, org.elasticsearch.repositories.ShardGeneration, java.lang.String, org.elasticsearch.repositories.ShardSnapshotResult)
diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion
index 876e3136ea819..9c57ca327c7b7 100644
--- a/build-tools-internal/src/main/resources/minimumGradleVersion
+++ b/build-tools-internal/src/main/resources/minimumGradleVersion
@@ -1 +1 @@
-8.11.1
\ No newline at end of file
+8.12
\ No newline at end of file
diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/info/BuildParameterExtensionSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/info/BuildParameterExtensionSpec.groovy
index 343268b9b4d47..ce63069a873ab 100644
--- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/info/BuildParameterExtensionSpec.groovy
+++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/info/BuildParameterExtensionSpec.groovy
@@ -9,9 +9,9 @@
 
 package org.elasticsearch.gradle.internal.info
 
+import spock.lang.Ignore
 import spock.lang.Specification
 
-import org.elasticsearch.gradle.internal.BwcVersions
 import org.gradle.api.JavaVersion
 import org.gradle.api.Project
 import org.gradle.api.provider.Provider
@@ -31,6 +31,7 @@ class BuildParameterExtensionSpec extends Specification {
 
     ProjectBuilder projectBuilder = new ProjectBuilder()
 
+    @Ignore
     def "#getterName is cached anc concurrently accessible"() {
         given:
         def project = projectBuilder.build()
@@ -85,8 +86,8 @@ class BuildParameterExtensionSpec extends Specification {
             JavaVersion.VERSION_11,
             JavaVersion.VERSION_11,
             JavaVersion.VERSION_11,
-            "gitRevision",
-            "gitOrigin",
+            providerMock(),
+            providerMock(),
             "testSeed",
             false,
             5,
diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AbstractToolchainResolverSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AbstractToolchainResolverSpec.groovy
index ad0db8b1b7de7..cea96437129a6 100644
--- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AbstractToolchainResolverSpec.groovy
+++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AbstractToolchainResolverSpec.groovy
@@ -9,6 +9,8 @@
 
 package org.elasticsearch.gradle.internal.toolchain
 
+import spock.lang.Unroll
+
 import org.gradle.api.provider.Property
 import org.gradle.jvm.toolchain.JavaLanguageVersion
 import org.gradle.jvm.toolchain.JavaToolchainDownload
@@ -26,6 +28,7 @@ import static org.gradle.platform.OperatingSystem.MAC_OS
 
 abstract class AbstractToolchainResolverSpec extends Specification {
 
+    @Unroll
     def "resolves #os #arch #vendor jdk #langVersion"() {
         given:
         def resolver = resolverImplementation()
diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy
index 9c55bbc4674e9..4993bf00f2af5 100644
--- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy
+++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy
@@ -9,15 +9,20 @@
 
 package org.elasticsearch.gradle.internal.toolchain
 
+import spock.util.environment.RestoreSystemProperties
 
 import org.gradle.api.services.BuildServiceParameters
 import org.gradle.jvm.toolchain.JavaLanguageVersion
+import org.gradle.jvm.toolchain.JavaToolchainDownload
+
 import static org.gradle.jvm.toolchain.JvmVendorSpec.ORACLE
-import static org.gradle.platform.Architecture.*
+import static org.gradle.platform.Architecture.AARCH64
+import static org.gradle.platform.Architecture.X86_64
 import static org.gradle.platform.OperatingSystem.*
 
 class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
 
+
     OracleOpenJdkToolchainResolver resolverImplementation() {
         var toolChain = new OracleOpenJdkToolchainResolver() {
             @Override
@@ -25,10 +30,13 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
                 return null
             }
         }
-        toolChain.builds = [
-            new OracleOpenJdkToolchainResolver.ReleasedJdkBuild(JavaLanguageVersion.of(20), "20", "36", "bdc68b4b9cbc4ebcb30745c85038d91d"),
-            new OracleOpenJdkToolchainResolver.EarlyAccessJdkBuild(JavaLanguageVersion.of(21), "21", "6")
-        ]
+        toolChain.builds = toolChain.builds.findAll { it instanceof OracleOpenJdkToolchainResolver.EarlyAccessJdkBuild } + [
+            new OracleOpenJdkToolchainResolver.ReleasedJdkBuild(
+                JavaLanguageVersion.of(20),
+                "20",
+                "36",
+                "bdc68b4b9cbc4ebcb30745c85038d91d"
+            )]
         toolChain
     }
 
@@ -44,23 +52,67 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
          [20, anyVendor(), LINUX, AARCH64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_linux-aarch64_bin.tar.gz"],
          [20, anyVendor(), WINDOWS, X86_64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_windows-x64_bin.zip"],
          // https://download.java.net/java/early_access/jdk23/23/GPL/openjdk-23-ea+23_macos-aarch64_bin.tar.gz
-         [21, ORACLE, MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-x64_bin.tar.gz"],
-         [21, ORACLE, MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-aarch64_bin.tar.gz"],
-         [21, ORACLE, LINUX, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-x64_bin.tar.gz"],
-         [21, ORACLE, LINUX, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-aarch64_bin.tar.gz"],
-         [21, ORACLE, WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_windows-x64_bin.zip"],
-         [21, anyVendor(), MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-x64_bin.tar.gz"],
-         [21, anyVendor(), MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-aarch64_bin.tar.gz"],
-         [21, anyVendor(), LINUX, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-x64_bin.tar.gz"],
-         [21, anyVendor(), LINUX, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-aarch64_bin.tar.gz"],
-         [21, anyVendor(), WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_windows-x64_bin.zip"]
-        ]
+         [24, ORACLE, MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_macos-x64_bin.tar.gz"],
+         [24, ORACLE, MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_macos-aarch64_bin.tar.gz"],
+         [24, ORACLE, LINUX, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_linux-x64_bin.tar.gz"],
+         [24, ORACLE, LINUX, AARCH64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_linux-aarch64_bin.tar.gz"],
+         [24, ORACLE, WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_windows-x64_bin.zip"],
+         [24, anyVendor(), MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_macos-x64_bin.tar.gz"],
+         [24, anyVendor(), MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_macos-aarch64_bin.tar.gz"],
+         [24, anyVendor(), LINUX, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_linux-x64_bin.tar.gz"],
+         [24, anyVendor(), LINUX, AARCH64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_linux-aarch64_bin.tar.gz"],
+         [24, anyVendor(), WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_windows-x64_bin.zip"]]
+    }
+
+    @RestoreSystemProperties
+    def "can provide build number for ea versions"() {
+        given:
+        System.setProperty('runtime.java.build', "42")
+        System.setProperty('runtime.java.25.build', "13")
+        def resolver = resolverImplementation()
+
+        when:
+        Optional<JavaToolchainDownload> download = resolver.resolve(
+            request(
+                JavaLanguageVersion.of(version),
+                vendor,
+                platform(os, arch)
+            )
+        )
+
+        then:
+        download.get().uri == URI.create(expectedUrl)
+
+        where:
+        version | vendor      | os      | arch    | expectedUrl
+        24      | ORACLE      | MAC_OS  | X86_64  | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_macos-x64_bin.tar.gz"
+        24      | ORACLE      | MAC_OS  | AARCH64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_macos-aarch64_bin.tar.gz"
+        24      | ORACLE      | LINUX   | X86_64  | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_linux-x64_bin.tar.gz"
+        24      | ORACLE      | LINUX   | AARCH64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_linux-aarch64_bin.tar.gz"
+        24      | ORACLE      | WINDOWS | X86_64  | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_windows-x64_bin.zip"
+        24      | anyVendor() | MAC_OS  | X86_64  | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_macos-x64_bin.tar.gz"
+        24      | anyVendor() | MAC_OS  | AARCH64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_macos-aarch64_bin.tar.gz"
+        24      | anyVendor() | LINUX   | X86_64  | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_linux-x64_bin.tar.gz"
+        24      | anyVendor() | LINUX   | AARCH64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_linux-aarch64_bin.tar.gz"
+        24      | anyVendor() | WINDOWS | X86_64  | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_windows-x64_bin.zip"
+        25      | ORACLE      | MAC_OS  | X86_64  | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_macos-x64_bin.tar.gz"
+        25      | ORACLE      | MAC_OS  | AARCH64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_macos-aarch64_bin.tar.gz"
+        25      | ORACLE      | LINUX   | X86_64  | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_linux-x64_bin.tar.gz"
+        25      | ORACLE      | LINUX   | AARCH64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_linux-aarch64_bin.tar.gz"
+        25      | ORACLE      | WINDOWS | X86_64  | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_windows-x64_bin.zip"
+        25      | anyVendor() | MAC_OS  | X86_64  | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_macos-x64_bin.tar.gz"
+        25      | anyVendor() | MAC_OS  | AARCH64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_macos-aarch64_bin.tar.gz"
+        25      | anyVendor() | LINUX   | X86_64  | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_linux-x64_bin.tar.gz"
+        25      | anyVendor() | LINUX   | AARCH64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_linux-aarch64_bin.tar.gz"
+        25      | anyVendor() | WINDOWS | X86_64  | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_windows-x64_bin.zip"
+    }
+
+    private static String urlPrefix(int i) {
+        return "https://download.java.net/java/early_access/jdk" + i + "/"
     }
 
     def unsupportedRequests() {
-        [
-                [20, ORACLE, WINDOWS, AARCH64]
-        ]
+        [[20, ORACLE, WINDOWS, AARCH64]]
     }
 
 }
diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
index ede1b392b8a41..57882fa842b41 100644
--- a/build-tools-internal/version.properties
+++ b/build-tools-internal/version.properties
@@ -19,6 +19,7 @@ commons_lang3     = 3.9
 google_oauth_client = 1.34.1
 awsv1sdk            = 1.12.270
 awsv2sdk            = 2.28.13
+reactive_streams    = 1.0.4
 
 antlr4            = 4.13.1
 # bouncy castle version for non-fips. fips jars use a different version
diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy
index 2575f80ad9da0..bbc47bedeffc6 100644
--- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy
+++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy
@@ -135,7 +135,7 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
 
             esplugin {
                 name = 'test-$pluginType'
-                classname 'org.acme.TestModule'
+                classname = 'org.acme.TestModule'
                 description = "test $pluginType description"
             }
 
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/distribution/ElasticsearchDistributionTypes.java b/build-tools/src/main/java/org/elasticsearch/gradle/distribution/ElasticsearchDistributionTypes.java
index 8a53d56e3c4ce..68eef25df5be1 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/distribution/ElasticsearchDistributionTypes.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/distribution/ElasticsearchDistributionTypes.java
@@ -12,6 +12,6 @@
 import org.elasticsearch.gradle.ElasticsearchDistributionType;
 
 public class ElasticsearchDistributionTypes {
-    public static ElasticsearchDistributionType ARCHIVE = new ArchiveElasticsearchDistributionType();
-    public static ElasticsearchDistributionType INTEG_TEST_ZIP = new IntegTestZipElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType ARCHIVE = new ArchiveElasticsearchDistributionType();
+    public static final ElasticsearchDistributionType INTEG_TEST_ZIP = new IntegTestZipElasticsearchDistributionType();
 }
diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy
index fe23204d5601c..72d8134869037 100644
--- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy
+++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy
@@ -107,7 +107,7 @@ abstract class AbstractGradleFuncTest extends Specification {
                                 .forwardOutput()
             ), configurationCacheCompatible,
                 buildApiRestrictionsDisabled)
-        ).withArguments(arguments.collect { it.toString() })
+        ).withArguments(arguments.collect { it.toString() } + "--full-stacktrace")
     }
 
     def assertOutputContains(String givenOutput, String expected) {
diff --git a/build.gradle b/build.gradle
index b95e34640cb5f..e6fc1f4eba28c 100644
--- a/build.gradle
+++ b/build.gradle
@@ -221,25 +221,11 @@ tasks.register("verifyVersions") {
   }
 }
 
-/*
- * When adding backcompat behavior that spans major versions, temporarily
- * disabling the backcompat tests is necessary. This flag controls
- * the enabled state of every bwc task. It should be set back to true
- * after the backport of the backcompat code is complete.
- */
-
+// TODO: This flag existed as a mechanism to disable bwc tests during a backport. It is no
+// longer used for that purpose, but instead a way to run only functional tests. We should
+// rework the functionalTests task to be more explicit about which tasks it wants to run
+// so that that this flag is no longer needed.
 boolean bwc_tests_enabled = true
-// place a PR link here when committing bwc changes:
-String bwc_tests_disabled_issue = ""
-if (bwc_tests_enabled == false) {
-  if (bwc_tests_disabled_issue.isEmpty()) {
-    throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
-  }
-  println "========================= WARNING ========================="
-  println "         Backwards compatibility tests are disabled!"
-  println "See ${bwc_tests_disabled_issue}"
-  println "==========================================================="
-}
 if (project.gradle.startParameter.taskNames.any { it.startsWith("checkPart") || it == 'functionalTests' }) {
   // Disable BWC tests for checkPart* tasks and platform support tests as it's expected that this will run on it's own check
   bwc_tests_enabled = false
@@ -378,24 +364,24 @@ tasks.register("verifyBwcTestsEnabled") {
 }
 
 tasks.register("branchConsistency") {
-  description 'Ensures this branch is internally consistent. For example, that versions constants match released versions.'
-  group 'Verification'
+  description = 'Ensures this branch is internally consistent. For example, that versions constants match released versions.'
+  group = 'Verification'
   dependsOn ":verifyVersions", ":verifyBwcTestsEnabled"
 }
 
 tasks.named("wrapper").configure {
   distributionType = 'ALL'
+  def minimumGradleVersionFile = project.file('build-tools-internal/src/main/resources/minimumGradleVersion')
   doLast {
     // copy wrapper properties file to build-tools-internal to allow seamless idea integration
     def file = new File("build-tools-internal/gradle/wrapper/gradle-wrapper.properties")
-    Files.copy(wrapper.getPropertiesFile().toPath(), file.toPath(), REPLACE_EXISTING)
+    Files.copy(getPropertiesFile().toPath(), file.toPath(), REPLACE_EXISTING)
     // copy wrapper properties file to plugins/examples to allow seamless idea integration
     def examplePluginsWrapperProperties = new File("plugins/examples/gradle/wrapper/gradle-wrapper.properties")
-    Files.copy(wrapper.getPropertiesFile().toPath(), examplePluginsWrapperProperties.toPath(), REPLACE_EXISTING)
-
+    Files.copy(getPropertiesFile().toPath(), examplePluginsWrapperProperties.toPath(), REPLACE_EXISTING)
     // Update build-tools to reflect the Gradle upgrade
     // TODO: we can remove this once we have tests to make sure older versions work.
-    project.file('build-tools-internal/src/main/resources/minimumGradleVersion').text = gradleVersion
+    minimumGradleVersionFile.text = gradleVersion
     println "Updated minimum Gradle Version"
   }
 }
diff --git a/client/client-benchmark-noop-api-plugin/build.gradle b/client/client-benchmark-noop-api-plugin/build.gradle
index 7f03450c406dc..9a93cb38b2881 100644
--- a/client/client-benchmark-noop-api-plugin/build.gradle
+++ b/client/client-benchmark-noop-api-plugin/build.gradle
@@ -12,9 +12,9 @@ group = 'org.elasticsearch.plugin'
 apply plugin: 'elasticsearch.internal-es-plugin'
 
 esplugin {
-  name 'client-benchmark-noop-api'
-  description 'Stubbed out Elasticsearch actions that can be used for client-side benchmarking'
-  classname 'org.elasticsearch.plugin.noop.NoopPlugin'
+  name = 'client-benchmark-noop-api'
+  description = 'Stubbed out Elasticsearch actions that can be used for client-side benchmarking'
+  classname ='org.elasticsearch.plugin.noop.NoopPlugin'
 }
 
 // Not published so no need to assemble
diff --git a/client/rest/build.gradle b/client/rest/build.gradle
index 003c251186510..3fb2aa6595869 100644
--- a/client/rest/build.gradle
+++ b/client/rest/build.gradle
@@ -36,7 +36,7 @@ base {
 }
 
 // LLRC is licenses under Apache 2.0
-projectLicenses.set(['The Apache Software License, Version 2.0': 'http://www.apache.org/licenses/LICENSE-2.0'])
+projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')])
 licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt'))
 
 dependencies {
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java
index 265bd52eabe83..916823fd91b61 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java
@@ -89,7 +89,6 @@ public static void stopHttpServers() throws IOException {
     }
 
     public void testBuilderUsesDefaultSSLContext() throws Exception {
-        assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm());
         final SSLContext defaultSSLContext = SSLContext.getDefault();
         try {
             try (RestClient client = buildRestClient()) {
@@ -97,10 +96,15 @@ public void testBuilderUsesDefaultSSLContext() throws Exception {
                     client.performRequest(new Request("GET", "/"));
                     fail("connection should have been rejected due to SSL handshake");
                 } catch (Exception e) {
-                    assertThat(e, instanceOf(SSLHandshakeException.class));
+                    if (inFipsJvm()) {
+                        // Bouncy Castle throw a different exception
+                        assertThat(e, instanceOf(IOException.class));
+                        assertThat(e.getCause(), instanceOf(javax.net.ssl.SSLException.class));
+                    } else {
+                        assertThat(e, instanceOf(SSLHandshakeException.class));
+                    }
                 }
             }
-
             SSLContext.setDefault(getSslContext());
             try (RestClient client = buildRestClient()) {
                 Response response = client.performRequest(new Request("GET", "/"));
@@ -112,7 +116,6 @@ public void testBuilderUsesDefaultSSLContext() throws Exception {
     }
 
     public void testBuilderSetsThreadName() throws Exception {
-        assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm());
         final SSLContext defaultSSLContext = SSLContext.getDefault();
         try {
             SSLContext.setDefault(getSslContext());
diff --git a/client/sniffer/build.gradle b/client/sniffer/build.gradle
index f6f26c8f7c0d5..9b1cb1140311b 100644
--- a/client/sniffer/build.gradle
+++ b/client/sniffer/build.gradle
@@ -32,7 +32,7 @@ base {
 }
 
 // rest client sniffer is licenses under Apache 2.0
-projectLicenses.set(['The Apache Software License, Version 2.0': 'http://www.apache.org/licenses/LICENSE-2.0'])
+projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')])
 licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt'))
 
 dependencies {
diff --git a/client/test/build.gradle b/client/test/build.gradle
index 8de6b3dbf92be..e39b7587b69d5 100644
--- a/client/test/build.gradle
+++ b/client/test/build.gradle
@@ -18,7 +18,7 @@ java {
 group = "${group}.client.test"
 
 // rest client sniffer is licenses under Apache 2.0
-projectLicenses.set(['The Apache Software License, Version 2.0': 'http://www.apache.org/licenses/LICENSE-2.0'])
+projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')])
 licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt'))
 
 dependencies {
diff --git a/distribution/build.gradle b/distribution/build.gradle
index e65d07dcfc2b4..e0302a081ce68 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -647,8 +647,8 @@ subprojects {
 // in the final log4j2.properties configuration, as it appears in the
 // archive distribution.
 artifacts.add('log4jConfig', file("${defaultOutputs}/log4j2.properties")) {
-  type 'file'
-  name 'log4j2.properties'
+  type = 'file'
+  name = 'log4j2.properties'
   builtBy 'buildDefaultLog4jConfig'
 }
 
diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle
index f5b94fb9dfd94..204cfc18950a8 100644
--- a/distribution/docker/build.gradle
+++ b/distribution/docker/build.gradle
@@ -28,7 +28,7 @@ repositories {
   // other Docker variants, the need for the main image to be rebuildable by Docker Hub
   // means that the Dockerfile itself has to fetch the binary.
   ivy {
-    url 'https://github.com/'
+    url = 'https://github.com/'
     patternLayout {
       artifact '/[organisation]/[module]/releases/download/v[revision]/[module]-[classifier]'
     }
@@ -45,12 +45,12 @@ if (useDra == false) {
         ivy {
           name = 'beats'
           if (useLocalArtifacts) {
-            url getLayout().getBuildDirectory().dir("artifacts").get().asFile
+            url = getLayout().getBuildDirectory().dir("artifacts").get().asFile
             patternLayout {
               artifact '/[organisation]/[module]-[revision]-[classifier].[ext]'
             }
           } else {
-            url "https://artifacts-snapshot.elastic.co/"
+            url = "https://artifacts-snapshot.elastic.co/"
             patternLayout {
               if (VersionProperties.isElasticsearchSnapshot()) {
                 artifact '/[organization]/[revision]/downloads/[organization]/[module]/[module]-[revision]-[classifier].[ext]'
@@ -127,7 +127,7 @@ ext.expansions = { Architecture architecture, DockerBase base ->
     'bin_dir'            : base == DockerBase.IRON_BANK ? 'scripts' : 'bin',
     'build_date'         : buildDate,
     'config_dir'         : base == DockerBase.IRON_BANK ? 'scripts' : 'config',
-    'git_revision'       : buildParams.gitRevision,
+    'git_revision'       : buildParams.gitRevision.get(),
     'license'            : base == DockerBase.IRON_BANK ? 'Elastic License 2.0' : 'Elastic-License-2.0',
     'package_manager'    : base.packageManager,
     'docker_base'        : base.name().toLowerCase(),
@@ -551,6 +551,7 @@ subprojects { Project subProject ->
       inputs.file("${parent.projectDir}/build/markers/${buildTaskName}.marker")
       executable = 'docker'
       outputs.file(tarFile)
+      outputs.doNotCacheIf("Build cache is disabled for export tasks") { true }
       args "save",
         "-o",
         tarFile,
@@ -583,8 +584,8 @@ subprojects { Project subProject ->
     }
 
     artifacts.add('default', file(tarFile)) {
-      type 'tar'
-      name artifactName
+      type = 'tar'
+      name = artifactName
       builtBy exportTaskName
     }
   }
diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile
index 6cb030565d9d2..48881660b30fe 100644
--- a/distribution/docker/src/docker/Dockerfile
+++ b/distribution/docker/src/docker/Dockerfile
@@ -22,7 +22,7 @@
 <% if (docker_base == 'iron_bank') { %>
 ARG BASE_REGISTRY=registry1.dso.mil
 ARG BASE_IMAGE=ironbank/redhat/ubi/ubi9
-ARG BASE_TAG=9.4
+ARG BASE_TAG=9.5
 <% } %>
 
 ################################################################################
diff --git a/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml b/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml
index f4364c5008c09..19b4a13dc9f22 100644
--- a/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml
+++ b/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml
@@ -1,21 +1,17 @@
 ---
 apiVersion: v1
-
 # The repository name in registry1, excluding /ironbank/
 name: "elastic/elasticsearch/elasticsearch"
-
 # List of tags to push for the repository in registry1
 # The most specific version should be the first tag and will be shown
 # on ironbank.dsop.io
 tags:
 - "${version}"
 - "latest"
-
 # Build args passed to Dockerfile ARGs
 args:
   BASE_IMAGE: "redhat/ubi/ubi9"
-  BASE_TAG: "9.4"
-
+  BASE_TAG: "9.5"
 # Docker image labels
 labels:
   org.opencontainers.image.title: "elasticsearch"
@@ -34,7 +30,6 @@ labels:
   mil.dso.ironbank.image.type: "commercial"
   # Product the image belongs to for grouping multiple images
   mil.dso.ironbank.product.name: "elasticsearch"
-
 # List of resources to make available to the offline build context
 resources:
   - filename: "elasticsearch-${version}-linux-x86_64.tar.gz"
@@ -47,12 +42,14 @@ resources:
     validation:
       type: "sha256"
       value: "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c"
-
 # List of project maintainers
 maintainers:
-  - name: "Rory Hunter"
-    email: "rory.hunter@elastic.co"
-    username: "rory"
+  - name: "Mark Vieira"
+    email: "mark.vieira@elastic.co"
+    username: "mark-vieira"
+  - name: "Rene Gröschke"
+    email: "rene.groschke@elastic.co"
+    username: "breskeby"
   - email: "klepal_alexander@bah.com"
     name: "Alexander Klepal"
     username: "alexander.klepal"
diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle
index 5f45b4b72974f..b7ba4e32edae3 100644
--- a/distribution/packages/build.gradle
+++ b/distribution/packages/build.gradle
@@ -43,7 +43,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "com.netflix.nebula.ospackage-base" version "11.10.0"
+  id "com.netflix.nebula.ospackage-base" version "11.10.1"
 }
 
 ['deb', 'rpm'].each { type ->
@@ -87,21 +87,21 @@ def commonPackageConfig(String type, String architecture) {
       OS.current().equals(OS.WINDOWS) == false
     }
     dependsOn "process${type.capitalize()}Files"
-    packageName "elasticsearch"
+    packageName = "elasticsearch"
     if (type == 'deb') {
       if (architecture == 'x64') {
-        arch('amd64')
+        arch = 'amd64'
       } else {
         assert architecture == 'aarch64': architecture
-        arch('arm64')
+        arch = 'arm64'
       }
     } else {
       assert type == 'rpm': type
       if (architecture == 'x64') {
-        arch('X86_64')
+        arch = 'X86_64'
       } else {
         assert architecture == 'aarch64': architecture
-        arch('aarch64')
+        arch = 'aarch64'
       }
     }
     // Follow elasticsearch's file naming convention
@@ -200,7 +200,7 @@ def commonPackageConfig(String type, String architecture) {
       into('/etc')
       permissionGroup 'elasticsearch'
       setgid true
-      includeEmptyDirs true
+      includeEmptyDirs = true
       createDirectoryEntry true
       include("elasticsearch") // empty dir, just to add directory entry
       include("elasticsearch/jvm.options.d") // empty dir, just to add directory entry
@@ -215,7 +215,7 @@ def commonPackageConfig(String type, String architecture) {
         unix(0660)
       }
       permissionGroup 'elasticsearch'
-      includeEmptyDirs true
+      includeEmptyDirs = true
       createDirectoryEntry true
       fileType CONFIG | NOREPLACE
     }
@@ -265,7 +265,7 @@ def commonPackageConfig(String type, String architecture) {
       into(file.parent) {
         from "${packagingFiles}/${file.parent}"
         include file.name
-        includeEmptyDirs true
+        includeEmptyDirs = true
         createDirectoryEntry true
         user u
         permissionGroup g
@@ -289,15 +289,15 @@ def commonPackageConfig(String type, String architecture) {
 
 // this is package independent configuration
 ospackage {
-  maintainer 'Elasticsearch Team <info@elastic.co>'
-  summary 'Distributed RESTful search engine built for the cloud'
-  packageDescription '''
+  maintainer = 'Elasticsearch Team <info@elastic.co>'
+  summary = 'Distributed RESTful search engine built for the cloud'
+  packageDescription = '''
     Reference documentation can be found at
     https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html
     and the 'Elasticsearch: The Definitive Guide' book can be found at
     https://www.elastic.co/guide/en/elasticsearch/guide/current/index.html
   '''.stripIndent().trim()
-  url 'https://www.elastic.co/'
+  url = 'https://www.elastic.co/'
 
   // signing setup
   if (project.hasProperty('signing.password') && buildParams.isSnapshotBuild() == false) {
@@ -311,10 +311,10 @@ ospackage {
   // version found on oldest supported distro, centos-6
   requires('coreutils', '8.4', GREATER | EQUAL)
 
-  fileMode 0644
-  dirMode 0755
-  user 'root'
-  permissionGroup 'root'
+  fileMode = 0644
+  dirMode = 0755
+  user = 'root'
+  permissionGroup = 'root'
 
   into '/usr/share/elasticsearch'
 }
@@ -330,7 +330,7 @@ Closure commonDebConfig(String architecture) {
     customFields['License'] = 'Elastic-License'
 
     archiveVersion = project.version.replace('-', '~')
-    packageGroup 'web'
+    packageGroup = 'web'
 
     // versions found on oldest supported distro, centos-6
     requires('bash', '4.1', GREATER | EQUAL)
@@ -358,24 +358,24 @@ Closure commonRpmConfig(String architecture) {
   return {
     configure(commonPackageConfig('rpm', architecture))
 
-    license 'Elastic License'
+    license = 'Elastic License'
 
-    packageGroup 'Application/Internet'
+    packageGroup = 'Application/Internet'
     requires '/bin/bash'
 
     obsoletes packageName, '7.0.0', Flags.LESS
 
     prefix '/usr'
-    packager 'Elasticsearch'
+    packager = 'Elasticsearch'
     archiveVersion = project.version.replace('-', '_')
     release = '1'
-    os 'LINUX'
-    distribution 'Elasticsearch'
-    vendor 'Elasticsearch'
+    os = 'LINUX'
+    distribution = 'Elasticsearch'
+    vendor = 'Elasticsearch'
     // TODO ospackage doesn't support icon but we used to have one
 
     // without this the rpm will have parent dirs of any files we copy in, eg /etc/elasticsearch
-    addParentDirs false
+    addParentDirs = false
   }
 }
 
diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options
index f55d90933ed61..94fc6f2cb9025 100644
--- a/distribution/src/config/jvm.options
+++ b/distribution/src/config/jvm.options
@@ -9,7 +9,7 @@
 ## should create one or more files in the jvm.options.d
 ## directory containing your adjustments.
 ##
-## See https://www.elastic.co/guide/en/elasticsearch/reference/@project.minor.version@/jvm-options.html
+## See https://www.elastic.co/guide/en/elasticsearch/reference/@project.minor.version@/advanced-configuration.html#set-jvm-options
 ## for more information.
 ##
 ################################################################
diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java
index 38bb7d592f7c0..5ab27bac3998a 100644
--- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java
+++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java
@@ -58,6 +58,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
 public class KeyStoreWrapperTests extends ESTestCase {
@@ -436,17 +437,8 @@ public void testStringAndFileDistinction() throws Exception {
     public void testLegacyV3() throws GeneralSecurityException, IOException {
         assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm());
         final Path configDir = createTempDir();
-        final Path keystore = configDir.resolve("elasticsearch.keystore");
-        try (
-            InputStream is = KeyStoreWrapperTests.class.getResourceAsStream("/format-v3-elasticsearch.keystore");
-            OutputStream os = Files.newOutputStream(keystore)
-        ) {
-            final byte[] buffer = new byte[4096];
-            int readBytes;
-            while ((readBytes = is.read(buffer)) > 0) {
-                os.write(buffer, 0, readBytes);
-            }
-        }
+        copyKeyStoreFromResourceToConfigDir(configDir, "/format-v3-elasticsearch.keystore");
+
         final KeyStoreWrapper wrapper = KeyStoreWrapper.load(configDir);
         assertNotNull(wrapper);
         wrapper.decrypt(new char[0]);
@@ -460,9 +452,31 @@ public void testLegacyV3() throws GeneralSecurityException, IOException {
 
     public void testLegacyV5() throws GeneralSecurityException, IOException {
         final Path configDir = createTempDir();
+        copyKeyStoreFromResourceToConfigDir(configDir, "/format-v5-with-password-elasticsearch.keystore");
+
+        final KeyStoreWrapper wrapper = KeyStoreWrapper.load(configDir);
+        assertNotNull(wrapper);
+        wrapper.decrypt("keystorepassword".toCharArray());
+        assertThat(wrapper.getFormatVersion(), equalTo(5));
+        assertThat(wrapper.getSettingNames(), equalTo(Set.of("keystore.seed")));
+    }
+
+    public void testLegacyV6() throws GeneralSecurityException, IOException {
+        final Path configDir = createTempDir();
+        copyKeyStoreFromResourceToConfigDir(configDir, "/format-v6-elasticsearch.keystore");
+
+        final KeyStoreWrapper wrapper = KeyStoreWrapper.load(configDir);
+        assertNotNull(wrapper);
+        wrapper.decrypt("keystorepassword".toCharArray());
+        assertThat(wrapper.getFormatVersion(), equalTo(6));
+        assertThat(wrapper.getSettingNames(), equalTo(Set.of("keystore.seed", "string")));
+        assertThat(wrapper.getString("string"), equalTo("value"));
+    }
+
+    private void copyKeyStoreFromResourceToConfigDir(Path configDir, String name) throws IOException {
         final Path keystore = configDir.resolve("elasticsearch.keystore");
         try (
-            InputStream is = KeyStoreWrapperTests.class.getResourceAsStream("/format-v5-with-password-elasticsearch.keystore");
+            InputStream is = KeyStoreWrapperTests.class.getResourceAsStream(name); //
             OutputStream os = Files.newOutputStream(keystore)
         ) {
             final byte[] buffer = new byte[4096];
@@ -471,11 +485,6 @@ public void testLegacyV5() throws GeneralSecurityException, IOException {
                 os.write(buffer, 0, readBytes);
             }
         }
-        final KeyStoreWrapper wrapper = KeyStoreWrapper.load(configDir);
-        assertNotNull(wrapper);
-        wrapper.decrypt("keystorepassword".toCharArray());
-        assertThat(wrapper.getFormatVersion(), equalTo(5));
-        assertThat(wrapper.getSettingNames(), equalTo(Set.of("keystore.seed")));
     }
 
     public void testSerializationNewlyCreated() throws Exception {
@@ -487,6 +496,7 @@ public void testSerializationNewlyCreated() throws Exception {
         wrapper.writeTo(out);
         final KeyStoreWrapper fromStream = new KeyStoreWrapper(out.bytes().streamInput());
 
+        assertThat(fromStream.getFormatVersion(), is(KeyStoreWrapper.CURRENT_VERSION));
         assertThat(fromStream.getSettingNames(), hasSize(2));
         assertThat(fromStream.getSettingNames(), containsInAnyOrder("string_setting", "keystore.seed"));
 
diff --git a/distribution/tools/keystore-cli/src/test/resources/format-v6-elasticsearch.keystore b/distribution/tools/keystore-cli/src/test/resources/format-v6-elasticsearch.keystore
new file mode 100644
index 0000000000000..0f680cc013563
Binary files /dev/null and b/distribution/tools/keystore-cli/src/test/resources/format-v6-elasticsearch.keystore differ
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java
index c3b9768946767..1e57d9fab7cfd 100644
--- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java
@@ -187,20 +187,12 @@ static String agentCommandLineOption(Path agentJar, Path tmpPropertiesFile) {
     static void extractSecureSettings(SecureSettings secrets, Map<String, String> propertiesMap) {
         final Set<String> settingNames = secrets.getSettingNames();
         for (String key : List.of("api_key", "secret_token")) {
-            for (String prefix : List.of("telemetry.", "tracing.apm.")) {
-                if (settingNames.contains(prefix + key)) {
-                    if (propertiesMap.containsKey(key)) {
-                        throw new IllegalStateException(
-                            Strings.format("Duplicate telemetry setting: [telemetry.%s] and [tracing.apm.%s]", key, key)
-                        );
-                    }
-
-                    try (SecureString token = secrets.getString(prefix + key)) {
-                        propertiesMap.put(key, token.toString());
-                    }
+            String prefix = "telemetry.";
+            if (settingNames.contains(prefix + key)) {
+                try (SecureString token = secrets.getString(prefix + key)) {
+                    propertiesMap.put(key, token.toString());
                 }
             }
-
         }
     }
 
@@ -227,44 +219,12 @@ private static Map<String, String> extractDynamicSettings(Map<String, String> pr
     static Map<String, String> extractApmSettings(Settings settings) throws UserException {
         final Map<String, String> propertiesMap = new HashMap<>();
 
-        // tracing.apm.agent. is deprecated by telemetry.agent.
         final String telemetryAgentPrefix = "telemetry.agent.";
-        final String deprecatedTelemetryAgentPrefix = "tracing.apm.agent.";
 
         final Settings telemetryAgentSettings = settings.getByPrefix(telemetryAgentPrefix);
         telemetryAgentSettings.keySet().forEach(key -> propertiesMap.put(key, String.valueOf(telemetryAgentSettings.get(key))));
 
-        final Settings apmAgentSettings = settings.getByPrefix(deprecatedTelemetryAgentPrefix);
-        for (String key : apmAgentSettings.keySet()) {
-            if (propertiesMap.containsKey(key)) {
-                throw new IllegalStateException(
-                    Strings.format(
-                        "Duplicate telemetry setting: [%s%s] and [%s%s]",
-                        telemetryAgentPrefix,
-                        key,
-                        deprecatedTelemetryAgentPrefix,
-                        key
-                    )
-                );
-            }
-            propertiesMap.put(key, String.valueOf(apmAgentSettings.get(key)));
-        }
-
         StringJoiner globalLabels = extractGlobalLabels(telemetryAgentPrefix, propertiesMap, settings);
-        if (globalLabels.length() == 0) {
-            globalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
-        } else {
-            StringJoiner tracingGlobalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
-            if (tracingGlobalLabels.length() != 0) {
-                throw new IllegalArgumentException(
-                    "Cannot have global labels with tracing.agent prefix ["
-                        + globalLabels
-                        + "] and telemetry.apm.agent prefix ["
-                        + tracingGlobalLabels
-                        + "]"
-                );
-            }
-        }
         if (globalLabels.length() > 0) {
             propertiesMap.put("global_labels", globalLabels.toString());
         }
@@ -274,7 +234,7 @@ static Map<String, String> extractApmSettings(Settings settings) throws UserExce
             if (propertiesMap.containsKey(key)) {
                 throw new UserException(
                     ExitCodes.CONFIG,
-                    "Do not set a value for [tracing.apm.agent." + key + "], as this is configured automatically by Elasticsearch"
+                    "Do not set a value for [telemetry.agent." + key + "], as this is configured automatically by Elasticsearch"
                 );
             }
         }
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java
index fe0f82560894c..d4d40e697470e 100644
--- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java
@@ -11,6 +11,7 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.jdk.RuntimeVersionFeature;
 
@@ -26,7 +27,9 @@ final class SystemJvmOptions {
     static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, String> sysprops) {
         String distroType = sysprops.get("es.distribution.type");
         boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot");
-        boolean useEntitlements = Boolean.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "false"));
+        boolean entitlementsExplicitlyEnabled = Booleans.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "false"));
+        // java 24+ only supports entitlements, but it may be enabled on earlier versions explicitly
+        boolean useEntitlements = RuntimeVersionFeature.isSecurityManagerAvailable() == false || entitlementsExplicitlyEnabled;
         return Stream.of(
             Stream.of(
                 /*
@@ -71,7 +74,7 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
             maybeSetActiveProcessorCount(nodeSettings),
             maybeSetReplayFile(distroType, isHotspot),
             maybeWorkaroundG1Bug(),
-            maybeAllowSecurityManager(),
+            maybeAllowSecurityManager(useEntitlements),
             maybeAttachEntitlementAgent(useEntitlements)
         ).flatMap(s -> s).toList();
     }
@@ -140,7 +143,7 @@ private static Stream<String> maybeWorkaroundG1Bug() {
     }
 
     @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
-    private static Stream<String> maybeAllowSecurityManager() {
+    private static Stream<String> maybeAllowSecurityManager(boolean useEntitlements) {
         if (RuntimeVersionFeature.isSecurityManagerAvailable()) {
             // Will become conditional on useEntitlements once entitlements can run without SM
             return Stream.of("-Djava.security.manager=allow");
@@ -167,12 +170,16 @@ private static Stream<String> maybeAttachEntitlementAgent(boolean useEntitlement
         } catch (IOException e) {
             throw new IllegalStateException("Failed to list entitlement jars in: " + dir, e);
         }
+        // We instrument classes in these modules to call the bridge. Because the bridge gets patched
+        // into java.base, we must export the bridge from java.base to these modules.
+        String modulesContainingEntitlementInstrumentation = "java.logging";
         return Stream.of(
             "-Des.entitlements.enabled=true",
             "-XX:+EnableDynamicAgentLoading",
             "-Djdk.attach.allowAttachSelf=true",
             "--patch-module=java.base=" + bridgeJar,
-            "--add-exports=java.base/org.elasticsearch.entitlement.bridge=org.elasticsearch.entitlement"
+            "--add-exports=java.base/org.elasticsearch.entitlement.bridge=org.elasticsearch.entitlement,"
+                + modulesContainingEntitlementInstrumentation
         );
     }
 }
diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java
index a7ba8eb11fbcc..0e067afc1aa73 100644
--- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java
+++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java
@@ -25,18 +25,15 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Function;
 
 import static org.elasticsearch.test.MapMatcher.matchesMap;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
@@ -82,109 +79,63 @@ public void testFileDeleteWorks() throws IOException {
     }
 
     public void testExtractSecureSettings() {
-        MockSecureSettings duplicateSecureSettings = new MockSecureSettings();
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("telemetry.secret_token", "token");
+        secureSettings.setString("telemetry.api_key", "key");
 
-        for (String prefix : List.of("telemetry.", "tracing.apm.")) {
-            MockSecureSettings secureSettings = new MockSecureSettings();
-            secureSettings.setString(prefix + "secret_token", "token");
-            secureSettings.setString(prefix + "api_key", "key");
-
-            duplicateSecureSettings.setString(prefix + "api_key", "secret");
-
-            Map<String, String> propertiesMap = new HashMap<>();
-            APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap);
-
-            assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key")));
-        }
-
-        Exception exception = expectThrows(
-            IllegalStateException.class,
-            () -> APMJvmOptions.extractSecureSettings(duplicateSecureSettings, new HashMap<>())
-        );
-        assertThat(exception.getMessage(), containsString("Duplicate telemetry setting"));
-        assertThat(exception.getMessage(), containsString("telemetry.api_key"));
-        assertThat(exception.getMessage(), containsString("tracing.apm.api_key"));
+        Map<String, String> propertiesMap = new HashMap<>();
+        APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap);
 
+        assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key")));
     }
 
     public void testExtractSettings() throws UserException {
-        Function<String, Settings.Builder> buildSettings = (prefix) -> Settings.builder()
-            .put(prefix + "server_url", "https://myurl:443")
-            .put(prefix + "service_node_name", "instance-0000000001");
-
-        for (String prefix : List.of("tracing.apm.agent.", "telemetry.agent.")) {
-            var name = "APM Tracing";
-            var deploy = "123";
-            var org = "456";
-            var extracted = APMJvmOptions.extractApmSettings(
-                buildSettings.apply(prefix)
-                    .put(prefix + "global_labels.deployment_name", name)
-                    .put(prefix + "global_labels.deployment_id", deploy)
-                    .put(prefix + "global_labels.organization_id", org)
-                    .build()
-            );
-
-            assertThat(
-                extracted,
-                allOf(
-                    hasEntry("server_url", "https://myurl:443"),
-                    hasEntry("service_node_name", "instance-0000000001"),
-                    hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one
-                    not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys
-                )
-            );
-
-            List<String> labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
-            assertThat(labels, hasSize(3));
-            assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy));
-
-            // test replacing with underscores and skipping empty
-            name = "APM=Tracing";
-            deploy = "";
-            org = ",456";
-            extracted = APMJvmOptions.extractApmSettings(
-                buildSettings.apply(prefix)
-                    .put(prefix + "global_labels.deployment_name", name)
-                    .put(prefix + "global_labels.deployment_id", deploy)
-                    .put(prefix + "global_labels.organization_id", org)
-                    .build()
-            );
-            labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
-            assertThat(labels, hasSize(2));
-            assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456"));
-        }
-
-        IllegalStateException err = expectThrows(
-            IllegalStateException.class,
-            () -> APMJvmOptions.extractApmSettings(
-                Settings.builder()
-                    .put("tracing.apm.agent.server_url", "https://myurl:443")
-                    .put("telemetry.agent.server_url", "https://myurl-2:443")
-                    .build()
-            )
-        );
-        assertThat(err.getMessage(), is("Duplicate telemetry setting: [telemetry.agent.server_url] and [tracing.apm.agent.server_url]"));
-    }
-
-    public void testNoMixedLabels() {
-        String telemetryAgent = "telemetry.agent.";
-        String tracingAgent = "tracing.apm.agent.";
-        Settings settings = Settings.builder()
-            .put("tracing.apm.enabled", true)
-            .put(telemetryAgent + "server_url", "https://myurl:443")
-            .put(telemetryAgent + "service_node_name", "instance-0000000001")
-            .put(tracingAgent + "global_labels.deployment_id", "123")
-            .put(telemetryAgent + "global_labels.organization_id", "456")
+        Settings defaults = Settings.builder()
+            .put("telemetry.agent.server_url", "https://myurl:443")
+            .put("telemetry.agent.service_node_name", "instance-0000000001")
             .build();
 
-        IllegalArgumentException err = assertThrows(IllegalArgumentException.class, () -> APMJvmOptions.extractApmSettings(settings));
+        var name = "APM Tracing";
+        var deploy = "123";
+        var org = "456";
+        var extracted = APMJvmOptions.extractApmSettings(
+            Settings.builder()
+                .put(defaults)
+                .put("telemetry.agent.global_labels.deployment_name", name)
+                .put("telemetry.agent.global_labels.deployment_id", deploy)
+                .put("telemetry.agent.global_labels.organization_id", org)
+                .build()
+        );
+
         assertThat(
-            err.getMessage(),
-            is(
-                "Cannot have global labels with tracing.agent prefix [organization_id=456] and"
-                    + " telemetry.apm.agent prefix [deployment_id=123]"
+            extracted,
+            allOf(
+                hasEntry("server_url", "https://myurl:443"),
+                hasEntry("service_node_name", "instance-0000000001"),
+                hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one
+                not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys
             )
         );
+
+        List<String> labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
+        assertThat(labels, hasSize(3));
+        assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy));
+
+        // test replacing with underscores and skipping empty
+        name = "APM=Tracing";
+        deploy = "";
+        org = ",456";
+        extracted = APMJvmOptions.extractApmSettings(
+            Settings.builder()
+                .put(defaults)
+                .put("telemetry.agent.global_labels.deployment_name", name)
+                .put("telemetry.agent.global_labels.deployment_id", deploy)
+                .put("telemetry.agent.global_labels.organization_id", org)
+                .build()
+        );
+        labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
+        assertThat(labels, hasSize(2));
+        assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456"));
     }
 
     private Path makeFakeAgentJar() throws IOException {
diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc
index bdb0704fcd880..f2e61861bd3a6 100644
--- a/docs/Versions.asciidoc
+++ b/docs/Versions.asciidoc
@@ -9,6 +9,7 @@ include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
 :docker-repo:       docker.elastic.co/elasticsearch/elasticsearch
 :docker-image:      {docker-repo}:{version}
+:docker-wolfi-image: {docker-repo}-wolfi:{version}
 :kib-docker-repo:   docker.elastic.co/kibana/kibana
 :kib-docker-image:  {kib-docker-repo}:{version}
 :plugin_url:      https://artifacts.elastic.co/downloads/elasticsearch-plugins
diff --git a/docs/build.gradle b/docs/build.gradle
index dec0de8ffa844..43b61ea97c089 100644
--- a/docs/build.gradle
+++ b/docs/build.gradle
@@ -130,8 +130,9 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.security.authc.api_key.enabled', 'true'
   setting 'xpack.security.authc.token.enabled', 'true'
-  // disable the ILM history for doc tests to avoid potential lingering tasks that'd cause test flakiness
+  // disable the ILM and SLM history for doc tests to avoid potential lingering tasks that'd cause test flakiness
   setting 'indices.lifecycle.history_index_enabled', 'false'
+  setting 'slm.history_index_enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'xpack.security.authc.realms.file.file.order', '0'
   setting 'xpack.security.authc.realms.native.native.order', '1'
@@ -177,8 +178,9 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
 }
 
 tasks.named("yamlRestTest").configure {
+  def repoFolder = "${layout.buildDirectory.asFile.get()}/cluster/shared/repo"
   doFirst {
-    delete("${buildDir}/cluster/shared/repo")
+    delete(repoFolder)
   }
 }
 
diff --git a/docs/changelog/116358.yaml b/docs/changelog/116358.yaml
deleted file mode 100644
index 58b44a1e9bcf5..0000000000000
--- a/docs/changelog/116358.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 116358
-summary: Update Deberta tokenizer
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/116388.yaml b/docs/changelog/116388.yaml
new file mode 100644
index 0000000000000..59cdafc9ec337
--- /dev/null
+++ b/docs/changelog/116388.yaml
@@ -0,0 +1,5 @@
+pr: 116388
+summary: Add support for partial shard results
+area: EQL
+type: enhancement
+issues: []
diff --git a/docs/changelog/116687.yaml b/docs/changelog/116687.yaml
new file mode 100644
index 0000000000000..f8c7f86eff04a
--- /dev/null
+++ b/docs/changelog/116687.yaml
@@ -0,0 +1,5 @@
+pr: 116687
+summary: Add LogsDB option to route on sort fields
+area: Logs
+type: enhancement
+issues: []
diff --git a/docs/changelog/116868.yaml b/docs/changelog/116868.yaml
new file mode 100644
index 0000000000000..4ca4d23306462
--- /dev/null
+++ b/docs/changelog/116868.yaml
@@ -0,0 +1,5 @@
+pr: 116868
+summary: Run `TransportGetComponentTemplateAction` on local node
+area: Indices APIs
+type: enhancement
+issues: []
diff --git a/docs/changelog/117153.yaml b/docs/changelog/117153.yaml
deleted file mode 100644
index f7640c0a7ed6a..0000000000000
--- a/docs/changelog/117153.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 117153
-summary: "ESQL: fix the column position in errors"
-area: ES|QL
-type: bug
-issues: []
diff --git a/docs/changelog/117214.yaml b/docs/changelog/117214.yaml
new file mode 100644
index 0000000000000..ba74197eb7634
--- /dev/null
+++ b/docs/changelog/117214.yaml
@@ -0,0 +1,5 @@
+pr: 117214
+summary: Returning ignored fields in the simulate ingest API
+area: Ingest Node
+type: enhancement
+issues: []
diff --git a/docs/changelog/117519.yaml b/docs/changelog/117519.yaml
new file mode 100644
index 0000000000000..f228278983785
--- /dev/null
+++ b/docs/changelog/117519.yaml
@@ -0,0 +1,20 @@
+pr: 117519
+summary: Remove `data_frame_transforms` roles
+area: Transform
+type: breaking
+issues: []
+breaking:
+  title: Remove `data_frame_transforms` roles
+  area: Transform
+  details: >-
+    `data_frame_transforms_admin` and `data_frame_transforms_user` were deprecated in
+    Elasticsearch 7 and are being removed in Elasticsearch 9.
+    `data_frame_transforms_admin` is now `transform_admin`.
+    `data_frame_transforms_user` is now `transform_user`.
+    Users must call the `_update` API to replace the permissions on the Transform before the
+    Transform can be started.
+  impact: >-
+    Transforms created with either the `data_frame_transforms_admin` or the
+    `data_frame_transforms_user` role will fail to start. The Transform will remain
+    in a `stopped` state, and its health will be red while displaying permission failures.
+  notable: false
diff --git a/docs/changelog/117581.yaml b/docs/changelog/117581.yaml
new file mode 100644
index 0000000000000..b88017f45e9c9
--- /dev/null
+++ b/docs/changelog/117581.yaml
@@ -0,0 +1,5 @@
+pr: 117581
+summary: Make reserved built-in roles queryable
+area: Authorization
+type: enhancement
+issues: []
diff --git a/docs/changelog/117778.yaml b/docs/changelog/117778.yaml
new file mode 100644
index 0000000000000..880d4f831e533
--- /dev/null
+++ b/docs/changelog/117778.yaml
@@ -0,0 +1,5 @@
+pr: 117778
+summary: "[Connector APIs] Enforce index prefix for managed connectors"
+area: Extract&Transform
+type: feature
+issues: []
diff --git a/docs/changelog/117858.yaml b/docs/changelog/117858.yaml
new file mode 100644
index 0000000000000..70f12dc40027f
--- /dev/null
+++ b/docs/changelog/117858.yaml
@@ -0,0 +1,5 @@
+pr: 117858
+summary: Create upgrade mode
+area: Transform
+type: enhancement
+issues: []
diff --git a/docs/changelog/117949.yaml b/docs/changelog/117949.yaml
new file mode 100644
index 0000000000000..b67f36a224094
--- /dev/null
+++ b/docs/changelog/117949.yaml
@@ -0,0 +1,5 @@
+pr: 117949
+summary: Move `SlowLogFieldProvider` instantiation to node construction
+area: Infra/Logging
+type: bug
+issues: []
diff --git a/docs/changelog/117989.yaml b/docs/changelog/117989.yaml
new file mode 100644
index 0000000000000..e4967141b3ebd
--- /dev/null
+++ b/docs/changelog/117989.yaml
@@ -0,0 +1,5 @@
+pr: 117989
+summary: ESQL Add esql hash function
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/118016.yaml b/docs/changelog/118016.yaml
new file mode 100644
index 0000000000000..7ee78b901b199
--- /dev/null
+++ b/docs/changelog/118016.yaml
@@ -0,0 +1,6 @@
+pr: 118016
+summary: Propagate status codes from shard failures appropriately
+area: Search
+type: enhancement
+issues:
+ - 118482
diff --git a/docs/changelog/118143.yaml b/docs/changelog/118143.yaml
new file mode 100644
index 0000000000000..4dcbf4b4b6c2c
--- /dev/null
+++ b/docs/changelog/118143.yaml
@@ -0,0 +1,5 @@
+pr: 118143
+summary: Infrastructure for assuming cluster features in the next major version
+area: "Infra/Core"
+type: feature
+issues: []
diff --git a/docs/changelog/118266.yaml b/docs/changelog/118266.yaml
new file mode 100644
index 0000000000000..1b14b12b973c5
--- /dev/null
+++ b/docs/changelog/118266.yaml
@@ -0,0 +1,5 @@
+pr: 118266
+summary: Prevent data nodes from sending stack traces to coordinator when `error_trace=false`
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/118324.yaml b/docs/changelog/118324.yaml
new file mode 100644
index 0000000000000..729ff56f6a253
--- /dev/null
+++ b/docs/changelog/118324.yaml
@@ -0,0 +1,6 @@
+pr: 118324
+summary: Allow the data type of `null` in filters
+area: ES|QL
+type: bug
+issues:
+  - 116351
diff --git a/docs/changelog/118353.yaml b/docs/changelog/118353.yaml
new file mode 100644
index 0000000000000..7be62a4a60c7e
--- /dev/null
+++ b/docs/changelog/118353.yaml
@@ -0,0 +1,5 @@
+pr: 118353
+summary: Epoch Millis Rounding Down and Not Up 2
+area: Infra/Core
+type: bug
+issues: []
diff --git a/docs/changelog/118366.yaml b/docs/changelog/118366.yaml
new file mode 100644
index 0000000000000..cfeab1937738b
--- /dev/null
+++ b/docs/changelog/118366.yaml
@@ -0,0 +1,22 @@
+pr: 118366
+summary: |-
+  Configuring a bind DN in an LDAP or Active Directory (AD) realm without a corresponding bind password
+  will prevent node from starting
+area: Authentication
+type: breaking
+issues: []
+breaking:
+  title: -|
+    Configuring a bind DN in an LDAP or Active Directory (AD) realm without
+    a corresponding bind password will prevent node from starting
+  area: Cluster and node setting
+  details: -|
+    For LDAP or AD authentication realms, setting a bind DN (via the
+    `xpack.security.authc.realms.ldap.*.bind_dn` or `xpack.security.authc.realms.active_directory.*.bind_dn`
+    realm settings) without a bind password is a misconfiguration that may prevent successful authentication
+    to the node. Nodes will fail to start if a bind DN is specified without a password.
+  impact: -|
+    If you have a bind DN configured for an LDAP or AD authentication
+    realm, set a bind password for {ref}/ldap-realm.html#ldap-realm-configuration[LDAP]
+    or {ref}/active-directory-realm.html#ad-realm-configuration[Active Directory].
+    Configuring a bind DN without a password prevents the misconfigured node from starting.
diff --git a/docs/changelog/118380.yaml b/docs/changelog/118380.yaml
deleted file mode 100644
index 8b26c871fb172..0000000000000
--- a/docs/changelog/118380.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 118380
-summary: Restore original "is within leaf" value in `SparseVectorFieldMapper`
-area: Mapping
-type: bug
-issues: []
diff --git a/docs/changelog/118484.yaml b/docs/changelog/118484.yaml
new file mode 100644
index 0000000000000..41db476a42523
--- /dev/null
+++ b/docs/changelog/118484.yaml
@@ -0,0 +1,14 @@
+pr: 118484
+summary: Remove date histogram boolean support
+area: Aggregations
+type: breaking
+issues: []
+breaking:
+  title: Remove date histogram boolean support
+  area: Aggregations
+  details: Elasticsearch no longer allows running Date Histogram aggregations
+           over boolean fields. Instead, use Terms aggregation for boolean
+           fields.
+  impact: We expect the impact to be minimal, as this never produced good
+          results, and has been deprecated for years.
+  notable: false
diff --git a/docs/changelog/118544.yaml b/docs/changelog/118544.yaml
new file mode 100644
index 0000000000000..d59783c4e6194
--- /dev/null
+++ b/docs/changelog/118544.yaml
@@ -0,0 +1,5 @@
+pr: 118544
+summary: ESQL - Remove restrictions for disjunctions in full text functions
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/118562.yaml b/docs/changelog/118562.yaml
new file mode 100644
index 0000000000000..a6b00b326151f
--- /dev/null
+++ b/docs/changelog/118562.yaml
@@ -0,0 +1,6 @@
+pr: 118562
+summary: Update data stream deprecations warnings to new format and filter searchable
+  snapshots from response
+area: Data streams
+type: enhancement
+issues: []
diff --git a/docs/changelog/118585.yaml b/docs/changelog/118585.yaml
new file mode 100644
index 0000000000000..4caa5efabbd33
--- /dev/null
+++ b/docs/changelog/118585.yaml
@@ -0,0 +1,7 @@
+pr: 118585
+summary: Add a generic `rescorer` retriever based on the search request's rescore
+  functionality
+area: Ranking
+type: feature
+issues:
+ - 118327
diff --git a/docs/changelog/118602.yaml b/docs/changelog/118602.yaml
new file mode 100644
index 0000000000000..a75c5dcf11da3
--- /dev/null
+++ b/docs/changelog/118602.yaml
@@ -0,0 +1,5 @@
+pr: 118602
+summary: Limit memory usage of `fold`
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/118603.yaml b/docs/changelog/118603.yaml
new file mode 100644
index 0000000000000..d61619adfa5f6
--- /dev/null
+++ b/docs/changelog/118603.yaml
@@ -0,0 +1,6 @@
+pr: 118603
+summary: Allow DATE_PARSE to read the timezones
+area: ES|QL
+type: bug
+issues:
+ - 117680
diff --git a/docs/changelog/118617.yaml b/docs/changelog/118617.yaml
new file mode 100644
index 0000000000000..a8793a114e913
--- /dev/null
+++ b/docs/changelog/118617.yaml
@@ -0,0 +1,5 @@
+pr: 118617
+summary: Add support for `sparse_vector` queries against `semantic_text` fields
+area: "Search"
+type: enhancement
+issues: []
diff --git a/docs/changelog/118652.yaml b/docs/changelog/118652.yaml
new file mode 100644
index 0000000000000..0b08686230405
--- /dev/null
+++ b/docs/changelog/118652.yaml
@@ -0,0 +1,5 @@
+pr: 118652
+summary: Add Jina AI API to do inference for Embedding and Rerank models
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/118669.yaml b/docs/changelog/118669.yaml
new file mode 100644
index 0000000000000..4e0d10aaac816
--- /dev/null
+++ b/docs/changelog/118669.yaml
@@ -0,0 +1,5 @@
+pr: 118669
+summary: "[Connector API] Support soft-deletes of connectors"
+area: Extract&Transform
+type: feature
+issues: []
diff --git a/docs/changelog/118671.yaml b/docs/changelog/118671.yaml
new file mode 100644
index 0000000000000..3931cc4179037
--- /dev/null
+++ b/docs/changelog/118671.yaml
@@ -0,0 +1,11 @@
+pr: 118671
+summary: Adjust `random_score` default field to `_seq_no` field
+area: Search
+type: breaking
+issues: []
+breaking:
+  title: Adjust `random_score` default field to `_seq_no` field
+  area: Search
+  details: When providing a 'seed' parameter to a 'random_score' function in the 'function_score' query but NOT providing a 'field', the default 'field' is switched from '_id' to '_seq_no'.
+  impact: The random scoring and ordering may change when providing a 'seed' and not providing a 'field' to a 'random_score' function. 
+  notable: false
diff --git a/docs/changelog/118674.yaml b/docs/changelog/118674.yaml
new file mode 100644
index 0000000000000..eeb90a3b38f66
--- /dev/null
+++ b/docs/changelog/118674.yaml
@@ -0,0 +1,5 @@
+pr: 118674
+summary: Ignore failures from renormalizing buckets in read-only index
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/118681.yaml b/docs/changelog/118681.yaml
new file mode 100644
index 0000000000000..a186c05e6cd7d
--- /dev/null
+++ b/docs/changelog/118681.yaml
@@ -0,0 +1,6 @@
+pr: 118681
+summary: '`ConnectTransportException` returns retryable BAD_GATEWAY'
+area: Network
+type: enhancement
+issues:
+ - 118320
diff --git a/docs/changelog/118697.yaml b/docs/changelog/118697.yaml
new file mode 100644
index 0000000000000..6e24e6ae4b47f
--- /dev/null
+++ b/docs/changelog/118697.yaml
@@ -0,0 +1,6 @@
+pr: 118697
+summary: Esql implicit casting for date nanos
+area: ES|QL
+type: enhancement
+issues:
+ - 118476
diff --git a/docs/changelog/118757.yaml b/docs/changelog/118757.yaml
new file mode 100644
index 0000000000000..956e220f21aeb
--- /dev/null
+++ b/docs/changelog/118757.yaml
@@ -0,0 +1,5 @@
+pr: 118757
+summary: Improve handling of nested fields in index reader wrappers
+area: Authorization
+type: enhancement
+issues: []
diff --git a/docs/changelog/118774.yaml b/docs/changelog/118774.yaml
new file mode 100644
index 0000000000000..cbd1ca82d1c59
--- /dev/null
+++ b/docs/changelog/118774.yaml
@@ -0,0 +1,5 @@
+pr: 118774
+summary: Apply default k for knn query eagerly
+area: Vector Search
+type: bug
+issues: []
diff --git a/docs/changelog/118802.yaml b/docs/changelog/118802.yaml
new file mode 100644
index 0000000000000..600c4b6a1e203
--- /dev/null
+++ b/docs/changelog/118802.yaml
@@ -0,0 +1,5 @@
+pr: 118802
+summary: ST_EXTENT_AGG optimize envelope extraction from doc-values for cartesian_shape
+area: "ES|QL"
+type: enhancement
+issues: []
diff --git a/docs/changelog/118804.yaml b/docs/changelog/118804.yaml
new file mode 100644
index 0000000000000..1548367a5485f
--- /dev/null
+++ b/docs/changelog/118804.yaml
@@ -0,0 +1,15 @@
+pr: 118804
+summary: Add new experimental `rank_vectors` mapping for late-interaction second order
+  ranking
+area: Vector Search
+type: feature
+issues: []
+highlight:
+  title: Add new experimental `rank_vectors` mapping for late-interaction second order
+    ranking
+  body:
+      Late-interaction models are powerful rerankers. While their size and overall
+      cost doesn't lend itself for HNSW indexing, utilizing them as second order reranking
+      can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring
+      over new and novel multi-vector late-interaction models like ColBERT or ColPali.
+  notable: true
diff --git a/docs/changelog/118816.yaml b/docs/changelog/118816.yaml
new file mode 100644
index 0000000000000..f1c1eac90dbcf
--- /dev/null
+++ b/docs/changelog/118816.yaml
@@ -0,0 +1,6 @@
+pr: 118816
+summary: Support flattened field with downsampling
+area: Downsampling
+type: bug
+issues:
+ - 116319
diff --git a/docs/changelog/118823.yaml b/docs/changelog/118823.yaml
new file mode 100644
index 0000000000000..b1afe1c873c17
--- /dev/null
+++ b/docs/changelog/118823.yaml
@@ -0,0 +1,5 @@
+pr: 118823
+summary: Fix attribute set equals
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/116944.yaml b/docs/changelog/118825.yaml
similarity index 84%
rename from docs/changelog/116944.yaml
rename to docs/changelog/118825.yaml
index e7833e49cf965..23170ec4705da 100644
--- a/docs/changelog/116944.yaml
+++ b/docs/changelog/118825.yaml
@@ -1,4 +1,4 @@
-pr: 116944
+pr: 118825
 summary: "Remove support for type, fields, `copy_to` and boost in metadata field definition"
 area: Mapping
 type: breaking
@@ -6,6 +6,6 @@ issues: []
 breaking:
   title: "Remove support for type, fields, copy_to and boost in metadata field definition"
   area: Mapping
-  details: The type, fields, copy_to and boost parameters are no longer supported in metadata field definition
+  details: The type, fields, copy_to and boost parameters are no longer supported in metadata field definition starting with version 9.
   impact: Users providing type, fields, copy_to or boost as part of metadata field definition should remove them from their mappings.
   notable: false
diff --git a/docs/changelog/118837.yaml b/docs/changelog/118837.yaml
new file mode 100644
index 0000000000000..38cd32f3a3513
--- /dev/null
+++ b/docs/changelog/118837.yaml
@@ -0,0 +1,5 @@
+pr: 118837
+summary: Add missing timeouts to rest-api-spec ILM APIs
+area: "ILM+SLM"
+type: bug
+issues: []
diff --git a/docs/changelog/118844.yaml b/docs/changelog/118844.yaml
new file mode 100644
index 0000000000000..f9f92bcaeb8cb
--- /dev/null
+++ b/docs/changelog/118844.yaml
@@ -0,0 +1,5 @@
+pr: 118844
+summary: Add missing timeouts to rest-api-spec ingest APIs
+area: Ingest Node
+type: bug
+issues: []
diff --git a/docs/changelog/118858.yaml b/docs/changelog/118858.yaml
new file mode 100644
index 0000000000000..a2161df1c84c7
--- /dev/null
+++ b/docs/changelog/118858.yaml
@@ -0,0 +1,5 @@
+pr: 118858
+summary: Lookup join on multiple join fields not yet supported
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/118870.yaml b/docs/changelog/118870.yaml
new file mode 100644
index 0000000000000..ce3692d5454ae
--- /dev/null
+++ b/docs/changelog/118870.yaml
@@ -0,0 +1,6 @@
+pr: 118870
+summary: Rewrite TO_UPPER/TO_LOWER comparisons
+area: ES|QL
+type: enhancement
+issues:
+ - 118304
diff --git a/docs/changelog/118871.yaml b/docs/changelog/118871.yaml
new file mode 100644
index 0000000000000..3c1a06d450f39
--- /dev/null
+++ b/docs/changelog/118871.yaml
@@ -0,0 +1,5 @@
+pr: 118871
+summary: "[Elastic Inference Service] Add ElasticInferenceService Unified ChatCompletions Integration"
+area: Inference
+type: enhancement
+issues: []
diff --git a/docs/changelog/118890.yaml b/docs/changelog/118890.yaml
new file mode 100644
index 0000000000000..d3fc17157f130
--- /dev/null
+++ b/docs/changelog/118890.yaml
@@ -0,0 +1,5 @@
+pr: 118890
+summary: Add action to create index from a source index
+area: Data streams
+type: enhancement
+issues: []
diff --git a/docs/changelog/118919.yaml b/docs/changelog/118919.yaml
new file mode 100644
index 0000000000000..832fd86fe08ba
--- /dev/null
+++ b/docs/changelog/118919.yaml
@@ -0,0 +1,5 @@
+pr: 118919
+summary: Remove unsupported timeout from rest-api-spec license API
+area: License
+type: bug
+issues: []
diff --git a/docs/changelog/118921.yaml b/docs/changelog/118921.yaml
new file mode 100644
index 0000000000000..bd341616d8a14
--- /dev/null
+++ b/docs/changelog/118921.yaml
@@ -0,0 +1,5 @@
+pr: 118921
+summary: Add missing timeouts to rest-api-spec shutdown APIs
+area: Infra/Node Lifecycle
+type: bug
+issues: []
diff --git a/docs/changelog/118931.yaml b/docs/changelog/118931.yaml
new file mode 100644
index 0000000000000..81e9b3cb16521
--- /dev/null
+++ b/docs/changelog/118931.yaml
@@ -0,0 +1,6 @@
+pr: 118931
+summary: Add a `LicenseAware` interface for licensed Nodes
+area: ES|QL
+type: enhancement
+issues:
+ - 117405
diff --git a/docs/changelog/118938.yaml b/docs/changelog/118938.yaml
new file mode 100644
index 0000000000000..395da7912fd4b
--- /dev/null
+++ b/docs/changelog/118938.yaml
@@ -0,0 +1,5 @@
+pr: 118938
+summary: Hash functions
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/118941.yaml b/docs/changelog/118941.yaml
new file mode 100644
index 0000000000000..4f0099bb32704
--- /dev/null
+++ b/docs/changelog/118941.yaml
@@ -0,0 +1,5 @@
+pr: 118941
+summary: Allow archive and searchable snapshots indices in N-2 version
+area: Recovery
+type: enhancement
+issues: []
diff --git a/docs/changelog/118954.yaml b/docs/changelog/118954.yaml
new file mode 100644
index 0000000000000..ab2f2cda5c11e
--- /dev/null
+++ b/docs/changelog/118954.yaml
@@ -0,0 +1,5 @@
+pr: 118954
+summary: Add missing parameter to `xpack.info` rest-api-spec
+area: Infra/REST API
+type: bug
+issues: []
diff --git a/docs/changelog/118958.yaml b/docs/changelog/118958.yaml
new file mode 100644
index 0000000000000..fb0fd6388ab61
--- /dev/null
+++ b/docs/changelog/118958.yaml
@@ -0,0 +1,5 @@
+pr: 118958
+summary: Add missing timeouts to rest-api-spec SLM APIs
+area: ILM+SLM
+type: bug
+issues: []
diff --git a/docs/changelog/118959.yaml b/docs/changelog/118959.yaml
new file mode 100644
index 0000000000000..95a9c146ae672
--- /dev/null
+++ b/docs/changelog/118959.yaml
@@ -0,0 +1,5 @@
+pr: 118959
+summary: Allow kibana_system user to manage .reindexed-v8-internal.alerts indices
+area: Authorization
+type: enhancement
+issues: []
diff --git a/docs/changelog/118968.yaml b/docs/changelog/118968.yaml
new file mode 100644
index 0000000000000..799cd32471f2f
--- /dev/null
+++ b/docs/changelog/118968.yaml
@@ -0,0 +1,6 @@
+pr: 118968
+summary: Configure index sorting through index settings for logsdb
+area: Logs
+type: enhancement
+issues:
+ - 118686
diff --git a/docs/changelog/118999.yaml b/docs/changelog/118999.yaml
new file mode 100644
index 0000000000000..0188cebbd7685
--- /dev/null
+++ b/docs/changelog/118999.yaml
@@ -0,0 +1,6 @@
+pr: 118999
+summary: Fix loss of context in the inference API for streaming APIs
+area: Machine Learning
+type: bug
+issues:
+ - 119000
diff --git a/docs/changelog/119007.yaml b/docs/changelog/119007.yaml
new file mode 100644
index 0000000000000..458101b68d454
--- /dev/null
+++ b/docs/changelog/119007.yaml
@@ -0,0 +1,6 @@
+pr: 119007
+summary: Block-writes cannot be added after read-only
+area: Data streams
+type: bug
+issues:
+ - 119002
diff --git a/docs/changelog/119011.yaml b/docs/changelog/119011.yaml
new file mode 100644
index 0000000000000..0dbb683ceb2fd
--- /dev/null
+++ b/docs/changelog/119011.yaml
@@ -0,0 +1,5 @@
+pr: 119011
+summary: "Add support for knn vector queries on `semantic_text` fields"
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/119054.yaml b/docs/changelog/119054.yaml
new file mode 100644
index 0000000000000..720f2e0ab02ed
--- /dev/null
+++ b/docs/changelog/119054.yaml
@@ -0,0 +1,6 @@
+pr: 119054
+summary: "[Security Solution] allows `kibana_system` user to manage .reindexed-v8-*\
+  \ Security Solution indices"
+area: Authorization
+type: enhancement
+issues: []
diff --git a/docs/changelog/119067.yaml b/docs/changelog/119067.yaml
new file mode 100644
index 0000000000000..c7ddd570bea18
--- /dev/null
+++ b/docs/changelog/119067.yaml
@@ -0,0 +1,5 @@
+pr: 119067
+summary: Metrics for indexing failures due to version conflicts
+area: CRUD
+type: feature
+issues: []
diff --git a/docs/changelog/119131.yaml b/docs/changelog/119131.yaml
new file mode 100644
index 0000000000000..2628b6184f90d
--- /dev/null
+++ b/docs/changelog/119131.yaml
@@ -0,0 +1,5 @@
+pr: 119131
+summary: Expose BwC enrich cache setting in plugin
+area: Ingest Node
+type: bug
+issues: []
diff --git a/docs/changelog/119134.yaml b/docs/changelog/119134.yaml
new file mode 100644
index 0000000000000..c4aefac91c701
--- /dev/null
+++ b/docs/changelog/119134.yaml
@@ -0,0 +1,6 @@
+pr: 119134
+summary: Handle `index.mapping.ignore_malformed` in downsampling
+area: Downsampling
+type: bug
+issues:
+ - 119075
diff --git a/docs/changelog/119227.yaml b/docs/changelog/119227.yaml
new file mode 100644
index 0000000000000..1e3d4f97a3d27
--- /dev/null
+++ b/docs/changelog/119227.yaml
@@ -0,0 +1,13 @@
+pr: 119227
+summary: Remove unfreeze REST endpoint
+area: Indices APIs
+type: breaking
+issues: []
+breaking:
+  title: Remove unfreeze REST endpoint
+  area: REST API
+  details: >-
+    The `/{index}/_unfreeze` REST endpoint is no longer supported. This API was deprecated, and the corresponding
+    `/{index}/_freeze` endpoint was removed in 8.0.
+  impact: None, since it is not possible to have a frozen index in a version which is readable by Elasticsearch 9.0
+  notable: false
diff --git a/docs/changelog/119233.yaml b/docs/changelog/119233.yaml
new file mode 100644
index 0000000000000..ef89c011ce4f6
--- /dev/null
+++ b/docs/changelog/119233.yaml
@@ -0,0 +1,5 @@
+pr: 119233
+summary: Fixing `GetDatabaseConfigurationAction` response serialization
+area: Ingest Node
+type: bug
+issues: []
diff --git a/docs/changelog/119250.yaml b/docs/changelog/119250.yaml
new file mode 100644
index 0000000000000..9db36957d8050
--- /dev/null
+++ b/docs/changelog/119250.yaml
@@ -0,0 +1,5 @@
+pr: 119250
+summary: Add rest endpoint for `create_from_source_index`
+area: Data streams
+type: enhancement
+issues: []
diff --git a/docs/changelog/119265.yaml b/docs/changelog/119265.yaml
new file mode 100644
index 0000000000000..296106b9c01c6
--- /dev/null
+++ b/docs/changelog/119265.yaml
@@ -0,0 +1,6 @@
+pr: 119265
+summary: Fix `AbstractShapeGeometryFieldMapperTests`
+area: "ES|QL"
+type: bug
+issues:
+ - 119201
diff --git a/docs/changelog/119291.yaml b/docs/changelog/119291.yaml
new file mode 100644
index 0000000000000..89a6b6118049c
--- /dev/null
+++ b/docs/changelog/119291.yaml
@@ -0,0 +1,5 @@
+pr: 119291
+summary: Register mustache size limit setting
+area: Infra/Scripting
+type: bug
+issues: []
diff --git a/docs/changelog/119296.yaml b/docs/changelog/119296.yaml
new file mode 100644
index 0000000000000..f1a92cffb5131
--- /dev/null
+++ b/docs/changelog/119296.yaml
@@ -0,0 +1,6 @@
+pr: 119296
+summary: Fix writing for LOOKUP status
+area: ES|QL
+type: bug
+issues:
+ - 119086
diff --git a/docs/changelog/119310.yaml b/docs/changelog/119310.yaml
new file mode 100644
index 0000000000000..4e09e1a5c39d1
--- /dev/null
+++ b/docs/changelog/119310.yaml
@@ -0,0 +1,6 @@
+pr: 119310
+summary: Remove ChunkedToXContentBuilder
+area: "Network"
+type: bug
+issues:
+ - 118647
diff --git a/docs/changelog/119348.yaml b/docs/changelog/119348.yaml
new file mode 100644
index 0000000000000..05d2d7499ba03
--- /dev/null
+++ b/docs/changelog/119348.yaml
@@ -0,0 +1,5 @@
+pr: 119348
+summary: Auto-migrate `max_page_search_size`
+area: Transform
+type: enhancement
+issues: []
diff --git a/docs/changelog/119449.yaml b/docs/changelog/119449.yaml
new file mode 100644
index 0000000000000..f02bfa6d16d60
--- /dev/null
+++ b/docs/changelog/119449.yaml
@@ -0,0 +1,5 @@
+pr: 119449
+summary: Add missing traces ilm policy for OTel traces data streams
+area: Data streams
+type: bug
+issues: []
diff --git a/docs/changelog/119474.yaml b/docs/changelog/119474.yaml
new file mode 100644
index 0000000000000..e37561277d220
--- /dev/null
+++ b/docs/changelog/119474.yaml
@@ -0,0 +1,5 @@
+pr: 119474
+summary: "Add ES|QL cross-cluster query telemetry collection"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/119476.yaml b/docs/changelog/119476.yaml
new file mode 100644
index 0000000000000..c275e6965d4a1
--- /dev/null
+++ b/docs/changelog/119476.yaml
@@ -0,0 +1,6 @@
+pr: 119476
+summary: Fix TopN row size estimate
+area: ES|QL
+type: bug
+issues:
+ - 106956
diff --git a/docs/changelog/119495.yaml b/docs/changelog/119495.yaml
new file mode 100644
index 0000000000000..b3e8f7e79d984
--- /dev/null
+++ b/docs/changelog/119495.yaml
@@ -0,0 +1,5 @@
+pr: 119495
+summary: Add mapping for `event_name` for OTel logs
+area: Data streams
+type: enhancement
+issues: []
diff --git a/docs/changelog/119504.yaml b/docs/changelog/119504.yaml
new file mode 100644
index 0000000000000..f63e422face10
--- /dev/null
+++ b/docs/changelog/119504.yaml
@@ -0,0 +1,5 @@
+pr: 119504
+summary: Optimized index sorting for OTel logs
+area: Data streams
+type: enhancement
+issues: []
diff --git a/docs/changelog/119516.yaml b/docs/changelog/119516.yaml
new file mode 100644
index 0000000000000..06dd5168a0823
--- /dev/null
+++ b/docs/changelog/119516.yaml
@@ -0,0 +1,5 @@
+pr: 119516
+summary: "Fix: do not let `_resolve/cluster` hang if remote is unresponsive"
+area: Search
+type: bug
+issues: []
diff --git a/docs/changelog/119542.yaml b/docs/changelog/119542.yaml
new file mode 100644
index 0000000000000..aaf26c7dc4b0f
--- /dev/null
+++ b/docs/changelog/119542.yaml
@@ -0,0 +1,5 @@
+pr: 119542
+summary: Wait while index is blocked
+area: Transform
+type: enhancement
+issues: []
diff --git a/docs/changelog/119543.yaml b/docs/changelog/119543.yaml
new file mode 100644
index 0000000000000..7027ea2a49672
--- /dev/null
+++ b/docs/changelog/119543.yaml
@@ -0,0 +1,7 @@
+pr: 119543
+summary: "[Inference API] Fix unique ID message for inference ID matches trained model\
+  \ ID"
+area: Machine Learning
+type: bug
+issues:
+ - 111312
diff --git a/docs/changelog/119564.yaml b/docs/changelog/119564.yaml
new file mode 100644
index 0000000000000..175eff75c8218
--- /dev/null
+++ b/docs/changelog/119564.yaml
@@ -0,0 +1,5 @@
+pr: 119564
+summary: Http stream activity tracker and exceptions handling
+area: Network
+type: enhancement
+issues: []
diff --git a/docs/changelog/119621.yaml b/docs/changelog/119621.yaml
new file mode 100644
index 0000000000000..66c679572cfc0
--- /dev/null
+++ b/docs/changelog/119621.yaml
@@ -0,0 +1,5 @@
+pr: 119621
+summary: Enable node-level reduction by default
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/119637.yaml b/docs/changelog/119637.yaml
new file mode 100644
index 0000000000000..c2fd6dc51f068
--- /dev/null
+++ b/docs/changelog/119637.yaml
@@ -0,0 +1,5 @@
+pr: 119637
+summary: Fix spike detection for short spikes at the tail of the data
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/changelog/116423.yaml b/docs/changelog/119679.yaml
similarity index 92%
rename from docs/changelog/116423.yaml
rename to docs/changelog/119679.yaml
index d6d10eab410e4..a3fb36bcd01c3 100644
--- a/docs/changelog/116423.yaml
+++ b/docs/changelog/119679.yaml
@@ -1,4 +1,4 @@
-pr: 116423
+pr: 119679
 summary: Support mTLS for the Elastic Inference Service integration inside the inference API
 area: Machine Learning
 type: feature
diff --git a/docs/changelog/119691.yaml b/docs/changelog/119691.yaml
new file mode 100644
index 0000000000000..186944394908d
--- /dev/null
+++ b/docs/changelog/119691.yaml
@@ -0,0 +1,6 @@
+pr: 119691
+summary: Fix `bbq_hnsw` merge file cleanup on random IO exceptions
+area: Vector Search
+type: bug
+issues:
+ - 119392
diff --git a/docs/changelog/119730.yaml b/docs/changelog/119730.yaml
new file mode 100644
index 0000000000000..09ff1362b3b1f
--- /dev/null
+++ b/docs/changelog/119730.yaml
@@ -0,0 +1,5 @@
+pr: 119730
+summary: Enable KQL function as a tech preview
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/119748.yaml b/docs/changelog/119748.yaml
new file mode 100644
index 0000000000000..8b29fb7c1a647
--- /dev/null
+++ b/docs/changelog/119748.yaml
@@ -0,0 +1,6 @@
+pr: 119748
+summary: Issue S3 web identity token refresh call with sufficient permissions
+area: Snapshot/Restore
+type: bug
+issues:
+ - 119747
diff --git a/docs/changelog/119749.yaml b/docs/changelog/119749.yaml
new file mode 100644
index 0000000000000..aa2b16ceda5ea
--- /dev/null
+++ b/docs/changelog/119749.yaml
@@ -0,0 +1,5 @@
+pr: 119749
+summary: Strengthen encryption for elasticsearch-keystore tool to AES 256
+area: Infra/CLI
+type: enhancement
+issues: []
diff --git a/docs/changelog/119750.yaml b/docs/changelog/119750.yaml
new file mode 100644
index 0000000000000..2ec5c298d0eb1
--- /dev/null
+++ b/docs/changelog/119750.yaml
@@ -0,0 +1,6 @@
+pr: 119750
+summary: "ESQL: `connect_transport_exception` should be thrown instead of `verification_exception`\
+  \ when ENRICH-ing if remote is disconnected"
+area: Search
+type: bug
+issues: []
diff --git a/docs/changelog/119772.yaml b/docs/changelog/119772.yaml
new file mode 100644
index 0000000000000..58d483566b109
--- /dev/null
+++ b/docs/changelog/119772.yaml
@@ -0,0 +1,6 @@
+pr: 119772
+summary: ESQL Support IN operator for Date nanos
+area: ES|QL
+type: enhancement
+issues:
+ - 118578
diff --git a/docs/changelog/119780.yaml b/docs/changelog/119780.yaml
new file mode 100644
index 0000000000000..5b7226741a416
--- /dev/null
+++ b/docs/changelog/119780.yaml
@@ -0,0 +1,5 @@
+pr: 119780
+summary: Add index and reindex request settings to speed up reindex
+area: Data streams
+type: enhancement
+issues: []
diff --git a/docs/changelog/119792.yaml b/docs/changelog/119792.yaml
new file mode 100644
index 0000000000000..02b17c203f69d
--- /dev/null
+++ b/docs/changelog/119792.yaml
@@ -0,0 +1,5 @@
+pr: 119792
+summary: Make semantic text part of the text family
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/119793.yaml b/docs/changelog/119793.yaml
new file mode 100644
index 0000000000000..80330c25c2f30
--- /dev/null
+++ b/docs/changelog/119793.yaml
@@ -0,0 +1,6 @@
+pr: 119793
+summary: Resolve/cluster should mark remotes as not connected when a security exception
+  is thrown
+area: CCS
+type: bug
+issues: []
diff --git a/docs/changelog/119797.yaml b/docs/changelog/119797.yaml
new file mode 100644
index 0000000000000..992c2078e0caa
--- /dev/null
+++ b/docs/changelog/119797.yaml
@@ -0,0 +1,5 @@
+pr: 119797
+summary: "[Inference API] Fix bug checking for e5 or reranker default IDs"
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/changelog/119798.yaml b/docs/changelog/119798.yaml
new file mode 100644
index 0000000000000..e21ba83d1e10f
--- /dev/null
+++ b/docs/changelog/119798.yaml
@@ -0,0 +1,5 @@
+pr: 119798
+summary: "Add a `PostAnalysisAware,` distribute verification"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/119830.yaml b/docs/changelog/119830.yaml
new file mode 100644
index 0000000000000..cf852ae03c1db
--- /dev/null
+++ b/docs/changelog/119830.yaml
@@ -0,0 +1,5 @@
+pr: 119830
+summary: Run `TransportGetComposableIndexTemplate` on local node
+area: Indices APIs
+type: enhancement
+issues: []
diff --git a/docs/changelog/119831.yaml b/docs/changelog/119831.yaml
new file mode 100644
index 0000000000000..61c09d7d54de0
--- /dev/null
+++ b/docs/changelog/119831.yaml
@@ -0,0 +1,5 @@
+pr: 119831
+summary: Run `TransportClusterGetSettingsAction` on local node
+area: Infra/Settings
+type: enhancement
+issues: []
diff --git a/docs/changelog/119837.yaml b/docs/changelog/119837.yaml
new file mode 100644
index 0000000000000..3c0a930aa3544
--- /dev/null
+++ b/docs/changelog/119837.yaml
@@ -0,0 +1,5 @@
+pr: 119837
+summary: Run `TransportGetIndexTemplateAction` on local node
+area: Indices APIs
+type: enhancement
+issues: []
diff --git a/docs/changelog/119846.yaml b/docs/changelog/119846.yaml
new file mode 100644
index 0000000000000..9e7d99fe1be13
--- /dev/null
+++ b/docs/changelog/119846.yaml
@@ -0,0 +1,12 @@
+pr: 119846
+summary: Drop support for brackets from METADATA syntax
+area: ES|QL
+type: deprecation
+issues:
+ - 115401
+deprecation:
+  title: Drop support for brackets from METADATA syntax
+  area: ES|QL
+  details: Please describe the details of this change for the release notes. You can
+    use asciidoc.
+  impact: Please describe the impact of this change to users
diff --git a/docs/changelog/119863.yaml b/docs/changelog/119863.yaml
new file mode 100644
index 0000000000000..63cbf1ba07851
--- /dev/null
+++ b/docs/changelog/119863.yaml
@@ -0,0 +1,11 @@
+pr: 119863
+summary: Restrict Connector APIs to manage/monitor_connector privileges
+area: Extract&Transform
+type: breaking
+issues: []
+breaking:
+  title: Restrict Connector APIs to manage/monitor_connector privileges
+  area: REST API
+  details: Connector APIs now enforce the manage_connector and monitor_connector privileges (introduced in 8.15), replacing the previous reliance on index-level permissions for .elastic-connectors and .elastic-connectors-sync-jobs in API calls.
+  impact: Connector APIs now require manage_connector and monitor_connector privileges
+  notable: false
diff --git a/docs/changelog/119893.yaml b/docs/changelog/119893.yaml
new file mode 100644
index 0000000000000..35a46ce0940d3
--- /dev/null
+++ b/docs/changelog/119893.yaml
@@ -0,0 +1,5 @@
+pr: 119893
+summary: Add enterprise license check for Inference API actions
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/119897.yaml b/docs/changelog/119897.yaml
new file mode 100644
index 0000000000000..87c5890f9fde1
--- /dev/null
+++ b/docs/changelog/119897.yaml
@@ -0,0 +1,5 @@
+pr: 119897
+summary: Fix ESQL async get while task is being cancelled
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/119922.yaml b/docs/changelog/119922.yaml
new file mode 100644
index 0000000000000..2fc9d9529c968
--- /dev/null
+++ b/docs/changelog/119922.yaml
@@ -0,0 +1,5 @@
+pr: 119922
+summary: "[Inference API] fix spell words: covertToString to convertToString"
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/119926.yaml b/docs/changelog/119926.yaml
new file mode 100644
index 0000000000000..3afafd5b2117f
--- /dev/null
+++ b/docs/changelog/119926.yaml
@@ -0,0 +1,11 @@
+pr: 119926
+summary: "Deprecated tracing.apm.* settings got removed."
+area: Infra/Metrics
+type: breaking
+issues: []
+breaking:
+  title: "Deprecated tracing.apm.* settings got removed."
+  area: Cluster and node setting
+  details: Deprecated `tracing.apm.*` settings got removed, use respective `telemetry.*` / `telemetry.tracing.*` settings instead.
+  impact: 9.x nodes will refuse to start if any such setting (including secret settings) is still present.
+  notable: false
diff --git a/docs/changelog/120014.yaml b/docs/changelog/120014.yaml
new file mode 100644
index 0000000000000..bef1f3ba49939
--- /dev/null
+++ b/docs/changelog/120014.yaml
@@ -0,0 +1,6 @@
+pr: 120014
+summary: Fix potential file leak in ES816BinaryQuantizedVectorsWriter
+area: Search
+type: bug
+issues:
+ - 119981
diff --git a/docs/changelog/120038.yaml b/docs/changelog/120038.yaml
new file mode 100644
index 0000000000000..fe3a2ccccc095
--- /dev/null
+++ b/docs/changelog/120038.yaml
@@ -0,0 +1,5 @@
+pr: 120038
+summary: Run template simulation actions on local node
+area: Ingest Node
+type: enhancement
+issues: []
diff --git a/docs/changelog/120042.yaml b/docs/changelog/120042.yaml
new file mode 100644
index 0000000000000..0093068ae9894
--- /dev/null
+++ b/docs/changelog/120042.yaml
@@ -0,0 +1,5 @@
+pr: 120042
+summary: Match dot prefix of migrated DS backing index with the source index
+area: Data streams
+type: bug
+issues: []
diff --git a/docs/changelog/120062.yaml b/docs/changelog/120062.yaml
new file mode 100644
index 0000000000000..42e8d97f17444
--- /dev/null
+++ b/docs/changelog/120062.yaml
@@ -0,0 +1,6 @@
+pr: 120062
+summary: Update Text Similarity Reranker to Properly Handle Aliases
+area: Ranking
+type: bug
+issues:
+ - 119617
diff --git a/docs/changelog/120084.yaml b/docs/changelog/120084.yaml
new file mode 100644
index 0000000000000..aafe490d79f1e
--- /dev/null
+++ b/docs/changelog/120084.yaml
@@ -0,0 +1,5 @@
+pr: 120084
+summary: Improve how reindex data stream index action handles api blocks
+area: Data streams
+type: enhancement
+issues: []
diff --git a/docs/changelog/120087.yaml b/docs/changelog/120087.yaml
new file mode 100644
index 0000000000000..8539640809b04
--- /dev/null
+++ b/docs/changelog/120087.yaml
@@ -0,0 +1,5 @@
+pr: 120087
+summary: Include `clusterApplyListener` in long cluster apply warnings
+area: Cluster Coordination
+type: enhancement
+issues: []
diff --git a/docs/changelog/120133.yaml b/docs/changelog/120133.yaml
new file mode 100644
index 0000000000000..4ec88267a1bf8
--- /dev/null
+++ b/docs/changelog/120133.yaml
@@ -0,0 +1,6 @@
+pr: 120133
+summary: Use approximation to advance matched queries
+area: Search
+type: bug
+issues:
+ - 120130
diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc
index 44acba4752aaa..164e3398d7a4f 100644
--- a/docs/plugins/discovery-ec2.asciidoc
+++ b/docs/plugins/discovery-ec2.asciidoc
@@ -241,7 +241,7 @@ The `discovery-ec2` plugin can automatically set the `aws_availability_zone`
 node attribute to the availability zone of each node. This node attribute
 allows you to ensure that each shard has copies allocated redundantly across
 multiple availability zones by using the
-{ref}/modules-cluster.html#shard-allocation-awareness[Allocation Awareness]
+{ref}/shard-allocation-awareness.html#[Allocation Awareness]
 feature.
 
 In order to enable the automatic definition of the `aws_availability_zone`
@@ -333,7 +333,7 @@ labelled as `Moderate` or `Low`.
 
 * It is a good idea to distribute your nodes across multiple
 https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html[availability
-zones] and use {ref}/modules-cluster.html#shard-allocation-awareness[shard
+zones] and use {ref}/shard-allocation-awareness.html[shard
 allocation awareness] to ensure that each shard has copies in more than one
 availability zone.
 
diff --git a/docs/reference/alias.asciidoc b/docs/reference/alias.asciidoc
index 9d784f530d63c..f676644c4ec48 100644
--- a/docs/reference/alias.asciidoc
+++ b/docs/reference/alias.asciidoc
@@ -407,3 +407,24 @@ POST _aliases
 }
 ----
 // TEST[s/^/PUT my-index-2099.05.06-000001\n/]
+
+[discrete]
+[[remove-index]]
+=== Remove an index
+
+To remove an index, use the aliases API's `remove_index` action.
+
+[source,console]
+----
+POST _aliases
+{
+  "actions": [
+    {
+      "remove_index": {
+        "index": "my-index-2099.05.06-000001"
+      }
+    }
+  ]
+}
+----
+// TEST[s/^/PUT my-index-2099.05.06-000001\n/]
diff --git a/docs/reference/analysis.asciidoc b/docs/reference/analysis.asciidoc
index 72ab42d22b911..e8fbc3bd81b6d 100644
--- a/docs/reference/analysis.asciidoc
+++ b/docs/reference/analysis.asciidoc
@@ -9,8 +9,7 @@
 --
 
 _Text analysis_ is the process of converting unstructured text, like
-the body of an email or a product description, into a structured format that's
-optimized for search.
+the body of an email or a product description, into a structured format that's <<full-text-search,optimized for search>>.
 
 [discrete]
 [[when-to-configure-analysis]]
diff --git a/docs/reference/analysis/tokenizers.asciidoc b/docs/reference/analysis/tokenizers.asciidoc
index 38e4ebfcabc39..89928f07b5638 100644
--- a/docs/reference/analysis/tokenizers.asciidoc
+++ b/docs/reference/analysis/tokenizers.asciidoc
@@ -1,6 +1,14 @@
 [[analysis-tokenizers]]
 == Tokenizer reference
 
+.Difference between {es} tokenization and neural tokenization
+[NOTE]
+====
+{es}'s tokenization process produces linguistic tokens, optimized for search and retrieval.
+This differs from neural tokenization in the context of machine learning and natural language processing. Neural tokenizers translate strings into smaller, subword tokens, which are encoded into vectors for consumptions by neural networks.
+{es} does not have built-in neural tokenizers.
+====
+
 A _tokenizer_ receives a stream of characters, breaks it up into individual
 _tokens_ (usually individual words), and outputs a stream of _tokens_. For
 instance, a <<analysis-whitespace-tokenizer,`whitespace`>> tokenizer breaks
diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc
index f8d925945401e..28933eb75050d 100644
--- a/docs/reference/api-conventions.asciidoc
+++ b/docs/reference/api-conventions.asciidoc
@@ -28,7 +28,7 @@ You can pass an `X-Opaque-Id` HTTP header to track the origin of a request in
 
 * Response of any request that includes the header
 * <<_identifying_running_tasks,Task management API>> response
-* <<_identifying_search_slow_log_origin,Slow logs>>
+* <<search-slow-log,Slow logs>>
 * <<deprecation-logging,Deprecation logs>>
 
 For the deprecation logs, {es} also uses the `X-Opaque-Id` value to throttle
@@ -52,7 +52,7 @@ safely generate a unique `traceparent` header for each request.
 If provided, {es} surfaces the header's `trace-id` value as `trace.id` in the:
 
 * <<logging,JSON {es} server logs>>
-* <<_identifying_search_slow_log_origin,Slow logs>>
+* <<search-slow-log,Slow logs>>
 * <<deprecation-logging,Deprecation logs>>
 
 For example, the following `traceparent` value would produce the following
diff --git a/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc b/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc
index e4da2c45ee978..87de3818bfaf8 100644
--- a/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc
+++ b/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc
@@ -4,6 +4,13 @@
 
 NOTE: {cloud-only}
 
+
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-autoscaling[Autoscaling APIs].
+--
+
 You can use the following APIs to perform {cloud}/ec-autoscaling.html[autoscaling operations].
 
 [discrete]
diff --git a/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc b/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc
index 190428485a003..349e40aab0540 100644
--- a/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc
+++ b/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc
@@ -7,6 +7,12 @@
 
 NOTE: {cloud-only}
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-autoscaling[Autoscaling APIs].
+--
+
 Delete {cloud}/ec-autoscaling.html[autoscaling] policy.
 
 [[autoscaling-delete-autoscaling-policy-request]]
diff --git a/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc b/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc
index d635d8c8f7bd0..d45f7cbacc242 100644
--- a/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc
+++ b/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc
@@ -7,6 +7,12 @@
 
 NOTE: {cloud-only}
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-autoscaling[Autoscaling APIs].
+--
+
 Get {cloud}/ec-autoscaling.html[autoscaling] capacity.
 
 [[autoscaling-get-autoscaling-capacity-request]]
diff --git a/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc b/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc
index 973eedcb361c9..9962b266fb662 100644
--- a/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc
+++ b/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc
@@ -7,6 +7,13 @@
 
 NOTE: {cloud-only}
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-autoscaling[Autoscaling APIs].
+--
+
+
 Get {cloud}/ec-autoscaling.html[autoscaling] policy.
 
 [[autoscaling-get-autoscaling-policy-request]]
diff --git a/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc b/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc
index e564f83411eb4..97c6a54fab03a 100644
--- a/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc
+++ b/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc
@@ -7,6 +7,12 @@
 
 NOTE: {cloud-only}
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-autoscaling[Autoscaling APIs].
+--
+
 Creates or updates an {cloud}/ec-autoscaling.html[autoscaling] policy.
 
 [[autoscaling-put-autoscaling-policy-request]]
diff --git a/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc
index a6894a933b460..19c1b5437ef0c 100644
--- a/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc
+++ b/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc
@@ -8,6 +8,12 @@ beta::[]
 <titleabbrev>Delete Analytics Collection</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-analytics[Behavioral analytics APIs].
+--
+
 ////
 [source,console]
 ----
diff --git a/docs/reference/behavioral-analytics/apis/index.asciidoc b/docs/reference/behavioral-analytics/apis/index.asciidoc
index 692d3374f89f5..6dc12599c2297 100644
--- a/docs/reference/behavioral-analytics/apis/index.asciidoc
+++ b/docs/reference/behavioral-analytics/apis/index.asciidoc
@@ -9,6 +9,12 @@ beta::[]
 
 ---
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-analytics[Behavioral analytics APIs].
+--
+
 Use the following APIs to manage tasks and resources related to <<behavioral-analytics-overview,Behavioral Analytics>>:
 
 * <<put-analytics-collection>>
diff --git a/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc
index 14511a1258278..46ee8296f3ebe 100644
--- a/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc
+++ b/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc
@@ -8,6 +8,12 @@ beta::[]
 <titleabbrev>List Analytics Collections</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-analytics[Behavioral analytics APIs].
+--
+
 ////
 [source,console]
 ----
diff --git a/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc b/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc
index f82717e22ed34..60985cd50d3d1 100644
--- a/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc
+++ b/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc
@@ -8,6 +8,12 @@ beta::[]
 <titleabbrev>Post Analytics Collection Event</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-analytics[Behavioral analytics APIs].
+--
+
 ////
 [source,console]
 ----
diff --git a/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc
index cbbab2ae3e26c..412277afa2076 100644
--- a/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc
+++ b/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc
@@ -8,6 +8,12 @@ beta::[]
 <titleabbrev>Put Analytics Collection</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-analytics[Behavioral analytics APIs].
+--
+
 ////
 [source,console]
 ----
diff --git a/docs/reference/cat.asciidoc b/docs/reference/cat.asciidoc
index 328970e1d9dc4..61ec9f7680f7f 100644
--- a/docs/reference/cat.asciidoc
+++ b/docs/reference/cat.asciidoc
@@ -1,6 +1,12 @@
 [[cat]]
 == Compact and aligned text (CAT) APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].
+--
+
 ["float",id="intro"]
 === Introduction
 
diff --git a/docs/reference/cat/alias.asciidoc b/docs/reference/cat/alias.asciidoc
index aab0c9df25ed4..0a7c1828d4876 100644
--- a/docs/reference/cat/alias.asciidoc
+++ b/docs/reference/cat/alias.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>cat aliases</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or the
diff --git a/docs/reference/cat/allocation.asciidoc b/docs/reference/cat/allocation.asciidoc
index bbd044c4b8e5e..34b8069b91e27 100644
--- a/docs/reference/cat/allocation.asciidoc
+++ b/docs/reference/cat/allocation.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>cat allocation</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib}
diff --git a/docs/reference/cat/anomaly-detectors.asciidoc b/docs/reference/cat/anomaly-detectors.asciidoc
index 68d952d2a8532..03cd824092cf6 100644
--- a/docs/reference/cat/anomaly-detectors.asciidoc
+++ b/docs/reference/cat/anomaly-detectors.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>cat anomaly detectors</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib}
diff --git a/docs/reference/cat/component-templates.asciidoc b/docs/reference/cat/component-templates.asciidoc
index 596c86befd1b7..8be1096a215f3 100644
--- a/docs/reference/cat/component-templates.asciidoc
+++ b/docs/reference/cat/component-templates.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>cat component templates</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib} 
diff --git a/docs/reference/cat/count.asciidoc b/docs/reference/cat/count.asciidoc
index 37e602c759020..7adcd1464dab1 100644
--- a/docs/reference/cat/count.asciidoc
+++ b/docs/reference/cat/count.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>cat count</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib} 
diff --git a/docs/reference/cat/datafeeds.asciidoc b/docs/reference/cat/datafeeds.asciidoc
index 506812fedabad..29f5bc8150af1 100644
--- a/docs/reference/cat/datafeeds.asciidoc
+++ b/docs/reference/cat/datafeeds.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>cat {dfeeds}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib} 
diff --git a/docs/reference/cat/dataframeanalytics.asciidoc b/docs/reference/cat/dataframeanalytics.asciidoc
index ed0f697c36d50..f00a9826ee5bd 100644
--- a/docs/reference/cat/dataframeanalytics.asciidoc
+++ b/docs/reference/cat/dataframeanalytics.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>cat {dfanalytics}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib} 
diff --git a/docs/reference/cat/fielddata.asciidoc b/docs/reference/cat/fielddata.asciidoc
index 376ef1d97057c..f11e40263ec2b 100644
--- a/docs/reference/cat/fielddata.asciidoc
+++ b/docs/reference/cat/fielddata.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>cat fielddata</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib} 
diff --git a/docs/reference/cat/health.asciidoc b/docs/reference/cat/health.asciidoc
index ad39ace310807..7ffc170ec8515 100644
--- a/docs/reference/cat/health.asciidoc
+++ b/docs/reference/cat/health.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>cat health</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib}
diff --git a/docs/reference/cat/indices.asciidoc b/docs/reference/cat/indices.asciidoc
index b8dda01c2eae0..3397c05f49735 100644
--- a/docs/reference/cat/indices.asciidoc
+++ b/docs/reference/cat/indices.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>cat indices</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib}
diff --git a/docs/reference/cat/master.asciidoc b/docs/reference/cat/master.asciidoc
index bcf2b876e4506..4ac40ff50be60 100644
--- a/docs/reference/cat/master.asciidoc
+++ b/docs/reference/cat/master.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>cat master</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib} 
diff --git a/docs/reference/cat/nodeattrs.asciidoc b/docs/reference/cat/nodeattrs.asciidoc
index ff37b430956aa..6c8093846030c 100644
--- a/docs/reference/cat/nodeattrs.asciidoc
+++ b/docs/reference/cat/nodeattrs.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>cat nodeattrs</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib} 
@@ -11,7 +17,7 @@ console. They are _not_ intended for use by applications. For application
 consumption, use the <<cluster-nodes-info,nodes info API>>.
 ====
 
-Returns information about <<shard-allocation-filtering,custom node attributes>>.
+Returns information about <<custom-node-attributes,custom node attributes>>.
 
 [[cat-nodeattrs-api-request]]
 ==== {api-request-title}
diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc
index 651c4ef3c7c2a..a5a813e8d37d5 100644
--- a/docs/reference/cat/nodes.asciidoc
+++ b/docs/reference/cat/nodes.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>cat nodes</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib}
@@ -233,6 +239,9 @@ Number of indexing operations, such as `1`.
 `indexing.index_failed`, `iif`, `indexingIndexFailed`::
 Number of failed indexing operations, such as `0`.
 
+`indexing.index_failed_due_to_version_conflict`, `iifvc`, `indexingIndexFailedDueToVersionConflict`::
+Number of failed indexing operations due to version conflict, such as `0`.
+
 `merges.current`, `mc`, `mergesCurrent`::
 Number of current merge operations, such as `0`.
 
diff --git a/docs/reference/cat/pending_tasks.asciidoc b/docs/reference/cat/pending_tasks.asciidoc
index f772cdb66d889..081a74da07552 100644
--- a/docs/reference/cat/pending_tasks.asciidoc
+++ b/docs/reference/cat/pending_tasks.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>cat pending tasks</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib} 
diff --git a/docs/reference/cat/plugins.asciidoc b/docs/reference/cat/plugins.asciidoc
index ae360862a0edb..c4d830ee52a76 100644
--- a/docs/reference/cat/plugins.asciidoc
+++ b/docs/reference/cat/plugins.asciidoc
@@ -4,6 +4,11 @@
 <titleabbrev>cat plugins</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
 
 [IMPORTANT]
 ====
diff --git a/docs/reference/cat/recovery.asciidoc b/docs/reference/cat/recovery.asciidoc
index 7393a8b719089..9df46f6fe93f6 100644
--- a/docs/reference/cat/recovery.asciidoc
+++ b/docs/reference/cat/recovery.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>cat recovery</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib} 
diff --git a/docs/reference/cat/repositories.asciidoc b/docs/reference/cat/repositories.asciidoc
index ec2f243c27bec..7e6283336e17a 100644
--- a/docs/reference/cat/repositories.asciidoc
+++ b/docs/reference/cat/repositories.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>cat repositories</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib} 
diff --git a/docs/reference/cat/segments.asciidoc b/docs/reference/cat/segments.asciidoc
index 82dc7298a0783..70b5e597eb95f 100644
--- a/docs/reference/cat/segments.asciidoc
+++ b/docs/reference/cat/segments.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>cat segments</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib} 
diff --git a/docs/reference/cat/shards.asciidoc b/docs/reference/cat/shards.asciidoc
index 87dcb01838bfd..2d3859e74c87e 100644
--- a/docs/reference/cat/shards.asciidoc
+++ b/docs/reference/cat/shards.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>cat shards</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib}
@@ -156,6 +162,9 @@ Number of indexing operations, such as `1`.
 `indexing.index_failed`, `iif`, `indexingIndexFailed`::
 Number of failed indexing operations, such as `0`.
 
+`indexing.index_failed_due_to_version_conflict`, `iifvc`, `indexingIndexFailedDueToVersionConflict`::
+Number of failed indexing operations due to version conflict, such as `0`.
+
 `merges.current`, `mc`, `mergesCurrent`::
 Number of current merge operations, such as `0`.
 
diff --git a/docs/reference/cat/snapshots.asciidoc b/docs/reference/cat/snapshots.asciidoc
index 820c4b56c783d..1da739b20272f 100644
--- a/docs/reference/cat/snapshots.asciidoc
+++ b/docs/reference/cat/snapshots.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>cat snapshots</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib} 
diff --git a/docs/reference/cat/tasks.asciidoc b/docs/reference/cat/tasks.asciidoc
index 91d67baa72d70..ff654b30de992 100644
--- a/docs/reference/cat/tasks.asciidoc
+++ b/docs/reference/cat/tasks.asciidoc
@@ -6,6 +6,12 @@
 
 beta::["The cat task management API is new and should still be considered a beta feature. The API may change in ways that are not backwards compatible.",{es-issue}51628]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib} 
diff --git a/docs/reference/cat/templates.asciidoc b/docs/reference/cat/templates.asciidoc
index bcc8e9e4f5dc4..78ff60c663d2f 100644
--- a/docs/reference/cat/templates.asciidoc
+++ b/docs/reference/cat/templates.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>cat templates</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib} 
diff --git a/docs/reference/cat/thread_pool.asciidoc b/docs/reference/cat/thread_pool.asciidoc
index 948ed9a1a7a30..1d8517f170aed 100644
--- a/docs/reference/cat/thread_pool.asciidoc
+++ b/docs/reference/cat/thread_pool.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>cat thread pool</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib} 
diff --git a/docs/reference/cat/trainedmodel.asciidoc b/docs/reference/cat/trainedmodel.asciidoc
index 5b20a0b6e842f..378238323f50b 100644
--- a/docs/reference/cat/trainedmodel.asciidoc
+++ b/docs/reference/cat/trainedmodel.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>cat trained model</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib} 
diff --git a/docs/reference/cat/transforms.asciidoc b/docs/reference/cat/transforms.asciidoc
index 53f22f02fbdbf..8e5becc5fa76d 100644
--- a/docs/reference/cat/transforms.asciidoc
+++ b/docs/reference/cat/transforms.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>cat transforms</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]..
+--
+
 [IMPORTANT]
 ====
 cat APIs are only intended for human consumption using the command line or {kib} 
diff --git a/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc
index b510163bab50b..2ad50d68b923a 100644
--- a/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc
+++ b/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete auto-follow pattern</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs].
+--
+
 Delete {ccr} <<ccr-auto-follow,auto-follow patterns>>.
 
 [[ccr-delete-auto-follow-pattern-request]]
diff --git a/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc
index a2969e993ddfb..951185d14e920 100644
--- a/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc
+++ b/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get auto-follow pattern</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs].
+--
+
 Get {ccr} <<ccr-auto-follow,auto-follow patterns>>.
 
 [[ccr-get-auto-follow-pattern-request]]
diff --git a/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc
index c5ae5a7b4af9d..462ee213ed4e4 100644
--- a/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc
+++ b/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Pause auto-follow pattern</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs].
+--
+
 Pauses a {ccr} <<ccr-auto-follow,auto-follow pattern>>.
 
 [[ccr-pause-auto-follow-pattern-request]]
diff --git a/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc
index 6769f21ca5cef..672a11302fdd5 100644
--- a/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc
+++ b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Create auto-follow pattern</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs].
+--
+
 Creates a {ccr} <<ccr-auto-follow,auto-follow pattern>>.
 
 [[ccr-put-auto-follow-pattern-request]]
diff --git a/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc
index a580bb3838f9b..d97c41d67c1ea 100644
--- a/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc
+++ b/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Resume auto-follow pattern</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs].
+--
+
 Resumes a {ccr} <<ccr-auto-follow,auto-follow pattern>>.
 
 [[ccr-resume-auto-follow-pattern-request]]
diff --git a/docs/reference/ccr/apis/ccr-apis.asciidoc b/docs/reference/ccr/apis/ccr-apis.asciidoc
index ae94e1931af85..0decb98197d31 100644
--- a/docs/reference/ccr/apis/ccr-apis.asciidoc
+++ b/docs/reference/ccr/apis/ccr-apis.asciidoc
@@ -2,6 +2,12 @@
 [[ccr-apis]]
 == {ccr-cap} APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs].
+--
+
 You can use the following APIs to perform <<xpack-ccr,{ccr}>> operations.
 
 [discrete]
diff --git a/docs/reference/ccr/apis/follow/get-follow-info.asciidoc b/docs/reference/ccr/apis/follow/get-follow-info.asciidoc
index 6c049d9c92b59..4c05faa0a7db8 100644
--- a/docs/reference/ccr/apis/follow/get-follow-info.asciidoc
+++ b/docs/reference/ccr/apis/follow/get-follow-info.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get follower info</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs].
+--
+
 Retrieves information about all <<xpack-ccr,{ccr}>> follower indices.
 
 [[ccr-get-follow-info-request]]
diff --git a/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc b/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc
index 4892f86b3523d..29000a98f64b4 100644
--- a/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc
+++ b/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get follower stats</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs].
+--
+
 Get <<xpack-ccr,{ccr}>> follower stats.
 
 [[ccr-get-follow-stats-request]]
diff --git a/docs/reference/ccr/apis/follow/post-forget-follower.asciidoc b/docs/reference/ccr/apis/follow/post-forget-follower.asciidoc
index 1917c08d6640d..8a7887072f6a2 100644
--- a/docs/reference/ccr/apis/follow/post-forget-follower.asciidoc
+++ b/docs/reference/ccr/apis/follow/post-forget-follower.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Forget follower</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs].
+--
+
 Removes the <<xpack-ccr,{ccr}>> follower retention leases from the leader.
 
 [[ccr-post-forget-follower-request]]
diff --git a/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc b/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc
index 6d4730d10efe6..c49e9a49b56c9 100644
--- a/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc
+++ b/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Pause follower</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs].
+--
+
 Pauses a <<xpack-ccr,{ccr}>> follower index.
 
 [[ccr-post-pause-follow-request]]
diff --git a/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc
index b023a8cb5cb70..f6da0110d5c24 100644
--- a/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc
+++ b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Resume follower</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs].
+--
+
 Resumes a <<xpack-ccr,{ccr}>> follower index.
 
 [[ccr-post-resume-follow-request]]
diff --git a/docs/reference/ccr/apis/follow/post-unfollow.asciidoc b/docs/reference/ccr/apis/follow/post-unfollow.asciidoc
index dab11ef9e7a54..56b3195e8a134 100644
--- a/docs/reference/ccr/apis/follow/post-unfollow.asciidoc
+++ b/docs/reference/ccr/apis/follow/post-unfollow.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Unfollow</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs].
+--
+
 Converts a <<xpack-ccr,{ccr}>> follower index to a regular index.
 
 [[ccr-post-unfollow-request]]
diff --git a/docs/reference/ccr/apis/follow/put-follow.asciidoc b/docs/reference/ccr/apis/follow/put-follow.asciidoc
index b7ae9ac987474..d9538fc436563 100644
--- a/docs/reference/ccr/apis/follow/put-follow.asciidoc
+++ b/docs/reference/ccr/apis/follow/put-follow.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Create follower</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs].
+--
+
 Creates a <<xpack-ccr,{ccr}>> follower index.
 
 [[ccr-put-follow-request]]
diff --git a/docs/reference/ccr/apis/get-ccr-stats.asciidoc b/docs/reference/ccr/apis/get-ccr-stats.asciidoc
index 92e6bae0bdce8..e92ad17e10437 100644
--- a/docs/reference/ccr/apis/get-ccr-stats.asciidoc
+++ b/docs/reference/ccr/apis/get-ccr-stats.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Get {ccr-init} stats</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs].
+--
+
 Get <<xpack-ccr,{ccr}>> stats.
 
 [[ccr-get-stats-request]]
diff --git a/docs/reference/cluster.asciidoc b/docs/reference/cluster.asciidoc
index 3de386d3288c6..398ece616fe07 100644
--- a/docs/reference/cluster.asciidoc
+++ b/docs/reference/cluster.asciidoc
@@ -1,6 +1,12 @@
 [[cluster]]
 == Cluster APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 ["float",id="cluster-nodes"]
 === Node specification
 
@@ -29,7 +35,7 @@ one of the following:
   master-eligible nodes, all data nodes, all ingest nodes, all voting-only
   nodes, all machine learning nodes, and all coordinating-only nodes.
 * a pair of patterns, using `*` wildcards, of the form `attrname:attrvalue`,
-  which adds to the subset all nodes with a custom node attribute whose name
+  which adds to the subset all nodes with a <<custom-node-attributes,custom node attribute>> whose name
   and value match the respective patterns. Custom node attributes are
   configured by setting properties in the configuration file of the form
   `node.attr.attrname: attrvalue`.
diff --git a/docs/reference/cluster/allocation-explain.asciidoc b/docs/reference/cluster/allocation-explain.asciidoc
index e640fa77c71ee..7099163cc98e9 100644
--- a/docs/reference/cluster/allocation-explain.asciidoc
+++ b/docs/reference/cluster/allocation-explain.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Cluster allocation explain</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 Provides an explanation for a shard's current <<index-modules-allocation,allocation>>.
 
 [source,console]
diff --git a/docs/reference/cluster/cluster-info.asciidoc b/docs/reference/cluster/cluster-info.asciidoc
index 7d67f1602aeaa..52ae637d8f46c 100644
--- a/docs/reference/cluster/cluster-info.asciidoc
+++ b/docs/reference/cluster/cluster-info.asciidoc
@@ -7,6 +7,12 @@ experimental::[]
 <titleabbrev>Cluster Info</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 Returns cluster information.
 
 [[cluster-info-api-request]]
diff --git a/docs/reference/cluster/delete-desired-balance.asciidoc b/docs/reference/cluster/delete-desired-balance.asciidoc
index c67834269e505..45fa147258b78 100644
--- a/docs/reference/cluster/delete-desired-balance.asciidoc
+++ b/docs/reference/cluster/delete-desired-balance.asciidoc
@@ -6,6 +6,12 @@
 
 NOTE: {cloud-only}
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 Discards the current <<shards-rebalancing-heuristics,desired balance>> and computes a new desired balance starting from the current allocation of shards.
 This can sometimes help {es} find a desired balance which needs fewer shard movements to achieve, especially if the
 cluster has experienced changes so substantial that the current desired balance is no longer optimal without {es} having
diff --git a/docs/reference/cluster/delete-desired-nodes.asciidoc b/docs/reference/cluster/delete-desired-nodes.asciidoc
index a58d19e2dfa3f..883bc22f6d964 100644
--- a/docs/reference/cluster/delete-desired-nodes.asciidoc
+++ b/docs/reference/cluster/delete-desired-nodes.asciidoc
@@ -6,6 +6,12 @@
 
 NOTE: {cloud-only}
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 Delete desired nodes.
 
 [[delete-desired-nodes-request]]
diff --git a/docs/reference/cluster/get-desired-balance.asciidoc b/docs/reference/cluster/get-desired-balance.asciidoc
index 74afdaa52daf1..3c6b1dc48719c 100644
--- a/docs/reference/cluster/get-desired-balance.asciidoc
+++ b/docs/reference/cluster/get-desired-balance.asciidoc
@@ -6,6 +6,12 @@
 
 NOTE: {cloud-only}
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 Exposes:
 
 * the <<shards-rebalancing-heuristics,desired balance>> computation and reconciliation stats
diff --git a/docs/reference/cluster/get-desired-nodes.asciidoc b/docs/reference/cluster/get-desired-nodes.asciidoc
index de27bd657b3ff..56af6913e34ac 100644
--- a/docs/reference/cluster/get-desired-nodes.asciidoc
+++ b/docs/reference/cluster/get-desired-nodes.asciidoc
@@ -6,6 +6,12 @@
 
 NOTE: {cloud-only}
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 Get desired nodes.
 
 [[get-desired-nodes-request]]
diff --git a/docs/reference/cluster/get-settings.asciidoc b/docs/reference/cluster/get-settings.asciidoc
index 32c186e4ef24c..5c0fe7a2026c7 100644
--- a/docs/reference/cluster/get-settings.asciidoc
+++ b/docs/reference/cluster/get-settings.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Cluster get settings</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 Returns cluster-wide settings.
 
 [source,console]
diff --git a/docs/reference/cluster/health.asciidoc b/docs/reference/cluster/health.asciidoc
index 94eb80a03d12e..374dd5d4a6f82 100644
--- a/docs/reference/cluster/health.asciidoc
+++ b/docs/reference/cluster/health.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Cluster health</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 Returns the health status of a cluster.
 
 [[cluster-health-api-request]]
diff --git a/docs/reference/cluster/nodes-hot-threads.asciidoc b/docs/reference/cluster/nodes-hot-threads.asciidoc
index 446a8fede69ec..f8b414453ae66 100644
--- a/docs/reference/cluster/nodes-hot-threads.asciidoc
+++ b/docs/reference/cluster/nodes-hot-threads.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Nodes hot threads</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 Returns the hot threads on each selected node in the cluster.
 
 [[cluster-nodes-hot-threads-api-request]]
diff --git a/docs/reference/cluster/nodes-info.asciidoc b/docs/reference/cluster/nodes-info.asciidoc
index 6f1d769e696c5..7ae6db7aa9a56 100644
--- a/docs/reference/cluster/nodes-info.asciidoc
+++ b/docs/reference/cluster/nodes-info.asciidoc
@@ -4,8 +4,13 @@
 <titleabbrev>Nodes info</titleabbrev>
 ++++
 
-Returns cluster nodes information.
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
 
+Returns cluster nodes information.
 
 [[cluster-nodes-info-api-request]]
 ==== {api-request-title}
diff --git a/docs/reference/cluster/nodes-reload-secure-settings.asciidoc b/docs/reference/cluster/nodes-reload-secure-settings.asciidoc
index f86304d2e9ba0..842ca30c335f9 100644
--- a/docs/reference/cluster/nodes-reload-secure-settings.asciidoc
+++ b/docs/reference/cluster/nodes-reload-secure-settings.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Nodes reload secure settings</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 Reloads the keystore on nodes in the cluster.
 
 [[cluster-nodes-reload-secure-settings-api-request]]
diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc
index adf8229712ecc..522983035079d 100644
--- a/docs/reference/cluster/nodes-stats.asciidoc
+++ b/docs/reference/cluster/nodes-stats.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Nodes stats</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 Returns cluster nodes statistics.
 
 [[cluster-nodes-stats-api-request]]
diff --git a/docs/reference/cluster/nodes-usage.asciidoc b/docs/reference/cluster/nodes-usage.asciidoc
index 486edf67bba87..c7994e32204a6 100644
--- a/docs/reference/cluster/nodes-usage.asciidoc
+++ b/docs/reference/cluster/nodes-usage.asciidoc
@@ -4,8 +4,13 @@
 <titleabbrev>Nodes feature usage</titleabbrev>
 ++++
 
-Returns information on the usage of features.
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
 
+Returns information on the usage of features.
 
 [[cluster-nodes-usage-api-request]]
 ==== {api-request-title}
diff --git a/docs/reference/cluster/pending.asciidoc b/docs/reference/cluster/pending.asciidoc
index 3e87234c7d26c..f5d42a6df76a6 100644
--- a/docs/reference/cluster/pending.asciidoc
+++ b/docs/reference/cluster/pending.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Pending cluster tasks</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 Returns cluster-level changes that have not yet been executed.
 
 
diff --git a/docs/reference/cluster/prevalidate-node-removal.asciidoc b/docs/reference/cluster/prevalidate-node-removal.asciidoc
index 16bf28c586687..0a09f1adda77c 100644
--- a/docs/reference/cluster/prevalidate-node-removal.asciidoc
+++ b/docs/reference/cluster/prevalidate-node-removal.asciidoc
@@ -6,6 +6,12 @@
 
 NOTE: {cloud-only}
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 Prevalidate node removal.
 
 [[prevalidate-node-removal-api-request]]
diff --git a/docs/reference/cluster/remote-info.asciidoc b/docs/reference/cluster/remote-info.asciidoc
index 8f2923846df0e..691acafd8ddbe 100644
--- a/docs/reference/cluster/remote-info.asciidoc
+++ b/docs/reference/cluster/remote-info.asciidoc
@@ -4,8 +4,13 @@
 <titleabbrev>Remote cluster info</titleabbrev>
 ++++
 
-Returns configured remote cluster information.
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
 
+Returns configured remote cluster information.
 
 [[cluster-remote-info-api-request]]
 ==== {api-request-title}
diff --git a/docs/reference/cluster/reroute.asciidoc b/docs/reference/cluster/reroute.asciidoc
index 429070f80b9bf..b393a9a68d2b2 100644
--- a/docs/reference/cluster/reroute.asciidoc
+++ b/docs/reference/cluster/reroute.asciidoc
@@ -4,8 +4,13 @@
 <titleabbrev>Cluster reroute</titleabbrev>
 ++++
 
-Changes the allocation of shards in a cluster.
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
 
+Changes the allocation of shards in a cluster.
 
 [[cluster-reroute-api-request]]
 ==== {api-request-title}
diff --git a/docs/reference/cluster/state.asciidoc b/docs/reference/cluster/state.asciidoc
index fcb2f5f2f5dcd..bf2863018893a 100644
--- a/docs/reference/cluster/state.asciidoc
+++ b/docs/reference/cluster/state.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Cluster state</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 Returns an internal representation of the cluster state for debugging or
 diagnostic purposes.
 
diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc
index d875417bde51a..f078fd2b7f2ee 100644
--- a/docs/reference/cluster/stats.asciidoc
+++ b/docs/reference/cluster/stats.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Cluster stats</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 Returns cluster statistics.
 
 [[cluster-stats-api-request]]
@@ -19,7 +25,6 @@ Returns cluster statistics.
 
 * If the {es} {security-features} are enabled, you must have the `monitor` or
 `manage` <<privileges-list-cluster,cluster privilege>> to use this API.
-
 [[cluster-stats-api-desc]]
 ==== {api-description-title}
 
@@ -1391,7 +1396,7 @@ as a human-readable string.
 
 
 `_search`:::
-(object) Contains the information about the <<modules-cross-cluster-search, {ccs}>> usage in the cluster.
+(object) Contains information about <<modules-cross-cluster-search, {ccs}>> usage.
 +
 .Properties of `_search`
 [%collapsible%open]
@@ -1522,7 +1527,11 @@ This may include requests where partial results were returned, but not requests
 
 =======
 
+
 ======
+`_esql`:::
+(object) Contains information about <<esql-cross-clusters,{esql} {ccs}>> usage.
+The structure of the object is the same as the `_search` object above.
 
 =====
 
diff --git a/docs/reference/cluster/tasks.asciidoc b/docs/reference/cluster/tasks.asciidoc
index 4b32d5f1b903a..79727d9a3078b 100644
--- a/docs/reference/cluster/tasks.asciidoc
+++ b/docs/reference/cluster/tasks.asciidoc
@@ -6,6 +6,12 @@
 
 beta::["The task management API is new and should still be considered a beta feature. The API may change in ways that are not backwards compatible.",{es-issue}51628]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-tasks[task management APIs].
+--
+
 Returns information about the tasks currently executing in the cluster.
 
 [[tasks-api-request]]
diff --git a/docs/reference/cluster/update-desired-nodes.asciidoc b/docs/reference/cluster/update-desired-nodes.asciidoc
index c72a2b53208e5..f83f551395134 100644
--- a/docs/reference/cluster/update-desired-nodes.asciidoc
+++ b/docs/reference/cluster/update-desired-nodes.asciidoc
@@ -6,6 +6,12 @@
 
 NOTE: {cloud-only}
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 Creates or updates the desired nodes.
 
 [[update-desired-nodes-request]]
diff --git a/docs/reference/cluster/update-settings.asciidoc b/docs/reference/cluster/update-settings.asciidoc
index 3d8bdcca07e2b..9a718ee413e64 100644
--- a/docs/reference/cluster/update-settings.asciidoc
+++ b/docs/reference/cluster/update-settings.asciidoc
@@ -4,8 +4,13 @@
 <titleabbrev>Cluster update settings</titleabbrev>
 ++++
 
-Configures <<dynamic-cluster-setting,dynamic cluster settings>>.
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
 
+Configures <<dynamic-cluster-setting,dynamic cluster settings>>.
 
 [[cluster-update-settings-api-request]]
 ==== {api-request-title}
diff --git a/docs/reference/cluster/voting-exclusions.asciidoc b/docs/reference/cluster/voting-exclusions.asciidoc
index 55587a7010f8f..e60b3be26508d 100644
--- a/docs/reference/cluster/voting-exclusions.asciidoc
+++ b/docs/reference/cluster/voting-exclusions.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Voting configuration exclusions</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 Adds or removes master-eligible nodes from the
 <<modules-discovery-voting,voting configuration exclusion list>>.
 
diff --git a/docs/reference/commands/node-tool.asciidoc b/docs/reference/commands/node-tool.asciidoc
index cdd2bb8f0f9d7..265006aa3df17 100644
--- a/docs/reference/commands/node-tool.asciidoc
+++ b/docs/reference/commands/node-tool.asciidoc
@@ -23,8 +23,8 @@ bin/elasticsearch-node repurpose|unsafe-bootstrap|detach-cluster|override-versio
 This tool has a number of modes:
 
 * `elasticsearch-node repurpose` can be used to delete unwanted data from a
-  node if it used to be a <<data-node,data node>> or a
-  <<master-node,master-eligible node>> but has been repurposed not to have one
+  node if it used to be a <<data-node-role,data node>> or a
+  <<master-node-role,master-eligible node>> but has been repurposed not to have one
   or other of these roles.
 
 * `elasticsearch-node remove-settings` can be used to remove persistent settings
diff --git a/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc
index 8cd8f71dc7aea..d4db8ce62bc46 100644
--- a/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc
+++ b/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc
@@ -6,6 +6,12 @@
 
 beta::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Cancels a connector sync job.
 
 To get started with Connector APIs, check out <<es-connectors-tutorial-api, our tutorial>>.
diff --git a/docs/reference/connector/apis/check-in-connector-api.asciidoc b/docs/reference/connector/apis/check-in-connector-api.asciidoc
index 15e65b10074d8..be7521e937316 100644
--- a/docs/reference/connector/apis/check-in-connector-api.asciidoc
+++ b/docs/reference/connector/apis/check-in-connector-api.asciidoc
@@ -6,6 +6,12 @@
 
 preview::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Updates the `last_seen` field of a connector with current timestamp.
 
 To get started with Connector APIs, check out <<es-connectors-tutorial-api, our tutorial>>.
diff --git a/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc
index 8d7d0a36ad88a..5f9b584621c2f 100644
--- a/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc
+++ b/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc
@@ -6,6 +6,12 @@
 
 preview::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Checks in a connector sync job (updates `last_seen` to the current time).
 
 To get started with Connector APIs, check out <<es-connectors-tutorial-api, our tutorial>>.
diff --git a/docs/reference/connector/apis/claim-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/claim-connector-sync-job-api.asciidoc
index 62491582ce757..565a39c2083af 100644
--- a/docs/reference/connector/apis/claim-connector-sync-job-api.asciidoc
+++ b/docs/reference/connector/apis/claim-connector-sync-job-api.asciidoc
@@ -6,6 +6,12 @@
 
 preview::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Claims a connector sync job.
 
 The `_claim` endpoint is not intended for direct connector management by users. It is there to support the implementation of services that utilize the https://github.com/elastic/connectors/blob/main/docs/CONNECTOR_PROTOCOL.md[Connector Protocol] to communicate with {es}.
diff --git a/docs/reference/connector/apis/connector-apis.asciidoc b/docs/reference/connector/apis/connector-apis.asciidoc
index 15ce31a605986..719db5a315714 100644
--- a/docs/reference/connector/apis/connector-apis.asciidoc
+++ b/docs/reference/connector/apis/connector-apis.asciidoc
@@ -1,6 +1,12 @@
 [[connector-apis]]
 == Connector APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 beta::[]
 
 The connector and sync jobs APIs provide a convenient way to create and manage Elastic <<es-connectors,connectors>>.
diff --git a/docs/reference/connector/apis/create-connector-api.asciidoc b/docs/reference/connector/apis/create-connector-api.asciidoc
index 3ecef6d302732..3300ce270c345 100644
--- a/docs/reference/connector/apis/create-connector-api.asciidoc
+++ b/docs/reference/connector/apis/create-connector-api.asciidoc
@@ -6,6 +6,12 @@
 
 beta::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Creates an Elastic connector.
 Connectors are {es} integrations that bring content from third-party data sources, which can be deployed on {ecloud} or hosted on your own infrastructure:
 
diff --git a/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc
index 240ab696954f3..1a66d47578a8b 100644
--- a/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc
+++ b/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc
@@ -6,6 +6,11 @@
 
 beta::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
 
 Creates a connector sync job.
 
diff --git a/docs/reference/connector/apis/delete-connector-api.asciidoc b/docs/reference/connector/apis/delete-connector-api.asciidoc
index 76621d7f1843b..f161a3c3b5933 100644
--- a/docs/reference/connector/apis/delete-connector-api.asciidoc
+++ b/docs/reference/connector/apis/delete-connector-api.asciidoc
@@ -6,8 +6,14 @@
 
 beta::[]
 
-Removes a connector and associated sync jobs.
-This is a destructive action that is not recoverable.
+
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
+Soft-deletes a connector and removes associated sync jobs.
 
 Note: this action doesn't delete any API key, ingest pipeline or data index associated with the connector. These need to be removed manually.
 
diff --git a/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc
index eeea40f430abd..bc906d12cae40 100644
--- a/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc
+++ b/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc
@@ -6,6 +6,12 @@
 
 beta::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Removes a connector sync job and its associated data.
 This is a destructive action that is not recoverable.
 
diff --git a/docs/reference/connector/apis/get-connector-api.asciidoc b/docs/reference/connector/apis/get-connector-api.asciidoc
index 302773e0af831..c8cbae668c261 100644
--- a/docs/reference/connector/apis/get-connector-api.asciidoc
+++ b/docs/reference/connector/apis/get-connector-api.asciidoc
@@ -6,6 +6,12 @@
 
 beta::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Retrieves the details about a connector.
 
 To get started with Connector APIs, check out <<es-connectors-tutorial-api, our tutorial>>.
@@ -27,6 +33,9 @@ To get started with Connector APIs, check out <<es-connectors-tutorial-api, our
 `<connector_id>`::
 (Required, string)
 
+`include_deleted`::
+(Optional, boolean) A flag indicating whether to also return connectors that have been soft-deleted. Defaults to `false`.
+
 [[get-connector-api-response-codes]]
 ==== {api-response-codes-title}
 
diff --git a/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc
index a524c1291c26a..f4ccc59e0315e 100644
--- a/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc
+++ b/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc
@@ -6,6 +6,12 @@
 
 beta::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Retrieves the details about a connector sync job.
 
 To get started with Connector APIs, check out <<es-connectors-tutorial-api, our tutorial>>.
diff --git a/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc b/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc
index 4a4fa5a22dcc1..b5f52e31ac296 100644
--- a/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc
+++ b/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc
@@ -7,6 +7,12 @@
 
 beta::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Returns information about all stored connector sync jobs ordered by their creation date in ascending order.
 
 To get started with Connector APIs, check out <<es-connectors-tutorial-api, our tutorial>>.
diff --git a/docs/reference/connector/apis/list-connectors-api.asciidoc b/docs/reference/connector/apis/list-connectors-api.asciidoc
index 4a93ecf2b0109..d334e5d92c232 100644
--- a/docs/reference/connector/apis/list-connectors-api.asciidoc
+++ b/docs/reference/connector/apis/list-connectors-api.asciidoc
@@ -7,6 +7,12 @@
 
 beta::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Returns information about all created connectors.
 
 To get started with Connector APIs, check out <<es-connectors-tutorial-api, our tutorial>>.
@@ -41,6 +47,9 @@ To get started with Connector APIs, check out <<es-connectors-tutorial-api, our
 `service_type`::
 (Optional, string) A comma-separated list of connector service types, used to filter search results.
 
+`include_deleted`::
+(Optional, boolean) A flag indicating whether to also return connectors that have been soft-deleted. Defaults to `false`.
+
 [[list-connector-api-example]]
 ==== {api-examples-title}
 
diff --git a/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc b/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc
index e6ad9e8cc93db..b5a0cd667f3ee 100644
--- a/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc
+++ b/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc
@@ -6,6 +6,12 @@
 
 preview::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Sets a connector sync job error.
 
 To get started with Connector APIs, check out <<es-connectors-tutorial-api, our tutorial>>.
diff --git a/docs/reference/connector/apis/set-connector-sync-job-stats-api.asciidoc b/docs/reference/connector/apis/set-connector-sync-job-stats-api.asciidoc
index 7e22f657ba6b6..5691280a30dd7 100644
--- a/docs/reference/connector/apis/set-connector-sync-job-stats-api.asciidoc
+++ b/docs/reference/connector/apis/set-connector-sync-job-stats-api.asciidoc
@@ -6,6 +6,12 @@
 
 preview::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Sets connector sync job stats.
 
 To get started with Connector APIs, check out <<es-connectors-tutorial-api, our tutorial>>.
diff --git a/docs/reference/connector/apis/update-connector-api-key-id-api.asciidoc b/docs/reference/connector/apis/update-connector-api-key-id-api.asciidoc
index fbd3f887758f2..8df49c43c128e 100644
--- a/docs/reference/connector/apis/update-connector-api-key-id-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-api-key-id-api.asciidoc
@@ -6,6 +6,12 @@
 
 beta::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Updates the `api_key_id` and/or `api_key_secret_id` field(s) of a connector, specifying:
 
 . The ID of the API key used for authorization
diff --git a/docs/reference/connector/apis/update-connector-configuration-api.asciidoc b/docs/reference/connector/apis/update-connector-configuration-api.asciidoc
index 4b25f9e71ae4b..d02c332d7f34b 100644
--- a/docs/reference/connector/apis/update-connector-configuration-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-configuration-api.asciidoc
@@ -6,6 +6,12 @@
 
 beta::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Updates a connector's `configuration`, allowing for config value updates within a registered configuration schema.
 
 To get started with Connector APIs, check out <<es-connectors-tutorial-api, our tutorial>>.
diff --git a/docs/reference/connector/apis/update-connector-error-api.asciidoc b/docs/reference/connector/apis/update-connector-error-api.asciidoc
index 29358b243041a..859a48c31c0ca 100644
--- a/docs/reference/connector/apis/update-connector-error-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-error-api.asciidoc
@@ -6,6 +6,12 @@
 
 preview::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Updates the `error` field of a connector.
 
 To get started with Connector APIs, check out <<es-connectors-tutorial-api, our tutorial>>.
diff --git a/docs/reference/connector/apis/update-connector-features-api.asciidoc b/docs/reference/connector/apis/update-connector-features-api.asciidoc
index 77571fcd7d5a0..74c512e42cd1b 100644
--- a/docs/reference/connector/apis/update-connector-features-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-features-api.asciidoc
@@ -6,6 +6,12 @@
 
 beta::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Manages the `features` of a connector. This endpoint can be used to control the following aspects of a connector:
 
 * document-level security
diff --git a/docs/reference/connector/apis/update-connector-filtering-api.asciidoc b/docs/reference/connector/apis/update-connector-filtering-api.asciidoc
index 4820fa151901d..7ba0080cde28f 100644
--- a/docs/reference/connector/apis/update-connector-filtering-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-filtering-api.asciidoc
@@ -6,6 +6,12 @@
 
 beta::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Updates the draft `filtering` configuration of a connector and marks the draft validation state as `edited`. The filtering draft is activated once validated by the running Elastic connector service.
 
 The filtering property is used to configure sync rules (both basic and advanced) for a connector. Learn more in the <<es-sync-rules>>.
diff --git a/docs/reference/connector/apis/update-connector-index-name-api.asciidoc b/docs/reference/connector/apis/update-connector-index-name-api.asciidoc
index 6222baf6a6caf..fbbef6e66ca40 100644
--- a/docs/reference/connector/apis/update-connector-index-name-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-index-name-api.asciidoc
@@ -6,6 +6,12 @@
 
 beta::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Updates the `index_name` field of a connector, specifying the index where the data ingested by the connector is stored.
 
 To get started with Connector APIs, check out <<es-connectors-tutorial-api, our tutorial>>.
diff --git a/docs/reference/connector/apis/update-connector-last-sync-api.asciidoc b/docs/reference/connector/apis/update-connector-last-sync-api.asciidoc
index 17f892d852f4a..d827d25c12b4e 100644
--- a/docs/reference/connector/apis/update-connector-last-sync-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-last-sync-api.asciidoc
@@ -6,6 +6,12 @@
 
 preview::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Updates the fields related to the last sync of a connector.
 
 This action is used for analytics and monitoring.
diff --git a/docs/reference/connector/apis/update-connector-name-description-api.asciidoc b/docs/reference/connector/apis/update-connector-name-description-api.asciidoc
index 384cec2c73e24..c0d0568baef35 100644
--- a/docs/reference/connector/apis/update-connector-name-description-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-name-description-api.asciidoc
@@ -6,6 +6,11 @@
 
 beta::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
 
 Updates the `name` and `description` fields of a connector.
 
diff --git a/docs/reference/connector/apis/update-connector-pipeline-api.asciidoc b/docs/reference/connector/apis/update-connector-pipeline-api.asciidoc
index e54b01ec47d01..a886fe6f20da7 100644
--- a/docs/reference/connector/apis/update-connector-pipeline-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-pipeline-api.asciidoc
@@ -6,6 +6,12 @@
 
 beta::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Updates the `pipeline` configuration of a connector.
 
 When you create a new connector, the configuration of an <<ingest-pipeline-search-details-generic-reference, ingest pipeline>> is populated with default settings.
diff --git a/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc b/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc
index 64302c26a7231..eed3d14ea1d97 100644
--- a/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc
@@ -6,6 +6,12 @@
 
 beta::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Updates the `scheduling` configuration of a connector.
 
 To get started with Connector APIs, check out <<es-connectors-tutorial-api, our tutorial>>.
diff --git a/docs/reference/connector/apis/update-connector-service-type-api.asciidoc b/docs/reference/connector/apis/update-connector-service-type-api.asciidoc
index c02967d03e2dd..28b4c72b682d2 100644
--- a/docs/reference/connector/apis/update-connector-service-type-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-service-type-api.asciidoc
@@ -6,6 +6,12 @@
 
 beta::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Updates the `service_type` of a connector.
 
 To get started with Connector APIs, check out <<es-connectors-tutorial-api, our tutorial>>.
diff --git a/docs/reference/connector/apis/update-connector-status-api.asciidoc b/docs/reference/connector/apis/update-connector-status-api.asciidoc
index dadd93fe5f9c4..7812cbff89d1b 100644
--- a/docs/reference/connector/apis/update-connector-status-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-status-api.asciidoc
@@ -6,6 +6,12 @@
 
 preview::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
+--
+
 Updates the `status` of a connector.
 
 To get started with Connector APIs, check out <<es-connectors-tutorial-api, our tutorial>>.
diff --git a/docs/reference/connector/docs/connectors-content-extraction.asciidoc b/docs/reference/connector/docs/connectors-content-extraction.asciidoc
index a87d38c9bf531..744fe1d87cb45 100644
--- a/docs/reference/connector/docs/connectors-content-extraction.asciidoc
+++ b/docs/reference/connector/docs/connectors-content-extraction.asciidoc
@@ -8,7 +8,7 @@ The logic for content extraction is defined in {connectors-python}/connectors/ut
 While intended primarily for PDF and Microsoft Office formats, you can use any of the <<es-connectors-content-extraction-supported-file-types, supported formats>>.
 
 Enterprise Search uses an {ref}/ingest.html[Elasticsearch ingest pipeline^] to power the web crawler's binary content extraction.
-The default pipeline, `ent-search-generic-ingestion`, is automatically created when Enterprise Search first starts.
+The default pipeline, `search-default-ingestion`, is automatically created when Enterprise Search first starts.
 
 You can {ref}/ingest.html#create-manage-ingest-pipelines[view^] this pipeline in Kibana.
 Customizing your pipeline usage is also an option.
diff --git a/docs/reference/connector/docs/connectors-filter-extract-transform.asciidoc b/docs/reference/connector/docs/connectors-filter-extract-transform.asciidoc
index 278478c908bf0..62a99928bfb46 100644
--- a/docs/reference/connector/docs/connectors-filter-extract-transform.asciidoc
+++ b/docs/reference/connector/docs/connectors-filter-extract-transform.asciidoc
@@ -13,7 +13,7 @@ The following diagram provides an overview of how content extraction, sync rules
 [.screenshot]
 image::images/pipelines-extraction-sync-rules.png[Architecture diagram of data pipeline with content extraction, sync rules, and ingest pipelines]
 
-By default, only the connector specific logic (2) and the default `ent-search-generic-ingestion` pipeline (6) extract and transform your data, as configured in your deployment.
+By default, only the connector specific logic (2) and the default `search-default-ingestion` pipeline (6) extract and transform your data, as configured in your deployment.
 
 The following tools are available for more advanced use cases:
 
@@ -50,4 +50,4 @@ Use ingest pipelines for data enrichment, normalization, and more.
 
 Elastic connectors use a default ingest pipeline, which you can copy and customize to meet your needs.
 
-Refer to {ref}/ingest-pipeline-search.html[ingest pipelines in Search] in the {es} documentation.
\ No newline at end of file
+Refer to {ref}/ingest-pipeline-search.html[ingest pipelines in Search] in the {es} documentation.
diff --git a/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc b/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc
index 02f598c16f63c..d09e089f194ad 100644
--- a/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc
+++ b/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc
@@ -75,12 +75,10 @@ Follow these steps:
 * Leave the *Redirect URIs* blank for now.
 * *Register* the application.
 * Find and keep the **Application (client) ID** and **Directory (tenant) ID** handy.
-* Locate the **Secret** by navigating to **Client credentials: Certificates & Secrets**.
-* Select **New client secret**
-* Pick a name for your client secret.
-Select an expiration date. (At this expiration date, you will need to generate a new secret and update your connector configuration.)
-** Save the client secret **Secret ID** before leaving this screen.
-** Save the client secret **Value** before leaving this screen.
+* Create a certificate and private key. This can, for example, be done by running `openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout azure_app.key -out azure_app.crt` command. Store both in a safe and secure place
+* Locate the **Certificates** by navigating to **Client credentials: Certificates & Secrets**.
+* Select **Upload certificate**
+* Upload the certificate created in one of previous steps: `azure_app.crt`
 * Set up the permissions the OAuth App will request from the Azure Portal service account.
 ** Navigate to **API Permissions** and click **Add Permission**.
 ** Add **application permissions** until the list looks like the following:
@@ -114,6 +112,24 @@ When entities are not available via the Graph API the connector falls back to us
 [discrete#es-connectors-sharepoint-online-oauth-app-permissions]
 ====== SharePoint permissions
 
+Microsoft is https://learn.microsoft.com/en-us/sharepoint/dev/sp-add-ins/retirement-announcement-for-azure-acs[retiring Azure Access Control Service (ACS)]. This affects permission configuration:
+
+* *Tenants created after November 1st, 2024*: Certificate authentication is required
+* *Tenants created before November 1st, 2024*: Secret-based authentication must be migrated to certificate authentication by April 2nd, 2026
+
+[discrete#es-connectors-sharepoint-online-oauth-app-certificate-auth]
+===== Certificate Authentication
+
+This authentication method does not require additional setup other than creating and uploading certificates to the OAuth App.
+
+[discrete#es-connectors-sharepoint-online-oauth-app-secret-auth]
+===== Secret Authentication
+
+[IMPORTANT]
+====
+This method is only applicable to tenants created before November 1st, 2024. This method will be fully retired as of April 2nd, 2026.
+====
+
 Refer to the following documentation for setting https://learn.microsoft.com/en-us/sharepoint/dev/solution-guidance/security-apponly-azureacs[SharePoint permissions^].
 
 * To set `DisableCustomAppAuthentication` to false, connect to SharePoint using PowerShell and run `set-spotenant -DisableCustomAppAuthentication $false`
@@ -133,6 +149,58 @@ The application name will appear in the Title box.
 </AppPermissionRequests>
 ----
 
+[discrete#es-connectors-sharepoint-online-sites-selected-permissions]
+====== Granting `Sites.Selected` permissions
+
+To configure `Sites.Selected` permissions, follow these steps in the Azure Active Directory portal. These permissions enable precise access control to specific SharePoint sites.
+
+. Sign in to the https://portal.azure.com/[Azure Active Directory portal^].
+. Navigate to **App registrations** and locate the application created for the connector.
+. Under **API permissions**, click **Add permission**.
+. Select **Microsoft Graph** > **Application permissions**, then add `Sites.Selected`.
+. Click **Grant admin consent** to approve the permission.
+
+[TIP]
+====
+Refer to the official https://learn.microsoft.com/en-us/graph/permissions-reference[Microsoft documentation] for managing permissions in Azure AD.
+====
+
+To assign access to specific SharePoint sites using `Sites.Selected`:
+
+. Use Microsoft Graph Explorer or PowerShell to grant access.
+. To fetch the site ID, run the following Graph API query:
++
+[source, http]
+----
+GET https://graph.microsoft.com/v1.0/sites?select=webUrl,Title,Id&$search="<Name of the site>*"
+----
++
+This will return the `id` of the site.
+
+. Use the `id` to assign read or write access:
++
+[source, http]
+----
+POST https://graph.microsoft.com/v1.0/sites/<siteId>/permissions
+{
+    "roles": ["read"], // or "write"
+    "grantedToIdentities": [
+        {
+            "application": {
+                "id": "<App_Client_ID>",
+                "displayName": "<App_Display_Name>"
+            }
+        }
+    ]
+}
+----
+
+[NOTE]
+====
+When using the `Comma-separated list of sites` configuration field, ensure the sites specified match those granted `Sites.Selected` permission in SharePoint.
+If the `Comma-separated list of sites` field is set to `*` or the `Enumerate all sites` toggle is enabled, the connector will attempt to access all sites. This requires broader permissions, which are not supported with `Sites.Selected`.
+====
+
 .Graph API permissions
 ****
 Microsoft recommends using Graph API for all operations with Sharepoint Online. Graph API is well-documented and more efficient at fetching data, which helps avoid throttling.
@@ -167,8 +235,17 @@ The tenant name for the Azure account hosting the Sharepoint Online instance.
 Client ID::
 The client id to authenticate with SharePoint Online.
 
+Authentication Method::
+Authentication method to use to connector to Sharepoint Online and Rest APIs. `secret` is deprecated and `certificate` is recommended.
+
 Secret value::
-The secret value to authenticate with SharePoint Online.
+The secret value to authenticate with SharePoint Online, if Authentication Method: `secret` is chosen.
+
+Content of certificate file::
+Content of certificate file if Authentication Method: `certificate` is chosen.
+
+Content of private key file::
+Content of private key file if Authentication Method: `certificate` is chosen.
 
 Comma-separated list of sites::
 List of site collection names or paths to fetch from SharePoint.
@@ -536,12 +613,10 @@ Follow these steps:
 * Leave the *Redirect URIs* blank for now.
 * *Register* the application.
 * Find and keep the **Application (client) ID** and **Directory (tenant) ID** handy.
-* Locate the **Secret** by navigating to **Client credentials: Certificates & Secrets**.
-* Select **New client secret**
-* Pick a name for your client secret.
-Select an expiration date. (At this expiration date, you will need to generate a new secret and update your connector configuration.)
-** Save the client secret **Secret ID** before leaving this screen.
-** Save the client secret **Value** before leaving this screen.
+* Create a certificate and private key. This can, for example, be done by running `openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout azure_app.key -out azure_app.crt` command. Store both in a safe and secure place
+* Locate the **Certificates** by navigating to **Client credentials: Certificates & Secrets**.
+* Select **Upload certificate**
+* Upload the certificate created in one of previous steps: `azure_app.crt`
 * Set up the permissions the OAuth App will request from the Azure Portal service account.
 ** Navigate to **API Permissions** and click **Add Permission**.
 ** Add **application permissions** until the list looks like the following:
@@ -575,6 +650,23 @@ When entities are not available via the Graph API the connector falls back to us
 [discrete#es-connectors-sharepoint-online-client-oauth-app-permissions]
 ====== SharePoint permissions
 
+Microsoft is https://learn.microsoft.com/en-us/sharepoint/dev/sp-add-ins/retirement-announcement-for-azure-acs[retiring Azure Access Control Service (ACS)]. This affects permission configuration:
+* *Tenants created after November 1st, 2024*: Certificate authentication is required
+* *Tenants created before November 1st, 2024*: Secret-based authentication must be migrated to certificate authentication by April 2nd, 2026
+
+[discrete#es-connectors-sharepoint-online-client-oauth-app-certificate-auth]
+===== Certificate Authentication
+
+This authentication method does not require additional setup other than creating and uploading certificates to the OAuth App.
+
+[discrete#es-connectors-sharepoint-online-client-oauth-app-secret-auth]
+===== Secret Authentication
+
+[IMPORTANT]
+====
+This method is only applicable to tenants created before November 1st, 2024. This method will be fully retired as of April 2nd, 2026.
+====
+
 Refer to the following documentation for setting https://learn.microsoft.com/en-us/sharepoint/dev/solution-guidance/security-apponly-azureacs[SharePoint permissions^].
 
 * To set `DisableCustomAppAuthentication` to false, connect to SharePoint using PowerShell and run `set-spotenant -DisableCustomAppAuthentication $false`
@@ -594,6 +686,59 @@ The application name will appear in the Title box.
 </AppPermissionRequests>
 ----
 
+[discrete#es-connectors-sharepoint-online-sites-selected-permissions-self-managed]
+====== Granting `Sites.Selected` permissions
+
+To configure `Sites.Selected` permissions, follow these steps in the Azure Active Directory portal. These permissions enable precise access control to specific SharePoint sites.
+
+. Sign in to the https://portal.azure.com/[Azure Active Directory portal^].
+. Navigate to **App registrations** and locate the application created for the connector.
+. Under **API permissions**, click **Add permission**.
+. Select **Microsoft Graph** > **Application permissions**, then add `Sites.Selected`.
+. Click **Grant admin consent** to approve the permission.
+
+[TIP]
+====
+Refer to the official https://learn.microsoft.com/en-us/graph/permissions-reference[Microsoft documentation] for managing permissions in Azure AD.
+====
+
+
+To assign access to specific SharePoint sites using `Sites.Selected`:
+
+. Use Microsoft Graph Explorer or PowerShell to grant access.
+. To fetch the site ID, run the following Graph API query:
++
+[source, http]
+----
+GET https://graph.microsoft.com/v1.0/sites?select=webUrl,Title,Id&$search="<Name of the site>*"
+----
++
+This will return the `id` of the site.
+
+. Use the `id` to assign read or write access:
++
+[source, http]
+----
+POST https://graph.microsoft.com/v1.0/sites/<siteId>/permissions
+{
+    "roles": ["read"], // or "write"
+    "grantedToIdentities": [
+        {
+            "application": {
+                "id": "<App_Client_ID>",
+                "displayName": "<App_Display_Name>"
+            }
+        }
+    ]
+}
+----
+
+[NOTE]
+====
+When using the `Comma-separated list of sites` configuration field, ensure the sites specified match those granted `Sites.Selected` permission in SharePoint.
+If the `Comma-separated list of sites` field is set to `*` or the `Enumerate all sites` toggle is enabled, the connector will attempt to access all sites. This requires broader permissions, which are not supported with `Sites.Selected`.
+====
+
 .Graph API permissions
 ****
 Microsoft recommends using Graph API for all operations with Sharepoint Online. Graph API is well-documented and more efficient at fetching data, which helps avoid throttling.
@@ -637,8 +782,17 @@ The tenant name for the Azure account hosting the Sharepoint Online instance.
 `client_id`::
 The client id to authenticate with SharePoint Online.
 
+`auth_method`::
+Authentication method to use to connector to Sharepoint Online and Rest APIs. `secret` is deprecated and `certificate` is recommended.
+
 `secret_value`::
-The secret value to authenticate with SharePoint Online.
+The secret value to authenticate with SharePoint Online, if auth_method: `secret` is chosen.
+
+`certificate`::
+Content of certificate file if auth_method: `certificate` is chosen.
+
+`private_key`::
+Content of private key file if auth_method: `certificate` is chosen.
 
 `site_collections`::
 List of site collection names or paths to fetch from SharePoint.
diff --git a/docs/reference/data-management.asciidoc b/docs/reference/data-management.asciidoc
index 4245227a1524d..4da62e5b2c7c0 100644
--- a/docs/reference/data-management.asciidoc
+++ b/docs/reference/data-management.asciidoc
@@ -6,29 +6,27 @@
 --
 The data you store in {es} generally falls into one of two categories:
 
-* Content: a collection of items you want to search, such as a catalog of products
-* Time series data: a stream of continuously-generated timestamped data, such as log entries
+* *Content*: a collection of items you want to search, such as a catalog of products
+* *Time series data*: a stream of continuously-generated timestamped data, such as log entries
 
-Content might be frequently updated,
+*Content* might be frequently updated,
 but the value of the content remains relatively constant over time.
 You want to be able to retrieve items quickly regardless of how old they are.
 
-Time series data keeps accumulating over time, so you need strategies for
+*Time series data* keeps accumulating over time, so you need strategies for
 balancing the value of the data against the cost of storing it.
 As it ages, it tends to become less important and less-frequently accessed,
 so you can move it to less expensive, less performant hardware.
 For your oldest data, what matters is that you have access to the data.
 It's ok if queries take longer to complete.
 
-To help you manage your data, {es} offers you:
+To help you manage your data, {es} offers you the following options:
 
-* <<index-lifecycle-management, {ilm-cap}>> ({ilm-init}) to manage both indices and data streams and it is fully customisable, and
-* <<data-stream-lifecycle, Data stream lifecycle>> which is the built-in lifecycle of data streams and addresses the most
-common lifecycle management needs.
+* <<index-lifecycle-management, {ilm-cap}>>
+* <<data-stream-lifecycle, Data stream lifecycle>>
+* {curator-ref-current}/about.html[Elastic Curator]
 
-preview::["The built-in data stream lifecycle is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but this feature is not subject to the support SLA of official GA features."]
-
-**{ilm-init}** can be used to manage both indices and data streams and it allows you to:
+**{ilm-init}** can be used to manage both indices and data streams. It allows you to do the following:
 
 * Define the retention period of your data. The retention period is the minimum time your data will be stored in {es}.
 Data older than this period can be deleted by {es}.
@@ -38,12 +36,24 @@ Data older than this period can be deleted by {es}.
 for your older indices while reducing operating costs and maintaining search performance.
 * Perform <<async-search-intro, asynchronous searches>> of data stored on less-performant hardware.
 
-**Data stream lifecycle** is less feature rich but is focused on simplicity, so it allows you to easily:
+**Data stream lifecycle** is less feature rich but is focused on simplicity. It allows you to do the following:
 
 * Define the retention period of your data. The retention period is the minimum time your data will be stored in {es}.
 Data older than this period can be deleted by {es} at a later time.
-* Improve the performance of your data stream by performing background operations that will optimise the way your data
-stream is stored.
+* Improve the performance of your data stream by performing background operations that will optimise the way your data stream is stored.
+
+**Elastic Curator** is a tool that allows you to manage your indices and snapshots using user-defined filters and predefined actions. If ILM provides the functionality to manage your index lifecycle, and you have at least a Basic license, consider using ILM in place of Curator. Many stack components make use of ILM by default. {curator-ref-current}/ilm.html[Learn more].
+
+NOTE: <<xpack-rollup,Data rollup>> is a deprecated {es} feature that allows you to manage the amount of data that is stored in your cluster, similar to the downsampling functionality of {ilm-init} and data stream lifecycle. This feature should not be used for new deployments.
+
+[TIP]
+====
+{ilm-init} is not available on {es-serverless}.
+
+In an {ecloud} or self-managed environment, ILM lets you automatically transition indices through data tiers according to your performance needs and retention requirements. This allows you to balance hardware costs with performance. {es-serverless} eliminates this complexity by optimizing your cluster performance for you.
+
+Data stream lifecycle is an optimized lifecycle tool that lets you focus on the most common lifecycle management needs, without unnecessary hardware-centric concepts like data tiers.
+====
 --
 
 include::ilm/index.asciidoc[]
diff --git a/docs/reference/data-management/migrate-index-allocation-filters.asciidoc b/docs/reference/data-management/migrate-index-allocation-filters.asciidoc
index 85d42e4105a92..ee7d5640d53df 100644
--- a/docs/reference/data-management/migrate-index-allocation-filters.asciidoc
+++ b/docs/reference/data-management/migrate-index-allocation-filters.asciidoc
@@ -2,7 +2,7 @@
 [[migrate-index-allocation-filters]]
 == Migrate index allocation filters to node roles
 
-If you currently use custom node attributes and
+If you currently use <<custom-node-attributes,custom node attributes>> and
 <<shard-allocation-filtering, attribute-based allocation filters>> to
 move indices through <<data-tiers, data tiers>> in a
 https://www.elastic.co/blog/implementing-hot-warm-cold-in-elasticsearch-with-index-lifecycle-management[hot-warm-cold architecture],
diff --git a/docs/reference/data-store-architecture.asciidoc b/docs/reference/data-store-architecture.asciidoc
new file mode 100644
index 0000000000000..a0d504eb117c8
--- /dev/null
+++ b/docs/reference/data-store-architecture.asciidoc
@@ -0,0 +1,24 @@
+= Data store architecture
+
+[partintro]
+--
+
+{es} is a distributed document store. Instead of storing information as rows of columnar data, {es} stores complex data structures that have been serialized as JSON documents. When you have multiple {es} nodes in a cluster, stored documents are distributed across the cluster and can be accessed immediately
+from any node.
+
+The topics in this section provides information about the architecture of {es} and how it stores and retrieves data: 
+
+* <<nodes-shards,Nodes and shards>>: Learn about the basic building blocks of an {es} cluster, including nodes, shards, primaries, and replicas.
+* <<node-roles-overview,Node roles>>: Learn about the different roles that nodes can have in an {es} cluster.
+* <<docs-replication,Reading and writing documents>>: Learn how {es} replicates read and write operations across shards and shard copies.
+* <<shard-allocation-relocation-recovery,Shard allocation, relocation, and recovery>>: Learn how {es} allocates and balances shards across nodes.
+** <<shard-allocation-awareness,Shard allocation awareness>>: Learn how to use custom node attributes to distribute shards across different racks or availability zones.
+* <<shard-request-cache,Shard request cache>>: Learn how {es} caches search requests to improve performance.
+--
+
+include::nodes-shards.asciidoc[]
+include::node-roles.asciidoc[]
+include::docs/data-replication.asciidoc[leveloffset=-1]
+include::modules/shard-ops.asciidoc[]
+include::modules/cluster/allocation_awareness.asciidoc[leveloffset=+1]
+include::shard-request-cache.asciidoc[leveloffset=-1]
diff --git a/docs/reference/data-streams/data-stream-apis.asciidoc b/docs/reference/data-streams/data-stream-apis.asciidoc
index c13703ab2a6ee..8b952fad59f8d 100644
--- a/docs/reference/data-streams/data-stream-apis.asciidoc
+++ b/docs/reference/data-streams/data-stream-apis.asciidoc
@@ -2,6 +2,12 @@
 [[data-stream-apis]]
 == Data stream APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs].
+--
+
 The following APIs are available for managing <<data-streams,data streams>>:
 
 * <<indices-create-data-stream>>
diff --git a/docs/reference/data-streams/downsampling.asciidoc b/docs/reference/data-streams/downsampling.asciidoc
index 0b08b0972f9a1..10a0241cf0732 100644
--- a/docs/reference/data-streams/downsampling.asciidoc
+++ b/docs/reference/data-streams/downsampling.asciidoc
@@ -72,6 +72,45 @@ the granularity of `cold` archival data to monthly or less.
 .Downsampled metrics series
 image::images/data-streams/time-series-downsampled.png[align="center"]
 
+[discrete]
+[[downsample-api-process]]
+==== The downsampling process
+
+The downsampling operation traverses the source TSDS index and performs the
+following steps:
+
+. Creates a new document for each value of the `_tsid` field and each
+`@timestamp` value, rounded to the `fixed_interval` defined in the downsample
+configuration.
+. For each new document, copies all <<time-series-dimension,time
+series dimensions>> from the source index to the target index. Dimensions in a
+TSDS are constant, so this is done only once per bucket.
+. For each <<time-series-metric,time series metric>> field, computes aggregations
+for all documents in the bucket. Depending on the metric type of each metric
+field a different set of pre-aggregated results is stored:
+
+** `gauge`: The `min`, `max`, `sum`, and `value_count` are stored; `value_count`
+is stored as type `aggregate_metric_double`.
+** `counter`: The `last_value` is stored.
+. For all other fields, the most recent value is copied to the target index.
+
+[discrete]
+[[downsample-api-mappings]]
+==== Source and target index field mappings
+
+Fields in the target, downsampled index are created based on fields in the
+original source index, as follows:
+
+. All fields mapped with the `time-series-dimension` parameter are created in
+the target downsample index with the same mapping as in the source index.
+. All fields mapped with the `time_series_metric` parameter are created
+in the target downsample index with the same mapping as in the source
+index. An exception is that for fields mapped as `time_series_metric: gauge`
+the field type is changed to `aggregate_metric_double`.
+. All other fields that are neither dimensions nor metrics (that is, label
+fields), are created in the target downsample index with the same mapping
+that they had in the source index.
+
 [discrete]
 [[running-downsampling]]
 === Running downsampling on time series data
diff --git a/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc
index 315f7fa85e45f..5222d33b5870b 100644
--- a/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc
+++ b/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Delete Data Stream Lifecycle</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs].
+--
+
 Deletes the <<data-stream-lifecycle,lifecycle>> from a set of data streams.
 
 [[delete-lifecycle-api-prereqs]]
diff --git a/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc
index 2b15886ebe192..8289fb54d51bd 100644
--- a/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc
+++ b/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Explain Data Stream Lifecycle</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs].
+--
+
 Retrieves the current <<data-stream-lifecycle,data stream lifecycle>> status for one or more data stream backing indices.
 
 [[explain-lifecycle-api-prereqs]]
diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc
index f48fa1eb52daa..0fbe7de287f7b 100644
--- a/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc
+++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get Data Stream Lifecycle</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs].
+--
+
 Gets stats about the execution of <<data-stream-lifecycle,data stream lifecycle>>.
 
 [[get-lifecycle-stats-api-prereqs]]
diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc
index 6323fac1eac2f..57d63fee2ddc1 100644
--- a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc
+++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get Data Stream Lifecycle</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs].
+--
+
 Gets the <<data-stream-lifecycle,lifecycle>> of a set of <<data-streams,data streams>>.
 
 [[get-lifecycle-api-prereqs]]
diff --git a/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc
index c60c105e818ab..c5002cf4882e2 100644
--- a/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc
+++ b/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Put Data Stream Lifecycle</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs].
+--
+
 Configures the data stream <<data-stream-lifecycle,lifecycle>> for the targeted <<data-streams,data streams>>.
 
 [[put-lifecycle-api-prereqs]]
diff --git a/docs/reference/data-streams/modify-data-streams-api.asciidoc b/docs/reference/data-streams/modify-data-streams-api.asciidoc
index 2da869083df22..2f717f9ec3b4b 100644
--- a/docs/reference/data-streams/modify-data-streams-api.asciidoc
+++ b/docs/reference/data-streams/modify-data-streams-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Modify data streams</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs].
+--
+
 Performs one or more <<data-streams,data stream>> modification actions in a single atomic
 operation.
 
diff --git a/docs/reference/data-streams/promote-data-stream-api.asciidoc b/docs/reference/data-streams/promote-data-stream-api.asciidoc
index 5ba9c4d9fad0e..33005e80e9408 100644
--- a/docs/reference/data-streams/promote-data-stream-api.asciidoc
+++ b/docs/reference/data-streams/promote-data-stream-api.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Promote data stream</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs].
+--
+
 The purpose of the promote <<data-streams,data stream>> API is to turn
 a data stream that is replicated by CCR into a regular
 data stream.
diff --git a/docs/reference/datatiers.asciidoc b/docs/reference/datatiers.asciidoc
index 65e029d876e6f..066765368ec5e 100644
--- a/docs/reference/datatiers.asciidoc
+++ b/docs/reference/datatiers.asciidoc
@@ -189,7 +189,7 @@ tier].
 [[configure-data-tiers-on-premise]]
 ==== Self-managed deployments
 
-For self-managed deployments, each node's <<data-node,data role>> is configured
+For self-managed deployments, each node's <<data-node-role,data role>> is configured
 in `elasticsearch.yml`. For example, the highest-performance nodes in a cluster
 might be assigned to both the hot and content tiers:
 
diff --git a/docs/reference/docs.asciidoc b/docs/reference/docs.asciidoc
index ff2d823410a6d..ccdbaaffb2b77 100644
--- a/docs/reference/docs.asciidoc
+++ b/docs/reference/docs.asciidoc
@@ -1,9 +1,13 @@
 [[docs]]
 == Document APIs
 
-This section starts with a short introduction to {es}'s <<docs-replication,data
-replication model>>, followed by a detailed description of the following CRUD
-APIs:
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs].
+--
+
+This section describes the following CRUD APIs:
 
 .Single document APIs
 * <<docs-index_>>
@@ -18,8 +22,6 @@ APIs:
 * <<docs-update-by-query>>
 * <<docs-reindex>>
 
-include::docs/data-replication.asciidoc[]
-
 include::docs/index_.asciidoc[]
 
 include::docs/get.asciidoc[]
diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc
index 6edccfcdb13f5..78169e841dab4 100644
--- a/docs/reference/docs/bulk.asciidoc
+++ b/docs/reference/docs/bulk.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Bulk</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs].
+--
+
 Performs multiple indexing or delete operations in a single API call.
 This reduces overhead and can greatly increase indexing speed.
 
diff --git a/docs/reference/docs/data-replication.asciidoc b/docs/reference/docs/data-replication.asciidoc
index 2c1a16c81d011..6ee266070e727 100644
--- a/docs/reference/docs/data-replication.asciidoc
+++ b/docs/reference/docs/data-replication.asciidoc
@@ -1,6 +1,6 @@
 
 [[docs-replication]]
-=== Reading and Writing documents
+=== Reading and writing documents
 
 [discrete]
 ==== Introduction
diff --git a/docs/reference/docs/delete.asciidoc b/docs/reference/docs/delete.asciidoc
index 452d7f7758bfa..046a20abdaffb 100644
--- a/docs/reference/docs/delete.asciidoc
+++ b/docs/reference/docs/delete.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Delete</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs].
+--
+
 Removes a JSON document from the specified index.
 
 [[docs-delete-api-request]]
diff --git a/docs/reference/docs/get.asciidoc b/docs/reference/docs/get.asciidoc
index c71215fff8d73..a3ff70fb95f6e 100644
--- a/docs/reference/docs/get.asciidoc
+++ b/docs/reference/docs/get.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs].
+--
+
 Retrieves the specified JSON document from an index.
 
 [source,console]
diff --git a/docs/reference/docs/multi-get.asciidoc b/docs/reference/docs/multi-get.asciidoc
index 7c3eafa9c79f3..293bd2568a34b 100644
--- a/docs/reference/docs/multi-get.asciidoc
+++ b/docs/reference/docs/multi-get.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Multi get</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs].
+--
+
 Retrieves multiple JSON documents by ID. 
 
 [source,console]
diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index dc27e40ecd90b..2f6ddd344eaab 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Reindex</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs].
+--
+
 Copies documents from a source to a destination.
 
 The source can be any existing index, alias, or data stream. The destination
diff --git a/docs/reference/docs/termvectors.asciidoc b/docs/reference/docs/termvectors.asciidoc
index 31dfba1ac2668..d40452fb4875a 100644
--- a/docs/reference/docs/termvectors.asciidoc
+++ b/docs/reference/docs/termvectors.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Term vectors</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs].
+--
+
 Retrieves information and statistics for terms in the fields of a particular document. 
 
 [source,console]
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index d470080fc602f..c8d68082c8ea1 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Update by query</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs].
+--
+
 Updates documents that match the specified query.
 If no query is specified, performs an update on every document in the data stream or index without
 modifying the source, which is useful for picking up mapping changes.
diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc
index a212c4e152b0e..ae9ae8fe73fc6 100644
--- a/docs/reference/docs/update.asciidoc
+++ b/docs/reference/docs/update.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Update</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs].
+--
+
 Updates a document using the specified script.
 
 [[docs-update-api-request]]
diff --git a/docs/reference/eql/delete-async-eql-search-api.asciidoc b/docs/reference/eql/delete-async-eql-search-api.asciidoc
index 2f52a1a9cb234..2783c9ac0b871 100644
--- a/docs/reference/eql/delete-async-eql-search-api.asciidoc
+++ b/docs/reference/eql/delete-async-eql-search-api.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Delete async EQL search</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-eql[EQL APIs].
+--
+
 Deletes an <<eql-search-async,async EQL search>> or a
 <<eql-search-store-sync-eql-search,stored synchronous EQL search>>. The API also
 deletes results for the search.
diff --git a/docs/reference/eql/eql-apis.asciidoc b/docs/reference/eql/eql-apis.asciidoc
index e8cc2b21492ae..4d6aafd2039df 100644
--- a/docs/reference/eql/eql-apis.asciidoc
+++ b/docs/reference/eql/eql-apis.asciidoc
@@ -1,6 +1,12 @@
 [[eql-apis]]
 == EQL APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-eql[EQL APIs].
+--
+
 <<eql,Event Query Language (EQL)>> is a query language for event-based time series data,
 such as logs, metrics, and traces. For an overview of EQL and related tutorials,
 see <<eql>>.
diff --git a/docs/reference/eql/eql-search-api.asciidoc b/docs/reference/eql/eql-search-api.asciidoc
index d7f10f4627f6c..544e4d7325c5b 100644
--- a/docs/reference/eql/eql-search-api.asciidoc
+++ b/docs/reference/eql/eql-search-api.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>EQL search</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-eql[EQL APIs].
+--
+
 Returns search results for an <<eql,Event Query Language (EQL)>> query.
 
 EQL assumes each document in a data stream or index corresponds to an
@@ -88,6 +94,53 @@ request that targets only `bar*` still returns an error.
 +
 Defaults to `true`.
 
+`allow_partial_search_results`::
+(Optional, Boolean)
+
+If `false`, the request returns an error if one or more shards involved in the query are unavailable.
++
+If `true`, the query is executed only on the available shards, ignoring shard request timeouts and
+<<shard-failures,shard failures>>.
++
+Defaults to `false`.
++
+To override the default for this field, set the
+`xpack.eql.default_allow_partial_results` cluster setting to `true`.
+
+
+[IMPORTANT]
+====
+You can also specify this value using the `allow_partial_search_results` request body parameter.
+If both parameters are specified, only the query parameter is used.
+====
+
+
+`allow_partial_sequence_results`::
+(Optional, Boolean)
+
+
+Used together with `allow_partial_search_results=true`, controls the behavior of sequence queries specifically
+(if `allow_partial_search_results=false`, this setting has no effect).
+If `true` and if some shards are unavailable, the sequences are calculated on available shards only.
++
+If `false` and if some shards are unavailable, the query only returns information about the shard failures,
+but no further results.
++
+Defaults to `false`.
++
+Consider that sequences calculated with `allow_partial_search_results=true` can return incorrect results
+(eg. if a <<eql-missing-events, missing event>> clause matches records in unavailable shards)
++
+To override the default for this field, set the
+`xpack.eql.default_allow_partial_sequence_results` cluster setting to `true`.
+
+
+[IMPORTANT]
+====
+You can also specify this value using the `allow_partial_sequence_results` request body parameter.
+If both parameters are specified, only the query parameter is used.
+====
+
 `ccs_minimize_roundtrips`::
 (Optional, Boolean) If `true`, network round-trips between the local and the
 remote cluster are minimized when running cross-cluster search (CCS) requests.
diff --git a/docs/reference/eql/get-async-eql-search-api.asciidoc b/docs/reference/eql/get-async-eql-search-api.asciidoc
index 2ac4271b5b986..ff4a07811b732 100644
--- a/docs/reference/eql/get-async-eql-search-api.asciidoc
+++ b/docs/reference/eql/get-async-eql-search-api.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Get async EQL search</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-eql[EQL APIs].
+--
+
 Returns the current status and available results for an <<eql-search-async,async
 EQL search>> or a <<eql-search-store-sync-eql-search,stored synchronous EQL
 search>>.
diff --git a/docs/reference/eql/get-async-eql-status-api.asciidoc b/docs/reference/eql/get-async-eql-status-api.asciidoc
index 908a65773f6aa..cb3904a644921 100644
--- a/docs/reference/eql/get-async-eql-status-api.asciidoc
+++ b/docs/reference/eql/get-async-eql-status-api.asciidoc
@@ -5,6 +5,13 @@
 ++++
 <titleabbrev>Get async EQL search status</titleabbrev>
 ++++
+
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-eql[EQL APIs].
+--
+
 Returns the current status for an <<eql-search-async,async EQL search>> or
 a <<eql-search-store-sync-eql-search,stored synchronous EQL search>>
 without returning results. This is a more lightweight API than
diff --git a/docs/reference/esql/esql-apis.asciidoc b/docs/reference/esql/esql-apis.asciidoc
index 8586cd1ae6bce..157f4e4357e78 100644
--- a/docs/reference/esql/esql-apis.asciidoc
+++ b/docs/reference/esql/esql-apis.asciidoc
@@ -1,6 +1,14 @@
 [[esql-apis]]
 == {esql} APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-esql[ES|QL APIs].
+--
+
+https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-query
+
 The <<esql,{es} Query Language ({esql})>> provides a powerful way to filter, transform,
 and analyze data stored in {es}, and in the future in other runtimes. For an
 overview of {esql} and related tutorials, see <<esql>>.
diff --git a/docs/reference/esql/esql-async-query-api.asciidoc b/docs/reference/esql/esql-async-query-api.asciidoc
index 6cd23fc524f96..8cb974cf6773b 100644
--- a/docs/reference/esql/esql-async-query-api.asciidoc
+++ b/docs/reference/esql/esql-async-query-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>{esql} async query API</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-esql[ES|QL APIs].
+--
+
 Runs an async <<esql,{esql} query>>.
 
 The async query API lets you asynchronously execute a query request,
diff --git a/docs/reference/esql/esql-async-query-delete-api.asciidoc b/docs/reference/esql/esql-async-query-delete-api.asciidoc
index 5cad566f7f9c0..421c59191f3bd 100644
--- a/docs/reference/esql/esql-async-query-delete-api.asciidoc
+++ b/docs/reference/esql/esql-async-query-delete-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>{esql} async query delete API</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-esql[ES|QL APIs].
+--
+
 The <<esql,{esql}>> async query delete API is used to manually delete an async query
 by ID. If the query is still running, the query will be cancelled. Otherwise,
 the stored results are deleted.
diff --git a/docs/reference/esql/esql-async-query-get-api.asciidoc b/docs/reference/esql/esql-async-query-get-api.asciidoc
index 82a6ae5b28b51..693e96861adb9 100644
--- a/docs/reference/esql/esql-async-query-get-api.asciidoc
+++ b/docs/reference/esql/esql-async-query-get-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>{esql} async query get API</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-esql[ES|QL APIs].
+--
+
 Returns the current status and available results for an <<esql-async-query-api,{esql}
 async query>> or a stored results.
 
diff --git a/docs/reference/esql/esql-limitations.asciidoc b/docs/reference/esql/esql-limitations.asciidoc
index c2849e4889f98..adfd38478ab21 100644
--- a/docs/reference/esql/esql-limitations.asciidoc
+++ b/docs/reference/esql/esql-limitations.asciidoc
@@ -30,11 +30,11 @@ include::processing-commands/limit.asciidoc[tag=limitation]
 ** You can use `to_datetime` to cast to millisecond dates to use unsupported functions
 * `double` (`float`, `half_float`, `scaled_float` are represented as `double`)
 * `ip`
-* `keyword` family including `keyword`, `constant_keyword`, and `wildcard`
+* `keyword` <<keyword, family>> including `keyword`, `constant_keyword`, and `wildcard`
 * `int` (`short` and `byte` are represented as `int`)
 * `long`
 * `null`
-* `text`
+* `text` <<text, family>> including `text`, `semantic_text` and `match_only_text`
 * experimental:[] `unsigned_long`
 * `version`
 * Spatial types
@@ -112,7 +112,7 @@ it is necessary to use the search function, like <<esql-match>>, in a <<esql-whe
 directly after the <<esql-from>> source command, or close enough to it.
 Otherwise, the query will fail with a validation error.
 Another limitation is that any <<esql-where>> command containing a full-text search function
-cannot also use disjunctions (`OR`).
+cannot also use disjunctions (`OR`) unless all functions used in the OR clauses are full-text functions themselves.
 
 For example, this query is valid:
 
@@ -139,9 +139,18 @@ FROM books
 | WHERE MATCH(author, "Faulkner") OR author LIKE "Hemingway"
 ----
 
+However this query will succeed because it uses full text functions on both `OR` clauses:
+
+[source,esql]
+----
+FROM books
+| WHERE MATCH(author, "Faulkner") OR QSTR("author: Hemingway")
+----
+
+
 Note that, because of <<esql-limitations-text-fields,the way {esql} treats `text` values>>,
 any queries on `text` fields that do not explicitly use the full-text functions,
-<<esql-match>> or <<esql-qstr>>, will behave as if the fields are actually `keyword` fields:
+<<esql-match>>, <<esql-qstr>> or <<esql-kql>>, will behave as if the fields are actually `keyword` fields:
 they are case-sensitive and need to match the full string.
 
 [discrete]
diff --git a/docs/reference/esql/esql-query-api.asciidoc b/docs/reference/esql/esql-query-api.asciidoc
index 8e07a627567df..eac66ecfde2dd 100644
--- a/docs/reference/esql/esql-query-api.asciidoc
+++ b/docs/reference/esql/esql-query-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>{esql} query API</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-esql[ES|QL APIs].
+--
+
 Returns search results for an <<esql,ES|QL ({es} query language)>> query.
 
 [source,console]
diff --git a/docs/reference/esql/functions/description/hash.asciidoc b/docs/reference/esql/functions/description/hash.asciidoc
new file mode 100644
index 0000000000000..e074915c5132a
--- /dev/null
+++ b/docs/reference/esql/functions/description/hash.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Computes the hash of the input using various algorithms such as MD5, SHA, SHA-224, SHA-256, SHA-384, SHA-512.
diff --git a/docs/reference/esql/functions/description/match.asciidoc b/docs/reference/esql/functions/description/match.asciidoc
index 25f0571878d47..0724f0f108e3c 100644
--- a/docs/reference/esql/functions/description/match.asciidoc
+++ b/docs/reference/esql/functions/description/match.asciidoc
@@ -2,4 +2,4 @@
 
 *Description*
 
-Performs a <<query-dsl-match-query,match query>> on the specified field. Returns true if the provided query matches the row.
+Use `MATCH` to perform a <<query-dsl-match-query,match query>> on the specified field. Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL.  Match can be used on fields from the text family like <<text, text>> and <<semantic-text, semantic_text>>, as well as other field types like keyword, boolean, dates, and numeric types.  For a simplified syntax, you can use the <<esql-search-operators,match operator>> `:` operator instead of `MATCH`.  `MATCH` returns true if the provided query matches the row.
diff --git a/docs/reference/esql/functions/description/md5.asciidoc b/docs/reference/esql/functions/description/md5.asciidoc
new file mode 100644
index 0000000000000..2ad847c0ce0e3
--- /dev/null
+++ b/docs/reference/esql/functions/description/md5.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Computes the MD5 hash of the input.
diff --git a/docs/reference/esql/functions/description/sha1.asciidoc b/docs/reference/esql/functions/description/sha1.asciidoc
new file mode 100644
index 0000000000000..5bc29f86cc591
--- /dev/null
+++ b/docs/reference/esql/functions/description/sha1.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Computes the SHA1 hash of the input.
diff --git a/docs/reference/esql/functions/description/sha256.asciidoc b/docs/reference/esql/functions/description/sha256.asciidoc
new file mode 100644
index 0000000000000..b2a7ef01e1069
--- /dev/null
+++ b/docs/reference/esql/functions/description/sha256.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Computes the SHA256 hash of the input.
diff --git a/docs/reference/esql/functions/description/to_date_nanos.asciidoc b/docs/reference/esql/functions/description/to_date_nanos.asciidoc
index 3fac7295f1bed..955c19b43a12f 100644
--- a/docs/reference/esql/functions/description/to_date_nanos.asciidoc
+++ b/docs/reference/esql/functions/description/to_date_nanos.asciidoc
@@ -4,4 +4,4 @@
 
 Converts an input to a nanosecond-resolution date value (aka date_nanos).
 
-NOTE: The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z.  Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.
+NOTE: The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z, attepting to convertvalues outside of that range will result in null with a warning..  Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.
diff --git a/docs/reference/esql/functions/examples/bucket.asciidoc b/docs/reference/esql/functions/examples/bucket.asciidoc
index 4afea30660339..264efc191748f 100644
--- a/docs/reference/esql/functions/examples/bucket.asciidoc
+++ b/docs/reference/esql/functions/examples/bucket.asciidoc
@@ -116,4 +116,18 @@ include::{esql-specs}/bucket.csv-spec[tag=reuseGroupingFunctionWithExpression]
 |===
 include::{esql-specs}/bucket.csv-spec[tag=reuseGroupingFunctionWithExpression-result]
 |===
+Sometimes you need to change the start value of each bucket by a given duration (similar to date histogram
+aggregation's <<search-aggregations-bucket-histogram-aggregation,`offset`>> parameter). To do so, you will need to
+take into account how the language handles expressions within the `STATS` command: if these contain functions or
+arithmetic operators, a virtual `EVAL` is inserted before and/or after the `STATS` command. Consequently, a double
+compensation is needed to adjust the bucketed date value before the aggregation and then again after. For instance,
+inserting a negative offset of `1 hour` to buckets of `1 year` looks like this:
+[source.merge.styled,esql]
+----
+include::{esql-specs}/bucket.csv-spec[tag=bucketWithOffset]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/bucket.csv-spec[tag=bucketWithOffset-result]
+|===
 
diff --git a/docs/reference/esql/functions/examples/hash.asciidoc b/docs/reference/esql/functions/examples/hash.asciidoc
new file mode 100644
index 0000000000000..492e466eb395e
--- /dev/null
+++ b/docs/reference/esql/functions/examples/hash.asciidoc
@@ -0,0 +1,13 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Example*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/hash.csv-spec[tag=hash]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/hash.csv-spec[tag=hash-result]
+|===
+
diff --git a/docs/reference/esql/functions/examples/md5.asciidoc b/docs/reference/esql/functions/examples/md5.asciidoc
new file mode 100644
index 0000000000000..0b43bc5b791c9
--- /dev/null
+++ b/docs/reference/esql/functions/examples/md5.asciidoc
@@ -0,0 +1,13 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Example*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/hash.csv-spec[tag=md5]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/hash.csv-spec[tag=md5-result]
+|===
+
diff --git a/docs/reference/esql/functions/examples/sha1.asciidoc b/docs/reference/esql/functions/examples/sha1.asciidoc
new file mode 100644
index 0000000000000..77786431a738a
--- /dev/null
+++ b/docs/reference/esql/functions/examples/sha1.asciidoc
@@ -0,0 +1,13 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Example*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/hash.csv-spec[tag=sha1]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/hash.csv-spec[tag=sha1-result]
+|===
+
diff --git a/docs/reference/esql/functions/examples/sha256.asciidoc b/docs/reference/esql/functions/examples/sha256.asciidoc
new file mode 100644
index 0000000000000..801c36d8effc8
--- /dev/null
+++ b/docs/reference/esql/functions/examples/sha256.asciidoc
@@ -0,0 +1,13 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Example*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/hash.csv-spec[tag=sha256]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/hash.csv-spec[tag=sha256-result]
+|===
+
diff --git a/docs/reference/esql/functions/kibana/definition/bucket.json b/docs/reference/esql/functions/kibana/definition/bucket.json
index 18802f5ff8fef..f9c7f2f27d6f9 100644
--- a/docs/reference/esql/functions/kibana/definition/bucket.json
+++ b/docs/reference/esql/functions/kibana/definition/bucket.json
@@ -1598,7 +1598,8 @@
     "FROM employees\n| WHERE hire_date >= \"1985-01-01T00:00:00Z\" AND hire_date < \"1986-01-01T00:00:00Z\"\n| STATS c = COUNT(1) BY b = BUCKET(salary, 5000.)\n| SORT b",
     "FROM sample_data \n| WHERE @timestamp >= NOW() - 1 day and @timestamp < NOW()\n| STATS COUNT(*) BY bucket = BUCKET(@timestamp, 25, NOW() - 1 day, NOW())",
     "FROM employees\n| WHERE hire_date >= \"1985-01-01T00:00:00Z\" AND hire_date < \"1986-01-01T00:00:00Z\"\n| STATS AVG(salary) BY bucket = BUCKET(hire_date, 20, \"1985-01-01T00:00:00Z\", \"1986-01-01T00:00:00Z\")\n| SORT bucket",
-    "FROM employees\n| STATS s1 = b1 + 1, s2 = BUCKET(salary / 1000 + 999, 50.) + 2 BY b1 = BUCKET(salary / 100 + 99, 50.), b2 = BUCKET(salary / 1000 + 999, 50.)\n| SORT b1, b2\n| KEEP s1, b1, s2, b2"
+    "FROM employees\n| STATS s1 = b1 + 1, s2 = BUCKET(salary / 1000 + 999, 50.) + 2 BY b1 = BUCKET(salary / 100 + 99, 50.), b2 = BUCKET(salary / 1000 + 999, 50.)\n| SORT b1, b2\n| KEEP s1, b1, s2, b2",
+    "FROM employees\n| STATS dates = MV_SORT(VALUES(birth_date)) BY b = BUCKET(birth_date + 1 HOUR, 1 YEAR) - 1 HOUR\n| EVAL d_count = MV_COUNT(dates)\n| SORT d_count, b\n| LIMIT 3"
   ],
   "preview" : false,
   "snapshot_only" : false
diff --git a/docs/reference/esql/functions/kibana/definition/date_format.json b/docs/reference/esql/functions/kibana/definition/date_format.json
index 6e2738fafb964..629415da30fa2 100644
--- a/docs/reference/esql/functions/kibana/definition/date_format.json
+++ b/docs/reference/esql/functions/kibana/definition/date_format.json
@@ -4,6 +4,18 @@
   "name" : "date_format",
   "description" : "Returns a string representation of a date, in the provided format.",
   "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "dateFormat",
+          "type" : "date",
+          "optional" : true,
+          "description" : "Date format (optional).  If no format is specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/hash.json b/docs/reference/esql/functions/kibana/definition/hash.json
new file mode 100644
index 0000000000000..dbf4a2542afc5
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/hash.json
@@ -0,0 +1,85 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "eval",
+  "name" : "hash",
+  "description" : "Computes the hash of the input using various algorithms such as MD5, SHA, SHA-224, SHA-256, SHA-384, SHA-512.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "algorithm",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Hash algorithm to use."
+        },
+        {
+          "name" : "input",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Input to hash."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "algorithm",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Hash algorithm to use."
+        },
+        {
+          "name" : "input",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Input to hash."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "algorithm",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Hash algorithm to use."
+        },
+        {
+          "name" : "input",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Input to hash."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "algorithm",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Hash algorithm to use."
+        },
+        {
+          "name" : "input",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Input to hash."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    }
+  ],
+  "examples" : [
+    "FROM sample_data \n| WHERE message != \"Connection error\"\n| EVAL md5 = hash(\"md5\", message), sha256 = hash(\"sha256\", message) \n| KEEP message, md5, sha256;"
+  ],
+  "preview" : false,
+  "snapshot_only" : false
+}
diff --git a/docs/reference/esql/functions/kibana/definition/kql.json b/docs/reference/esql/functions/kibana/definition/kql.json
index 6960681fbbf0d..440786ec63e77 100644
--- a/docs/reference/esql/functions/kibana/definition/kql.json
+++ b/docs/reference/esql/functions/kibana/definition/kql.json
@@ -33,5 +33,5 @@
     "FROM books \n| WHERE KQL(\"author: Faulkner\")\n| KEEP book_no, author \n| SORT book_no \n| LIMIT 5;"
   ],
   "preview" : true,
-  "snapshot_only" : true
+  "snapshot_only" : false
 }
diff --git a/docs/reference/esql/functions/kibana/definition/match.json b/docs/reference/esql/functions/kibana/definition/match.json
index 7f2a8239cc0d0..eb206cb9ddf4d 100644
--- a/docs/reference/esql/functions/kibana/definition/match.json
+++ b/docs/reference/esql/functions/kibana/definition/match.json
@@ -2,7 +2,7 @@
   "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
   "type" : "eval",
   "name" : "match",
-  "description" : "Performs a <<query-dsl-match-query,match query>> on the specified field. Returns true if the provided query matches the row.",
+  "description" : "Use `MATCH` to perform a <<query-dsl-match-query,match query>> on the specified field.\nUsing `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL.\n\nMatch can be used on fields from the text family like <<text, text>> and <<semantic-text, semantic_text>>,\nas well as other field types like keyword, boolean, dates, and numeric types.\n\nFor a simplified syntax, you can use the <<esql-search-operators,match operator>> `:` operator instead of `MATCH`.\n\n`MATCH` returns true if the provided query matches the row.",
   "signatures" : [
     {
       "params" : [
diff --git a/docs/reference/esql/functions/kibana/definition/match_operator.json b/docs/reference/esql/functions/kibana/definition/match_operator.json
index 44233bbddb653..c8cbf1cf9d966 100644
--- a/docs/reference/esql/functions/kibana/definition/match_operator.json
+++ b/docs/reference/esql/functions/kibana/definition/match_operator.json
@@ -2,7 +2,7 @@
   "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
   "type" : "operator",
   "name" : "match_operator",
-  "description" : "Performs a <<query-dsl-match-query,match query>> on the specified field. Returns true if the provided query matches the row.",
+  "description" : "Use `MATCH` to perform a <<query-dsl-match-query,match query>> on the specified field.\nUsing `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL.\n\nMatch can be used on text fields, as well as other field types like boolean, dates, and numeric types.\n\nFor a simplified syntax, you can use the <<esql-search-operators,match operator>> `:` operator instead of `MATCH`.\n\n`MATCH` returns true if the provided query matches the row.",
   "signatures" : [
     {
       "params" : [
diff --git a/docs/reference/esql/functions/kibana/definition/md5.json b/docs/reference/esql/functions/kibana/definition/md5.json
new file mode 100644
index 0000000000000..4d3a88e123ff4
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/md5.json
@@ -0,0 +1,37 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "eval",
+  "name" : "md5",
+  "description" : "Computes the MD5 hash of the input.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "input",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Input to hash."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "input",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Input to hash."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    }
+  ],
+  "examples" : [
+    "FROM sample_data \n| WHERE message != \"Connection error\"\n| EVAL md5 = md5(message)\n| KEEP message, md5;"
+  ],
+  "preview" : false,
+  "snapshot_only" : false
+}
diff --git a/docs/reference/esql/functions/kibana/definition/sha1.json b/docs/reference/esql/functions/kibana/definition/sha1.json
new file mode 100644
index 0000000000000..a6abb31368bb3
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/sha1.json
@@ -0,0 +1,37 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "eval",
+  "name" : "sha1",
+  "description" : "Computes the SHA1 hash of the input.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "input",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Input to hash."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "input",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Input to hash."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    }
+  ],
+  "examples" : [
+    "FROM sample_data \n| WHERE message != \"Connection error\"\n| EVAL sha1 = sha1(message)\n| KEEP message, sha1;"
+  ],
+  "preview" : false,
+  "snapshot_only" : false
+}
diff --git a/docs/reference/esql/functions/kibana/definition/sha256.json b/docs/reference/esql/functions/kibana/definition/sha256.json
new file mode 100644
index 0000000000000..700425d485b61
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/sha256.json
@@ -0,0 +1,37 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "eval",
+  "name" : "sha256",
+  "description" : "Computes the SHA256 hash of the input.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "input",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Input to hash."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "input",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Input to hash."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    }
+  ],
+  "examples" : [
+    "FROM sample_data \n| WHERE message != \"Connection error\"\n| EVAL sha256 = sha256(message)\n| KEEP message, sha256;"
+  ],
+  "preview" : false,
+  "snapshot_only" : false
+}
diff --git a/docs/reference/esql/functions/kibana/definition/to_date_nanos.json b/docs/reference/esql/functions/kibana/definition/to_date_nanos.json
index d9409bceb8e6f..210b9608f9eff 100644
--- a/docs/reference/esql/functions/kibana/definition/to_date_nanos.json
+++ b/docs/reference/esql/functions/kibana/definition/to_date_nanos.json
@@ -3,7 +3,7 @@
   "type" : "eval",
   "name" : "to_date_nanos",
   "description" : "Converts an input to a nanosecond-resolution date value (aka date_nanos).",
-  "note" : "The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z.  Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.",
+  "note" : "The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z, attepting to convertvalues outside of that range will result in null with a warning..  Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.",
   "signatures" : [
     {
       "params" : [
@@ -90,6 +90,6 @@
       "returnType" : "date_nanos"
     }
   ],
-  "preview" : true,
+  "preview" : false,
   "snapshot_only" : false
 }
diff --git a/docs/reference/esql/functions/kibana/docs/hash.md b/docs/reference/esql/functions/kibana/docs/hash.md
new file mode 100644
index 0000000000000..4e937778ba67a
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/hash.md
@@ -0,0 +1,13 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### HASH
+Computes the hash of the input using various algorithms such as MD5, SHA, SHA-224, SHA-256, SHA-384, SHA-512.
+
+```
+FROM sample_data 
+| WHERE message != "Connection error"
+| EVAL md5 = hash("md5", message), sha256 = hash("sha256", message) 
+| KEEP message, md5, sha256;
+```
diff --git a/docs/reference/esql/functions/kibana/docs/match.md b/docs/reference/esql/functions/kibana/docs/match.md
index adf6de91c90f1..80bf84351c188 100644
--- a/docs/reference/esql/functions/kibana/docs/match.md
+++ b/docs/reference/esql/functions/kibana/docs/match.md
@@ -3,7 +3,15 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ
 -->
 
 ### MATCH
-Performs a <<query-dsl-match-query,match query>> on the specified field. Returns true if the provided query matches the row.
+Use `MATCH` to perform a <<query-dsl-match-query,match query>> on the specified field.
+Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL.
+
+Match can be used on fields from the text family like <<text, text>> and <<semantic-text, semantic_text>>,
+as well as other field types like keyword, boolean, dates, and numeric types.
+
+For a simplified syntax, you can use the <<esql-search-operators,match operator>> `:` operator instead of `MATCH`.
+
+`MATCH` returns true if the provided query matches the row.
 
 ```
 FROM books 
diff --git a/docs/reference/esql/functions/kibana/docs/match_operator.md b/docs/reference/esql/functions/kibana/docs/match_operator.md
index b0b6196798087..7681c2d1ce231 100644
--- a/docs/reference/esql/functions/kibana/docs/match_operator.md
+++ b/docs/reference/esql/functions/kibana/docs/match_operator.md
@@ -3,7 +3,14 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ
 -->
 
 ### MATCH_OPERATOR
-Performs a <<query-dsl-match-query,match query>> on the specified field. Returns true if the provided query matches the row.
+Use `MATCH` to perform a <<query-dsl-match-query,match query>> on the specified field.
+Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL.
+
+Match can be used on text fields, as well as other field types like boolean, dates, and numeric types.
+
+For a simplified syntax, you can use the <<esql-search-operators,match operator>> `:` operator instead of `MATCH`.
+
+`MATCH` returns true if the provided query matches the row.
 
 ```
 FROM books 
diff --git a/docs/reference/esql/functions/kibana/docs/md5.md b/docs/reference/esql/functions/kibana/docs/md5.md
new file mode 100644
index 0000000000000..aacb8a3960165
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/md5.md
@@ -0,0 +1,13 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### MD5
+Computes the MD5 hash of the input.
+
+```
+FROM sample_data 
+| WHERE message != "Connection error"
+| EVAL md5 = md5(message)
+| KEEP message, md5;
+```
diff --git a/docs/reference/esql/functions/kibana/docs/sha1.md b/docs/reference/esql/functions/kibana/docs/sha1.md
new file mode 100644
index 0000000000000..a940aa133f06e
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/sha1.md
@@ -0,0 +1,13 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### SHA1
+Computes the SHA1 hash of the input.
+
+```
+FROM sample_data 
+| WHERE message != "Connection error"
+| EVAL sha1 = sha1(message)
+| KEEP message, sha1;
+```
diff --git a/docs/reference/esql/functions/kibana/docs/sha256.md b/docs/reference/esql/functions/kibana/docs/sha256.md
new file mode 100644
index 0000000000000..fbe576c7c20d6
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/sha256.md
@@ -0,0 +1,13 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### SHA256
+Computes the SHA256 hash of the input.
+
+```
+FROM sample_data 
+| WHERE message != "Connection error"
+| EVAL sha256 = sha256(message)
+| KEEP message, sha256;
+```
diff --git a/docs/reference/esql/functions/kibana/docs/to_date_nanos.md b/docs/reference/esql/functions/kibana/docs/to_date_nanos.md
index 0294802485ccb..1bce8d4fca832 100644
--- a/docs/reference/esql/functions/kibana/docs/to_date_nanos.md
+++ b/docs/reference/esql/functions/kibana/docs/to_date_nanos.md
@@ -5,4 +5,4 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ
 ### TO_DATE_NANOS
 Converts an input to a nanosecond-resolution date value (aka date_nanos).
 
-Note: The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z.  Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.
+Note: The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z, attepting to convertvalues outside of that range will result in null with a warning..  Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.
diff --git a/docs/reference/esql/functions/layout/hash.asciidoc b/docs/reference/esql/functions/layout/hash.asciidoc
new file mode 100644
index 0000000000000..daf7fbf1170b2
--- /dev/null
+++ b/docs/reference/esql/functions/layout/hash.asciidoc
@@ -0,0 +1,15 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-hash]]
+=== `HASH`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/hash.svg[Embedded,opts=inline]
+
+include::../parameters/hash.asciidoc[]
+include::../description/hash.asciidoc[]
+include::../types/hash.asciidoc[]
+include::../examples/hash.asciidoc[]
diff --git a/docs/reference/esql/functions/layout/md5.asciidoc b/docs/reference/esql/functions/layout/md5.asciidoc
new file mode 100644
index 0000000000000..82d3031d6bdfd
--- /dev/null
+++ b/docs/reference/esql/functions/layout/md5.asciidoc
@@ -0,0 +1,15 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-md5]]
+=== `MD5`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/md5.svg[Embedded,opts=inline]
+
+include::../parameters/md5.asciidoc[]
+include::../description/md5.asciidoc[]
+include::../types/md5.asciidoc[]
+include::../examples/md5.asciidoc[]
diff --git a/docs/reference/esql/functions/layout/sha1.asciidoc b/docs/reference/esql/functions/layout/sha1.asciidoc
new file mode 100644
index 0000000000000..23e1e0e9ac2ab
--- /dev/null
+++ b/docs/reference/esql/functions/layout/sha1.asciidoc
@@ -0,0 +1,15 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-sha1]]
+=== `SHA1`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/sha1.svg[Embedded,opts=inline]
+
+include::../parameters/sha1.asciidoc[]
+include::../description/sha1.asciidoc[]
+include::../types/sha1.asciidoc[]
+include::../examples/sha1.asciidoc[]
diff --git a/docs/reference/esql/functions/layout/sha256.asciidoc b/docs/reference/esql/functions/layout/sha256.asciidoc
new file mode 100644
index 0000000000000..d36a1345271f5
--- /dev/null
+++ b/docs/reference/esql/functions/layout/sha256.asciidoc
@@ -0,0 +1,15 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-sha256]]
+=== `SHA256`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/sha256.svg[Embedded,opts=inline]
+
+include::../parameters/sha256.asciidoc[]
+include::../description/sha256.asciidoc[]
+include::../types/sha256.asciidoc[]
+include::../examples/sha256.asciidoc[]
diff --git a/docs/reference/esql/functions/layout/to_date_nanos.asciidoc b/docs/reference/esql/functions/layout/to_date_nanos.asciidoc
index 977a0ac969e5d..2dfd13dac7e20 100644
--- a/docs/reference/esql/functions/layout/to_date_nanos.asciidoc
+++ b/docs/reference/esql/functions/layout/to_date_nanos.asciidoc
@@ -4,8 +4,6 @@
 [[esql-to_date_nanos]]
 === `TO_DATE_NANOS`
 
-preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."]
-
 *Syntax*
 
 [.text-center]
diff --git a/docs/reference/esql/functions/parameters/hash.asciidoc b/docs/reference/esql/functions/parameters/hash.asciidoc
new file mode 100644
index 0000000000000..d47a82d4ab214
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/hash.asciidoc
@@ -0,0 +1,9 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`algorithm`::
+Hash algorithm to use.
+
+`input`::
+Input to hash.
diff --git a/docs/reference/esql/functions/parameters/md5.asciidoc b/docs/reference/esql/functions/parameters/md5.asciidoc
new file mode 100644
index 0000000000000..99eba4dc2cb3d
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/md5.asciidoc
@@ -0,0 +1,6 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`input`::
+Input to hash.
diff --git a/docs/reference/esql/functions/parameters/sha1.asciidoc b/docs/reference/esql/functions/parameters/sha1.asciidoc
new file mode 100644
index 0000000000000..99eba4dc2cb3d
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/sha1.asciidoc
@@ -0,0 +1,6 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`input`::
+Input to hash.
diff --git a/docs/reference/esql/functions/parameters/sha256.asciidoc b/docs/reference/esql/functions/parameters/sha256.asciidoc
new file mode 100644
index 0000000000000..99eba4dc2cb3d
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/sha256.asciidoc
@@ -0,0 +1,6 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`input`::
+Input to hash.
diff --git a/docs/reference/esql/functions/search-functions.asciidoc b/docs/reference/esql/functions/search-functions.asciidoc
index 238813c382c8c..a36473a7e8869 100644
--- a/docs/reference/esql/functions/search-functions.asciidoc
+++ b/docs/reference/esql/functions/search-functions.asciidoc
@@ -6,19 +6,23 @@
 ++++
 
 Full text functions are used to search for text in fields.
-<<analysis, Text analysiss>> is used to analyze the query before it is searched.
+<<analysis, Text analysis>> is used to analyze the query before it is searched.
 
 Full text functions can be used to match <<esql-multivalued-fields,multivalued fields>>.
 A multivalued field that contains a value that matches a full text query is considered to match the query.
 
+Full text functions are significantly more performant for text search use cases on large data sets than using pattern matching or regular expressions with `LIKE` or `RLIKE`
+
 See <<esql-limitations-full-text-search,full text search limitations>> for information on the limitations of full text search.
 
 {esql} supports these full-text search functions:
 
 // tag::search_list[]
+* experimental:[] <<esql-kql>>
 * experimental:[] <<esql-match>>
 * experimental:[] <<esql-qstr>>
 // end::search_list[]
 
+include::layout/kql.asciidoc[]
 include::layout/match.asciidoc[]
 include::layout/qstr.asciidoc[]
diff --git a/docs/reference/esql/functions/signature/hash.svg b/docs/reference/esql/functions/signature/hash.svg
new file mode 100644
index 0000000000000..f819e14c9d1a4
--- /dev/null
+++ b/docs/reference/esql/functions/signature/hash.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="432" height="46" viewbox="0 0 432 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m68 0h10m32 0h10m128 0h10m32 0h10m80 0h10m32 0h5"/><rect class="s" x="5" y="5" width="68" height="36"/><text class="k" x="15" y="31">HASH</text><rect class="s" x="83" y="5" width="32" height="36" rx="7"/><text class="syn" x="93" y="31">(</text><rect class="s" x="125" y="5" width="128" height="36" rx="7"/><text class="k" x="135" y="31">algorithm</text><rect class="s" x="263" y="5" width="32" height="36" rx="7"/><text class="syn" x="273" y="31">,</text><rect class="s" x="305" y="5" width="80" height="36" rx="7"/><text class="k" x="315" y="31">input</text><rect class="s" x="395" y="5" width="32" height="36" rx="7"/><text class="syn" x="405" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/signature/md5.svg b/docs/reference/esql/functions/signature/md5.svg
new file mode 100644
index 0000000000000..419af764a212e
--- /dev/null
+++ b/docs/reference/esql/functions/signature/md5.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="240" height="46" viewbox="0 0 240 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m56 0h10m32 0h10m80 0h10m32 0h5"/><rect class="s" x="5" y="5" width="56" height="36"/><text class="k" x="15" y="31">MD5</text><rect class="s" x="71" y="5" width="32" height="36" rx="7"/><text class="syn" x="81" y="31">(</text><rect class="s" x="113" y="5" width="80" height="36" rx="7"/><text class="k" x="123" y="31">input</text><rect class="s" x="203" y="5" width="32" height="36" rx="7"/><text class="syn" x="213" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/signature/sha1.svg b/docs/reference/esql/functions/signature/sha1.svg
new file mode 100644
index 0000000000000..bab03a7eb88c8
--- /dev/null
+++ b/docs/reference/esql/functions/signature/sha1.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="252" height="46" viewbox="0 0 252 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m68 0h10m32 0h10m80 0h10m32 0h5"/><rect class="s" x="5" y="5" width="68" height="36"/><text class="k" x="15" y="31">SHA1</text><rect class="s" x="83" y="5" width="32" height="36" rx="7"/><text class="syn" x="93" y="31">(</text><rect class="s" x="125" y="5" width="80" height="36" rx="7"/><text class="k" x="135" y="31">input</text><rect class="s" x="215" y="5" width="32" height="36" rx="7"/><text class="syn" x="225" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/signature/sha256.svg b/docs/reference/esql/functions/signature/sha256.svg
new file mode 100644
index 0000000000000..b77126bbefbd8
--- /dev/null
+++ b/docs/reference/esql/functions/signature/sha256.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="276" height="46" viewbox="0 0 276 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m92 0h10m32 0h10m80 0h10m32 0h5"/><rect class="s" x="5" y="5" width="92" height="36"/><text class="k" x="15" y="31">SHA256</text><rect class="s" x="107" y="5" width="32" height="36" rx="7"/><text class="syn" x="117" y="31">(</text><rect class="s" x="149" y="5" width="80" height="36" rx="7"/><text class="k" x="159" y="31">input</text><rect class="s" x="239" y="5" width="32" height="36" rx="7"/><text class="syn" x="249" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/string-functions.asciidoc b/docs/reference/esql/functions/string-functions.asciidoc
index ce9636f5c5a3a..dd10e4c77581e 100644
--- a/docs/reference/esql/functions/string-functions.asciidoc
+++ b/docs/reference/esql/functions/string-functions.asciidoc
@@ -13,15 +13,19 @@
 * <<esql-concat>>
 * <<esql-ends_with>>
 * <<esql-from_base64>>
+* <<esql-hash>>
 * <<esql-left>>
 * <<esql-length>>
 * <<esql-locate>>
 * <<esql-ltrim>>
+* <<esql-md5>>
 * <<esql-repeat>>
 * <<esql-replace>>
 * <<esql-reverse>>
 * <<esql-right>>
 * <<esql-rtrim>>
+* <<esql-sha1>>
+* <<esql-sha256>>
 * <<esql-space>>
 * <<esql-split>>
 * <<esql-starts_with>>
@@ -37,15 +41,19 @@ include::layout/byte_length.asciidoc[]
 include::layout/concat.asciidoc[]
 include::layout/ends_with.asciidoc[]
 include::layout/from_base64.asciidoc[]
+include::layout/hash.asciidoc[]
 include::layout/left.asciidoc[]
 include::layout/length.asciidoc[]
 include::layout/locate.asciidoc[]
 include::layout/ltrim.asciidoc[]
+include::layout/md5.asciidoc[]
 include::layout/repeat.asciidoc[]
 include::layout/replace.asciidoc[]
 include::layout/reverse.asciidoc[]
 include::layout/right.asciidoc[]
 include::layout/rtrim.asciidoc[]
+include::layout/sha1.asciidoc[]
+include::layout/sha256.asciidoc[]
 include::layout/space.asciidoc[]
 include::layout/split.asciidoc[]
 include::layout/starts_with.asciidoc[]
diff --git a/docs/reference/esql/functions/type-conversion-functions.asciidoc b/docs/reference/esql/functions/type-conversion-functions.asciidoc
index 9ac9ec290c07b..bd70c2789dfa2 100644
--- a/docs/reference/esql/functions/type-conversion-functions.asciidoc
+++ b/docs/reference/esql/functions/type-conversion-functions.asciidoc
@@ -18,6 +18,7 @@
 * <<esql-to_cartesianshape>>
 * experimental:[] <<esql-to_dateperiod>>
 * <<esql-to_datetime>>
+* <<esql-to_date_nanos>>
 * <<esql-to_degrees>>
 * <<esql-to_double>>
 * <<esql-to_geopoint>>
@@ -37,6 +38,7 @@ include::layout/to_cartesianpoint.asciidoc[]
 include::layout/to_cartesianshape.asciidoc[]
 include::layout/to_dateperiod.asciidoc[]
 include::layout/to_datetime.asciidoc[]
+include::layout/to_date_nanos.asciidoc[]
 include::layout/to_degrees.asciidoc[]
 include::layout/to_double.asciidoc[]
 include::layout/to_geopoint.asciidoc[]
diff --git a/docs/reference/esql/functions/types/date_format.asciidoc b/docs/reference/esql/functions/types/date_format.asciidoc
index b2e97dfa8835a..580094e9be906 100644
--- a/docs/reference/esql/functions/types/date_format.asciidoc
+++ b/docs/reference/esql/functions/types/date_format.asciidoc
@@ -5,6 +5,7 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 dateFormat | date | result
+date | | keyword
 keyword | date | keyword
 text | date | keyword
 |===
diff --git a/docs/reference/esql/functions/types/hash.asciidoc b/docs/reference/esql/functions/types/hash.asciidoc
new file mode 100644
index 0000000000000..786ba03b2aa60
--- /dev/null
+++ b/docs/reference/esql/functions/types/hash.asciidoc
@@ -0,0 +1,12 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+algorithm | input | result
+keyword | keyword | keyword
+keyword | text | keyword
+text | keyword | keyword
+text | text | keyword
+|===
diff --git a/docs/reference/esql/functions/types/md5.asciidoc b/docs/reference/esql/functions/types/md5.asciidoc
new file mode 100644
index 0000000000000..049a553397bbd
--- /dev/null
+++ b/docs/reference/esql/functions/types/md5.asciidoc
@@ -0,0 +1,10 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+input | result
+keyword | keyword
+text | keyword
+|===
diff --git a/docs/reference/esql/functions/types/sha1.asciidoc b/docs/reference/esql/functions/types/sha1.asciidoc
new file mode 100644
index 0000000000000..049a553397bbd
--- /dev/null
+++ b/docs/reference/esql/functions/types/sha1.asciidoc
@@ -0,0 +1,10 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+input | result
+keyword | keyword
+text | keyword
+|===
diff --git a/docs/reference/esql/functions/types/sha256.asciidoc b/docs/reference/esql/functions/types/sha256.asciidoc
new file mode 100644
index 0000000000000..049a553397bbd
--- /dev/null
+++ b/docs/reference/esql/functions/types/sha256.asciidoc
@@ -0,0 +1,10 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+input | result
+keyword | keyword
+text | keyword
+|===
diff --git a/docs/reference/esql/processing-commands/where.asciidoc b/docs/reference/esql/processing-commands/where.asciidoc
index 1d6fc1e90d595..68336d5358eaf 100644
--- a/docs/reference/esql/processing-commands/where.asciidoc
+++ b/docs/reference/esql/processing-commands/where.asciidoc
@@ -7,7 +7,7 @@ the input table for which the provided condition evaluates to `true`.
 
 [TIP]
 ====
-In case of value exclusions, fields with `null` values will be excluded from search results. 
+In case of value exclusions, fields with `null` values will be excluded from search results.
 In this context a `null` means either there is an explicit `null` value in the document or there is no value at all.
 For example: `WHERE field != "value"` will be interpreted as `WHERE field != "value" AND field IS NOT NULL`.
 ====
@@ -58,6 +58,26 @@ For a complete list of all functions, refer to <<esql-functions>>.
 
 include::../functions/predicates.asciidoc[tag=body]
 
+For matching text, you can use <<esql-search-functions,full text search functions>> like `MATCH`.
+
+Use <<esql-match,`MATCH`>> to perform a <<query-dsl-match-query,match query>> on a specified field.
+
+Match can be used on text fields, as well as other field types like boolean, dates, and numeric types.
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/match-function.csv-spec[tag=match-with-field]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/match-function.csv-spec[tag=match-with-field-result]
+|===
+
+[TIP]
+====
+You can also use the shorthand <<esql-search-operators,match operator>> `:` instead of `MATCH`.
+====
+
 include::../functions/like.asciidoc[tag=body]
 
 include::../functions/rlike.asciidoc[tag=body]
diff --git a/docs/reference/features/apis/features-apis.asciidoc b/docs/reference/features/apis/features-apis.asciidoc
index fe06471cff0df..2582446340f15 100644
--- a/docs/reference/features/apis/features-apis.asciidoc
+++ b/docs/reference/features/apis/features-apis.asciidoc
@@ -1,6 +1,12 @@
 [[features-apis]]
 == Features APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-features[Features APIs].
+--
+
 You can use the following APIs to introspect and manage Features provided
 by Elasticsearch and Elasticsearch plugins.
 
diff --git a/docs/reference/features/apis/get-features-api.asciidoc b/docs/reference/features/apis/get-features-api.asciidoc
index 676ec3c41da24..62986d7728ca7 100644
--- a/docs/reference/features/apis/get-features-api.asciidoc
+++ b/docs/reference/features/apis/get-features-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get features</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-features[Features APIs].
+--
+
 Gets a list of features which can be included in snapshots using the
 <<create-snapshot-api-feature-states,`feature_states` field>> when creating a
 snapshot.
diff --git a/docs/reference/features/apis/reset-features-api.asciidoc b/docs/reference/features/apis/reset-features-api.asciidoc
index 36ff12cf0fc33..e2d3f249304bf 100644
--- a/docs/reference/features/apis/reset-features-api.asciidoc
+++ b/docs/reference/features/apis/reset-features-api.asciidoc
@@ -6,6 +6,12 @@
 
 experimental::[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-features[Features APIs].
+--
+
 Clears all of the state information stored in system indices by {es} features, including the security and machine learning indices.
 
 WARNING: Intended for development and testing use only. Do not reset features on a production cluster.
diff --git a/docs/reference/fleet/fleet-multi-search.asciidoc b/docs/reference/fleet/fleet-multi-search.asciidoc
index 3ee6b67b06ba4..5673e1abbbd1d 100644
--- a/docs/reference/fleet/fleet-multi-search.asciidoc
+++ b/docs/reference/fleet/fleet-multi-search.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Fleet multi search</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-fleet[{fleet} APIs].
+--
+
 Executes several <<fleet-search,fleet searches>> with a single API request.
 
 The API follows the same structure as the <<search-multi-search, multi search>> API. However,
diff --git a/docs/reference/fleet/fleet-search.asciidoc b/docs/reference/fleet/fleet-search.asciidoc
index 961846385969a..81ad0c9a7aa95 100644
--- a/docs/reference/fleet/fleet-search.asciidoc
+++ b/docs/reference/fleet/fleet-search.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Fleet search</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-fleet[{fleet} APIs].
+--
+
 The purpose of the fleet search api is to provide a search api where the search
 will only be executed after provided checkpoint has been processed and is visible
 for searches inside of Elasticsearch.
diff --git a/docs/reference/fleet/index.asciidoc b/docs/reference/fleet/index.asciidoc
index b22609aff32e7..8cd3cd59f21b0 100644
--- a/docs/reference/fleet/index.asciidoc
+++ b/docs/reference/fleet/index.asciidoc
@@ -2,6 +2,12 @@
 [[fleet-apis]]
 == Fleet APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-fleet[{fleet} APIs].
+--
+
 TIP: For the {kib} {fleet} APIs, see the
 {fleet-guide}/fleet-api-docs.html[Fleet API Documentation].
 
diff --git a/docs/reference/graph/explore.asciidoc b/docs/reference/graph/explore.asciidoc
index 34ac367125ade..60e9edb3b0f38 100644
--- a/docs/reference/graph/explore.asciidoc
+++ b/docs/reference/graph/explore.asciidoc
@@ -2,6 +2,12 @@
 [[graph-explore-api]]
 == Graph explore API
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-graph[Graph explore APIs].
+--
+
 The graph explore API enables you to extract and summarize information about
 the documents and terms in an {es} data stream or index.
 
diff --git a/docs/reference/health/health.asciidoc b/docs/reference/health/health.asciidoc
index 34714e80e1b18..606804a83bbda 100644
--- a/docs/reference/health/health.asciidoc
+++ b/docs/reference/health/health.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Health</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-health_report[Cluster health APIs].
+--
+
 An API that reports the health status of an {es} cluster.
 
 [[health-api-request]]
diff --git a/docs/reference/high-availability.asciidoc b/docs/reference/high-availability.asciidoc
index 2f34b6bc1bb21..37e2a38aa0f2c 100644
--- a/docs/reference/high-availability.asciidoc
+++ b/docs/reference/high-availability.asciidoc
@@ -3,28 +3,28 @@
 
 [partintro]
 --
-Your data is important to you. Keeping it safe and available is important
-to {es}. Sometimes your cluster may experience hardware failure or a power
-loss. To help you plan for this, {es} offers a number of features
-to achieve high availability despite failures.
+Your data is important to you. Keeping it safe and available is important to Elastic. Sometimes your cluster may experience hardware failure or a power loss. To help you plan for this, {es} offers a number of features to achieve high availability despite failures. Depending on your deployment type, you might need to provision servers in different zones or configure external repositories to meet your organization's availability needs.
 
-* With proper planning, a cluster can be
-  <<high-availability-cluster-design,designed for resilience>> to many of the
-  things that commonly go wrong, from the loss of a single node or network
-  connection right up to a zone-wide outage such as power loss.
+* *<<high-availability-cluster-design,Design for resilience>>* 
++
+Distributed systems like Elasticsearch are designed to keep working even if some of their components have failed. An Elasticsearch cluster can continue operating normally if some of its nodes are unavailable or disconnected, as long as there are enough well-connected nodes to take over the unavailable node's responsibilities.
++
+If you're designing a smaller cluster, you might focus on making your cluster resilient to single-node failures. Designers of larger clusters must also consider cases where multiple nodes fail at the same time.
+// need to improve connections to ECE, EC hosted, ECK pod/zone docs in the child topics
 
-* You can use <<xpack-ccr,{ccr}>> to replicate data to a remote _follower_
-  cluster which may be in a different data centre or even on a different
-  continent from the leader cluster. The follower cluster acts as a hot
-  standby, ready for you to fail over in the event of a disaster so severe that
-  the leader cluster fails. The follower cluster can also act as a geo-replica
-  to serve searches from nearby clients.
+* *<<xpack-ccr,Cross-cluster replication>>*
++
+To effectively distribute read and write operations across nodes, the nodes in a cluster need good, reliable connections to each other. To provide better connections, you typically co-locate the nodes in the same data center or nearby data centers.
++
+Co-locating nodes in a single location exposes you to the risk of a single outage taking your entire cluster offline. To maintain high availability, you can prepare a second cluster that can take over in case of disaster by implementing {ccr} (CCR).
++
+CCR provides a way to automatically synchronize indices from a leader cluster to a follower cluster. This cluster could be in a different data center or even a different content from the leader cluster. If the primary cluster fails, the secondary cluster can take over.
++
+TIP: You can also use CCR to create secondary clusters to serve read requests in geo-proximity to your users.
 
-* The last line of defence against data loss is to take
-  <<snapshots-take-snapshot,regular snapshots>> of your cluster so that you can
-  restore a completely fresh copy of it elsewhere if needed.
+* *<<snapshot-restore,Snapshots>>* 
++
+Take snapshots of your cluster that can be restored in case of failure.
 --
 
 include::high-availability/cluster-design.asciidoc[]
-
-include::ccr/index.asciidoc[]
diff --git a/docs/reference/high-availability/cluster-design.asciidoc b/docs/reference/high-availability/cluster-design.asciidoc
index 105c8b236b0b1..d187db83c43f9 100644
--- a/docs/reference/high-availability/cluster-design.asciidoc
+++ b/docs/reference/high-availability/cluster-design.asciidoc
@@ -87,7 +87,7 @@ the same thing, but it's not necessary to use this feature in such a small
 cluster.
 
 We recommend you set only one of your two nodes to be
-<<master-node,master-eligible>>. This means you can be certain which of your
+<<master-node-role,master-eligible>>. This means you can be certain which of your
 nodes is the elected master of the cluster. The cluster can tolerate the loss of
 the other master-ineligible node. If you set both nodes to master-eligible, two
 nodes are required for a master election. Since the election will fail if either
@@ -164,12 +164,12 @@ cluster that is suitable for production deployments.
 [[high-availability-cluster-design-three-nodes]]
 ==== Three-node clusters
 
-If you have three nodes, we recommend they all be <<data-node,data nodes>> and
+If you have three nodes, we recommend they all be <<data-node-role,data nodes>> and
 every index that is not a <<searchable-snapshots,searchable snapshot index>>
 should have at least one replica. Nodes are data nodes by default. You may
 prefer for some indices to have two replicas so that each node has a copy of
 each shard in those indices. You should also configure each node to be
-<<master-node,master-eligible>> so that any two of them can hold a master
+<<master-node-role,master-eligible>> so that any two of them can hold a master
 election without needing to communicate with the third node. Nodes are
 master-eligible by default. This cluster will be resilient to the loss of any
 single node.
@@ -188,8 +188,8 @@ service provides such a load balancer.
 
 Once your cluster grows to more than three nodes, you can start to specialise
 these nodes according to their responsibilities, allowing you to scale their
-resources independently as needed. You can have as many <<data-node,data
-nodes>>, <<ingest,ingest nodes>>, <<ml-node,{ml} nodes>>, etc. as needed to
+resources independently as needed. You can have as many <<data-node-role,data
+nodes>>, <<ingest,ingest nodes>>, <<ml-node-role,{ml} nodes>>, etc. as needed to
 support your workload. As your cluster grows larger, we recommend using
 dedicated nodes for each role. This allows you to independently scale resources
 for each task.
diff --git a/docs/reference/how-to/search-speed.asciidoc b/docs/reference/how-to/search-speed.asciidoc
index 0ef55d7808873..2db03a19b3532 100644
--- a/docs/reference/how-to/search-speed.asciidoc
+++ b/docs/reference/how-to/search-speed.asciidoc
@@ -567,3 +567,8 @@ also possible to update the client-side logic in order to route queries to the
 relevant indices based on filters. However `constant_keyword` makes it
 transparently and allows to decouple search requests from the index topology in
 exchange of very little overhead.
+
+[discrete]
+=== Default search timeout
+
+By default, search requests don't time out. You can set a timeout using the <<search-timeout,`search.default_search_timeout`>> setting.
diff --git a/docs/reference/ilm/apis/delete-lifecycle.asciidoc b/docs/reference/ilm/apis/delete-lifecycle.asciidoc
index fc9a35e4ef570..64a421f091eff 100644
--- a/docs/reference/ilm/apis/delete-lifecycle.asciidoc
+++ b/docs/reference/ilm/apis/delete-lifecycle.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete policy</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs].
+--
+
 Deletes an index <<index-lifecycle-management,lifecycle>> policy.
 
 [[ilm-delete-lifecycle-request]]
diff --git a/docs/reference/ilm/apis/explain.asciidoc b/docs/reference/ilm/apis/explain.asciidoc
index 31c6ae9e82ec7..4cc2667186442 100644
--- a/docs/reference/ilm/apis/explain.asciidoc
+++ b/docs/reference/ilm/apis/explain.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Explain lifecycle</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs].
+--
+
 Retrieves the current <<index-lifecycle-management,lifecycle>> status for one or more indices. For data
 streams, the API retrieves the current lifecycle status for the stream's backing
 indices.
diff --git a/docs/reference/ilm/apis/get-lifecycle.asciidoc b/docs/reference/ilm/apis/get-lifecycle.asciidoc
index b4e07389a9fb7..b02d129ebe734 100644
--- a/docs/reference/ilm/apis/get-lifecycle.asciidoc
+++ b/docs/reference/ilm/apis/get-lifecycle.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get policy</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs].
+--
+
 Retrieves a <<index-lifecycle-management,lifecycle>> policy.
 
 [[ilm-get-lifecycle-request]]
diff --git a/docs/reference/ilm/apis/get-status.asciidoc b/docs/reference/ilm/apis/get-status.asciidoc
index f2ab8d65ec9a1..648080f26b79b 100644
--- a/docs/reference/ilm/apis/get-status.asciidoc
+++ b/docs/reference/ilm/apis/get-status.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Get {ilm} status</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs].
+--
+
 Retrieves the current <<index-lifecycle-management,{ilm}>> ({ilm-init}) status.
 
 You can start or stop {ilm-init} with the <<ilm-start,start {ilm-init}>> and
diff --git a/docs/reference/ilm/apis/ilm-api.asciidoc b/docs/reference/ilm/apis/ilm-api.asciidoc
index 149ba2a6b4491..9cc594be1bea7 100644
--- a/docs/reference/ilm/apis/ilm-api.asciidoc
+++ b/docs/reference/ilm/apis/ilm-api.asciidoc
@@ -1,6 +1,12 @@
 [[index-lifecycle-management-api]]
 == {ilm-cap} APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs].
+--
+
 You use the following APIs to set up policies to automatically manage the index lifecycle.
 For more information about {ilm} ({ilm-init}), see <<index-lifecycle-management>>.
 
diff --git a/docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc b/docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc
index 8ba57120a8a65..bbcdd71c45f1a 100644
--- a/docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc
+++ b/docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc
@@ -5,7 +5,13 @@
 <titleabbrev>Migrate indices, ILM policies, and legacy, composable and component templates to data tiers routing</titleabbrev>
 ++++
 
-Switches the indices, ILM policies, and legacy, composable and component templates from using custom node attributes and
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs].
+--
+
+Switches the indices, ILM policies, and legacy, composable and component templates from using <<custom-node-attributes,custom node attributes>> and
 <<shard-allocation-filtering, attribute-based allocation filters>> to using <<data-tiers, data tiers>>, and
 optionally deletes one legacy index template.
 Using node roles enables {ilm-init} to <<data-tier-migration, automatically move the indices>> between
diff --git a/docs/reference/ilm/apis/move-to-step.asciidoc b/docs/reference/ilm/apis/move-to-step.asciidoc
index f3441fa997cff..a7a8cacf551db 100644
--- a/docs/reference/ilm/apis/move-to-step.asciidoc
+++ b/docs/reference/ilm/apis/move-to-step.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Move to step</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs].
+--
+
 Triggers execution of a specific step in the <<index-lifecycle-management,lifecycle>> policy.
 
 [[ilm-move-to-step-request]]
diff --git a/docs/reference/ilm/apis/put-lifecycle.asciidoc b/docs/reference/ilm/apis/put-lifecycle.asciidoc
index 390f6b1bb4d15..3aa691ab06da9 100644
--- a/docs/reference/ilm/apis/put-lifecycle.asciidoc
+++ b/docs/reference/ilm/apis/put-lifecycle.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Create or update lifecycle policy</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs].
+--
+
 Creates or updates <<index-lifecycle-management,lifecycle>> policy. See <<ilm-index-lifecycle>> for
 definitions of policy components.
 
diff --git a/docs/reference/ilm/apis/remove-policy-from-index.asciidoc b/docs/reference/ilm/apis/remove-policy-from-index.asciidoc
index 107cab4d5aa19..5b5842f28619c 100644
--- a/docs/reference/ilm/apis/remove-policy-from-index.asciidoc
+++ b/docs/reference/ilm/apis/remove-policy-from-index.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Remove policy</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs].
+--
+
 Removes assigned <<index-lifecycle-management,lifecycle>> policies from an index or a data stream's backing
 indices.
 
diff --git a/docs/reference/ilm/apis/retry-policy.asciidoc b/docs/reference/ilm/apis/retry-policy.asciidoc
index 8f01f15e0c3ad..a41d064b73400 100644
--- a/docs/reference/ilm/apis/retry-policy.asciidoc
+++ b/docs/reference/ilm/apis/retry-policy.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Retry policy</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs].
+--
+
 Retry executing the <<index-lifecycle-management,lifecycle>> policy for an index that is in the ERROR step.
 
 [[ilm-retry-policy-request]]
diff --git a/docs/reference/ilm/apis/start.asciidoc b/docs/reference/ilm/apis/start.asciidoc
index c38b3d9ca8831..ce9b64455be05 100644
--- a/docs/reference/ilm/apis/start.asciidoc
+++ b/docs/reference/ilm/apis/start.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Start {ilm}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs].
+--
+
 Start the <<index-lifecycle-management,{ilm}>> ({ilm-init}) plugin.
 
 [[ilm-start-request]]
diff --git a/docs/reference/ilm/apis/stop.asciidoc b/docs/reference/ilm/apis/stop.asciidoc
index a6100d794c2d3..50d8aaf4e1fd6 100644
--- a/docs/reference/ilm/apis/stop.asciidoc
+++ b/docs/reference/ilm/apis/stop.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Stop {ilm}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs].
+--
+
 Stop the <<index-lifecycle-management,{ilm}>> ({ilm-init}) plugin.
 
 [[ilm-stop-request]]
diff --git a/docs/reference/ilm/example-index-lifecycle-policy.asciidoc b/docs/reference/ilm/example-index-lifecycle-policy.asciidoc
index 6ec261fabc448..0b3c17fb2caae 100644
--- a/docs/reference/ilm/example-index-lifecycle-policy.asciidoc
+++ b/docs/reference/ilm/example-index-lifecycle-policy.asciidoc
@@ -24,7 +24,7 @@ and retention requirements.
 You want to send log files to an {es} cluster so you can visualize and analyze
 the data. This data has the following retention requirements:
 
-* When the write index reaches 50GB or is 30 days old, roll over to a new index.
+* When the primary shard size of the write index reaches 50GB or the index is 30 days old, roll over to a new index.
 * After rollover, keep indices in the hot data tier for 30 days.
 * 30 days after rollover:
 ** Move indices to the warm data tier.
@@ -84,7 +84,7 @@ To save the `logs@lifecycle` policy as a new policy in {kib}:
 . On the **Edit policy logs** page, toggle **Save as new policy**, and then provide a new name for the policy, for example, `logs-custom`.
 
 The `logs@lifecycle` policy uses the recommended rollover defaults: Start writing to a new
-index when the current write index reaches 50GB or becomes 30 days old.
+index when the primary shard size of the current write index reaches 50GB or the index becomes 30 days old.
 
 To view or change the rollover settings, click **Advanced settings** for the hot
 phase. Then disable **Use recommended defaults** to display the rollover
diff --git a/docs/reference/images/ilm/tutorial-ilm-hotphaserollover-default.png b/docs/reference/images/ilm/tutorial-ilm-hotphaserollover-default.png
index 14ff66e410835..d7f314cedb261 100644
Binary files a/docs/reference/images/ilm/tutorial-ilm-hotphaserollover-default.png and b/docs/reference/images/ilm/tutorial-ilm-hotphaserollover-default.png differ
diff --git a/docs/reference/images/search/full-text-search-overview.svg b/docs/reference/images/search/full-text-search-overview.svg
new file mode 100644
index 0000000000000..e7a1c5ba14cfa
--- /dev/null
+++ b/docs/reference/images/search/full-text-search-overview.svg
@@ -0,0 +1,81 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 700 900">
+    <!-- Styles -->
+    <defs>
+        <marker id="arrowhead" markerWidth="10" markerHeight="7" refX="9" refY="3.5" orient="auto">
+            <polygon points="0 0, 10 3.5, 0 7" fill="#666"/>
+        </marker>
+        <style type="text/css">
+            text {
+                font-family: Inter, Arial, sans-serif;
+            }
+            .title {
+                font-size: 24px;
+                font-weight: bold;
+                fill: #333;
+            }
+            .box-title {
+                font-size: 16px;
+                fill: #333;
+            }
+            .box-subtitle {
+                font-size: 12px;
+                fill: #666;
+            }
+            .search-text {
+                fill: #2d5a27;
+            }
+            .doc-text {
+                fill: #1a3d66;
+            }
+        </style>
+    </defs>
+    
+    <!-- Background -->
+    <rect width="700" height="900" fill="#ffffff"/>
+    
+    <!-- Title -->
+    <text x="350" y="50" text-anchor="middle" class="title">
+        Full-text search with Elasticsearch
+    </text>
+
+    <!-- Document Input Box -->
+    <rect x="50" y="100" width="240" height="80" rx="10" fill="#e8f0f9" stroke="#1a3d66" stroke-width="2"/>
+    <text x="170" y="145" text-anchor="middle" class="box-title doc-text">Source documents</text>
+
+    <!-- Analysis Pipeline -->
+    <rect x="230" y="250" width="240" height="80" rx="10" fill="#f4f4f4" stroke="#666" stroke-width="2"/>
+    <text x="350" y="285" text-anchor="middle" class="box-title">Analysis pipeline</text>
+    <text x="350" y="305" text-anchor="middle" class="box-subtitle">Transforms text to normalized terms</text>
+
+    <!-- Inverted Index -->
+    <rect x="230" y="420" width="240" height="80" rx="10" fill="#f4f4f4" stroke="#666" stroke-width="2"/>
+    <text x="350" y="455" text-anchor="middle" class="box-title">Inverted index</text>
+    <text x="350" y="475" text-anchor="middle" class="box-subtitle">Search-optimized data structure</text>
+
+    <!-- Search Query Box -->
+    <rect x="410" y="100" width="240" height="80" rx="10" fill="#edf7ec" stroke="#2d5a27" stroke-width="2"/>
+    <text x="530" y="145" text-anchor="middle" class="box-title search-text">Search query</text>
+
+    <!-- Scoring Engine -->
+    <rect x="230" y="590" width="240" height="80" rx="10" fill="#f4f4f4" stroke="#666" stroke-width="2"/>
+    <text x="350" y="625" text-anchor="middle" class="box-title">Relevance scoring</text>
+    <text x="350" y="645" text-anchor="middle" class="box-subtitle">Similarity algorithm scores documents</text>
+
+    <!-- Results Box (using circle shape) -->
+    <circle cx="350" cy="800" r="70" fill="#edf7ec" stroke="#2d5a27" stroke-width="2"/>
+    <text x="350" y="790" text-anchor="middle" class="box-title search-text">Search results</text>
+    <text x="350" y="810" text-anchor="middle" class="box-subtitle">Most relevant first</text>
+
+    <!-- Arrows -->
+    <line x1="170" y1="180" x2="170" y2="300" stroke="#666" stroke-width="2"/>
+    <line x1="170" y1="300" x2="230" y2="300" stroke="#666" stroke-width="2" marker-end="url(#arrowhead)"/>
+    
+    <line x1="530" y1="180" x2="530" y2="300" stroke="#666" stroke-width="2"/>
+    <line x1="530" y1="300" x2="470" y2="300" stroke="#666" stroke-width="2" marker-end="url(#arrowhead)"/>
+    
+    <line x1="350" y1="330" x2="350" y2="420" stroke="#666" stroke-width="2" marker-end="url(#arrowhead)"/>
+    
+    <line x1="350" y1="500" x2="350" y2="590" stroke="#666" stroke-width="2" marker-end="url(#arrowhead)"/>
+    
+    <line x1="350" y1="670" x2="350" y2="730" stroke="#666" stroke-width="2" marker-end="url(#arrowhead)"/>
+</svg>
\ No newline at end of file
diff --git a/docs/reference/images/search/rag-schema.svg b/docs/reference/images/search/rag-schema.svg
new file mode 100644
index 0000000000000..f26edac6c0077
--- /dev/null
+++ b/docs/reference/images/search/rag-schema.svg
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg viewBox="-20 -80 840 380" xmlns="http://www.w3.org/2000/svg">
+    <!-- Arrow markers definition -->
+    <defs>
+        <marker id="arrowhead" markerWidth="10" markerHeight="7" refX="9" refY="3.5" orient="auto">
+            <polygon points="0 0, 10 3.5, 0 7" fill="#666"/>
+        </marker>
+        <marker id="arrowhead-blue" markerWidth="10" markerHeight="7" refX="9" refY="3.5" orient="auto">
+            <polygon points="0 0, 10 3.5, 0 7" fill="#0066cc"/>
+        </marker>
+    </defs>
+    
+    <!-- Custom Instructions component -->
+    <g>
+        <text x="470" y="-15" text-anchor="middle" font-family="Arial" font-size="11" fill="#666">Define how the model should</text>
+        <text x="470" y="-5" text-anchor="middle" font-family="Arial" font-size="11" fill="#666">parse and render information</text>
+        <rect x="410" y="5" width="120" height="35" rx="10" fill="#e8f0f9" stroke="#0066cc"/>
+        <text x="470" y="25" text-anchor="middle" font-family="Arial" font-size="12" fill="#333">Custom instructions</text>
+        <line x1="470" y1="40" x2="470" y2="95" stroke="#0066cc" stroke-width="2" stroke-dasharray="2" marker-end="url(#arrowhead-blue)"/>
+    </g>
+
+    <!-- Search Strategy component -->
+    <g>
+        <text x="310" y="-15" text-anchor="middle" font-family="Arial" font-size="11" fill="#666">Full-text, semantic</text>
+        <text x="310" y="-5" text-anchor="middle" font-family="Arial" font-size="11" fill="#666">or hybrid search</text>
+        <rect x="250" y="5" width="120" height="35" rx="10" fill="#e8f0f9" stroke="#0066cc"/>
+        <text x="310" y="25" text-anchor="middle" font-family="Arial" font-size="12" fill="#333">Search strategy</text>
+        <line x1="310" y1="40" x2="310" y2="95" stroke="#0066cc" stroke-width="2" stroke-dasharray="2" marker-end="url(#arrowhead-blue)"/>
+    </g>
+
+    <!-- Number circles -->
+    <g>
+        <circle cx="150" cy="85" r="10" fill="#333"/>
+        <text x="150" y="89" text-anchor="middle" font-family="Arial" font-size="12" fill="white">1</text>
+        
+        <circle cx="240" cy="85" r="10" fill="#333"/>
+        <text x="240" y="89" text-anchor="middle" font-family="Arial" font-size="12" fill="white">2</text>
+        
+        <circle cx="400" cy="85" r="10" fill="#333"/>
+        <text x="400" y="89" text-anchor="middle" font-family="Arial" font-size="12" fill="white">3</text>
+        
+        <circle cx="580" cy="85" r="10" fill="#333"/>
+        <text x="580" y="89" text-anchor="middle" font-family="Arial" font-size="12" fill="white">4</text>
+    </g>
+
+    <!-- Main flow components -->
+    <!-- Input component -->
+    <path d="M150,115 h45 a15,15 0 0 1 0,30 h-45 a15,15 0 0 1 0,-30" fill="#edf7ec" stroke="#006400"/>
+    <text x="172" y="135" text-anchor="middle" font-family="Arial" font-size="12" fill="#006400">User query</text>
+
+    <!-- Search component -->
+    <rect x="250" y="100" width="120" height="60" rx="10" fill="#f0f0f0" stroke="#000"/>
+    <text x="310" y="135" text-anchor="middle" font-family="Arial" font-size="12" fill="#333">Elasticsearch</text>
+    <text x="310" y="180" text-anchor="middle" font-family="Arial" font-size="12" fill="#666">
+        <tspan font-weight="bold">Retrieves</tspan> relevant</text>
+    <text x="310" y="195" text-anchor="middle" font-family="Arial" font-size="12" fill="#666">documents</text>
+
+    <!-- Processing component -->
+    <rect x="410" y="100" width="120" height="60" rx="10" fill="#f0f0f0" stroke="#000"/>
+    <text x="470" y="135" text-anchor="middle" font-family="Arial" font-size="12" fill="#333">Language model</text>
+    <text x="470" y="180" text-anchor="middle" font-family="Arial" font-size="12" fill="#666">Processes context &amp;</text>
+    <text x="470" y="195" text-anchor="middle" font-family="Arial" font-size="12" fill="#666">
+        <tspan font-weight="bold">generates</tspan> answer</text>
+
+    <!-- Flow connections (arrows) -->
+    <line x1="210" y1="130" x2="250" y2="130" stroke="#666" stroke-width="2" marker-end="url(#arrowhead)"/>
+    <line x1="370" y1="130" x2="410" y2="130" stroke="#666" stroke-width="2" marker-end="url(#arrowhead)"/>
+    <line x1="530" y1="130" x2="570" y2="130" stroke="#666" stroke-width="2" marker-end="url(#arrowhead)"/>
+
+    <!-- Output component -->
+    <path d="M585,115 h45 a15,15 0 0 1 0,30 h-45 a15,15 0 0 1 0,-30" fill="#edf7ec" stroke="#006400"/>
+    <text x="607" y="135" text-anchor="middle" font-family="Arial" font-size="12" fill="#006400">Response</text>
+</svg>
\ No newline at end of file
diff --git a/docs/reference/images/search/rag-venn-diagram.svg b/docs/reference/images/search/rag-venn-diagram.svg
new file mode 100644
index 0000000000000..9906aaefaba0c
--- /dev/null
+++ b/docs/reference/images/search/rag-venn-diagram.svg
@@ -0,0 +1,19 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 600 400">
+  <!-- Left circle (Information Retrieval) -->
+  <circle cx="220" cy="200" r="150" fill="#4A90E2" opacity="0.6"/>
+  
+  <!-- Right circle (Generative AI) -->
+  <circle cx="380" cy="200" r="150" fill="#50C878" opacity="0.6"/>
+  
+  <!-- Text labels -->
+  <text x="160" y="210" font-family="Arial" font-size="20" fill="#2C3E50" text-anchor="middle">Information
+    <tspan x="160" y="235">retrieval</tspan>
+  </text>
+  
+  <text x="440" y="200" font-family="Arial" font-size="20" fill="#2C3E50" text-anchor="middle">Generative
+    <tspan x="440" y="225">AI</tspan>
+  </text>
+  
+  <!-- RAG label in intersection -->
+  <text x="300" y="200" font-family="Arial" font-size="28" font-weight="bold" fill="#2C3E50" text-anchor="middle">RAG</text>
+</svg>
\ No newline at end of file
diff --git a/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc b/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc
index d08af21007622..2d59e9be31cd4 100644
--- a/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc
+++ b/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc
@@ -13,7 +13,7 @@ This setting corresponds to the data node roles:
 * <<data-cold-node, data_cold>>
 * <<data-frozen-node, data_frozen>>
 
-NOTE: The <<data-node, data>> role is not a valid data tier and cannot be used
+NOTE: The <<data-node-role, data>> role is not a valid data tier and cannot be used
 with the `_tier_preference` setting. The frozen tier stores <<partially-mounted,partially
 mounted indices>> exclusively.
 
diff --git a/docs/reference/index-modules/allocation/filtering.asciidoc b/docs/reference/index-modules/allocation/filtering.asciidoc
index 07a2455ca1eff..5da8e254cb4f2 100644
--- a/docs/reference/index-modules/allocation/filtering.asciidoc
+++ b/docs/reference/index-modules/allocation/filtering.asciidoc
@@ -6,7 +6,7 @@ a particular index. These per-index filters are applied in conjunction with
 <<cluster-shard-allocation-filtering, cluster-wide allocation filtering>> and
 <<shard-allocation-awareness, allocation awareness>>.
 
-Shard allocation filters can be based on custom node attributes or the built-in
+Shard allocation filters can be based on <<custom-node-attributes,custom node attributes>> or the built-in
 `_name`, `_host_ip`, `_publish_ip`, `_ip`, `_host`, `_id`, `_tier` and `_tier_preference`
 attributes. <<index-lifecycle-management, Index lifecycle management>> uses filters based
 on custom node attributes to determine how to reallocate shards when moving
@@ -114,7 +114,7 @@ The index allocation settings support the following built-in attributes:
 
 NOTE: `_tier` filtering is based on <<modules-node, node>> roles. Only
 a subset of roles are <<data-tiers, data tier>> roles, and the generic
-<<data-node, data role>> will match any tier filtering.
+<<data-node-role, data role>> will match any tier filtering.
 
 You can use wildcards when specifying attribute values, for example:
 
diff --git a/docs/reference/index-modules/slowlog.asciidoc b/docs/reference/index-modules/slowlog.asciidoc
index c29296b59ad4a..e848668c1a66d 100644
--- a/docs/reference/index-modules/slowlog.asciidoc
+++ b/docs/reference/index-modules/slowlog.asciidoc
@@ -1,15 +1,118 @@
 [[index-modules-slowlog]]
-== Slow Log
+== Slow log
+
+The slow log records database searching and indexing events that have execution durations above specified thresholds. You can use these logs to investigate analyze or troubleshoot your cluster's historical search and indexing performance.
+
+Slow logs report task duration at the shard level for searches, and at the index level
+for indexing, but might not encompass the full task execution time observed on the client. For example, slow logs don't surface HTTP network delays or the impact of <<task-queue-backlog,task queues>>. 
+
+Events that meet the specified threshold are emitted into <<logging,{es} logging>> under the `fileset.name` of `slowlog`. These logs can be viewed in the following locations:
+
+* If <<monitoring-overview,{es} monitoring>> is enabled, from
+{kibana-ref}/xpack-monitoring.html[Stack Monitoring]. Slow log events have a `logger` value of `index.search.slowlog` or `index.indexing.slowlog`.
+
+* From local {es} service logs directory. Slow log files have a suffix of `_index_search_slowlog.json` or `_index_indexing_slowlog.json`.
+
+[discrete]
+[[slow-log-format]]
+=== Slow log format
+
+The following is an example of a search event in the slow log:
+
+TIP: If a call was initiated with an `X-Opaque-ID` header, then the ID is automatically included in Search slow logs in the **elasticsearch.slowlog.id** field. See <<x-opaque-id,X-Opaque-Id HTTP header>> for details and best practices.
+
+[source,js]
+---------------------------
+{
+  "@timestamp": "2024-12-21T12:42:37.255Z",
+  "auth.type": "REALM",
+  "ecs.version": "1.2.0",
+  "elasticsearch.cluster.name": "distribution_run",
+  "elasticsearch.cluster.uuid": "Ui23kfF1SHKJwu_hI1iPPQ",
+  "elasticsearch.node.id": "JK-jn-XpQ3OsDUsq5ZtfGg",
+  "elasticsearch.node.name": "node-0",
+  "elasticsearch.slowlog.id": "tomcat-123",
+  "elasticsearch.slowlog.message": "[index6][0]",
+  "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH",
+  "elasticsearch.slowlog.source": "{\"query\":{\"match_all\":{\"boost\":1.0}}}",
+  "elasticsearch.slowlog.stats": "[]",
+  "elasticsearch.slowlog.took": "747.3micros",
+  "elasticsearch.slowlog.took_millis": 0,
+  "elasticsearch.slowlog.total_hits": "1 hits",
+  "elasticsearch.slowlog.total_shards": 1,
+  "event.dataset": "elasticsearch.index_search_slowlog",
+  "fileset.name" : "slowlog",
+  "log.level": "WARN",
+  "log.logger": "index.search.slowlog.query",
+  "process.thread.name": "elasticsearch[runTask-0][search][T#5]",
+  "service.name": "ES_ECS",
+  "user.name": "elastic",
+  "user.realm": "reserved"
+}
+
+---------------------------
+// NOTCONSOLE
+
+
+The following is an example of an indexing event in the slow log:
+
+[source,js]
+---------------------------
+{
+  "@timestamp" : "2024-12-11T22:34:22.613Z",
+  "auth.type": "REALM",
+  "ecs.version": "1.2.0",
+  "elasticsearch.cluster.name" : "41bd111609d849fc9bf9d25b5df9ce96",
+  "elasticsearch.cluster.uuid" : "BZTn4I9URXSK26imlia0QA",
+  "elasticsearch.index.id" : "3VfGR7wRRRKmMCEn7Ii58g",
+  "elasticsearch.index.name": "my-index-000001",
+  "elasticsearch.node.id" : "GGiBgg21S3eqPDHzQiCMvQ",
+  "elasticsearch.node.name" : "instance-0000000001",
+  "elasticsearch.slowlog.id" : "RCHbt5MBT0oSsCOu54AJ",
+  "elasticsearch.slowlog.source": "{\"key\":\"value\"}"
+  "elasticsearch.slowlog.took" : "0.01ms",
+  "event.dataset": "elasticsearch.index_indexing_slowlog",
+  "fileset.name" : "slowlog",
+  "log.level" : "TRACE",
+  "log.logger" : "index.indexing.slowlog.index",
+  "service.name" : "ES_ECS",
+  "user.name": "elastic",
+  "user.realm": "reserved"
+}
+
+---------------------------
+// NOTCONSOLE
+
+[discrete]
+[[enable-slow-log]]
+=== Enable slow logging
+
+You can enable slow logging at two levels: 
+
+* For all indices under the <<settings,{es} `log4j2.properties` configuration file>>. This method requires a node restart.
+* At the index level, using the <<indices-update-settings,update indices settings API>>
+
+By default, all thresholds are set to `-1`, which results in no events being logged. 
+
+Slow log thresholds can be enabled for the four logging levels: `trace`, `debug`, `info`, and `warn`. You can mimic setting log level thresholds by disabling more verbose levels.
+
+To view the current slow log settings, use the <<indices-get-settings,get index settings API>>:
+
+[source,console]
+--------------------------------------------------
+GET _all/_settings?expand_wildcards=all&filter_path=*.settings.index.*.slowlog
+--------------------------------------------------
 
 [discrete]
 [[search-slow-log]]
-=== Search Slow Log
+==== Enable slow logging for search events
+
+Search slow logs emit per shard. They must be enabled separately for the shard's link:https://www.elastic.co/blog/understanding-query-then-fetch-vs-dfs-query-then-fetch[query and fetch search phases].
 
-Shard level slow search log allows to log slow search (query and fetch
-phases) into a dedicated log file.
+You can use the `index.search.slowlog.include.user` setting to append `user.*` and `auth.type` fields to slow log entries. These fields contain information about the user who triggered the request.
 
-Thresholds can be set for both the query phase of the execution, and
-fetch phase, here is a sample:
+The following snippet adjusts all available search slow log settings across all indices using the 
+<<settings,`log4j2.properties` configuration file>>:
 
 [source,yaml]
 --------------------------------------------------
@@ -22,10 +125,11 @@ index.search.slowlog.threshold.fetch.warn: 1s
 index.search.slowlog.threshold.fetch.info: 800ms
 index.search.slowlog.threshold.fetch.debug: 500ms
 index.search.slowlog.threshold.fetch.trace: 200ms
+
+index.search.slowlog.include.user: true
 --------------------------------------------------
 
-All of the above settings are _dynamic_ and can be set for each index using the
-<<indices-update-settings, update indices settings>> API. For example:
+The following snippet adjusts the same settings for a single index using the <<indices-update-settings,update indices settings API>>:
 
 [source,console]
 --------------------------------------------------
@@ -38,138 +142,109 @@ PUT /my-index-000001/_settings
   "index.search.slowlog.threshold.fetch.warn": "1s",
   "index.search.slowlog.threshold.fetch.info": "800ms",
   "index.search.slowlog.threshold.fetch.debug": "500ms",
-  "index.search.slowlog.threshold.fetch.trace": "200ms"
+  "index.search.slowlog.threshold.fetch.trace": "200ms",
+  "index.search.slowlog.include.user": true
 }
 --------------------------------------------------
 // TEST[setup:my_index]
 
-By default thresholds are disabled (set to `-1`).
 
-The logging is done on the shard level scope, meaning the execution of a
-search request within a specific shard. It does not encompass the whole
-search request, which can be broadcast to several shards in order to
-execute. Some of the benefits of shard level logging is the association
-of the actual execution on the specific machine, compared with request
-level.
+[discrete]
+[[index-slow-log]]
+==== Enable slow logging for indexing events
 
+Indexing slow logs emit per index document. 
 
-The search slow log file is configured in the `log4j2.properties` file.
+You can use the `index.indexing.slowlog.include.user` setting to append `user.*` and `auth.type` fields to slow log entries. These fields contain information about the user who triggered the request.
 
-[discrete]
-==== Identifying search slow log origin
+The following snippet adjusts all available indexing slow log settings across all indices using the 
+<<settings,`log4j2.properties` configuration file>>:
 
-It is often useful to identify what triggered a slow running query.
-To include information about the user that triggered a slow search,
-use the `index.search.slowlog.include.user` setting.
+[source,yaml]
+--------------------------------------------------
+index.indexing.slowlog.threshold.index.warn: 10s
+index.indexing.slowlog.threshold.index.info: 5s
+index.indexing.slowlog.threshold.index.debug: 2s
+index.indexing.slowlog.threshold.index.trace: 500ms
+
+index.indexing.slowlog.source: 1000
+index.indexing.slowlog.reformat: true
+
+index.indexing.slowlog.include.user: true
+--------------------------------------------------
+
+
+The following snippet adjusts the same settings for a single index using the <<indices-update-settings,update indices settings API>>:
 
 [source,console]
 --------------------------------------------------
 PUT /my-index-000001/_settings
 {
-  "index.search.slowlog.include.user": true
+  "index.indexing.slowlog.threshold.index.warn": "10s",
+  "index.indexing.slowlog.threshold.index.info": "5s",
+  "index.indexing.slowlog.threshold.index.debug": "2s",
+  "index.indexing.slowlog.threshold.index.trace": "500ms",
+  "index.indexing.slowlog.source": "1000",
+  "index.indexing.slowlog.reformat": true,
+  "index.indexing.slowlog.include.user": true
 }
 --------------------------------------------------
 // TEST[setup:my_index]
 
-This will result in user information being included in the slow log.
+[discrete]
+===== Logging the `_source` field
 
-[source,js]
----------------------------
-{
-  "@timestamp": "2024-02-21T12:42:37.255Z",
-  "log.level": "WARN",
-  "auth.type": "REALM",
-  "elasticsearch.slowlog.id": "tomcat-123",
-  "elasticsearch.slowlog.message": "[index6][0]",
-  "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH",
-  "elasticsearch.slowlog.source": "{\"query\":{\"match_all\":{\"boost\":1.0}}}",
-  "elasticsearch.slowlog.stats": "[]",
-  "elasticsearch.slowlog.took": "747.3micros",
-  "elasticsearch.slowlog.took_millis": 0,
-  "elasticsearch.slowlog.total_hits": "1 hits",
-  "elasticsearch.slowlog.total_shards": 1,
-  "user.name": "elastic",
-  "user.realm": "reserved",
-  "ecs.version": "1.2.0",
-  "service.name": "ES_ECS",
-  "event.dataset": "elasticsearch.index_search_slowlog",
-  "process.thread.name": "elasticsearch[runTask-0][search][T#5]",
-  "log.logger": "index.search.slowlog.query",
-  "elasticsearch.cluster.uuid": "Ui23kfF1SHKJwu_hI1iPPQ",
-  "elasticsearch.node.id": "JK-jn-XpQ3OsDUsq5ZtfGg",
-  "elasticsearch.node.name": "node-0",
-  "elasticsearch.cluster.name": "distribution_run"
-}
+By default, {es} logs the first 1000 characters of the `_source` in the slow log. You can adjust how `_source` is logged using the `index.indexing.slowlog.source` setting. Set `index.indexing.slowlog.source` to `false` or `0` to skip logging the source entirely. Set `index.indexing.slowlog.source` to `true` to log the entire source regardless of size. 
 
----------------------------
-// NOTCONSOLE
+The original `_source` is reformatted by default to make sure that it fits on a single log line. If preserving the original document format is important, then you can turn off reformatting by setting `index.indexing.slowlog.reformat` to `false`. This causes source to be logged with the original formatting intact, potentially spanning multiple log lines.
 
-If a call was initiated with an `X-Opaque-ID` header, then the ID is included
-in Search Slow logs in the **elasticsearch.slowlog.id** field. See
-<<x-opaque-id, X-Opaque-Id HTTP header>> for details and best practices.
+[discrete]
+[[slow-log-fields]]
 
 [discrete]
-[[index-slow-log]]
-=== Index Slow log
+[[troubleshoot-slow-log]]
+=== Best practices for slow logging
 
-The indexing slow log, similar in functionality to the search slow
-log. The log file name ends with `_index_indexing_slowlog.json`. Log and
-the thresholds are configured in the same way as the search slowlog.
-Index slowlog sample:
+Logging slow requests can be resource intensive to your {es} cluster depending on the qualifying traffic's volume. For example, emitted logs might increase the index disk usage of your <<monitoring-overview,{es} monitoring>> cluster. To reduce the impact of slow logs, consider the following:
 
-[source,yaml]
---------------------------------------------------
-index.indexing.slowlog.threshold.index.warn: 10s
-index.indexing.slowlog.threshold.index.info: 5s
-index.indexing.slowlog.threshold.index.debug: 2s
-index.indexing.slowlog.threshold.index.trace: 500ms
-index.indexing.slowlog.source: 1000
---------------------------------------------------
+* Enable slow logs against specific indices rather than across all indices.
+* Set high thresholds to reduce the number of logged events.
+* Enable slow logs only when troubleshooting. 
 
-All of the above settings are _dynamic_ and can be set for each index using the
-<<indices-update-settings, update indices settings>> API. For example:
+If you aren't sure how to start investigating traffic issues, consider enabling the `warn` threshold with a high `30s` threshold at the index level using the <<indices-update-settings,update indices settings API>>: 
 
+* Enable for search requests: 
++
 [source,console]
 --------------------------------------------------
-PUT /my-index-000001/_settings
+PUT /*/_settings
 {
-  "index.indexing.slowlog.threshold.index.warn": "10s",
-  "index.indexing.slowlog.threshold.index.info": "5s",
-  "index.indexing.slowlog.threshold.index.debug": "2s",
-  "index.indexing.slowlog.threshold.index.trace": "500ms",
-  "index.indexing.slowlog.source": "1000"
+  "index.search.slowlog.include.user": true,
+  "index.search.slowlog.threshold.fetch.warn": "30s",
+  "index.search.slowlog.threshold.query.warn": "30s"
 }
 --------------------------------------------------
 // TEST[setup:my_index]
 
-To include information about the user that triggered a slow indexing event,
-use the `index.indexing.slowlog.include.user` setting.
-
+* Enable for indexing requests:
++
 [source,console]
 --------------------------------------------------
-PUT /my-index-000001/_settings
+PUT /*/_settings
 {
-  "index.indexing.slowlog.include.user": true
+  "index.indexing.slowlog.include.user": true,
+  "index.indexing.slowlog.threshold.index.warn": "30s"
 }
 --------------------------------------------------
 // TEST[setup:my_index]
 
-By default Elasticsearch will log the first 1000 characters of the _source in
-the slowlog. You can change that with `index.indexing.slowlog.source`. Setting
-it to `false` or `0` will skip logging the source entirely, while setting it to
-`true` will log the entire source regardless of size. The original `_source` is
-reformatted by default to make sure that it fits on a single log line. If preserving
-the original document format is important, you can turn off reformatting by setting
-`index.indexing.slowlog.reformat` to `false`, which will cause the source to be
-logged "as is" and can potentially span multiple log lines.
+Slow log thresholds being met does not guarantee cluster performance issues. In the event that symptoms are noticed, slow logs can provide helpful data to diagnose upstream traffic patterns or sources to resolve client-side issues. For example, you can use data included in `X-Opaque-ID`, the `_source` request body, or `user.*` fields to identify the source of your issue. This is similar to troubleshooting <<task-queue-backlog,live expensive tasks>>.
+
+If you're experiencing search performance issues, then you might also consider investigating searches flagged for their query durations using the <<search-profile,profile API>>. You can then use the profiled query to investigate optimization options using the link:{kibana-ref}/xpack-profiler.html[query profiler]. This type of investigation should usually take place in a non-production environment.
 
-The index slow log file is configured in the `log4j2.properties` file.
+Slow logging checks each event against the reporting threshold when the event is complete. This means that it can't report if events trigger <<circuit-breaker-errors,circuit breaker errors>>. If suspect circuit breaker errors, then you should also consider enabling <<enable-audit-logging,audit logging>>, which logs events before they are executed.
 
 [discrete]
-=== Slow log levels
+=== Learn more
 
-You can mimic the search or indexing slow log level by setting appropriate
-threshold making "more verbose" loggers to be switched off.
-If for instance we want to simulate `index.indexing.slowlog.level: INFO`
-then all we need to do is to set
-`index.indexing.slowlog.threshold.index.debug` and `index.indexing.slowlog.threshold.index.trace` to `-1`.
+To learn about other ways to optimize your search and indexing requests, refer to <<tune-for-search-speed,tune for search speed>> and <<tune-for-indexing-speed,tune for indexing speed>>.
\ No newline at end of file
diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc
index 18052cfb64e8f..8e1c211eb9426 100644
--- a/docs/reference/index.asciidoc
+++ b/docs/reference/index.asciidoc
@@ -76,8 +76,12 @@ include::autoscaling/index.asciidoc[]
 
 include::snapshot-restore/index.asciidoc[]
 
+include::ccr/index.asciidoc[leveloffset=-1]
+
 // reference
 
+include::data-store-architecture.asciidoc[]
+
 include::rest-api/index.asciidoc[]
 
 include::commands/index.asciidoc[]
diff --git a/docs/reference/indices.asciidoc b/docs/reference/indices.asciidoc
index eaef54a1effb1..b6b82422cbb4a 100644
--- a/docs/reference/indices.asciidoc
+++ b/docs/reference/indices.asciidoc
@@ -1,6 +1,12 @@
 [[indices]]
 == Index APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Index APIs are used to manage individual indices,
 index settings, aliases, mappings, and index templates.
 
@@ -18,7 +24,6 @@ index settings, aliases, mappings, and index templates.
 * <<indices-split-index>>
 * <<indices-clone-index>>
 * <<indices-rollover-index>>
-* <<unfreeze-index-api>>
 * <<indices-resolve-index-api>>
 * <<indices-resolve-cluster-api>>
 * <<indices-downsample-data-stream>>
@@ -137,6 +142,5 @@ include::indices/shrink-index.asciidoc[]
 include::indices/simulate-index.asciidoc[]
 include::indices/simulate-template.asciidoc[]
 include::indices/split-index.asciidoc[]
-include::indices/apis/unfreeze.asciidoc[]
 include::indices/update-settings.asciidoc[]
 include::indices/put-mapping.asciidoc[]
diff --git a/docs/reference/indices/add-alias.asciidoc b/docs/reference/indices/add-alias.asciidoc
index e14af6a64a2ec..13b49f5a47dd1 100644
--- a/docs/reference/indices/add-alias.asciidoc
+++ b/docs/reference/indices/add-alias.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Create or update alias</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Adds a data stream or index to an <<aliases,alias>>.
 
 [source,console]
diff --git a/docs/reference/indices/alias-exists.asciidoc b/docs/reference/indices/alias-exists.asciidoc
index a514d36a1bfef..c086143ed3f1c 100644
--- a/docs/reference/indices/alias-exists.asciidoc
+++ b/docs/reference/indices/alias-exists.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Alias exists</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Checks if an <<aliases,alias>> exists.
 
 [source,console]
diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc
index 1df9e0a4883b8..243f436f55b5a 100644
--- a/docs/reference/indices/aliases.asciidoc
+++ b/docs/reference/indices/aliases.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Aliases</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Performs one or more <<aliases,alias>> actions in a single atomic operation.
 
 [source,console]
diff --git a/docs/reference/indices/analyze.asciidoc b/docs/reference/indices/analyze.asciidoc
index eb4d877b463cd..a211a44730a64 100644
--- a/docs/reference/indices/analyze.asciidoc
+++ b/docs/reference/indices/analyze.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Analyze</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Performs <<analysis,analysis>> on a text string
 and returns the resulting tokens.
 
diff --git a/docs/reference/indices/apis/reload-analyzers.asciidoc b/docs/reference/indices/apis/reload-analyzers.asciidoc
index ca5f540564f8e..a27a3bb859cf1 100644
--- a/docs/reference/indices/apis/reload-analyzers.asciidoc
+++ b/docs/reference/indices/apis/reload-analyzers.asciidoc
@@ -2,6 +2,12 @@
 [[indices-reload-analyzers]]
 == Reload search analyzers API
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/operation/operation-indices-reload-search-analyzers[Reload search analyzers].
+--
+
 Reloads an index's <<search-analyzer,search analyzers>> and their resources.
 For data streams, the API reloads search analyzers and resources for the
 stream's backing indices.
diff --git a/docs/reference/indices/apis/unfreeze.asciidoc b/docs/reference/indices/apis/unfreeze.asciidoc
deleted file mode 100644
index 450efc03c492a..0000000000000
--- a/docs/reference/indices/apis/unfreeze.asciidoc
+++ /dev/null
@@ -1,55 +0,0 @@
-[role="xpack"]
-[[unfreeze-index-api]]
-=== Unfreeze index API
-++++
-<titleabbrev>Unfreeze index</titleabbrev>
-++++
-
-[WARNING]
-.Deprecated in 7.14
-====
-In 8.0, we removed the ability to freeze an index. In previous versions,
-freezing an index reduced its memory overhead. However, frozen indices are no
-longer useful due to
-https://www.elastic.co/blog/significantly-decrease-your-elasticsearch-heap-memory-usage[recent
-improvements in heap memory usage].
-You can use this API to unfreeze indices that were frozen in 7.x. Frozen indices
-are not related to the frozen data tier.
-====
-
-Unfreezes an index.
-
-[[unfreeze-index-api-request]]
-==== {api-request-title}
-
-`POST /<index>/_unfreeze`
-
-[[unfreeze-index-api-prereqs]]
-==== {api-prereq-title}
-
-* If the {es} {security-features} are enabled, you must have the `manage`
-<<privileges-list-indices,index privilege>> for the target index or index alias.
-
-[[unfreeze-index-api-desc]]
-==== {api-description-title}
-
-When a frozen index is unfrozen, the index goes through the normal recovery
-process and becomes writeable again.
-
-[[unfreeze-index-api-path-parms]]
-==== {api-path-parms-title}
-
-`<index>`::
-  (Required, string) Identifier for the index.
-
-[[unfreeze-index-api-examples]]
-==== {api-examples-title}
-
-The following example unfreezes an index:
-
-[source,console]
---------------------------------------------------
-POST /my-index-000001/_unfreeze
---------------------------------------------------
-// TEST[s/^/PUT my-index-000001\n/]
-// TEST[skip:unable to ignore deprecation warning]
diff --git a/docs/reference/indices/clearcache.asciidoc b/docs/reference/indices/clearcache.asciidoc
index a3150ec6f72ee..f981a6d69dd63 100644
--- a/docs/reference/indices/clearcache.asciidoc
+++ b/docs/reference/indices/clearcache.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Clear cache</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Clears the caches of one or more indices. For data streams, the API clears the
 caches of the stream's backing indices.
 
diff --git a/docs/reference/indices/clone-index.asciidoc b/docs/reference/indices/clone-index.asciidoc
index c8e5d2e200f2e..734ff33e48215 100644
--- a/docs/reference/indices/clone-index.asciidoc
+++ b/docs/reference/indices/clone-index.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Clone index</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Clones an existing index.
 
 [source,console]
diff --git a/docs/reference/indices/close.asciidoc b/docs/reference/indices/close.asciidoc
index a4bf1742fdea8..56d318aad2f7a 100644
--- a/docs/reference/indices/close.asciidoc
+++ b/docs/reference/indices/close.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Close index</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Closes an index.
 
 [source,console]
diff --git a/docs/reference/indices/create-data-stream.asciidoc b/docs/reference/indices/create-data-stream.asciidoc
index e97e9973f1063..dd2d471117949 100644
--- a/docs/reference/indices/create-data-stream.asciidoc
+++ b/docs/reference/indices/create-data-stream.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Create data stream</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs].
+--
+
 Creates a new <<data-streams,data stream>>.
 
 ////
diff --git a/docs/reference/indices/create-index.asciidoc b/docs/reference/indices/create-index.asciidoc
index 2e66f3d6030cb..a210b05b2bcf0 100644
--- a/docs/reference/indices/create-index.asciidoc
+++ b/docs/reference/indices/create-index.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Create index</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Creates a new index.
 
 [source,console]
diff --git a/docs/reference/indices/dangling-index-delete.asciidoc b/docs/reference/indices/dangling-index-delete.asciidoc
index 6af35031e9e61..b1fd6790972c4 100644
--- a/docs/reference/indices/dangling-index-delete.asciidoc
+++ b/docs/reference/indices/dangling-index-delete.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Delete dangling index</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Deletes a dangling index.
 
 [[dangling-index-delete-api-request]]
diff --git a/docs/reference/indices/dangling-index-import.asciidoc b/docs/reference/indices/dangling-index-import.asciidoc
index 44cde56de8c95..e266ecd2c9c70 100644
--- a/docs/reference/indices/dangling-index-import.asciidoc
+++ b/docs/reference/indices/dangling-index-import.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Import dangling index</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Imports a dangling index.
 
 [[dangling-index-import-api-request]]
diff --git a/docs/reference/indices/dangling-indices-list.asciidoc b/docs/reference/indices/dangling-indices-list.asciidoc
index b7774843e36d9..2b7dcdc497ebf 100644
--- a/docs/reference/indices/dangling-indices-list.asciidoc
+++ b/docs/reference/indices/dangling-indices-list.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>List dangling indices</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Lists dangling indices.
 
 [[dangling-indices-list-api-request]]
diff --git a/docs/reference/indices/data-stream-stats.asciidoc b/docs/reference/indices/data-stream-stats.asciidoc
index 3ed285abc035a..cce145b4dadcc 100644
--- a/docs/reference/indices/data-stream-stats.asciidoc
+++ b/docs/reference/indices/data-stream-stats.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Data stream stats</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs].
+--
+
 Retrieves statistics for one or more <<data-streams,data streams>>.
 
 ////
diff --git a/docs/reference/indices/delete-alias.asciidoc b/docs/reference/indices/delete-alias.asciidoc
index 748862df06100..c204c89579352 100644
--- a/docs/reference/indices/delete-alias.asciidoc
+++ b/docs/reference/indices/delete-alias.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Delete alias</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Removes a data stream or index from an <<aliases,alias>>.
 
 [source,console]
diff --git a/docs/reference/indices/delete-component-template.asciidoc b/docs/reference/indices/delete-component-template.asciidoc
index 065a4adb90023..27c1e00a5d57a 100644
--- a/docs/reference/indices/delete-component-template.asciidoc
+++ b/docs/reference/indices/delete-component-template.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Delete component template</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Deletes an existing component template.
 
 ////
diff --git a/docs/reference/indices/delete-data-stream.asciidoc b/docs/reference/indices/delete-data-stream.asciidoc
index 38e7a00d451d8..1cb43a615ede8 100644
--- a/docs/reference/indices/delete-data-stream.asciidoc
+++ b/docs/reference/indices/delete-data-stream.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete data stream</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs].
+--
+
 Deletes one or more <<data-streams,data streams>> and their backing
 indices. See <<delete-data-stream>>.
 
diff --git a/docs/reference/indices/delete-index-template-v1.asciidoc b/docs/reference/indices/delete-index-template-v1.asciidoc
index 98b1e2fb255f1..e035c13a7bceb 100644
--- a/docs/reference/indices/delete-index-template-v1.asciidoc
+++ b/docs/reference/indices/delete-index-template-v1.asciidoc
@@ -9,6 +9,12 @@ templates>>, which are deprecated and will be replaced by the composable
 templates introduced in {es} 7.8. For information about composable templates,
 see <<index-templates>>.
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Deletes a legacy index template.
 
 ////
diff --git a/docs/reference/indices/delete-index-template.asciidoc b/docs/reference/indices/delete-index-template.asciidoc
index b828e4a536b71..23713f62fa031 100644
--- a/docs/reference/indices/delete-index-template.asciidoc
+++ b/docs/reference/indices/delete-index-template.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Delete index template</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Deletes an <<index-templates,index template>>.
 
 ////
diff --git a/docs/reference/indices/delete-index.asciidoc b/docs/reference/indices/delete-index.asciidoc
index d5d168154e44a..81ee7b502d377 100644
--- a/docs/reference/indices/delete-index.asciidoc
+++ b/docs/reference/indices/delete-index.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Delete index</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Deletes one or more indices.
 
 [source,console]
diff --git a/docs/reference/indices/diskusage.asciidoc b/docs/reference/indices/diskusage.asciidoc
index 3510ba346e5a7..6db122ab7d836 100644
--- a/docs/reference/indices/diskusage.asciidoc
+++ b/docs/reference/indices/diskusage.asciidoc
@@ -6,6 +6,12 @@
 
 experimental[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Analyzes the disk usage of each field of an index or data stream.
 This API might not support indices created in previous {es} versions.
 The result of a small index can be inaccurate as some parts of an index
diff --git a/docs/reference/indices/downsample-data-stream.asciidoc b/docs/reference/indices/downsample-data-stream.asciidoc
index 5ace4e03dfb66..6354f8e30d254 100644
--- a/docs/reference/indices/downsample-data-stream.asciidoc
+++ b/docs/reference/indices/downsample-data-stream.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Downsample</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs].
+--
+
 Aggregates a time series (TSDS) index and stores
 pre-computed statistical summaries (`min`, `max`, `sum`, `value_count` and
 `avg`) for each metric field grouped by a configured time interval. For example,
@@ -75,6 +81,8 @@ DELETE _index_template/*
 ////
 // end::downsample-example[]
 
+Check the <<downsampling,Downsampling>> documentation for an overview, details about the downsampling process, and examples of running downsampling manually and as part of an ILM policy.
+
 [[downsample-api-request]]
 ==== {api-request-title}
 
@@ -115,44 +123,4 @@ to aggregate the original time series index. For example, `60m` produces a
 document for each 60 minute (hourly) interval. This follows standard time
 formatting syntax as used elsewhere in {es}.
 +
-NOTE: Smaller, more granular intervals take up proportionally more space.
-
-[[downsample-api-process]]
-==== The downsampling process
-
-The downsampling operation traverses the source TSDS index and performs the
-following steps:
-
-. Creates a new document for each value of the `_tsid` field and each
-`@timestamp` value, rounded to the `fixed_interval` defined in the downsample
-configuration.
-. For each new document, copies all <<time-series-dimension,time
-series dimensions>> from the source index to the target index. Dimensions in a
-TSDS are constant, so this is done only once per bucket.
-. For each <<time-series-metric,time series metric>> field, computes aggregations
-for all documents in the bucket. Depending on the metric type of each metric
-field a different set of pre-aggregated results is stored:
-
-** `gauge`: The `min`, `max`, `sum`, and `value_count` are stored; `value_count`
-is stored as type `aggregate_metric_double`.
-** `counter`: The `last_value` is stored.
-. For all other fields, the most recent value is copied to the target index.
-
-[[downsample-api-mappings]]
-==== Source and target index field mappings
-
-Fields in the target, downsampled index are created based on fields in the
-original source index, as follows:
-
-. All fields mapped with the `time-series-dimension` parameter are created in
-the target downsample index with the same mapping as in the source index.
-. All fields mapped with the `time_series_metric` parameter are created
-in the target downsample index with the same mapping as in the source
-index. An exception is that for fields mapped as `time_series_metric: gauge`
-the field type is changed to `aggregate_metric_double`.
-. All other fields that are neither dimensions nor metrics (that is, label
-fields), are created in the target downsample index with the same mapping
-that they had in the source index.
-
-Check the <<downsampling,Downsampling>> documentation for an overview and
-examples of running downsampling manually and as part of an ILM policy.
+NOTE: Smaller, more granular intervals take up proportionally more space.
\ No newline at end of file
diff --git a/docs/reference/indices/field-usage-stats.asciidoc b/docs/reference/indices/field-usage-stats.asciidoc
index a4856092834e5..cbb65f8c2effb 100644
--- a/docs/reference/indices/field-usage-stats.asciidoc
+++ b/docs/reference/indices/field-usage-stats.asciidoc
@@ -6,6 +6,12 @@
 
 experimental[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Returns field usage information for each shard and field
 of an index.
 Field usage statistics are automatically captured when
diff --git a/docs/reference/indices/flush.asciidoc b/docs/reference/indices/flush.asciidoc
index 61c44f157da95..a458e6906b62f 100644
--- a/docs/reference/indices/flush.asciidoc
+++ b/docs/reference/indices/flush.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Flush</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Flushes one or more data streams or indices.
 
 [source,console]
diff --git a/docs/reference/indices/forcemerge.asciidoc b/docs/reference/indices/forcemerge.asciidoc
index e1581a3cfa632..cddf368ce2398 100644
--- a/docs/reference/indices/forcemerge.asciidoc
+++ b/docs/reference/indices/forcemerge.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Force merge</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Forces a <<index-modules-merge,merge>> on the shards of one or more indices.
 For data streams, the API forces a merge on the shards of the stream's backing
 indices.
diff --git a/docs/reference/indices/get-alias.asciidoc b/docs/reference/indices/get-alias.asciidoc
index d4c5b92116949..d305bf02e746a 100644
--- a/docs/reference/indices/get-alias.asciidoc
+++ b/docs/reference/indices/get-alias.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get alias</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Retrieves information for one or more <<aliases,aliases>>.
 
 [source,console]
diff --git a/docs/reference/indices/get-component-template.asciidoc b/docs/reference/indices/get-component-template.asciidoc
index f35192ca448db..33676c0cf2a71 100644
--- a/docs/reference/indices/get-component-template.asciidoc
+++ b/docs/reference/indices/get-component-template.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get component template</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Retrieves information about one or more component templates.
 
 //////////////////////////
@@ -67,7 +73,7 @@ Wildcard (`*`) expressions are supported.
 
 include::{docdir}/rest-api/common-parms.asciidoc[tag=flat-settings]
 
-include::{docdir}/rest-api/common-parms.asciidoc[tag=local]
+include::{docdir}/rest-api/common-parms.asciidoc[tag=local-deprecated-9.0.0]
 
 include::{docdir}/rest-api/common-parms.asciidoc[tag=master-timeout]
 
diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc
index 6bf150897acab..ccab53f020e5f 100644
--- a/docs/reference/indices/get-data-stream.asciidoc
+++ b/docs/reference/indices/get-data-stream.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get data stream</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs].
+--
+
 Retrieves information about one or more <<data-streams,data streams>>.
 See <<get-info-about-data-stream>>.
 
diff --git a/docs/reference/indices/get-field-mapping.asciidoc b/docs/reference/indices/get-field-mapping.asciidoc
index ac5895872fbb5..4b25e0e925541 100644
--- a/docs/reference/indices/get-field-mapping.asciidoc
+++ b/docs/reference/indices/get-field-mapping.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get field mapping</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Retrieves <<mapping,mapping definitions>> for one or more fields. For data
 streams, the API retrieves field mappings for the stream's backing indices.
 
diff --git a/docs/reference/indices/get-index-template-v1.asciidoc b/docs/reference/indices/get-index-template-v1.asciidoc
index 602ca2fe454ad..d1b652e3987d8 100644
--- a/docs/reference/indices/get-index-template-v1.asciidoc
+++ b/docs/reference/indices/get-index-template-v1.asciidoc
@@ -4,10 +4,16 @@
 <titleabbrev>Get index template (legacy)</titleabbrev>
 ++++
 
-IMPORTANT: This documentation is about legacy index templates, 
-which are deprecated and will be replaced by the composable templates introduced in {es} 7.8. 
+IMPORTANT: This documentation is about legacy index templates,
+which are deprecated and will be replaced by the composable templates introduced in {es} 7.8.
 For information about composable templates, see <<index-templates>>.
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Retrieves information about one or more index templates.
 
 ////
@@ -63,7 +69,7 @@ or use a value of `_all` or `*`.
 
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=flat-settings]
 
-include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local]
+include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local-deprecated-9.0.0]
 
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
 
diff --git a/docs/reference/indices/get-index-template.asciidoc b/docs/reference/indices/get-index-template.asciidoc
index 2cde5adc8ae23..4cbf2b313f9de 100644
--- a/docs/reference/indices/get-index-template.asciidoc
+++ b/docs/reference/indices/get-index-template.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get index template</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Returns information about one or more index templates.
 
 ////
@@ -59,7 +65,7 @@ expressions. If omitted, all templates are returned.
 
 include::{docdir}/rest-api/common-parms.asciidoc[tag=flat-settings]
 
-include::{docdir}/rest-api/common-parms.asciidoc[tag=local]
+include::{docdir}/rest-api/common-parms.asciidoc[tag=local-deprecated-9.0.0]
 
 include::{docdir}/rest-api/common-parms.asciidoc[tag=master-timeout]
 
diff --git a/docs/reference/indices/get-index.asciidoc b/docs/reference/indices/get-index.asciidoc
index 2551d25801d70..4e026f4ff07a3 100644
--- a/docs/reference/indices/get-index.asciidoc
+++ b/docs/reference/indices/get-index.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get index</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Returns information about one or more indices. For data streams, the API
 returns information about the stream's backing indices.
 
diff --git a/docs/reference/indices/get-mapping.asciidoc b/docs/reference/indices/get-mapping.asciidoc
index 16dc8c66d0715..543bda6cb3242 100644
--- a/docs/reference/indices/get-mapping.asciidoc
+++ b/docs/reference/indices/get-mapping.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get mapping</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Retrieves <<mapping,mapping definitions>> for one or more indices. For data
 streams, the API retrieves mappings for the stream's backing indices.
 
diff --git a/docs/reference/indices/get-settings.asciidoc b/docs/reference/indices/get-settings.asciidoc
index b6cb6d2926387..d48514b9a3493 100644
--- a/docs/reference/indices/get-settings.asciidoc
+++ b/docs/reference/indices/get-settings.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get index settings</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Returns setting information for one or more indices. For data streams, the API
 returns setting information for the stream's backing indices.
 
diff --git a/docs/reference/indices/index-mgmt.asciidoc b/docs/reference/indices/index-mgmt.asciidoc
index 73643dbfd4b3b..131bc79faa40c 100644
--- a/docs/reference/indices/index-mgmt.asciidoc
+++ b/docs/reference/indices/index-mgmt.asciidoc
@@ -43,7 +43,7 @@ For more information on managing indices, refer to <<indices, Index APIs>>.
 
 * To filter the list of indices, use the search bar or click a badge.
 Badges indicate if an index is a <<ccr-put-follow,follower index>>, a
-<<rollup-get-rollup-index-caps,rollup index>>, or <<unfreeze-index-api,frozen>>.
+<<rollup-get-rollup-index-caps,rollup index>>, or <<frozen-indices,frozen>>.
 
 * To drill down into the index
 <<mapping,mappings>>, <<index-modules-settings,settings>>, and statistics,
diff --git a/docs/reference/indices/index-template-exists-v1.asciidoc b/docs/reference/indices/index-template-exists-v1.asciidoc
index 2358f0b1a376e..f7b3a4416715d 100644
--- a/docs/reference/indices/index-template-exists-v1.asciidoc
+++ b/docs/reference/indices/index-template-exists-v1.asciidoc
@@ -9,6 +9,12 @@ templates>>, which are deprecated and will be replaced by the composable
 templates introduced in {es} 7.8. For information about composable templates,
 see <<index-templates>>.
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Checks if an <<indices-templates-v1,legacy index template>> exists.
 
 
diff --git a/docs/reference/indices/index-templates.asciidoc b/docs/reference/indices/index-templates.asciidoc
index 5b152ecf177ec..90c4a6952446e 100644
--- a/docs/reference/indices/index-templates.asciidoc
+++ b/docs/reference/indices/index-templates.asciidoc
@@ -61,7 +61,7 @@ applying the templates, do one or more of the following:
 
 - Use a non-overlapping index pattern.
 
-- Assign templates with an overlapping pattern a `priority` higher than `200`.
+- Assign templates with an overlapping pattern a `priority` higher than `500`.
 For example, if you don't use {fleet} or {agent} and want to create a template
 for the `logs-*` index pattern, assign your template a priority of `500`. This
 ensures your template is applied instead of the built-in template for
diff --git a/docs/reference/indices/indices-exists.asciidoc b/docs/reference/indices/indices-exists.asciidoc
index d699d36add03b..11da491adf8e5 100644
--- a/docs/reference/indices/indices-exists.asciidoc
+++ b/docs/reference/indices/indices-exists.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Exists</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Checks if a data stream, index, or alias exists.
 
 [source,console]
diff --git a/docs/reference/indices/migrate-to-data-stream.asciidoc b/docs/reference/indices/migrate-to-data-stream.asciidoc
index 48302761c7dc9..745e3e28683c4 100644
--- a/docs/reference/indices/migrate-to-data-stream.asciidoc
+++ b/docs/reference/indices/migrate-to-data-stream.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Migrate to data stream</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs].
+--
+
 Converts an <<aliases,index alias>> to a <<data-streams,data stream>>.
 
 ////
diff --git a/docs/reference/indices/open-close.asciidoc b/docs/reference/indices/open-close.asciidoc
index a077c4d19fd56..fe00e5a968d96 100644
--- a/docs/reference/indices/open-close.asciidoc
+++ b/docs/reference/indices/open-close.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Open index</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Opens a closed index. For data streams, the API
 opens any closed backing indices.
 
diff --git a/docs/reference/indices/put-component-template.asciidoc b/docs/reference/indices/put-component-template.asciidoc
index d880edfe42b8c..9f129c3507d87 100644
--- a/docs/reference/indices/put-component-template.asciidoc
+++ b/docs/reference/indices/put-component-template.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Create or update component template</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Creates or updates a component template. Component templates are building blocks
 for constructing <<index-templates,index templates>> that specify index
 <<mapping,mappings>>, <<index-modules-settings,settings>>, and
diff --git a/docs/reference/indices/put-index-template-v1.asciidoc b/docs/reference/indices/put-index-template-v1.asciidoc
index 86a8a54edd97f..8b07acfdadba8 100644
--- a/docs/reference/indices/put-index-template-v1.asciidoc
+++ b/docs/reference/indices/put-index-template-v1.asciidoc
@@ -8,6 +8,12 @@ IMPORTANT: This documentation is about legacy index templates,
 which are deprecated and will be replaced by the composable templates introduced in {es} 7.8. 
 For information about composable templates, see <<index-templates>>.
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Creates or updates an index template.
 
 [source,console]
diff --git a/docs/reference/indices/put-index-template.asciidoc b/docs/reference/indices/put-index-template.asciidoc
index 9a31037546796..fac928c4a61b0 100644
--- a/docs/reference/indices/put-index-template.asciidoc
+++ b/docs/reference/indices/put-index-template.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Create or update index template</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Creates or updates an index template. Index templates define
 <<index-modules-settings,settings>>, <<mapping,mappings>>, and <<aliases,aliases>>
 that can be applied automatically to new indices.
diff --git a/docs/reference/indices/put-mapping.asciidoc b/docs/reference/indices/put-mapping.asciidoc
index dc6dbff1df42c..479bdff22a80c 100644
--- a/docs/reference/indices/put-mapping.asciidoc
+++ b/docs/reference/indices/put-mapping.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Update mapping</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Adds new fields to an existing data stream or index. You can also use this
 API to change the search settings of existing fields.
 
diff --git a/docs/reference/indices/recovery.asciidoc b/docs/reference/indices/recovery.asciidoc
index 06b4d9d92e49f..032ffc5c5a082 100644
--- a/docs/reference/indices/recovery.asciidoc
+++ b/docs/reference/indices/recovery.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Index recovery</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Returns information about ongoing and completed shard recoveries for one or
 more indices. For data streams, the API returns information for the stream's
 backing indices.
diff --git a/docs/reference/indices/refresh.asciidoc b/docs/reference/indices/refresh.asciidoc
index bd8e821ff56b9..6cc78b22323c1 100644
--- a/docs/reference/indices/refresh.asciidoc
+++ b/docs/reference/indices/refresh.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Refresh</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 A refresh makes recent operations performed on one or more indices available for
 search. For data streams, the API runs the refresh operation on the stream's
 backing indices. For more information about the refresh operation, see
diff --git a/docs/reference/indices/resolve-cluster.asciidoc b/docs/reference/indices/resolve-cluster.asciidoc
index 48e6bfac4af10..a0583f4d7beaf 100644
--- a/docs/reference/indices/resolve-cluster.asciidoc
+++ b/docs/reference/indices/resolve-cluster.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Resolve cluster</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Resolves the specified index expressions to return information about
 each cluster, including the local cluster, if included.
 
@@ -22,8 +28,19 @@ For each cluster in the index expression, information is returned about:
 3. whether there are any indices, aliases or data streams on that cluster that match
    the index expression
 4. whether the search is likely to have errors returned when you do the {ccs} (including any
-   authorization errors if your user does not have permission to query the index)
-5. cluster version information, including the Elasticsearch server version
+   authorization errors if your user does not have permission to query a remote cluster or
+   the indices on that cluster)
+5. (in some cases) cluster version information, including the Elasticsearch server version
+
+[TIP]
+====
+Whenever a security exception is returned for a remote cluster, that remote
+will always be marked as connected=false in the response, since your user does not have
+permissions to access that cluster (or perhaps the remote index) you are querying.
+Once the proper security permissions are obtained, then you can rely on the `connected` field
+in the response to determine whether the remote cluster is available and ready for querying.
+====
+
 
 ////
 [source,console]
diff --git a/docs/reference/indices/resolve.asciidoc b/docs/reference/indices/resolve.asciidoc
index 856546b037fea..f0b91695b0aa1 100644
--- a/docs/reference/indices/resolve.asciidoc
+++ b/docs/reference/indices/resolve.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Resolve index</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Resolves the specified name(s) and/or index patterns for indices, aliases, and
 data streams. Multiple patterns and remote clusters are supported.
 
diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc
index 2a47d28e5358d..cc93204469b80 100644
--- a/docs/reference/indices/rollover-index.asciidoc
+++ b/docs/reference/indices/rollover-index.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Rollover</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Creates a new index for a <<data-streams,data stream>> or <<aliases,index alias>>.
 
 [source,console]
diff --git a/docs/reference/indices/segments.asciidoc b/docs/reference/indices/segments.asciidoc
index 54f0b6c0cd6de..b2b7b244321f4 100644
--- a/docs/reference/indices/segments.asciidoc
+++ b/docs/reference/indices/segments.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Index segments</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Returns low-level information about the https://lucene.apache.org/core/[Lucene]
 segments in index shards. For data streams, the API returns information about
 the stream's backing indices.
diff --git a/docs/reference/indices/shard-stores.asciidoc b/docs/reference/indices/shard-stores.asciidoc
index 1b001a3175b8c..35f6a0915caa0 100644
--- a/docs/reference/indices/shard-stores.asciidoc
+++ b/docs/reference/indices/shard-stores.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Index shard stores</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Retrieves store information
 about replica shards in one or more indices.
 For data streams, the API retrieves store
diff --git a/docs/reference/indices/shrink-index.asciidoc b/docs/reference/indices/shrink-index.asciidoc
index 244733282d46e..931731fceb49d 100644
--- a/docs/reference/indices/shrink-index.asciidoc
+++ b/docs/reference/indices/shrink-index.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Shrink index</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Shrinks an existing index into a new index with fewer primary shards.
 
 
diff --git a/docs/reference/indices/simulate-index.asciidoc b/docs/reference/indices/simulate-index.asciidoc
index 5e5709a2d82fc..414b54aa5f414 100644
--- a/docs/reference/indices/simulate-index.asciidoc
+++ b/docs/reference/indices/simulate-index.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Simulate index</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Returns the index configuration that would be applied to the specified index from an
 existing <<index-templates, index template>>.
 
diff --git a/docs/reference/indices/simulate-template.asciidoc b/docs/reference/indices/simulate-template.asciidoc
index c7397ace97886..1ea72970b647a 100644
--- a/docs/reference/indices/simulate-template.asciidoc
+++ b/docs/reference/indices/simulate-template.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Simulate template</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Returns the index configuration that would be applied by a particular
 <<index-templates, index template>>.
 
diff --git a/docs/reference/indices/split-index.asciidoc b/docs/reference/indices/split-index.asciidoc
index 0c93b572639db..4bda778839735 100644
--- a/docs/reference/indices/split-index.asciidoc
+++ b/docs/reference/indices/split-index.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Split index</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Splits an existing index into a new index with more primary shards.
 
 [source,console]
diff --git a/docs/reference/indices/stats.asciidoc b/docs/reference/indices/stats.asciidoc
index 088d65c37ec6e..1370935a7cde7 100644
--- a/docs/reference/indices/stats.asciidoc
+++ b/docs/reference/indices/stats.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Index stats</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Returns statistics for one or more indices. For data streams, the API retrieves
 statistics for the stream's backing indices.
 
diff --git a/docs/reference/indices/update-settings.asciidoc b/docs/reference/indices/update-settings.asciidoc
index 3b29946d5ed7d..dd92a922ec655 100644
--- a/docs/reference/indices/update-settings.asciidoc
+++ b/docs/reference/indices/update-settings.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Update index settings</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
 Changes a <<index-modules-settings,dynamic index setting>> in real time.
 
 For data streams, index setting changes are applied to all backing indices by
diff --git a/docs/reference/inference/chat-completion-inference.asciidoc b/docs/reference/inference/chat-completion-inference.asciidoc
new file mode 100644
index 0000000000000..83a8f94634f2f
--- /dev/null
+++ b/docs/reference/inference/chat-completion-inference.asciidoc
@@ -0,0 +1,417 @@
+[role="xpack"]
+[[chat-completion-inference-api]]
+=== Chat completion inference API
+
+Streams a chat completion response.
+
+IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face.
+For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models.
+However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <<ml-df-trained-models-apis>>.
+
+
+[discrete]
+[[chat-completion-inference-api-request]]
+==== {api-request-title}
+
+`POST /_inference/<inference_id>/_unified`
+
+`POST /_inference/chat_completion/<inference_id>/_unified`
+
+
+[discrete]
+[[chat-completion-inference-api-prereqs]]
+==== {api-prereq-title}
+
+* Requires the `monitor_inference` <<privileges-list-cluster,cluster privilege>>
+(the built-in `inference_admin` and `inference_user` roles grant this privilege)
+* You must use a client that supports streaming.
+
+
+[discrete]
+[[chat-completion-inference-api-desc]]
+==== {api-description-title}
+
+The chat completion {infer} API enables real-time responses for chat completion tasks by delivering answers incrementally, reducing response times during computation.
+It only works with the `chat_completion` task type for `openai` and `elastic` {infer} services.
+
+[NOTE]
+====
+The `chat_completion` task type is only available within the _unified API and only supports streaming.
+====
+
+[discrete]
+[[chat-completion-inference-api-path-params]]
+==== {api-path-parms-title}
+
+`<inference_id>`::
+(Required, string)
+The unique identifier of the {infer} endpoint.
+
+
+`<task_type>`::
+(Optional, string)
+The type of {infer} task that the model performs. If included, this must be set to the value `chat_completion`.
+
+
+[discrete]
+[[chat-completion-inference-api-request-body]]
+==== {api-request-body-title}
+
+`messages`::
+(Required, array of objects) A list of objects representing the conversation.
+Requests should generally only add new messages from the user (role `user`). The other message roles (`assistant`, `system`, or `tool`) should generally only be copied from the response to a previous completion request, such that the messages array is built up throughout a conversation.
++
+.Assistant message
+[%collapsible%closed]
+=====
+`content`::
+(Required unless `tool_calls` is specified, string or array of objects)
+The contents of the message.
++
+include::inference-shared.asciidoc[tag=chat-completion-schema-content-with-examples]
++
+`role`::
+(Required, string)
+The role of the message author. This should be set to `assistant` for this type of message.
++
+`tool_calls`::
+(Optional, array of objects)
+The tool calls generated by the model.
++
+.Examples
+[%collapsible%closed]
+======
+[source,js]
+------------------------------------------------------------
+{
+    "tool_calls": [
+        {
+            "id": "call_KcAjWtAww20AihPHphUh46Gd",
+            "type": "function",
+            "function": {
+                "name": "get_current_weather",
+                "arguments": "{\"location\":\"Boston, MA\"}"
+            }
+        }
+    ]
+}
+------------------------------------------------------------
+// NOTCONSOLE
+======
++
+`id`:::
+(Required, string)
+The identifier of the tool call.
++
+`type`:::
+(Required, string)
+The type of tool call. This must be set to the value `function`.
++
+`function`:::
+(Required, object)
+The function that the model called.
++
+`name`::::
+(Required, string)
+The name of the function to call.
++
+`arguments`::::
+(Required, string)
+The arguments to call the function with in JSON format.
+=====
++
+.System message
+[%collapsible%closed]
+=====
+`content`:::
+(Required, string or array of objects)
+The contents of the message.
++
+include::inference-shared.asciidoc[tag=chat-completion-schema-content-with-examples]
++
+`role`:::
+(Required, string)
+The role of the message author. This should be set to `system` for this type of message.
+=====
++
+.Tool message
+[%collapsible%closed]
+=====
+`content`::
+(Required, string or array of objects)
+The contents of the message.
++
+include::inference-shared.asciidoc[tag=chat-completion-schema-content-with-examples]
++
+`role`::
+(Required, string)
+The role of the message author. This should be set to `tool` for this type of message.
++
+`tool_call_id`::
+(Required, string)
+The tool call that this message is responding to.
+=====
++
+.User message
+[%collapsible%closed]
+=====
+`content`::
+(Required, string or array of objects)
+The contents of the message.
++
+include::inference-shared.asciidoc[tag=chat-completion-schema-content-with-examples]
++
+`role`::
+(Required, string)
+The role of the message author. This should be set to `user` for this type of message.
+=====
+
+`model`::
+(Optional, string)
+The ID of the model to use. By default, the model ID is set to the value included when creating the inference endpoint.
+
+`max_completion_tokens`::
+(Optional, integer)
+The upper bound limit for the number of tokens that can be generated for a completion request.
+
+`stop`::
+(Optional, array of strings)
+A sequence of strings to control when the model should stop generating additional tokens.
+
+`temperature`::
+(Optional, float)
+The sampling temperature to use.
+
+`tools`::
+(Optional, array of objects)
+A list of tools that the model can call.
++
+.Structure
+[%collapsible%closed]
+=====
+`type`::
+(Required, string)
+The type of tool, must be set to the value `function`.
++
+`function`::
+(Required, object)
+The function definition.
++
+`description`:::
+(Optional, string)
+A description of what the function does. This is used by the model to choose when and how to call the function.
++
+`name`:::
+(Required, string)
+The name of the function.
++
+`parameters`:::
+(Optional, object)
+The parameters the functional accepts. This should be formatted as a JSON object.
++
+`strict`:::
+(Optional, boolean)
+Whether to enable schema adherence when generating the function call.
+=====
++
+.Examples
+[%collapsible%closed]
+======
+[source,js]
+------------------------------------------------------------
+{
+    "tools": [
+        {
+            "type": "function",
+            "function": {
+                "name": "get_price_of_item",
+                "description": "Get the current price of an item",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "item": {
+                            "id": "12345"
+                        },
+                        "unit": {
+                            "type": "currency"
+                        }
+                    }
+                }
+            }
+        }
+    ]
+}
+------------------------------------------------------------
+// NOTCONSOLE
+======
+
+`tool_choice`::
+(Optional, string or object)
+Controls which tool is called by the model.
++
+String representation:::
+One of `auto`, `none`, or `requrired`. `auto` allows the model to choose between calling tools and generating a message. `none` causes the model to not call any tools. `required` forces the model to call one or more tools.
++
+Object representation:::
++
+.Structure
+[%collapsible%closed]
+=====
+`type`::
+(Required, string)
+The type of the tool. This must be set to the value `function`.
++
+`function`::
+(Required, object)
++
+`name`:::
+(Required, string)
+The name of the function to call.
+=====
++
+.Examples
+[%collapsible%closed]
+=====
+[source,js]
+------------------------------------------------------------
+{
+    "tool_choice": {
+        "type": "function",
+        "function": {
+            "name": "get_current_weather"
+        }
+    }
+}
+------------------------------------------------------------
+// NOTCONSOLE
+=====
+
+`top_p`::
+(Optional, float)
+Nucleus sampling, an alternative to sampling with temperature.
+
+[discrete]
+[[chat-completion-inference-api-example]]
+==== {api-examples-title}
+
+The following example performs a chat completion on the example question with streaming.
+
+
+[source,console]
+------------------------------------------------------------
+POST _inference/chat_completion/openai-completion/_stream
+{
+    "model": "gpt-4o",
+    "messages": [
+        {
+            "role": "user",
+            "content": "What is Elastic?"
+        }
+    ]
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+The following example performs a chat completion using an Assistant message with `tool_calls`.
+
+[source,console]
+------------------------------------------------------------
+POST _inference/chat_completion/openai-completion/_stream
+{
+    "messages": [
+        {
+            "role": "assistant",
+            "content": "Let's find out what the weather is",
+            "tool_calls": [ <1>
+                {
+                    "id": "call_KcAjWtAww20AihPHphUh46Gd",
+                    "type": "function",
+                    "function": {
+                        "name": "get_current_weather",
+                        "arguments": "{\"location\":\"Boston, MA\"}"
+                    }
+                }
+            ]
+        },
+        { <2>
+            "role": "tool",
+            "content": "The weather is cold",
+            "tool_call_id": "call_KcAjWtAww20AihPHphUh46Gd"
+        }
+    ]
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+<1> Each tool call needs a corresponding Tool message.
+<2> The corresponding Tool message.
+
+The following example performs a chat completion using a User message with `tools` and `tool_choice`.
+
+[source,console]
+------------------------------------------------------------
+POST _inference/chat_completion/openai-completion/_stream
+{
+    "messages": [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "What's the price of a scarf?"
+                }
+            ]
+        }
+    ],
+    "tools": [
+        {
+            "type": "function",
+            "function": {
+                "name": "get_current_price",
+                "description": "Get the current price of a item",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "item": {
+                            "id": "123"
+                        }
+                    }
+                }
+            }
+        }
+    ],
+    "tool_choice": {
+        "type": "function",
+        "function": {
+            "name": "get_current_price"
+        }
+    }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+The API returns the following response when a request is made to the OpenAI service:
+
+
+[source,txt]
+------------------------------------------------------------
+event: message
+data: {"chat_completion":{"id":"chatcmpl-Ae0TWsy2VPnSfBbv5UztnSdYUMFP3","choices":[{"delta":{"content":"","role":"assistant"},"index":0}],"model":"gpt-4o-2024-08-06","object":"chat.completion.chunk"}}
+
+event: message
+data: {"chat_completion":{"id":"chatcmpl-Ae0TWsy2VPnSfBbv5UztnSdYUMFP3","choices":[{"delta":{"content":Elastic"},"index":0}],"model":"gpt-4o-2024-08-06","object":"chat.completion.chunk"}}
+
+event: message
+data: {"chat_completion":{"id":"chatcmpl-Ae0TWsy2VPnSfBbv5UztnSdYUMFP3","choices":[{"delta":{"content":" is"},"index":0}],"model":"gpt-4o-2024-08-06","object":"chat.completion.chunk"}}
+
+(...)
+
+event: message
+data: {"chat_completion":{"id":"chatcmpl-Ae0TWsy2VPnSfBbv5UztnSdYUMFP3","choices":[],"model":"gpt-4o-2024-08-06","object":"chat.completion.chunk","usage":{"completion_tokens":28,"prompt_tokens":16,"total_tokens":44}}} <1>
+
+event: message
+data: [DONE]
+------------------------------------------------------------
+// NOTCONSOLE
+
+<1> The last object message of the stream contains the token usage information.
diff --git a/docs/reference/inference/delete-inference.asciidoc b/docs/reference/inference/delete-inference.asciidoc
index a83fb1a516b80..a9dfeabcdf259 100644
--- a/docs/reference/inference/delete-inference.asciidoc
+++ b/docs/reference/inference/delete-inference.asciidoc
@@ -2,6 +2,12 @@
 [[delete-inference-api]]
 === Delete {infer} API
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Deletes an {infer} endpoint.
 
 IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face.
diff --git a/docs/reference/inference/get-inference.asciidoc b/docs/reference/inference/get-inference.asciidoc
index 16e38d2aa148b..74220514eeb98 100644
--- a/docs/reference/inference/get-inference.asciidoc
+++ b/docs/reference/inference/get-inference.asciidoc
@@ -2,6 +2,12 @@
 [[get-inference-api]]
 === Get {infer} API
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Retrieves {infer} endpoint information.
 
 IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face.
@@ -78,12 +84,17 @@ The API returns the following response:
 {
   "inference_id": "my-elser-model",
   "task_type": "sparse_embedding",
-  "service": "elser",
+  "service": "elasticsearch",
   "service_settings": {
     "num_allocations": 1,
-    "num_threads": 1
+    "num_threads": 1,
+    "model_id": ".elser_model_2"
   },
-  "task_settings": {}
+  "chunking_settings": {
+    "strategy": "sentence",
+    "max_chunk_size": 250,
+    "sentence_overlap": 1
+  }
 }
 ------------------------------------------------------------
 // NOTCONSOLE
diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc
index 8d5ee1b7d6ba5..4f27409973ca2 100644
--- a/docs/reference/inference/inference-apis.asciidoc
+++ b/docs/reference/inference/inference-apis.asciidoc
@@ -10,6 +10,12 @@ trained models. However, if you do not plan to use the {infer} APIs to use these
 models or if you want to use non-NLP models, use the
 <<ml-df-trained-models-apis>>.
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 The {infer} APIs enable you to create {infer} endpoints and use {ml} models of
 different providers - such as Amazon Bedrock, Anthropic, Azure AI Studio,
 Cohere, Google AI, Mistral, OpenAI, or HuggingFace - as a service. Use
@@ -20,6 +26,7 @@ the following APIs to manage {infer} models and perform {infer}:
 * <<post-inference-api>>
 * <<put-inference-api>>
 * <<stream-inference-api>>
+* <<chat-completion-inference-api>>
 * <<update-inference-api>>
 
 [[inference-landscape]]
@@ -28,9 +35,9 @@ image::images/inference-landscape.jpg[A representation of the Elastic inference
 
 An {infer} endpoint enables you to use the corresponding {ml} model without
 manual deployment and apply it to your data at ingestion time through
-<<semantic-search-semantic-text, semantic text>>. 
+<<semantic-search-semantic-text, semantic text>>.
 
-Choose a model from your provider or use ELSER – a retrieval model trained by 
+Choose a model from your provider or use ELSER – a retrieval model trained by
 Elastic –, then create an {infer} endpoint by the <<put-inference-api>>.
 Now use <<semantic-search-semantic-text, semantic text>> to perform
 <<semantic-search, semantic search>> on your data.
@@ -61,7 +68,7 @@ The following list contains the default {infer} endpoints listed by `inference_i
 Use the `inference_id` of the endpoint in a <<semantic-text,`semantic_text`>> field definition or when creating an <<inference-processor,{infer} processor>>.
 The API call will automatically download and deploy the model which might take a couple of minutes.
 Default {infer} enpoints have {ml-docs}/ml-nlp-auto-scale.html#nlp-model-adaptive-allocations[adaptive allocations] enabled.
-For these models, the minimum number of allocations is `0`. 
+For these models, the minimum number of allocations is `0`.
 If there is no {infer} activity that uses the endpoint, the number of allocations will scale down to `0` automatically after 15 minutes.
 
 
@@ -78,7 +85,7 @@ Returning a long document in search results is less useful than providing the mo
 Each chunk will include the text subpassage and the corresponding embedding generated from it.
 
 By default, documents are split into sentences and grouped in sections up to 250 words with 1 sentence overlap so that each chunk shares a sentence with the previous chunk.
-Overlapping ensures continuity and prevents vital contextual information in the input text from being lost by a hard break. 
+Overlapping ensures continuity and prevents vital contextual information in the input text from being lost by a hard break.
 
 {es} uses the https://unicode-org.github.io/icu-docs/[ICU4J] library to detect word and sentence boundaries for chunking.
 https://unicode-org.github.io/icu/userguide/boundaryanalysis/#word-boundary[Word boundaries] are identified by following a series of rules, not just the presence of a whitespace character.
@@ -129,6 +136,7 @@ PUT _inference/sparse_embedding/small_chunk_size
 include::delete-inference.asciidoc[]
 include::get-inference.asciidoc[]
 include::post-inference.asciidoc[]
+include::chat-completion-inference.asciidoc[]
 include::put-inference.asciidoc[]
 include::stream-inference.asciidoc[]
 include::update-inference.asciidoc[]
@@ -143,6 +151,7 @@ include::service-elser.asciidoc[]
 include::service-google-ai-studio.asciidoc[]
 include::service-google-vertex-ai.asciidoc[]
 include::service-hugging-face.asciidoc[]
+include::service-jinaai.asciidoc[]
 include::service-mistral.asciidoc[]
 include::service-openai.asciidoc[]
 include::service-watsonx-ai.asciidoc[]
diff --git a/docs/reference/inference/inference-shared.asciidoc b/docs/reference/inference/inference-shared.asciidoc
index da497c6581e5d..b133c54082810 100644
--- a/docs/reference/inference/inference-shared.asciidoc
+++ b/docs/reference/inference/inference-shared.asciidoc
@@ -41,7 +41,7 @@ end::chunking-settings[]
 tag::chunking-settings-max-chunking-size[]
 Specifies the maximum size of a chunk in words.
 Defaults to `250`.
-This value cannot be higher than `300` or lower than `20` (for `sentence` strategy) or `10` (for `word` strategy). 
+This value cannot be higher than `300` or lower than `20` (for `sentence` strategy) or `10` (for `word` strategy).
 end::chunking-settings-max-chunking-size[]
 
 tag::chunking-settings-overlap[]
@@ -63,4 +63,48 @@ Specifies the chunking strategy.
 It could be either `sentence` or `word`.
 end::chunking-settings-strategy[]
 
+tag::chat-completion-schema-content-with-examples[]
+.Examples
+[%collapsible%closed]
+======
+String example
+[source,js]
+------------------------------------------------------------
+{
+    "content": "Some string"
+}
+------------------------------------------------------------
+// NOTCONSOLE
+
+Object example
+[source,js]
+------------------------------------------------------------
+{
+    "content": [
+        {
+            "text": "Some text",
+            "type": "text"
+        }
+    ]
+}
+------------------------------------------------------------
+// NOTCONSOLE
+======
+
+String representation:::
+(Required, string)
+The text content.
++
+Object representation:::
+`text`::::
+(Required, string)
+The text content.
++
+`type`::::
+(Required, string)
+This must be set to the value `text`.
+end::chat-completion-schema-content-with-examples[]
 
+tag::chat-completion-docs[]
+For more information on how to use the `chat_completion` task type, please refer to the <<chat-completion-inference-api, chat completion documentation>>.
+end::chat-completion-docs[]
diff --git a/docs/reference/inference/post-inference.asciidoc b/docs/reference/inference/post-inference.asciidoc
index 4edefcc911e2e..07e557aa355bd 100644
--- a/docs/reference/inference/post-inference.asciidoc
+++ b/docs/reference/inference/post-inference.asciidoc
@@ -2,6 +2,12 @@
 [[post-inference-api]]
 === Perform inference API
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Performs an inference task on an input text by using an {infer} endpoint.
 
 IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face.
diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc
index 4f82889f562d8..da07d1d3e7d84 100644
--- a/docs/reference/inference/put-inference.asciidoc
+++ b/docs/reference/inference/put-inference.asciidoc
@@ -2,6 +2,12 @@
 [[put-inference-api]]
 === Create {infer} API
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Creates an {infer} endpoint to perform an {infer} task.
 
 [IMPORTANT]
@@ -36,7 +42,7 @@ include::inference-shared.asciidoc[tag=inference-id]
 include::inference-shared.asciidoc[tag=task-type]
 +
 --
-Refer to the service list in the <<put-inference-api-desc,API description section>> for the available task types. 
+Refer to the service list in the <<put-inference-api-desc,API description section>> for the available task types.
 --
 
 
@@ -55,7 +61,7 @@ The create {infer} API enables you to create an {infer} endpoint and configure a
 
 
 The following services are available through the {infer} API.
-You can find the available task types next to the service name. 
+You can find the available task types next to the service name.
 Click the links to review the configuration details of the services:
 
 * <<infer-service-alibabacloud-ai-search,AlibabaCloud AI Search>> (`completion`, `rerank`, `sparse_embedding`, `text_embedding`)
@@ -67,11 +73,12 @@ Click the links to review the configuration details of the services:
 * <<infer-service-elasticsearch,Elasticsearch>> (`rerank`, `sparse_embedding`, `text_embedding` - this service is for built-in models and models uploaded through Eland)
 * <<infer-service-elser,ELSER>> (`sparse_embedding`)
 * <<infer-service-google-ai-studio,Google AI Studio>> (`completion`, `text_embedding`)
-* <<infer-service-google-vertex-ai,Google Vertex AI>> (`rerank`, `text_embedding`) 
+* <<infer-service-google-vertex-ai,Google Vertex AI>> (`rerank`, `text_embedding`)
 * <<infer-service-hugging-face,Hugging Face>> (`text_embedding`)
 * <<infer-service-mistral,Mistral>> (`text_embedding`)
-* <<infer-service-openai,OpenAI>> (`completion`, `text_embedding`)
+* <<infer-service-openai,OpenAI>> (`chat_completion`, `completion`, `text_embedding`)
 * <<infer-service-watsonx-ai>> (`text_embedding`)
+* <<infer-service-jinaai,JinaAI>> (`text_embedding`, `rerank`)
 
 The {es} and ELSER services run on a {ml} node in your {es} cluster. The rest of
 the services connect to external providers.
@@ -87,4 +94,4 @@ When adaptive allocations are enabled:
 - The number of allocations scales up automatically when the load increases.
 - Allocations scale down to a minimum of 0 when the load decreases, saving resources.
 
-For more information about adaptive allocations and resources, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] documentation.
\ No newline at end of file
+For more information about adaptive allocations and resources, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] documentation.
diff --git a/docs/reference/inference/service-alibabacloud-ai-search.asciidoc b/docs/reference/inference/service-alibabacloud-ai-search.asciidoc
index c3ff40a39cd86..5a11190549ce6 100644
--- a/docs/reference/inference/service-alibabacloud-ai-search.asciidoc
+++ b/docs/reference/inference/service-alibabacloud-ai-search.asciidoc
@@ -1,6 +1,12 @@
 [[infer-service-alibabacloud-ai-search]]
 === AlibabaCloud AI Search {infer} service
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Creates an {infer} endpoint to perform an {infer} task with the `alibabacloud-ai-search` service.
 
 [discrete]
diff --git a/docs/reference/inference/service-amazon-bedrock.asciidoc b/docs/reference/inference/service-amazon-bedrock.asciidoc
index 761777e32f8e0..ed25ce0d515b5 100644
--- a/docs/reference/inference/service-amazon-bedrock.asciidoc
+++ b/docs/reference/inference/service-amazon-bedrock.asciidoc
@@ -1,6 +1,12 @@
 [[infer-service-amazon-bedrock]]
 === Amazon Bedrock {infer} service
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Creates an {infer} endpoint to perform an {infer} task with the `amazonbedrock` service.
 
 [discrete]
diff --git a/docs/reference/inference/service-anthropic.asciidoc b/docs/reference/inference/service-anthropic.asciidoc
index 7fb3d1d5bea34..4ce76dc1d57bd 100644
--- a/docs/reference/inference/service-anthropic.asciidoc
+++ b/docs/reference/inference/service-anthropic.asciidoc
@@ -1,6 +1,12 @@
 [[infer-service-anthropic]]
 === Anthropic {infer} service
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Creates an {infer} endpoint to perform an {infer} task with the `anthropic` service.
 
 
diff --git a/docs/reference/inference/service-azure-ai-studio.asciidoc b/docs/reference/inference/service-azure-ai-studio.asciidoc
index dd13a3e59aae5..7ada8df1ecdaa 100644
--- a/docs/reference/inference/service-azure-ai-studio.asciidoc
+++ b/docs/reference/inference/service-azure-ai-studio.asciidoc
@@ -1,6 +1,12 @@
 [[infer-service-azure-ai-studio]]
 === Azure AI studio {infer} service
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Creates an {infer} endpoint to perform an {infer} task with the `azureaistudio` service.
 
 
diff --git a/docs/reference/inference/service-azure-openai.asciidoc b/docs/reference/inference/service-azure-openai.asciidoc
index b134e2b687f6c..170c0939166f7 100644
--- a/docs/reference/inference/service-azure-openai.asciidoc
+++ b/docs/reference/inference/service-azure-openai.asciidoc
@@ -1,6 +1,12 @@
 [[infer-service-azure-openai]]
 === Azure OpenAI {infer} service
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Creates an {infer} endpoint to perform an {infer} task with the `azureopenai` service.
 
 
diff --git a/docs/reference/inference/service-cohere.asciidoc b/docs/reference/inference/service-cohere.asciidoc
index 1a815e3c45f36..70e311c810cdd 100644
--- a/docs/reference/inference/service-cohere.asciidoc
+++ b/docs/reference/inference/service-cohere.asciidoc
@@ -1,6 +1,12 @@
 [[infer-service-cohere]]
 === Cohere {infer} service
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Creates an {infer} endpoint to perform an {infer} task with the `cohere` service.
 
 
diff --git a/docs/reference/inference/service-elasticsearch.asciidoc b/docs/reference/inference/service-elasticsearch.asciidoc
index bf7e2976bbe63..8870fbed357a6 100644
--- a/docs/reference/inference/service-elasticsearch.asciidoc
+++ b/docs/reference/inference/service-elasticsearch.asciidoc
@@ -1,10 +1,19 @@
 [[infer-service-elasticsearch]]
 === Elasticsearch {infer} service
 
-Creates an {infer} endpoint to perform an {infer} task with the `elasticsearch` service.
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
 
-NOTE: If you use the ELSER or the E5 model through the `elasticsearch` service, the API request will automatically download and deploy the model if it isn't downloaded yet.
+Creates an {infer} endpoint to perform an {infer} task with the `elasticsearch` service.
 
+[NOTE]
+====
+* Your {es} deployment contains <<default-enpoints,preconfigured ELSER and E5 {infer} endpoints>>, you only need to create the enpoints using the API if you want to customize the settings.
+* If you use the ELSER or the E5 model through the `elasticsearch` service, the API request will automatically download and deploy the model if it isn't downloaded yet.
+====
 
 [discrete]
 [[infer-service-elasticsearch-api-request]]
diff --git a/docs/reference/inference/service-elser.asciidoc b/docs/reference/inference/service-elser.asciidoc
index c1cc23c8c9adb..47aaa58814602 100644
--- a/docs/reference/inference/service-elser.asciidoc
+++ b/docs/reference/inference/service-elser.asciidoc
@@ -1,17 +1,26 @@
 [[infer-service-elser]]
 === ELSER {infer} service
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Creates an {infer} endpoint to perform an {infer} task with the `elser` service.
 You can also deploy ELSER by using the <<infer-service-elasticsearch>>.
 
-NOTE: The API request will automatically download and deploy the ELSER model if
-it isn't already downloaded.
+[NOTE]
+====
+* Your {es} deployment contains <<default-enpoints,a preconfigured ELSER {infer} endpoint>>, you only need to create the enpoint using the API if you want to customize the settings.
+* The API request will automatically download and deploy the ELSER model if it isn't already downloaded.
+====
 
 [WARNING]
 .Deprecated in 8.16
 ====
-The elser service is deprecated and will be removed in a future release. 
-Use the <<infer-service-elasticsearch>> instead, with model_id included in the service_settings.
+The `elser` service is deprecated and will be removed in a future release. 
+Use the <<infer-service-elasticsearch>> instead, with `model_id` included in the `service_settings`.
 ====
 
 [discrete]
diff --git a/docs/reference/inference/service-google-ai-studio.asciidoc b/docs/reference/inference/service-google-ai-studio.asciidoc
index 738fce3d53e9b..5b30292fb9beb 100644
--- a/docs/reference/inference/service-google-ai-studio.asciidoc
+++ b/docs/reference/inference/service-google-ai-studio.asciidoc
@@ -1,6 +1,12 @@
 [[infer-service-google-ai-studio]]
 === Google AI Studio {infer} service
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Creates an {infer} endpoint to perform an {infer} task with the `googleaistudio` service.
 
 
diff --git a/docs/reference/inference/service-google-vertex-ai.asciidoc b/docs/reference/inference/service-google-vertex-ai.asciidoc
index 34e14e05e072a..28fa65b6e5fcc 100644
--- a/docs/reference/inference/service-google-vertex-ai.asciidoc
+++ b/docs/reference/inference/service-google-vertex-ai.asciidoc
@@ -1,6 +1,12 @@
 [[infer-service-google-vertex-ai]]
 === Google Vertex AI {infer} service
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Creates an {infer} endpoint to perform an {infer} task with the `googlevertexai` service.
 
 
diff --git a/docs/reference/inference/service-hugging-face.asciidoc b/docs/reference/inference/service-hugging-face.asciidoc
index 6d8667351a6b4..862914c141740 100644
--- a/docs/reference/inference/service-hugging-face.asciidoc
+++ b/docs/reference/inference/service-hugging-face.asciidoc
@@ -1,6 +1,12 @@
 [[infer-service-hugging-face]]
 === HuggingFace {infer} service
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Creates an {infer} endpoint to perform an {infer} task with the `hugging_face` service.
 
 
diff --git a/docs/reference/inference/service-jinaai.asciidoc b/docs/reference/inference/service-jinaai.asciidoc
new file mode 100644
index 0000000000000..7c5aebe5bcf8e
--- /dev/null
+++ b/docs/reference/inference/service-jinaai.asciidoc
@@ -0,0 +1,255 @@
+[[infer-service-jinaai]]
+=== JinaAI {infer} service
+
+Creates an {infer} endpoint to perform an {infer} task with the `jinaai` service.
+
+
+[discrete]
+[[infer-service-jinaai-api-request]]
+==== {api-request-title}
+
+`PUT /_inference/<task_type>/<inference_id>`
+
+[discrete]
+[[infer-service-jinaai-api-path-params]]
+==== {api-path-parms-title}
+
+`<inference_id>`::
+(Required, string)
+include::inference-shared.asciidoc[tag=inference-id]
+
+`<task_type>`::
+(Required, string)
+include::inference-shared.asciidoc[tag=task-type]
++
+--
+Available task types:
+
+* `text_embedding`,
+* `rerank`.
+--
+
+[discrete]
+[[infer-service-jinaai-api-request-body]]
+==== {api-request-body-title}
+
+`chunking_settings`::
+(Optional, object)
+include::inference-shared.asciidoc[tag=chunking-settings]
+
+`max_chunking_size`:::
+(Optional, integer)
+include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size]
+
+`overlap`:::
+(Optional, integer)
+include::inference-shared.asciidoc[tag=chunking-settings-overlap]
+
+`sentence_overlap`:::
+(Optional, integer)
+include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap]
+
+`strategy`:::
+(Optional, string)
+include::inference-shared.asciidoc[tag=chunking-settings-strategy]
+
+`service`::
+(Required, string)
+The type of service supported for the specified task type. In this case, 
+`jinaai`.
+
+`service_settings`::
+(Required, object)
+include::inference-shared.asciidoc[tag=service-settings]
++
+--
+These settings are specific to the `jinaai` service.
+--
+
+`api_key`:::
+(Required, string)
+A valid API key for your JinaAI account.
+You can find it at https://jina.ai/embeddings/.
++
+--
+include::inference-shared.asciidoc[tag=api-key-admonition]
+--
+
+`rate_limit`:::
+(Optional, object)
+The default rate limit for the `jinaai` service is 2000 requests per minute for all task types. 
+You can modify this using the `requests_per_minute` setting in your service settings:
++
+--
+include::inference-shared.asciidoc[tag=request-per-minute-example]
+
+More information about JinaAI's rate limits can be found in https://jina.ai/contact-sales/#rate-limit.
+--
++
+.`service_settings` for the `rerank` task type
+[%collapsible%closed]
+=====
+`model_id`::
+(Required, string)
+The name of the model to use for the {infer} task.
+To review the available `rerank` compatible models, refer to https://jina.ai/reranker.
+=====
++
+.`service_settings` for the `text_embedding` task type
+[%collapsible%closed]
+=====
+`model_id`:::
+(Optional, string)
+The name of the model to use for the {infer} task.
+To review the available `text_embedding` models, refer to the
+https://jina.ai/embeddings/.
+
+`similarity`:::
+(Optional, string)
+Similarity measure. One of `cosine`, `dot_product`, `l2_norm`.
+Defaults based on the `embedding_type` (`float` -> `dot_product`, `int8/byte` -> `cosine`).
+=====
+
+
+
+`task_settings`::
+(Optional, object)
+include::inference-shared.asciidoc[tag=task-settings]
++
+.`task_settings` for the `rerank` task type
+[%collapsible%closed]
+=====
+`return_documents`::
+(Optional, boolean)
+Specify whether to return doc text within the results.
+
+`top_n`::
+(Optional, integer)
+The number of most relevant documents to return, defaults to the number of the documents.
+If this {infer} endpoint is used in a `text_similarity_reranker` retriever query and `top_n` is set, it must be greater than or equal to `rank_window_size` in the query.
+=====
++
+.`task_settings` for the `text_embedding` task type
+[%collapsible%closed]
+=====
+`task`:::
+(Optional, string)
+Specifies the task passed to the model.
+Valid values are:
+* `classification`: use it for embeddings passed through a text classifier.
+* `clustering`: use it for the embeddings run through a clustering algorithm.
+* `ingest`: use it for storing document embeddings in a vector database.
+* `search`: use it for storing embeddings of search queries run against a vector database to find relevant documents.
+=====
+
+
+[discrete]
+[[inference-example-jinaai]]
+==== JinaAI service examples
+
+The following examples demonstrate how to create {infer} endpoints for `text_embeddings` and `rerank` tasks using the JinaAI service and use them in search requests.
+
+First, we create the `embeddings` service:
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/text_embedding/jinaai-embeddings
+{
+    "service": "jinaai",
+    "service_settings": {
+        "model_id": "jina-embeddings-v3",
+        "api_key": "<api_key>"
+    }
+}
+------------------------------------------------------------
+// TEST[skip:uses ML]
+
+Then, we create the `rerank` service:
+[source,console]
+------------------------------------------------------------
+PUT _inference/rerank/jinaai-rerank
+{
+    "service": "jinaai",
+    "service_settings": {
+        "api_key": "<api_key>",
+        "model_id": "jina-reranker-v2-base-multilingual"
+    },
+    "task_settings": {
+        "top_n": 10,
+        "return_documents": true
+    }
+}
+------------------------------------------------------------
+// TEST[skip:uses ML]
+
+Now we can create an index that will use `jinaai-embeddings` service to index the documents.
+
+[source,console]
+------------------------------------------------------------
+PUT jinaai-index
+{
+  "mappings": {
+    "properties": {
+      "content": {
+        "type": "semantic_text",
+        "inference_id": "jinaai-embeddings"
+      }
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:uses ML]
+
+[source,console]
+------------------------------------------------------------
+PUT jinaai-index/_bulk
+{ "index" : { "_index" : "jinaai-index", "_id" : "1" } }
+{"content": "Sarah Johnson is a talented marine biologist working at the Oceanographic Institute. Her groundbreaking research on coral reef ecosystems has garnered international attention and numerous accolades."}
+{ "index" : { "_index" : "jinaai-index", "_id" : "2" } }
+{"content": "She spends months at a time diving in remote locations, meticulously documenting the intricate relationships between various marine species. "}
+{ "index" : { "_index" : "jinaai-index", "_id" : "3" } }
+{"content": "Her dedication to preserving these delicate underwater environments has inspired a new generation of conservationists."}
+------------------------------------------------------------
+// TEST[skip:uses ML]
+
+Now, with the index created, we can search with and without the reranker service.
+
+[source,console]
+------------------------------------------------------------
+GET jinaai-index/_search 
+{
+  "query": {
+    "semantic": {
+      "field": "content",
+      "query": "who inspired taking care of the sea?"
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:uses ML]
+
+[source,console]
+------------------------------------------------------------
+POST jinaai-index/_search
+{
+  "retriever": {
+    "text_similarity_reranker": {
+      "retriever": {
+        "standard": {
+          "query": {
+            "semantic": {
+              "field": "content",
+              "query": "who inspired taking care of the sea?"
+            }
+          }
+        }
+      },
+      "field": "content",
+      "rank_window_size": 100,
+      "inference_id": "jinaai-rerank",
+      "inference_text": "who inspired taking care of the sea?"
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:uses ML]
\ No newline at end of file
diff --git a/docs/reference/inference/service-mistral.asciidoc b/docs/reference/inference/service-mistral.asciidoc
index 244381d107161..326e8458be767 100644
--- a/docs/reference/inference/service-mistral.asciidoc
+++ b/docs/reference/inference/service-mistral.asciidoc
@@ -1,6 +1,12 @@
 [[infer-service-mistral]]
 === Mistral {infer} service
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Creates an {infer} endpoint to perform an {infer} task with the `mistral` service.
 
 
diff --git a/docs/reference/inference/service-openai.asciidoc b/docs/reference/inference/service-openai.asciidoc
index 9211e2d08e88b..590f280b1c494 100644
--- a/docs/reference/inference/service-openai.asciidoc
+++ b/docs/reference/inference/service-openai.asciidoc
@@ -1,6 +1,12 @@
 [[infer-service-openai]]
 === OpenAI {infer} service
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Creates an {infer} endpoint to perform an {infer} task with the `openai` service.
 
 
@@ -25,10 +31,18 @@ include::inference-shared.asciidoc[tag=task-type]
 --
 Available task types:
 
+* `chat_completion`,
 * `completion`,
 * `text_embedding`.
 --
 
+[NOTE]
+====
+The `chat_completion` task type only supports streaming and only through the `_unified` API.
+
+include::inference-shared.asciidoc[tag=chat-completion-docs]
+====
+
 [discrete]
 [[infer-service-openai-api-request-body]]
 ==== {api-request-body-title}
@@ -55,7 +69,7 @@ include::inference-shared.asciidoc[tag=chunking-settings-strategy]
 
 `service`::
 (Required, string)
-The type of service supported for the specified task type. In this case, 
+The type of service supported for the specified task type. In this case,
 `openai`.
 
 `service_settings`::
@@ -170,4 +184,4 @@ PUT _inference/completion/openai-completion
     }
 }
 ------------------------------------------------------------
-// TEST[skip:TBD]
\ No newline at end of file
+// TEST[skip:TBD]
diff --git a/docs/reference/inference/service-watsonx-ai.asciidoc b/docs/reference/inference/service-watsonx-ai.asciidoc
index 597afc27fd0cf..e7bba7b4e9a97 100644
--- a/docs/reference/inference/service-watsonx-ai.asciidoc
+++ b/docs/reference/inference/service-watsonx-ai.asciidoc
@@ -1,6 +1,12 @@
 [[infer-service-watsonx-ai]]
 === Watsonx {infer} service
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Creates an {infer} endpoint to perform an {infer} task with the `watsonxai` service.
 
 You need an https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-provisioning&interface=api[IBM Cloud® Databases for Elasticsearch deployment] to use the `watsonxai` {infer} service.
diff --git a/docs/reference/inference/stream-inference.asciidoc b/docs/reference/inference/stream-inference.asciidoc
index e66acd630cb3e..4a3ce31909712 100644
--- a/docs/reference/inference/stream-inference.asciidoc
+++ b/docs/reference/inference/stream-inference.asciidoc
@@ -2,6 +2,12 @@
 [[stream-inference-api]]
 === Stream inference API
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Streams a chat completion response.
 
 IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face.
@@ -32,8 +38,12 @@ However, if you do not plan to use the {infer} APIs to use these models or if yo
 ==== {api-description-title}
 
 The stream {infer} API enables real-time responses for completion tasks by delivering answers incrementally, reducing response times during computation.
-It only works with the `completion` task type.
+It only works with the `completion` and `chat_completion` task types.
 
+[NOTE]
+====
+include::inference-shared.asciidoc[tag=chat-completion-docs]
+====
 
 [discrete]
 [[stream-inference-api-path-params]]
diff --git a/docs/reference/inference/update-inference.asciidoc b/docs/reference/inference/update-inference.asciidoc
index efd29231ac12e..d3a90f5d84e65 100644
--- a/docs/reference/inference/update-inference.asciidoc
+++ b/docs/reference/inference/update-inference.asciidoc
@@ -2,6 +2,12 @@
 [[update-inference-api]]
 === Update inference API
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs].
+--
+
 Updates an {infer} endpoint.
 
 IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face.
diff --git a/docs/reference/ingest/apis/delete-ip-location-database.asciidoc b/docs/reference/ingest/apis/delete-ip-location-database.asciidoc
index c3a10a914d2f4..23f1d15bfa7ee 100644
--- a/docs/reference/ingest/apis/delete-ip-location-database.asciidoc
+++ b/docs/reference/ingest/apis/delete-ip-location-database.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Delete IP geolocation database configuration</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs].
+--
+
 Deletes a IP geolocation database configuration.
 
 [source,console]
diff --git a/docs/reference/ingest/apis/delete-pipeline.asciidoc b/docs/reference/ingest/apis/delete-pipeline.asciidoc
index 94ac87c61b56b..cd4cae1b636ca 100644
--- a/docs/reference/ingest/apis/delete-pipeline.asciidoc
+++ b/docs/reference/ingest/apis/delete-pipeline.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Delete pipeline</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs].
+--
+
 Deletes one or more existing ingest pipeline.
 
 ////
diff --git a/docs/reference/ingest/apis/enrich/delete-enrich-policy.asciidoc b/docs/reference/ingest/apis/enrich/delete-enrich-policy.asciidoc
index 1d4b7e770325a..cd68bd1fcc67b 100644
--- a/docs/reference/ingest/apis/enrich/delete-enrich-policy.asciidoc
+++ b/docs/reference/ingest/apis/enrich/delete-enrich-policy.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete enrich policy</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-enrich[Enrich APIs].
+--
+
 Deletes an existing <<enrich-policy,enrich policy>> and its
 <<enrich-index,enrich index>>.
 
diff --git a/docs/reference/ingest/apis/enrich/execute-enrich-policy.asciidoc b/docs/reference/ingest/apis/enrich/execute-enrich-policy.asciidoc
index ebad9f09250d3..8e72a51514a59 100644
--- a/docs/reference/ingest/apis/enrich/execute-enrich-policy.asciidoc
+++ b/docs/reference/ingest/apis/enrich/execute-enrich-policy.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Execute enrich policy</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-enrich[Enrich APIs].
+--
+
 Executes an existing <<enrich-policy,enrich policy>>.
 
 ////
diff --git a/docs/reference/ingest/apis/enrich/get-enrich-policy.asciidoc b/docs/reference/ingest/apis/enrich/get-enrich-policy.asciidoc
index 88cf2cfa7cf60..2f9e069d5e43d 100644
--- a/docs/reference/ingest/apis/enrich/get-enrich-policy.asciidoc
+++ b/docs/reference/ingest/apis/enrich/get-enrich-policy.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get enrich policy</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-enrich[Enrich APIs].
+--
+
 Returns information about an <<enrich-policy,enrich policy>>.
 
 ////
diff --git a/docs/reference/ingest/apis/enrich/index.asciidoc b/docs/reference/ingest/apis/enrich/index.asciidoc
index a17c8179af1b1..7566b8812ec3c 100644
--- a/docs/reference/ingest/apis/enrich/index.asciidoc
+++ b/docs/reference/ingest/apis/enrich/index.asciidoc
@@ -2,6 +2,12 @@
 [[enrich-apis]]
 == Enrich APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-enrich[Enrich APIs].
+--
+
 The following enrich APIs are available for managing <<enrich-policy,enrich
 policies>>:
 
diff --git a/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc b/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc
index ee33b0b320905..80223cb0ec711 100644
--- a/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc
+++ b/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc
@@ -5,6 +5,11 @@
 <titleabbrev>Create enrich policy</titleabbrev>
 ++++
 
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-enrich[Enrich APIs].
+--
+
 Creates an enrich policy.
 
 ////
diff --git a/docs/reference/ingest/apis/geoip-stats.asciidoc b/docs/reference/ingest/apis/geoip-stats.asciidoc
index 6135b7821f2a5..423fee1c5bceb 100644
--- a/docs/reference/ingest/apis/geoip-stats.asciidoc
+++ b/docs/reference/ingest/apis/geoip-stats.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>GeoIP stats</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs].
+--
+
 Gets statistics about the <<geoip-processor,`geoip` processor>>, including
 download statistics for GeoIP2 databases used with it.
 
diff --git a/docs/reference/ingest/apis/get-ip-location-database.asciidoc b/docs/reference/ingest/apis/get-ip-location-database.asciidoc
index 26e9ddc1eee50..25f1c7ffdf695 100644
--- a/docs/reference/ingest/apis/get-ip-location-database.asciidoc
+++ b/docs/reference/ingest/apis/get-ip-location-database.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get IP geolocation database configuration</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs].
+--
+
 Returns information about one or more IP geolocation database configurations.
 
 [source,console]
diff --git a/docs/reference/ingest/apis/get-pipeline.asciidoc b/docs/reference/ingest/apis/get-pipeline.asciidoc
index f2a1155bca12b..88fc22590004b 100644
--- a/docs/reference/ingest/apis/get-pipeline.asciidoc
+++ b/docs/reference/ingest/apis/get-pipeline.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get pipeline</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs].
+--
+
 Returns information about one or more ingest pipelines.
 This API returns a local reference of the pipeline.
 
diff --git a/docs/reference/ingest/apis/index.asciidoc b/docs/reference/ingest/apis/index.asciidoc
index 35adc47821978..e83131b554452 100644
--- a/docs/reference/ingest/apis/index.asciidoc
+++ b/docs/reference/ingest/apis/index.asciidoc
@@ -1,6 +1,12 @@
 [[ingest-apis]]
 == Ingest APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs].
+--
+
 Use ingest APIs to manage tasks and resources related to <<ingest,ingest
 pipelines>> and processors.
 
diff --git a/docs/reference/ingest/apis/put-ip-location-database.asciidoc b/docs/reference/ingest/apis/put-ip-location-database.asciidoc
index e42d84752694c..1cdd789008881 100644
--- a/docs/reference/ingest/apis/put-ip-location-database.asciidoc
+++ b/docs/reference/ingest/apis/put-ip-location-database.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Create or update IP geolocation database configuration</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs].
+--
+
 Creates or updates an IP geolocation database configuration.
 
 IMPORTANT: The Maxmind `account_id` shown below requires a license key. Because the license key is sensitive information,
diff --git a/docs/reference/ingest/apis/put-pipeline.asciidoc b/docs/reference/ingest/apis/put-pipeline.asciidoc
index 5b532dedf8e82..7d9854a55758d 100644
--- a/docs/reference/ingest/apis/put-pipeline.asciidoc
+++ b/docs/reference/ingest/apis/put-pipeline.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Create or update pipeline</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs].
+--
+
 Creates or updates an <<ingest,ingest pipeline>>. Changes made using this API
 take effect immediately.
 
diff --git a/docs/reference/ingest/apis/simulate-ingest.asciidoc b/docs/reference/ingest/apis/simulate-ingest.asciidoc
index 52ed09b1d32c2..b48faff876641 100644
--- a/docs/reference/ingest/apis/simulate-ingest.asciidoc
+++ b/docs/reference/ingest/apis/simulate-ingest.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Simulate ingest</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs].
+--
+
 Executes ingest pipelines against a set of provided documents, optionally
 with substitute pipeline definitions. This API is meant to be used for
 troubleshooting or pipeline development, as it does not actually index any
diff --git a/docs/reference/ingest/apis/simulate-pipeline.asciidoc b/docs/reference/ingest/apis/simulate-pipeline.asciidoc
index bb7c343383132..fe7e5f2e91c6a 100644
--- a/docs/reference/ingest/apis/simulate-pipeline.asciidoc
+++ b/docs/reference/ingest/apis/simulate-pipeline.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Simulate pipeline</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs].
+--
+
 Executes an ingest pipeline against
 a set of provided documents.
 
diff --git a/docs/reference/ingest/enrich.asciidoc b/docs/reference/ingest/enrich.asciidoc
index 4bd50641149c0..3c08384eb23db 100644
--- a/docs/reference/ingest/enrich.asciidoc
+++ b/docs/reference/ingest/enrich.asciidoc
@@ -98,7 +98,8 @@ and <<update-enrich-policies, update your enrich policies>>.
 [IMPORTANT]
 ====
 The enrich processor performs several operations and may impact the speed of
-your ingest pipeline.
+your ingest pipeline. We recommend <<modules-node,node roles>> co-locating
+ingest and data roles to minimize remote search operations.
 
 We strongly recommend testing and benchmarking your enrich processors
 before deploying them in production.
diff --git a/docs/reference/ingest/search-inference-processing.asciidoc b/docs/reference/ingest/search-inference-processing.asciidoc
index 006cc96294477..73642b3bb3447 100644
--- a/docs/reference/ingest/search-inference-processing.asciidoc
+++ b/docs/reference/ingest/search-inference-processing.asciidoc
@@ -88,7 +88,7 @@ The `monitor_ml` <<security-privileges, Elasticsearch cluster privilege>> is req
 
 To create the index-specific ML inference pipeline, go to *Search -> Content -> Indices -> <your index> -> Pipelines* in the Kibana UI.
 
-If you only see the `ent-search-generic-ingestion` pipeline, you will need to click *Copy and customize* to create index-specific pipelines.
+If you only see the `search-default-ingestion` pipeline, you will need to click *Copy and customize* to create index-specific pipelines.
 This will create the `{index_name}@ml-inference` pipeline.
 
 Once your index-specific ML inference pipeline is ready, you can add inference processors that use your ML trained models.
diff --git a/docs/reference/ingest/search-ingest-pipelines.asciidoc b/docs/reference/ingest/search-ingest-pipelines.asciidoc
index e414dacaab964..272c6ba2884b9 100644
--- a/docs/reference/ingest/search-ingest-pipelines.asciidoc
+++ b/docs/reference/ingest/search-ingest-pipelines.asciidoc
@@ -40,7 +40,7 @@ Considerations such as error handling, conditional execution, sequencing, versio
 To this end, when you create indices for search use cases, (including {enterprise-search-ref}/crawler.html[Elastic web crawler], <<es-connectors,connectors>>.
 , and API indices), each index already has a pipeline set up with several processors that optimize your content for search.
 
-This pipeline is called `ent-search-generic-ingestion`.
+This pipeline is called `search-default-ingestion`.
 While it is a "managed" pipeline (meaning it should not be tampered with), you can view its details via the Kibana UI or the Elasticsearch API.
 You can also <<ingest-pipeline-search-details-generic-reference,read more about its contents below>>.
 
@@ -56,14 +56,14 @@ This will not effect existing indices.
 
 Each index also provides the capability to easily create index-specific ingest pipelines with customizable processing.
 If you need that extra flexibility, you can create a custom pipeline by going to your pipeline settings and choosing to "copy and customize".
-This will replace the index's use of `ent-search-generic-ingestion` with 3 newly generated pipelines:
+This will replace the index's use of `search-default-ingestion` with 3 newly generated pipelines:
 
 1. `<index-name>`
 2. `<index-name>@custom`
 3. `<index-name>@ml-inference`
 
-Like `ent-search-generic-ingestion`, the first of these is "managed", but the other two can and should be modified to fit your needs.
-You can view these pipelines using the platform tools (Kibana UI, Elasticsearch API), and can also 
+Like `search-default-ingestion`, the first of these is "managed", but the other two can and should be modified to fit your needs.
+You can view these pipelines using the platform tools (Kibana UI, Elasticsearch API), and can also
 <<ingest-pipeline-search-details-specific,read more about their content below>>.
 
 [discrete#ingest-pipeline-search-pipeline-settings]
@@ -123,7 +123,7 @@ If the pipeline is not specified, the underscore-prefixed fields will actually b
 === Details
 
 [discrete#ingest-pipeline-search-details-generic-reference]
-==== `ent-search-generic-ingestion` Reference
+==== `search-default-ingestion` Reference
 
 You can access this pipeline with the <<get-pipeline-api, Elasticsearch Ingest Pipelines API>> or via Kibana's <<create-manage-ingest-pipelines,Stack Management > Ingest Pipelines>> UI.
 
@@ -149,7 +149,7 @@ If you want to make customizations, we recommend you utilize index-specific pipe
 [discrete#ingest-pipeline-search-details-generic-reference-params]
 ===== Control flow parameters
 
-The `ent-search-generic-ingestion` pipeline does not always run all processors.
+The `search-default-ingestion` pipeline does not always run all processors.
 It utilizes a feature of ingest pipelines to <<conditionally-run-processor,conditionally run processors>> based on the contents of each individual document.
 
 * `_extract_binary_content` - if this field is present and has a value of `true` on a source document, the pipeline will attempt to run the `attachment`, `set_body`, and `remove_replacement_chars` processors.
@@ -167,8 +167,8 @@ See <<ingest-pipeline-search-pipeline-settings>>.
 ==== Index-specific ingest pipelines
 
 In the Kibana UI for your index, by clicking on the Pipelines tab, then *Settings > Copy and customize*, you can quickly generate 3 pipelines which are specific to your index.
-These 3 pipelines replace `ent-search-generic-ingestion` for the index.
-There is nothing lost in this action, as the `<index-name>` pipeline is a superset of functionality over the `ent-search-generic-ingestion` pipeline.
+These 3 pipelines replace `search-default-ingestion` for the index.
+There is nothing lost in this action, as the `<index-name>` pipeline is a superset of functionality over the `search-default-ingestion` pipeline.
 
 [IMPORTANT]
 ====
@@ -179,7 +179,7 @@ Refer to the Elastic subscriptions pages for https://www.elastic.co/subscription
 [discrete#ingest-pipeline-search-details-specific-reference]
 ===== `<index-name>` Reference
 
-This pipeline looks and behaves a lot like the <<ingest-pipeline-search-details-generic-reference,`ent-search-generic-ingestion` pipeline>>, but with <<ingest-pipeline-search-details-specific-reference-processors,two additional processors>>.
+This pipeline looks and behaves a lot like the <<ingest-pipeline-search-details-generic-reference,`search-default-ingestion` pipeline>>, but with <<ingest-pipeline-search-details-specific-reference-processors,two additional processors>>.
 
 [WARNING]
 =========================
@@ -197,7 +197,7 @@ If you want to make customizations, we recommend you utilize <<ingest-pipeline-s
 [discrete#ingest-pipeline-search-details-specific-reference-processors]
 ====== Processors
 
-In addition to the processors inherited from the <<ingest-pipeline-search-details-generic-reference,`ent-search-generic-ingestion` pipeline>>, the index-specific pipeline also defines:
+In addition to the processors inherited from the <<ingest-pipeline-search-details-generic-reference,`search-default-ingestion` pipeline>>, the index-specific pipeline also defines:
 
 * `index_ml_inference_pipeline` - this uses the <<pipeline-processor, Pipeline>> processor to run the `<index-name>@ml-inference` pipeline.
   This processor will only be run if the source document includes a `_run_ml_inference` field with the value `true`.
@@ -206,7 +206,7 @@ In addition to the processors inherited from the <<ingest-pipeline-search-detail
 [discrete#ingest-pipeline-search-details-specific-reference-params]
 ====== Control flow parameters
 
-Like the `ent-search-generic-ingestion` pipeline, the `<index-name>` pipeline does not always run all processors.
+Like the `search-default-ingestion` pipeline, the `<index-name>` pipeline does not always run all processors.
 In addition to the `_extract_binary_content` and `_reduce_whitespace` control flow parameters, the `<index-name>` pipeline also supports:
 
 * `_run_ml_inference` - if this field is present and has a value of `true` on a source document, the pipeline will attempt to run the `index_ml_inference_pipeline` processor.
@@ -220,7 +220,7 @@ See <<ingest-pipeline-search-pipeline-settings>>.
 ===== `<index-name>@ml-inference` Reference
 
 This pipeline is empty to start (no processors), but can be added to via the Kibana UI either through the Pipelines tab of your index, or from the *Stack Management > Ingest Pipelines* page.
-Unlike the `ent-search-generic-ingestion` pipeline and the `<index-name>` pipeline, this pipeline is NOT "managed".
+Unlike the `search-default-ingestion` pipeline and the `<index-name>` pipeline, this pipeline is NOT "managed".
 
 It's possible to add one or more ML inference pipelines to an index in the *Content* UI.
 This pipeline will serve as a container for all of the ML inference pipelines configured for the index.
@@ -241,7 +241,7 @@ The `monitor_ml` Elasticsearch cluster permission is required in order to manage
 
 This pipeline is empty to start (no processors), but can be added to via the Kibana UI either through the Pipelines
 tab of your index, or from the *Stack Management > Ingest Pipelines* page.
-Unlike the `ent-search-generic-ingestion` pipeline and the `<index-name>` pipeline, this pipeline is NOT "managed".
+Unlike the `search-default-ingestion` pipeline and the `<index-name>` pipeline, this pipeline is NOT "managed".
 
 You are encouraged to make additions and edits to this pipeline, provided its name remains the same.
 This provides a convenient hook from which to add custom processing and transformations for your data.
@@ -272,9 +272,12 @@ extraction.
 These changes should be re-applied to each index's `<index-name>@custom` pipeline in order to ensure a consistent data processing experience.
   In 8.5+, the <<ingest-pipeline-search-pipeline-settings, index setting to enable binary content>> is required *in addition* to the configurations mentioned in the {enterprise-search-ref}/crawler-managing.html#crawler-managing-binary-content[Elastic web crawler Guide].
 
-* `ent-search-generic-ingestion` - Since 8.5, Native Connectors, Connector Clients, and new (>8.4) Elastic web crawler indices will all make use of this pipeline by default.
+* `ent-search-generic-ingestion` - Since 8.5, Native Connectors, Connector Clients, and new (>8.4) Elastic web crawler indices all made use of this pipeline by default.
+  This pipeline evolved into the `search-default-ingestion` pipeline.
+
+* `search-default-ingestion` - Since 9.0, Connectors have made use of this pipeline by default.
   You can <<ingest-pipeline-search-details-generic-reference, read more about this pipeline>> above.
-  As this pipeline is "managed", any modifications that were made to `app_search_crawler` and/or `ent_search_crawler` should NOT be made to `ent-search-generic-ingestion`.
+  As this pipeline is "managed", any modifications that were made to `app_search_crawler` and/or `ent_search_crawler` should NOT be made to `search-default-ingestion`.
   Instead, if such customizations are desired, you should utilize <<ingest-pipeline-search-details-specific>>, placing all modifications in the `<index-name>@custom` pipeline(s).
 =============
 
diff --git a/docs/reference/ingest/search-nlp-tutorial.asciidoc b/docs/reference/ingest/search-nlp-tutorial.asciidoc
index afdceeeb8bac2..b23a15c96b1a2 100644
--- a/docs/reference/ingest/search-nlp-tutorial.asciidoc
+++ b/docs/reference/ingest/search-nlp-tutorial.asciidoc
@@ -164,8 +164,8 @@ Now it's time to create an inference pipeline.
 
 1. From the overview page for your `search-photo-comments` index in "Search", click the *Pipelines* tab.
 By default, Elasticsearch does not create any index-specific ingest pipelines.
-2. Because we want to customize these pipelines, we need to *Copy and customize* the `ent-search-generic-ingestion` ingest pipeline.
-Find this option above the settings for the `ent-search-generic-ingestion` ingest pipeline.
+2. Because we want to customize these pipelines, we need to *Copy and customize* the `search-default-ingestion` ingest pipeline.
+Find this option above the settings for the `search-default-ingestion` ingest pipeline.
 This will create two new index-specific ingest pipelines.
 
 Next, we'll add an inference pipeline.
diff --git a/docs/reference/intro.asciidoc b/docs/reference/intro.asciidoc
index e0100b1c5640b..2fd0722bcd660 100644
--- a/docs/reference/intro.asciidoc
+++ b/docs/reference/intro.asciidoc
@@ -260,7 +260,7 @@ Refer to <<getting-started,first steps with Elasticsearch>> for a hands-on examp
 
 *{esql}* is a new piped query language and compute engine which was first added in version *8.11*.
 
-{esql} does not yet support all the features of Query DSL, like full-text search and semantic search.
+{esql} does not yet support all the features of Query DSL.
 Look forward to new {esql} features and functionalities in each release.
 
 Refer to <<search-analyze-query-languages>> for a full overview of the query languages available in {es}.
@@ -280,7 +280,7 @@ The <<search-your-data, `_search` endpoint>> accepts queries written in Query DS
 
 Query DSL support a wide range of search techniques, including the following:
 
-* <<full-text-queries,*Full-text search*>>: Search text that has been analyzed and indexed to support phrase or proximity queries, fuzzy matches, and more.
+* <<full-text-search,*Full-text search*>>: Search text that has been analyzed and indexed to support phrase or proximity queries, fuzzy matches, and more.
 * <<keyword,*Keyword search*>>: Search for exact matches using `keyword` fields.
 * <<semantic-search-semantic-text,*Semantic search*>>: Search `semantic_text` fields using dense or sparse vector search on embeddings generated in your {es} cluster.
 * <<knn-search,*Vector search*>>: Search for similar dense vectors using the kNN algorithm for embeddings generated outside of {es}.
@@ -328,8 +328,7 @@ directly executed within {es} itself.
 
 The <<esql-rest,`_query` endpoint>> accepts queries written in {esql} syntax.
 
-Today, it supports a subset of the features available in Query DSL, like aggregations, filters, and transformations.
-It does not yet support full-text search or semantic search.
+Today, it supports a subset of the features available in Query DSL, but it is rapidly evolving.
 
 It comes with a comprehensive set of <<esql-functions-operators,functions and operators>> for working with data and has robust integration with {kib}'s Discover, dashboards and visualizations.
 
@@ -397,51 +396,18 @@ geographic location of your users and your resources.
 [[use-multiple-nodes-shards]]
 ==== Use multiple nodes and shards
 
-[NOTE]
-====
-Nodes and shards are what make {es} distributed and scalable.
+When you move to production, you need to introduce multiple nodes and shards to your cluster. Nodes and shards are what make {es} distributed and scalable. The size and number of these nodes and shards depends on your data, your use case, and your budget.
 
-These concepts aren’t essential if you’re just getting started. How you <<elasticsearch-intro-deploy,deploy {es}>> in production determines what you need to know:
+These concepts aren't essential if you're just getting started. How you <<elasticsearch-intro-deploy,deploy {es}>> in production determines what you need to know:
 
 * *Self-managed {es}*: You are responsible for setting up and managing nodes, clusters, shards, and replicas. This includes
 managing the underlying infrastructure, scaling, and ensuring high availability through failover and backup strategies.
 * *Elastic Cloud*: Elastic can autoscale resources in response to workload changes. Choose from different deployment types
 to apply sensible defaults for your use case. A basic understanding of nodes, shards, and replicas is still important.
-* *Elastic Cloud Serverless*: You don’t need to worry about nodes, shards, or replicas. These resources are 100% automated
+* *Elastic Cloud Serverless*: You don't need to worry about nodes, shards, or replicas. These resources are 100% automated
 on the serverless platform, which is designed to scale with your workload.
-====
-
-You can add servers (_nodes_) to a cluster to increase capacity, and {es} automatically distributes your data and query load
-across all of the available nodes.
-
-Elastic is able to distribute your data across nodes by subdividing an index into _shards_. Each index in {es} is a grouping
-of one or more physical shards, where each shard is a self-contained Lucene index containing a subset of the documents in
-the index. By distributing the documents in an index across multiple shards, and distributing those shards across multiple
-nodes, {es} increases indexing and query capacity.
-
-There are two types of shards: _primaries_ and _replicas_. Each document in an index belongs to one primary shard. A replica
-shard is a copy of a primary shard. Replicas maintain redundant copies of your data across the nodes in your cluster. 
-This protects against hardware failure and increases capacity to serve read requests like searching or retrieving a document.
-
-[TIP]
-====
-The number of primary shards in an index is fixed at the time that an index is created, but the number of replica shards can
-be changed at any time, without interrupting indexing or query operations.
-====
-
-Shard copies in your cluster are automatically balanced across nodes to provide scale and high availability. All nodes are
-aware of all the other nodes in the cluster and can forward client requests to the appropriate node. This allows {es}
-to distribute indexing and query load across the cluster.
-
-If you’re exploring {es} for the first time or working in a development environment, then you can use a cluster with a single node and create indices
-with only one shard. However, in a production environment, you should build a cluster with multiple nodes and indices
-with multiple shards to increase performance and resilience.
-
-// TODO - diagram
 
-To learn about optimizing the number and size of shards in your cluster, refer to <<size-your-shards,Size your shards>>. 
-To learn about how read and write operations are replicated across shards and shard copies, refer to <<docs-replication,Reading and writing documents>>.
-To adjust how shards are allocated and balanced across nodes, refer to <<shard-allocation-relocation-recovery,Shard allocation, relocation, and recovery>>.
+Learn more about <<nodes-shards,nodes and shards>>.
 
 [discrete]
 [[ccr-disaster-recovery-geo-proximity]]
diff --git a/docs/reference/licensing/delete-license.asciidoc b/docs/reference/licensing/delete-license.asciidoc
index 0f0aac416ccf8..fd9236664e1c0 100644
--- a/docs/reference/licensing/delete-license.asciidoc
+++ b/docs/reference/licensing/delete-license.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete license</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-license[Licensing APIs].
+--
+
 This API enables you to delete licensing information.
 
 [discrete]
diff --git a/docs/reference/licensing/get-basic-status.asciidoc b/docs/reference/licensing/get-basic-status.asciidoc
index 44bc246e72bc3..0e354e0dc07cd 100644
--- a/docs/reference/licensing/get-basic-status.asciidoc
+++ b/docs/reference/licensing/get-basic-status.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get basic status</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-license[Licensing APIs].
+--
+
 This API enables you to check the status of your basic license.
 
 [discrete]
diff --git a/docs/reference/licensing/get-license.asciidoc b/docs/reference/licensing/get-license.asciidoc
index 816bdd36ff4bd..1b50a5be8fa20 100644
--- a/docs/reference/licensing/get-license.asciidoc
+++ b/docs/reference/licensing/get-license.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get license</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-license[Licensing APIs].
+--
+
 This API enables you to retrieve licensing information.
 
 [discrete]
diff --git a/docs/reference/licensing/get-trial-status.asciidoc b/docs/reference/licensing/get-trial-status.asciidoc
index 912d90bde850f..b0ae110faa8cc 100644
--- a/docs/reference/licensing/get-trial-status.asciidoc
+++ b/docs/reference/licensing/get-trial-status.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get trial status</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-license[Licensing APIs].
+--
+
 Enables you to check the status of your trial.
 
 [discrete]
diff --git a/docs/reference/licensing/index.asciidoc b/docs/reference/licensing/index.asciidoc
index a1dfd398acfe7..12df7b606c5bd 100644
--- a/docs/reference/licensing/index.asciidoc
+++ b/docs/reference/licensing/index.asciidoc
@@ -2,6 +2,12 @@
 [[licensing-apis]]
 == Licensing APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-license[Licensing APIs].
+--
+
 You can use the following APIs to manage your licenses:
 
 * <<delete-license>>
diff --git a/docs/reference/licensing/start-basic.asciidoc b/docs/reference/licensing/start-basic.asciidoc
index 8ae25e1940a62..291a0ec452caf 100644
--- a/docs/reference/licensing/start-basic.asciidoc
+++ b/docs/reference/licensing/start-basic.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Start basic</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-license[Licensing APIs].
+--
+
 This API starts an indefinite basic license.
 
 [discrete]
diff --git a/docs/reference/licensing/update-license.asciidoc b/docs/reference/licensing/update-license.asciidoc
index 4207dbb092fb0..03b6c3ce40edd 100644
--- a/docs/reference/licensing/update-license.asciidoc
+++ b/docs/reference/licensing/update-license.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Update license</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-license[Licensing APIs].
+--
+
 Updates the license for your {es} cluster.
 
 [[update-license-api-request]]
diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc
index babe4f508b5f0..e5155b7d4ce5b 100644
--- a/docs/reference/mapping/types.asciidoc
+++ b/docs/reference/mapping/types.asciidoc
@@ -180,6 +180,8 @@ include::types/rank-feature.asciidoc[]
 
 include::types/rank-features.asciidoc[]
 
+include::types/rank-vectors.asciidoc[]
+
 include::types/search-as-you-type.asciidoc[]
 
 include::types/semantic-text.asciidoc[]
diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc
index 199a59a5b143c..c16b979043a57 100644
--- a/docs/reference/mapping/types/dense-vector.asciidoc
+++ b/docs/reference/mapping/types/dense-vector.asciidoc
@@ -1,4 +1,3 @@
-[role="xpack"]
 [[dense-vector]]
 === Dense vector field type
 ++++
diff --git a/docs/reference/mapping/types/rank-vectors.asciidoc b/docs/reference/mapping/types/rank-vectors.asciidoc
new file mode 100644
index 0000000000000..a718a5e47ec85
--- /dev/null
+++ b/docs/reference/mapping/types/rank-vectors.asciidoc
@@ -0,0 +1,201 @@
+[role="xpack"]
+[[rank-vectors]]
+=== Rank Vectors
+++++
+<titleabbrev> Rank Vectors </titleabbrev>
+++++
+experimental::[]
+
+The `rank_vectors` field type enables late-interaction dense vector scoring in Elasticsearch. The number of vectors
+per field can vary, but they must all share the same number of dimensions and element type.
+
+The purpose of vectors stored in this field is second order ranking documents with max-sim similarity.
+
+Here is a simple example of using this field with `float` elements.
+
+[source,console]
+--------------------------------------------------
+PUT my-rank-vectors-float
+{
+  "mappings": {
+    "properties": {
+      "my_vector": {
+        "type": "rank_vectors"
+      }
+    }
+  }
+}
+
+PUT my-rank-vectors-float/_doc/1
+{
+  "my_vector" : [[0.5, 10, 6], [-0.5, 10, 10]]
+}
+
+--------------------------------------------------
+// TESTSETUP
+
+In addition to the `float` element type, `byte` and `bit` element types are also supported.
+
+Here is an example of using this field with `byte` elements.
+
+[source,console]
+--------------------------------------------------
+PUT my-rank-vectors-byte
+{
+  "mappings": {
+    "properties": {
+      "my_vector": {
+        "type": "rank_vectors",
+        "element_type": "byte"
+      }
+    }
+  }
+}
+
+PUT my-rank-vectors-byte/_doc/1
+{
+  "my_vector" : [[1, 2, 3], [4, 5, 6]]
+}
+--------------------------------------------------
+
+Here is an example of using this field with `bit` elements.
+
+[source,console]
+--------------------------------------------------
+PUT my-rank-vectors-bit
+{
+  "mappings": {
+    "properties": {
+      "my_vector": {
+        "type": "rank_vectors",
+        "element_type": "bit"
+      }
+    }
+  }
+}
+
+POST /my-rank-vectors-bit/_bulk?refresh
+{"index": {"_id" : "1"}}
+{"my_vector": [127, -127, 0, 1, 42]}
+{"index": {"_id" : "2"}}
+{"my_vector": "8100012a7f"}
+--------------------------------------------------
+
+[role="child_attributes"]
+[[rank-vectors-params]]
+==== Parameters for rank vectors fields
+
+The `rank_vectors` field type supports the following parameters:
+
+[[rank-vectors-element-type]]
+`element_type`::
+(Optional, string)
+The data type used to encode vectors. The supported data types are
+`float` (default), `byte`, and bit.
+
+.Valid values for `element_type`
+[%collapsible%open]
+====
+`float`:::
+indexes a 4-byte floating-point
+value per dimension. This is the default value.
+
+`byte`:::
+indexes a 1-byte integer value per dimension.
+
+`bit`:::
+indexes a single bit per dimension. Useful for very high-dimensional vectors or models that specifically support bit vectors.
+NOTE: when using `bit`, the number of dimensions must be a multiple of 8 and must represent the number of bits.
+
+====
+
+`dims`::
+(Optional, integer)
+Number of vector dimensions. Can't exceed `4096`. If `dims` is not specified,
+it will be set to the length of the first vector added to the field.
+
+[[rank-vectors-synthetic-source]]
+==== Synthetic `_source`
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will work to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
+
+`rank_vectors` fields support <<synthetic-source,synthetic `_source`>> .
+
+[[rank-vectors-scoring]]
+==== Scoring with rank vectors
+
+Rank vectors can be accessed and used in <<query-dsl-script-score-query,`script_score` queries>>.
+
+For example, the following query scores documents based on the maxSim similarity between the query vector and the vectors stored in the `my_vector` field:
+
+[source,console]
+--------------------------------------------------
+GET my-rank-vectors-float/_search
+{
+  "query": {
+    "script_score": {
+      "query": {
+        "match_all": {}
+      },
+      "script": {
+        "source": "maxSimDotProduct(params.query_vector, 'my_vector')",
+        "params": {
+          "query_vector": [[0.5, 10, 6], [-0.5, 10, 10]]
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+
+Additionally, asymmetric similarity functions can be used to score against `bit` vectors. For example, the following query scores documents based on the maxSimDotProduct similarity between a floating point query vector and bit vectors stored in the `my_vector` field:
+
+[source,console]
+--------------------------------------------------
+PUT my-rank-vectors-bit
+{
+  "mappings": {
+    "properties": {
+      "my_vector": {
+        "type": "rank_vectors",
+        "element_type": "bit"
+      }
+    }
+  }
+}
+
+POST /my-rank-vectors-bit/_bulk?refresh
+{"index": {"_id" : "1"}}
+{"my_vector": [127, -127, 0, 1, 42]}
+{"index": {"_id" : "2"}}
+{"my_vector": "8100012a7f"}
+
+GET my-rank-vectors-bit/_search
+{
+  "query": {
+    "script_score": {
+      "query": {
+        "match_all": {}
+      },
+      "script": {
+        "source": "maxSimDotProduct(params.query_vector, 'my_vector')",
+        "params": {
+          "query_vector": [
+            [0.35, 0.77, 0.95, 0.15, 0.11, 0.08, 0.58, 0.06, 0.44, 0.52, 0.21,
+       0.62, 0.65, 0.16, 0.64, 0.39, 0.93, 0.06, 0.93, 0.31, 0.92, 0.0,
+       0.66, 0.86, 0.92, 0.03, 0.81, 0.31, 0.2 , 0.92, 0.95, 0.64, 0.19,
+       0.26, 0.77, 0.64, 0.78, 0.32, 0.97, 0.84]
+           ] <1>
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+<1> Note that the query vector has 40 elements, matching the number of bits in the bit vectors.
+
diff --git a/docs/reference/mapping/types/semantic-text.asciidoc b/docs/reference/mapping/types/semantic-text.asciidoc
index 96dc402e10c60..4514c8b6756a8 100644
--- a/docs/reference/mapping/types/semantic-text.asciidoc
+++ b/docs/reference/mapping/types/semantic-text.asciidoc
@@ -13,6 +13,7 @@ Long passages are <<auto-text-chunking, automatically chunked>> to smaller secti
 The `semantic_text` field type specifies an inference endpoint identifier that will be used to generate embeddings.
 You can create the inference endpoint by using the <<put-inference-api>>.
 This field type and the <<query-dsl-semantic-query,`semantic` query>> type make it simpler to perform semantic search on your data. 
+The `semantic_text` field type may also be queried with <<query-dsl-match-query, match>>, <<query-dsl-sparse-vector-query, sparse_vector>> or <<query-dsl-knn-query, knn>> queries.
 
 If you don’t specify an inference endpoint, the `inference_id` field defaults to `.elser-2-elasticsearch`, a preconfigured endpoint for the elasticsearch service.
 
diff --git a/docs/reference/migration/apis/deprecation.asciidoc b/docs/reference/migration/apis/deprecation.asciidoc
index 67b4c113af2bc..71453a001eeeb 100644
--- a/docs/reference/migration/apis/deprecation.asciidoc
+++ b/docs/reference/migration/apis/deprecation.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Deprecation info</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-migration[Migration APIs].
+--
+
 include::{es-ref-dir}/migration/apis/shared-migration-apis-tip.asciidoc[]
 
 The deprecation API is to be used to retrieve information about different
diff --git a/docs/reference/migration/apis/feature-migration.asciidoc b/docs/reference/migration/apis/feature-migration.asciidoc
index e38639ac44531..717a46442bd92 100644
--- a/docs/reference/migration/apis/feature-migration.asciidoc
+++ b/docs/reference/migration/apis/feature-migration.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Feature migration</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-migration[Migration APIs].
+--
+
 include::{es-ref-dir}/migration/apis/shared-migration-apis-tip.asciidoc[]
 
 Version upgrades sometimes require changes to how features store configuration
diff --git a/docs/reference/migration/migration.asciidoc b/docs/reference/migration/migration.asciidoc
index ffb2ca7a7859d..57b6c88aefea4 100644
--- a/docs/reference/migration/migration.asciidoc
+++ b/docs/reference/migration/migration.asciidoc
@@ -2,6 +2,12 @@
 [[migration-api]]
 == Migration APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-migration[Migration APIs].
+--
+
 The migration APIs power {kib}'s **Upgrade Assistant** feature.
 
 include::apis/shared-migration-apis-tip.asciidoc[]
diff --git a/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc
index f0cb968e082c5..865213dad3f50 100644
--- a/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Close jobs</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Closes one or more {anomaly-jobs}.
 
 [[ml-close-job-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc
index b80a248038aea..d67f7cce9d1f6 100644
--- a/docs/reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete events from calendar</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Deletes scheduled events from a calendar.
 
 [[ml-delete-calendar-event-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/delete-calendar-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-calendar-job.asciidoc
index 6720e236fd635..17a581f964f54 100644
--- a/docs/reference/ml/anomaly-detection/apis/delete-calendar-job.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/delete-calendar-job.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete jobs from calendar</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Deletes {anomaly-jobs} from a calendar.
 
 [[ml-delete-calendar-job-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/delete-calendar.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-calendar.asciidoc
index 6684366c6f336..23ce74c8fb310 100644
--- a/docs/reference/ml/anomaly-detection/apis/delete-calendar.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/delete-calendar.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete calendars</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Deletes a calendar.
 
 [[ml-delete-calendar-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/delete-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-datafeed.asciidoc
index 64a1e4c336fe6..61d3624ef50dc 100644
--- a/docs/reference/ml/anomaly-detection/apis/delete-datafeed.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/delete-datafeed.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Delete {dfeeds}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Deletes an existing {dfeed}.
 
 [[ml-delete-datafeed-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/delete-expired-data.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-expired-data.asciidoc
index a8c1279ac1b47..bee453f8d2082 100644
--- a/docs/reference/ml/anomaly-detection/apis/delete-expired-data.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/delete-expired-data.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete expired data</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Deletes expired and unused machine learning data.
 
 [[ml-delete-expired-data-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/delete-filter.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-filter.asciidoc
index 4b41347543e8e..84346ae012300 100644
--- a/docs/reference/ml/anomaly-detection/apis/delete-filter.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/delete-filter.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete filters</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Deletes a filter.
 
 [[ml-delete-filter-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/delete-forecast.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-forecast.asciidoc
index 74e6ce27084ad..ee3167036f66d 100644
--- a/docs/reference/ml/anomaly-detection/apis/delete-forecast.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/delete-forecast.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete forecasts</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Deletes forecasts from a {ml} job.
 
 [[ml-delete-forecast-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/delete-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-job.asciidoc
index 1bbe07fd44f49..595c5acc041d6 100644
--- a/docs/reference/ml/anomaly-detection/apis/delete-job.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/delete-job.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete jobs</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Deletes an existing {anomaly-job}.
 
 [[ml-delete-job-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/delete-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-snapshot.asciidoc
index ad10de7a2ba0f..345b933a19622 100644
--- a/docs/reference/ml/anomaly-detection/apis/delete-snapshot.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/delete-snapshot.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete model snapshots</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Deletes an existing model snapshot.
 
 [[ml-delete-snapshot-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/estimate-model-memory.asciidoc b/docs/reference/ml/anomaly-detection/apis/estimate-model-memory.asciidoc
index 1cc4df42f083d..8be8e8df72d4e 100644
--- a/docs/reference/ml/anomaly-detection/apis/estimate-model-memory.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/estimate-model-memory.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Estimate model memory</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Makes an estimation of the memory usage for an {anomaly-job} model. It is based 
 on analysis configuration details for the job and cardinality estimates for the 
 fields it references.
diff --git a/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc
index 68ff601749b4b..dcf4d7a6b1d60 100644
--- a/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Flush jobs</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Forces any buffered data to be processed by the job.
 
 [[ml-flush-job-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/forecast.asciidoc b/docs/reference/ml/anomaly-detection/apis/forecast.asciidoc
index 3e6067ab05857..ac4e3eb4ed29a 100644
--- a/docs/reference/ml/anomaly-detection/apis/forecast.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/forecast.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Forecast jobs</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Predicts the future behavior of a time series by using its historical behavior. 
 
 [[ml-forecast-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/get-bucket.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-bucket.asciidoc
index bca839d1db318..d33d6f64021a3 100644
--- a/docs/reference/ml/anomaly-detection/apis/get-bucket.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/get-bucket.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get buckets</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Retrieves {anomaly-job} results for one or more buckets.
 
 [[ml-get-bucket-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/get-calendar-event.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-calendar-event.asciidoc
index fc06e286bf46c..565a553e18974 100644
--- a/docs/reference/ml/anomaly-detection/apis/get-calendar-event.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/get-calendar-event.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get scheduled events</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Retrieves information about the scheduled events in calendars.
 
 [[ml-get-calendar-event-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/get-calendar.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-calendar.asciidoc
index b2c46bbe16c0e..bf00458299633 100644
--- a/docs/reference/ml/anomaly-detection/apis/get-calendar.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/get-calendar.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get calendars</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Retrieves configuration information for calendars.
 
 [[ml-get-calendar-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/get-category.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-category.asciidoc
index 33de5e0f71a08..0f5fa1504c9fe 100644
--- a/docs/reference/ml/anomaly-detection/apis/get-category.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/get-category.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get categories</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Retrieves {anomaly-job} results for one or more categories.
 
 [[ml-get-category-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc
index 0a8f1e802715e..f9130bb78f5e7 100644
--- a/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Get {dfeed} statistics</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Retrieves usage information for {ml-docs}/ml-ad-run-jobs.html#ml-ad-datafeeds[{dfeeds}].
 
 [[ml-get-datafeed-stats-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc
index a986e2220f928..3b86c5cc22a87 100644
--- a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Get {dfeeds}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Retrieves configuration information for {dfeeds}.
 
 [[ml-get-datafeed-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/get-filter.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-filter.asciidoc
index f73dcd236f1af..dc167238701cd 100644
--- a/docs/reference/ml/anomaly-detection/apis/get-filter.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/get-filter.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get filters</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Retrieves filters.
 
 [[ml-get-filter-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/get-influencer.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-influencer.asciidoc
index 31489e361a848..f2e26344207a8 100644
--- a/docs/reference/ml/anomaly-detection/apis/get-influencer.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/get-influencer.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get influencers</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Retrieves {anomaly-job} results for one or more influencers.
 
 [[ml-get-influencer-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/get-job-model-snapshot-upgrade-stats.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job-model-snapshot-upgrade-stats.asciidoc
index 0939282a75916..32a9b67258b2d 100644
--- a/docs/reference/ml/anomaly-detection/apis/get-job-model-snapshot-upgrade-stats.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/get-job-model-snapshot-upgrade-stats.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Get model snapshot upgrade statistics</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Retrieves usage information for {anomaly-job} model snapshot upgrades.
 
 [[ml-get-job-model-snapshot-upgrade-stats-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc
index 627109d11e11e..1df41abdda27b 100644
--- a/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get job statistics</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Retrieves usage information for {ml-docs}/ml-ad-overview.html[{anomaly-jobs}].
 
 [[ml-get-job-stats-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc
index 33692fd182fa7..74fd1c7ecb54a 100644
--- a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get jobs</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Retrieves configuration information for {anomaly-jobs}.
 
 [[ml-get-job-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc
index b581b5c3a2eb2..45ae61647bfda 100644
--- a/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get overall buckets</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Retrieves overall bucket results that summarize the bucket results of multiple
 {anomaly-jobs}.
 
diff --git a/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc
index e74ab3ecb4b12..266ab64ba5e19 100644
--- a/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get records</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Retrieves anomaly records for an {anomaly-job}.
 
 [[ml-get-record-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc
index d94bd4060854a..47fdc3db46b60 100644
--- a/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get model snapshots</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Retrieves information about model snapshots.
 
 [[ml-get-snapshot-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/ml-ad-apis.asciidoc b/docs/reference/ml/anomaly-detection/apis/ml-ad-apis.asciidoc
index 856232c933432..951f9522db74b 100644
--- a/docs/reference/ml/anomaly-detection/apis/ml-ad-apis.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/ml-ad-apis.asciidoc
@@ -2,6 +2,12 @@
 [[ml-ad-apis]]
 = {ml-cap} {anomaly-detect} APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 You can use the following APIs to perform {ml} {anomaly-detect} activities.
 
 See also <<ml-apis>>, <<ml-df-analytics-apis>>,  <<ml-df-trained-models-apis>>.
diff --git a/docs/reference/ml/anomaly-detection/apis/open-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/open-job.asciidoc
index 385f672f467f6..9a5153952b5a2 100644
--- a/docs/reference/ml/anomaly-detection/apis/open-job.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/open-job.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Open jobs</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Opens one or more {anomaly-jobs}.
 
 [[ml-open-job-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/post-calendar-event.asciidoc b/docs/reference/ml/anomaly-detection/apis/post-calendar-event.asciidoc
index 46ffeab694fa3..429dca427a9d9 100644
--- a/docs/reference/ml/anomaly-detection/apis/post-calendar-event.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/post-calendar-event.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Add events to calendar</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Posts scheduled events in a calendar.
 
 [[ml-post-calendar-event-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc b/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc
index 931efcf8c2a52..83e5dd77e814a 100644
--- a/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc
@@ -7,6 +7,12 @@
 
 deprecated::[7.11.0, "Posting data directly to anomaly detection jobs is deprecated, in a future major version a <<ml-api-datafeed-endpoint,{dfeed}>> will be required."]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Sends data to an anomaly detection job for analysis.
 
 [[ml-post-data-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc
index 243cd2a5f32a3..c57965401e074 100644
--- a/docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Preview {dfeeds}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Previews a {dfeed}.
 
 [[ml-preview-datafeed-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/put-calendar-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-calendar-job.asciidoc
index 0c19a08cbd74b..612f472bba37d 100644
--- a/docs/reference/ml/anomaly-detection/apis/put-calendar-job.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/put-calendar-job.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Add jobs to calendar</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Adds an {anomaly-job} to a calendar.
 
 [[ml-put-calendar-job-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/put-calendar.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-calendar.asciidoc
index fd2b58a31737a..69a4498cc29da 100644
--- a/docs/reference/ml/anomaly-detection/apis/put-calendar.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/put-calendar.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Create calendars</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Instantiates a calendar.
 
 [[ml-put-calendar-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc
index 47e3059666d76..18260d3538e4b 100644
--- a/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Create {dfeeds}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Instantiates a {dfeed}.
 
 [[ml-put-datafeed-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/put-filter.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-filter.asciidoc
index b50ba8cb1e23b..3f3157f008e9f 100644
--- a/docs/reference/ml/anomaly-detection/apis/put-filter.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/put-filter.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Create filters</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Instantiates a filter.
 
 [[ml-put-filter-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc
index 7bf02e7a0dd6e..30a1039f93db0 100644
--- a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Create jobs</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Instantiates an {anomaly-job}.
 
 [[ml-put-job-request]]
@@ -551,4 +557,3 @@ The API returns the following results:
 // TESTRESPONSE[s/"job_version" : "8.4.0"/"job_version" : $body.job_version/]
 // TESTRESPONSE[s/1656087283340/$body.$_path/]
 // TESTRESPONSE[s/"superuser"/"_es_test_root"/]
-// TESTRESPONSE[s/"ignore_throttled" : true/"ignore_throttled" : true,"failure_store":"exclude"/]
diff --git a/docs/reference/ml/anomaly-detection/apis/reset-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/reset-job.asciidoc
index 9009d634a2e9c..2d9c4696cab4b 100644
--- a/docs/reference/ml/anomaly-detection/apis/reset-job.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/reset-job.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Reset jobs</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Resets an existing {anomaly-job}.
 
 [[ml-reset-job-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc
index c8d7a27ee2047..792a102ff2953 100644
--- a/docs/reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Revert model snapshots</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Reverts to a specific snapshot.
 
 [[ml-revert-snapshot-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc
index b54c80133d7dc..4281b3a02b826 100644
--- a/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Start {dfeeds}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Starts one or more {dfeeds}.
 
 [[ml-start-datafeed-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc
index bc15a1de8a056..91e5087715fb7 100644
--- a/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Stop {dfeeds}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Stops one or more {dfeeds}.
 
 [[ml-stop-datafeed-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc
index b3920d9d4f809..c03c921e7875a 100644
--- a/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Update {dfeeds}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Updates certain properties of a {dfeed}.
 
 
diff --git a/docs/reference/ml/anomaly-detection/apis/update-filter.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-filter.asciidoc
index a4221c37a438e..8fb5376f4acbc 100644
--- a/docs/reference/ml/anomaly-detection/apis/update-filter.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/update-filter.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Update filters</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Updates the description of a filter, adds items, or removes items. 
 
 [[ml-update-filter-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc
index ee13247fc8838..bed013c113c01 100644
--- a/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Update jobs</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Updates certain properties of an {anomaly-job}.
 
 [[ml-update-job-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/update-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-snapshot.asciidoc
index f8c0384860029..54276f62ff094 100644
--- a/docs/reference/ml/anomaly-detection/apis/update-snapshot.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/update-snapshot.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Update model snapshots</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Updates certain properties of a snapshot.
 
 [[ml-update-snapshot-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc
index 3a74e3b2296df..2538b4958ada1 100644
--- a/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Upgrade model snapshots</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Upgrades an {anomaly-detect} model snapshot to the latest major version.
 
 NOTE: From {es} 8.10.0,  a new version number is used to
diff --git a/docs/reference/ml/anomaly-detection/apis/validate-detector.asciidoc b/docs/reference/ml/anomaly-detection/apis/validate-detector.asciidoc
index c71673be7dc00..820845d0fd233 100644
--- a/docs/reference/ml/anomaly-detection/apis/validate-detector.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/validate-detector.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Validate detectors</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Validates detector configuration information.
 
 [[ml-valid-detector-request]]
diff --git a/docs/reference/ml/anomaly-detection/apis/validate-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/validate-job.asciidoc
index c77623566d4c8..382199e315695 100644
--- a/docs/reference/ml/anomaly-detection/apis/validate-job.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/validate-job.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Validate jobs</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap}  {anomaly-detect} APIs].
+--
+
 Validates {anomaly-job} configuration information.
 
 [[ml-valid-job-request]]
diff --git a/docs/reference/ml/common/apis/get-ml-info.asciidoc b/docs/reference/ml/common/apis/get-ml-info.asciidoc
index 104375bd641c8..62fae2a9a9184 100644
--- a/docs/reference/ml/common/apis/get-ml-info.asciidoc
+++ b/docs/reference/ml/common/apis/get-ml-info.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Get {ml} info</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml[{ml-cap} APIs].
+--
+
 Returns defaults and limits used by machine learning.
 
 [[get-ml-info-request]]
diff --git a/docs/reference/ml/common/apis/get-ml-memory.asciidoc b/docs/reference/ml/common/apis/get-ml-memory.asciidoc
index 81e0f59a97e58..8c1cf620b70eb 100644
--- a/docs/reference/ml/common/apis/get-ml-memory.asciidoc
+++ b/docs/reference/ml/common/apis/get-ml-memory.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Get {ml} memory stats</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml[{ml-cap} APIs].
+--
+
 Returns information on how {ml} is using memory.
 
 [[get-ml-memory-request]]
diff --git a/docs/reference/ml/common/apis/ml-apis.asciidoc b/docs/reference/ml/common/apis/ml-apis.asciidoc
index c4349f3eb7366..95f102ceecfa8 100644
--- a/docs/reference/ml/common/apis/ml-apis.asciidoc
+++ b/docs/reference/ml/common/apis/ml-apis.asciidoc
@@ -2,6 +2,12 @@
 [[ml-apis]]
 = {ml-cap} APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml[{ml-cap} APIs].
+--
+
 You can use the following APIs to retrieve information related to the
 {stack-ml-features}:
 
diff --git a/docs/reference/ml/common/apis/set-upgrade-mode.asciidoc b/docs/reference/ml/common/apis/set-upgrade-mode.asciidoc
index 2469753382f9b..52adcac9bacdc 100644
--- a/docs/reference/ml/common/apis/set-upgrade-mode.asciidoc
+++ b/docs/reference/ml/common/apis/set-upgrade-mode.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Set upgrade mode</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml[{ml-cap} APIs].
+--
+
 Sets a cluster wide upgrade_mode setting that prepares {ml} indices for an
 upgrade. 
 
diff --git a/docs/reference/ml/df-analytics/apis/delete-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/delete-dfanalytics.asciidoc
index b505da570244f..a6b5058eab887 100644
--- a/docs/reference/ml/df-analytics/apis/delete-dfanalytics.asciidoc
+++ b/docs/reference/ml/df-analytics/apis/delete-dfanalytics.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Delete {dfanalytics-jobs}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap}  {dfanalytics} APIs].
+--
+
 Deletes an existing {dfanalytics-job}.
 
 
diff --git a/docs/reference/ml/df-analytics/apis/evaluate-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/evaluate-dfanalytics.asciidoc
index 8d31fcd0107d9..fb0b6da0bc805 100644
--- a/docs/reference/ml/df-analytics/apis/evaluate-dfanalytics.asciidoc
+++ b/docs/reference/ml/df-analytics/apis/evaluate-dfanalytics.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Evaluate {dfanalytics}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap}  {dfanalytics} APIs].
+--
+
 Evaluates the {dfanalytics} for an annotated index.
 
 
diff --git a/docs/reference/ml/df-analytics/apis/explain-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/explain-dfanalytics.asciidoc
index 0ee7ec5634582..566cfc3ba3596 100644
--- a/docs/reference/ml/df-analytics/apis/explain-dfanalytics.asciidoc
+++ b/docs/reference/ml/df-analytics/apis/explain-dfanalytics.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Explain {dfanalytics}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap}  {dfanalytics} APIs].
+--
+
 Explains a {dataframe-analytics-config}.
 
 
diff --git a/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc b/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc
index 9037819c9f236..960388d8d4e33 100644
--- a/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc
+++ b/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Get {dfanalytics-jobs} stats</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap}  {dfanalytics} APIs].
+--
+
 Retrieves usage information for {ml-docs}/ml-dfanalytics.html[{dfanalytics-jobs}].
 
 
diff --git a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc
index c2a4caa981da1..6e65bdfe83a25 100644
--- a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc
+++ b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get {dfanalytics-jobs}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap}  {dfanalytics} APIs].
+--
+
 Retrieves configuration information for {dfanalytics-jobs}.
 
 
diff --git a/docs/reference/ml/df-analytics/apis/ml-df-analytics-apis.asciidoc b/docs/reference/ml/df-analytics/apis/ml-df-analytics-apis.asciidoc
index ba16f728e4b9a..a90f42b7b9757 100644
--- a/docs/reference/ml/df-analytics/apis/ml-df-analytics-apis.asciidoc
+++ b/docs/reference/ml/df-analytics/apis/ml-df-analytics-apis.asciidoc
@@ -2,6 +2,12 @@
 [[ml-df-analytics-apis]]
 = {ml-cap} {dfanalytics} APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap}  {dfanalytics} APIs].
+--
+
 You can use the following APIs to perform {ml} {dfanalytics} activities:
 
 * <<put-dfanalytics,Create {dfanalytics-jobs}>>
diff --git a/docs/reference/ml/df-analytics/apis/preview-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/preview-dfanalytics.asciidoc
index 2c61c3263992c..896cf5ca8eb8a 100644
--- a/docs/reference/ml/df-analytics/apis/preview-dfanalytics.asciidoc
+++ b/docs/reference/ml/df-analytics/apis/preview-dfanalytics.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Preview {dfanalytics}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap}  {dfanalytics} APIs].
+--
+
 Previews the features used by a {dataframe-analytics-config}.
 
 
diff --git a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc
index 54cbe78b34452..8c2e95e0744ed 100644
--- a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc
+++ b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Create {dfanalytics-jobs}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap}  {dfanalytics} APIs].
+--
+
 Instantiates a {dfanalytics-job}.
 
 
diff --git a/docs/reference/ml/df-analytics/apis/start-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/start-dfanalytics.asciidoc
index 70e996ef8dd0a..7409c443ff802 100644
--- a/docs/reference/ml/df-analytics/apis/start-dfanalytics.asciidoc
+++ b/docs/reference/ml/df-analytics/apis/start-dfanalytics.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Start {dfanalytics-jobs}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap}  {dfanalytics} APIs].
+--
+
 Starts a {dfanalytics-job}.
 
 
diff --git a/docs/reference/ml/df-analytics/apis/stop-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/stop-dfanalytics.asciidoc
index 2fa3bc4413d7a..95d39b98e3bbd 100644
--- a/docs/reference/ml/df-analytics/apis/stop-dfanalytics.asciidoc
+++ b/docs/reference/ml/df-analytics/apis/stop-dfanalytics.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Stop {dfanalytics-jobs}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap}  {dfanalytics} APIs].
+--
+
 Stops one or more {dfanalytics-jobs}.
 
 
diff --git a/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc
index 49cca176be69b..35e871761b21a 100644
--- a/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc
+++ b/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Update {dfanalytics-jobs}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap}  {dfanalytics} APIs].
+--
+
 Updates an existing {dfanalytics-job}.
 
 
diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc
index 4948db48664ed..af384c2c90011 100644
--- a/docs/reference/ml/ml-shared.asciidoc
+++ b/docs/reference/ml/ml-shared.asciidoc
@@ -1167,6 +1167,12 @@ tag::inference-config-text-embedding-size[]
 The number of dimensions in the embedding vector produced by the model.
 end::inference-config-text-embedding-size[]
 
+tag::inference-config-text-expansion[]
+The text expansion task works with sparse embedding models to transform an input sequence
+into a vector of weighted tokens. These embeddings capture semantic meanings and
+context and can be used in a <<sparse-vector,sparse vector>> field for powerful insights.
+end::inference-config-text-expansion[]
+
 tag::inference-config-text-similarity[]
 Text similarity takes an input sequence and compares it with another input sequence. This is commonly referred to
 as cross-encoding. This task is useful for ranking document text when comparing it to another provided text input.
diff --git a/docs/reference/ml/trained-models/apis/clear-trained-model-deployment-cache.asciidoc b/docs/reference/ml/trained-models/apis/clear-trained-model-deployment-cache.asciidoc
index f24379705fc75..1e1639ab5eee2 100644
--- a/docs/reference/ml/trained-models/apis/clear-trained-model-deployment-cache.asciidoc
+++ b/docs/reference/ml/trained-models/apis/clear-trained-model-deployment-cache.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Clear trained model deployment cache</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs].
+--
+
 Clears the {infer} cache on all nodes where the deployment is assigned.
 
 [[clear-trained-model-deployment-cache-request]]
diff --git a/docs/reference/ml/trained-models/apis/delete-trained-models-aliases.asciidoc b/docs/reference/ml/trained-models/apis/delete-trained-models-aliases.asciidoc
index 1b67a642f7b7a..22e0422df9fb7 100644
--- a/docs/reference/ml/trained-models/apis/delete-trained-models-aliases.asciidoc
+++ b/docs/reference/ml/trained-models/apis/delete-trained-models-aliases.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Delete trained model aliases</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs].
+--
+
 Deletes a trained model alias.
 
 
diff --git a/docs/reference/ml/trained-models/apis/delete-trained-models.asciidoc b/docs/reference/ml/trained-models/apis/delete-trained-models.asciidoc
index 1b54343d1f1c9..d738c6630a78f 100644
--- a/docs/reference/ml/trained-models/apis/delete-trained-models.asciidoc
+++ b/docs/reference/ml/trained-models/apis/delete-trained-models.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Delete trained models</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs].
+--
+
 Deletes an existing trained {infer} model.
 
 
diff --git a/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc b/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc
index b55f022a5d168..82263c98e9112 100644
--- a/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc
+++ b/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Get trained models stats</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs].
+--
+
 Retrieves usage information for trained models.
 
 
diff --git a/docs/reference/ml/trained-models/apis/get-trained-models.asciidoc b/docs/reference/ml/trained-models/apis/get-trained-models.asciidoc
index ffb46eaa1fe2e..4f583319ca383 100644
--- a/docs/reference/ml/trained-models/apis/get-trained-models.asciidoc
+++ b/docs/reference/ml/trained-models/apis/get-trained-models.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Get trained models</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs].
+--
+
 Retrieves configuration information about {ml-docs}/ml-nlp-deploy-models.html[{infer} trained models].
 
 
diff --git a/docs/reference/ml/trained-models/apis/infer-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/infer-trained-model-deployment.asciidoc
index 83bc56d18df63..8bb48eeb5cbf2 100644
--- a/docs/reference/ml/trained-models/apis/infer-trained-model-deployment.asciidoc
+++ b/docs/reference/ml/trained-models/apis/infer-trained-model-deployment.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Infer trained model deployment</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs].
+--
+
 Evaluates a trained model.
 
 deprecated::[8.3.0,Replaced by <<infer-trained-model>>.]
diff --git a/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc b/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc
index 7acbc0bd23859..93b8aa3422abc 100644
--- a/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc
+++ b/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Infer trained model</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs].
+--
+
 Evaluates a trained model. The model may be any supervised model either trained
 by {dfanalytics} or imported.
 
diff --git a/docs/reference/ml/trained-models/apis/ml-trained-models-apis.asciidoc b/docs/reference/ml/trained-models/apis/ml-trained-models-apis.asciidoc
index 83ef3c49fb5ed..5b0fc2b83afa5 100644
--- a/docs/reference/ml/trained-models/apis/ml-trained-models-apis.asciidoc
+++ b/docs/reference/ml/trained-models/apis/ml-trained-models-apis.asciidoc
@@ -2,6 +2,12 @@
 [[ml-df-trained-models-apis]]
 = {ml-cap} trained model APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs].
+--
+
 You can use the following APIs to perform model management operations:
 
 * <<clear-trained-model-deployment-cache>>
diff --git a/docs/reference/ml/trained-models/apis/put-trained-model-definition-part.asciidoc b/docs/reference/ml/trained-models/apis/put-trained-model-definition-part.asciidoc
index d1da29abffcd3..857b86892a6b9 100644
--- a/docs/reference/ml/trained-models/apis/put-trained-model-definition-part.asciidoc
+++ b/docs/reference/ml/trained-models/apis/put-trained-model-definition-part.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Create part of a trained model</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs].
+--
+
 Creates part of a trained model definition.
 
 [[ml-put-trained-model-definition-part-request]]
diff --git a/docs/reference/ml/trained-models/apis/put-trained-model-vocabulary.asciidoc b/docs/reference/ml/trained-models/apis/put-trained-model-vocabulary.asciidoc
index 2fdf86259388f..39f93f882235a 100644
--- a/docs/reference/ml/trained-models/apis/put-trained-model-vocabulary.asciidoc
+++ b/docs/reference/ml/trained-models/apis/put-trained-model-vocabulary.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Create trained model vocabulary</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs].
+--
+
 Creates a trained model vocabulary.
 This is supported only for natural language processing (NLP) models.
 
diff --git a/docs/reference/ml/trained-models/apis/put-trained-models-aliases.asciidoc b/docs/reference/ml/trained-models/apis/put-trained-models-aliases.asciidoc
index d8c00efe28015..3fcc7f5bb2da4 100644
--- a/docs/reference/ml/trained-models/apis/put-trained-models-aliases.asciidoc
+++ b/docs/reference/ml/trained-models/apis/put-trained-models-aliases.asciidoc
@@ -6,6 +6,11 @@
 <titleabbrev>Create or update trained model aliases</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs].
+--
 
 Creates or updates a trained model alias.
 
diff --git a/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc b/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc
index 32265af5f795b..ccd76b7095762 100644
--- a/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc
+++ b/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc
@@ -3,11 +3,15 @@
 = Create trained models API
 [subs="attributes"]
 ++++
-
 <titleabbrev>Create trained models</titleabbrev>
-
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs].
+--
+
 Creates a trained model.
 
 WARNING: Models created in version 7.8.0 are not backwards compatible
@@ -391,10 +395,10 @@ the model definition is not supplied.
 (Required, object)
 The default configuration for inference. This can be: `regression`,
 `classification`, `fill_mask`, `ner`, `question_answering`,
-`text_classification`, `text_embedding` or `zero_shot_classification`.
+`text_classification`, `text_embedding`, `text_expansion` or `zero_shot_classification`.
 If `regression` or `classification`, it must match the `target_type` of the
 underlying `definition.trained_model`. If `fill_mask`, `ner`,
-`question_answering`, `text_classification`, or `text_embedding`; the
+`question_answering`, `text_classification`, `text_embedding` or `text_expansion`; the
 `model_type` must be `pytorch`.
 +
 .Properties of `inference_config`
@@ -588,6 +592,25 @@ Refer to <<tokenization-properties>> to review the properties of the
 `tokenization` object.
 =====
 
+`text_expansion`:::
+(Object, optional)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-expansion]
++
+.Properties of text_expansion inference
+[%collapsible%open]
+=====
+`results_field`::::
+(Optional, string)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field]
+
+`tokenization`::::
+(Optional, object)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization]
++
+Refer to <<tokenization-properties>> to review the properties of the
+`tokenization` object.
+=====
+
 `text_similarity`:::
 (Object, optional)
 include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity]
diff --git a/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc
index 6f7e2a4d9f988..7eaa149976652 100644
--- a/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc
+++ b/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Start trained model deployment</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs].
+--
+
 Starts a new trained model deployment.
 
 [[start-trained-model-deployment-request]]
@@ -138,8 +144,8 @@ normal priority deployments.
 Controls how many inference requests are allowed in the queue at a time.
 Every machine learning node in the cluster where the model can be allocated
 has a queue of this size; when the number of requests exceeds the total value,
-new requests are rejected with a 429 error. Defaults to 1024. Max allowed value
-is 1000000.
+new requests are rejected with a 429 error. Defaults to 10000. Max allowed value
+is 100000.
 
 `threads_per_allocation`::
 (Optional, integer)
@@ -173,7 +179,7 @@ The API returns the following results:
             "model_bytes": 265632637,
             "threads_per_allocation" : 1,
             "number_of_allocations" : 1,
-            "queue_capacity" : 1024,
+            "queue_capacity" : 10000,
             "priority": "normal"
         },
         "routing_table": {
@@ -229,4 +235,4 @@ POST _ml/trained_models/my_model/deployment/_start?deployment_id=my_model_for_se
   }
 }
 --------------------------------------------------
-// TEST[skip:TBD]
\ No newline at end of file
+// TEST[skip:TBD]
diff --git a/docs/reference/ml/trained-models/apis/stop-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/stop-trained-model-deployment.asciidoc
index 622b440622cd3..2f179e2391b78 100644
--- a/docs/reference/ml/trained-models/apis/stop-trained-model-deployment.asciidoc
+++ b/docs/reference/ml/trained-models/apis/stop-trained-model-deployment.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Stop trained model deployment</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs].
+--
+
 Stops a trained model deployment.
 
 [[stop-trained-model-deployment-request]]
diff --git a/docs/reference/ml/trained-models/apis/update-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/update-trained-model-deployment.asciidoc
index d49ee3c6e872c..a986e412f1a8a 100644
--- a/docs/reference/ml/trained-models/apis/update-trained-model-deployment.asciidoc
+++ b/docs/reference/ml/trained-models/apis/update-trained-model-deployment.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Update trained model deployment</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs].
+--
+
 Updates certain properties of a trained model deployment.
 
 [[update-trained-model-deployment-request]]
diff --git a/docs/reference/modules/cluster.asciidoc b/docs/reference/modules/cluster.asciidoc
index b3eaa5b47c238..cf8e97de5e188 100644
--- a/docs/reference/modules/cluster.asciidoc
+++ b/docs/reference/modules/cluster.asciidoc
@@ -27,7 +27,23 @@ include::cluster/shards_allocation.asciidoc[]
 
 include::cluster/disk_allocator.asciidoc[]
 
-include::cluster/allocation_awareness.asciidoc[]
+[[shard-allocation-awareness-settings]]
+==== Shard allocation awareness settings
+
+You can use <<custom-node-attributes,custom node attributes>> as _awareness attributes_ to enable {es}
+to take your physical hardware configuration into account when allocating shards.
+If {es} knows which nodes are on the same physical server, in the same rack, or
+in the same zone, it can distribute the primary shard and its replica shards to
+minimize the risk of losing all shard copies in the event of a failure. <<shard-allocation-awareness,Learn more about shard allocation awareness>>.
+
+`cluster.routing.allocation.awareness.attributes`::
+      (<<dynamic-cluster-setting,Dynamic>>)
+      The node attributes that {es} should use as awareness attributes. For example, if you have a `rack_id` attribute that specifies the rack in which each node resides, you can set this setting to `rack_id` to ensure that primary and replica shards are not allocated on the same rack. You can specify multiple attributes as a comma-separated list.
+
+`cluster.routing.allocation.awareness.force.*`:: 
+        (<<dynamic-cluster-setting,Dynamic>>)
+        The shard allocation awareness values that must exist for shards to be reallocated in case of location failure. Learn more about <<forced-awareness,forced awareness>>.
+      
 
 include::cluster/allocation_filtering.asciidoc[]
 
diff --git a/docs/reference/modules/cluster/allocation_awareness.asciidoc b/docs/reference/modules/cluster/allocation_awareness.asciidoc
index 9c6197f9ba40d..34164cd364fc5 100644
--- a/docs/reference/modules/cluster/allocation_awareness.asciidoc
+++ b/docs/reference/modules/cluster/allocation_awareness.asciidoc
@@ -1,5 +1,5 @@
 [[shard-allocation-awareness]]
-==== Shard allocation awareness
+== Shard allocation awareness
 
 You can use custom node attributes as _awareness attributes_ to enable {es}
 to take your physical hardware configuration into account when allocating shards.
@@ -7,12 +7,7 @@ If {es} knows which nodes are on the same physical server, in the same rack, or
 in the same zone, it can distribute the primary shard and its replica shards to
 minimize the risk of losing all shard copies in the event of a failure.
 
-When shard allocation awareness is enabled with the
-<<dynamic-cluster-setting,dynamic>>
-`cluster.routing.allocation.awareness.attributes` setting, shards are only
-allocated to nodes that have values set for the specified awareness attributes.
-If you use multiple awareness attributes, {es} considers each attribute
-separately when allocating shards.
+When shard allocation awareness is enabled with the `cluster.routing.allocation.awareness.attributes` setting, shards are only allocated to nodes that have values set for the specified awareness attributes. If you use multiple awareness attributes, {es} considers each attribute separately when allocating shards.
 
 NOTE: The number of attribute values determines how many shard copies are
 allocated in each location. If the number of nodes in each location is
@@ -22,11 +17,11 @@ unassigned.
 TIP: Learn more about <<high-availability-cluster-design-large-clusters,designing resilient clusters>>.
 
 [[enabling-awareness]]
-===== Enabling shard allocation awareness
+=== Enabling shard allocation awareness
 
 To enable shard allocation awareness:
 
-. Specify the location of each node with a custom node attribute. For example, 
+. Specify the location of each node with a <<custom-node-attributes,custom node attribute>>. For example, 
 if you want Elasticsearch to distribute shards across different racks, you might 
 use an awareness attribute called `rack_id`. 
 +
@@ -94,7 +89,7 @@ copies of a particular shard from being allocated in the same location, you can
 enable forced awareness.
 
 [[forced-awareness]]
-===== Forced awareness
+=== Forced awareness
 
 By default, if one location fails, {es} spreads its shards across the remaining
 locations. This might be undesirable if the cluster does not have sufficient
diff --git a/docs/reference/modules/cluster/allocation_filtering.asciidoc b/docs/reference/modules/cluster/allocation_filtering.asciidoc
index e70e43682973b..d0d2652059048 100644
--- a/docs/reference/modules/cluster/allocation_filtering.asciidoc
+++ b/docs/reference/modules/cluster/allocation_filtering.asciidoc
@@ -6,7 +6,7 @@ allocates shards from any index. These cluster wide filters are applied in
 conjunction with <<shard-allocation-filtering, per-index allocation filtering>>
 and <<shard-allocation-awareness, allocation awareness>>.
 
-Shard allocation filters can be based on custom node attributes or the built-in
+Shard allocation filters can be based on <<custom-node-attributes,custom node attributes>> or the built-in
 `_name`, `_host_ip`, `_publish_ip`, `_ip`, `_host`, `_id` and `_tier` attributes.
 
 The `cluster.routing.allocation` settings are <<dynamic-cluster-setting,dynamic>>, enabling live indices to
@@ -59,9 +59,9 @@ The cluster allocation settings support the following built-in attributes:
 
 NOTE: `_tier` filtering is based on <<modules-node, node>> roles. Only
 a subset of roles are <<data-tiers, data tier>> roles, and the generic
-<<data-node, data role>> will match any tier filtering.
+<<data-node-role, data role>> will match any tier filtering.
 a subset of roles that are <<data-tiers, data tier>> roles, but the generic
-<<data-node, data role>> will match any tier filtering.
+<<data-node-role, data role>> will match any tier filtering.
 
 
 You can use wildcards when specifying attribute values, for example:
diff --git a/docs/reference/modules/cluster/disk_allocator.asciidoc b/docs/reference/modules/cluster/disk_allocator.asciidoc
index 02cc48c6e27fc..8efe4c0132e86 100644
--- a/docs/reference/modules/cluster/disk_allocator.asciidoc
+++ b/docs/reference/modules/cluster/disk_allocator.asciidoc
@@ -41,6 +41,23 @@ on the affected node drops below the high watermark, {es} automatically removes
 the write block. Refer to <<fix-watermark-errors,Fix watermark errors>> to 
 resolve persistent watermark errors.
 
+[NOTE]
+.Max headroom settings
+===================================================
+
+Max headroom settings apply only when watermark settings are percentages or ratios. 
+
+A max headroom value is intended to cap the required free disk space before hitting
+the respective watermark. This is useful for servers with larger disks, where a percentage or ratio watermark could translate to an overly large free disk space requirement. In this case, the max headroom can be used to cap the required free disk space amount.
+
+For example, where `cluster.routing.allocation.disk.watermark.flood_stage` is 95% and `cluster.routing.allocation.disk.watermark.flood_stage.max_headroom` is 100GB, this means that:
+
+* For a smaller disk, e.g., of 100GB, the flood watermark will hit at 95%, meaning at 5GB of free space, since 5GB is smaller than the 100GB max headroom value.
+* For a larger disk, e.g., of 100TB, the flood watermark will hit at 100GB of free space. That is because the 95% flood watermark alone would require 5TB of free disk space, but is capped by the max headroom setting to 100GB.
+
+Max headroom settings have their default values only if their respective watermark settings are not explicitly set. If watermarks are explicitly set, then the max headroom settings do not have their default values, and need to be explicitly set if they are needed.
+===================================================
+
 [[disk-based-shard-allocation-does-not-balance]]
 [TIP]
 ====
@@ -100,18 +117,7 @@ is now `true`. The setting will be removed in a future release.
 +
 --
 (<<dynamic-cluster-setting,Dynamic>>)
-Controls the flood stage watermark, which defaults to 95%. {es} enforces a read-only index block (`index.blocks.read_only_allow_delete`) on every index that has one or more shards allocated on the node, and that has at least one disk exceeding the flood stage. This setting is a last resort to prevent nodes from running out of disk space. The index block is automatically released when the disk utilization falls below the high watermark. Similarly to the low and high watermark values, it can alternatively be set to a ratio value, e.g., `0.95`, or an absolute byte value.
-
-An example of resetting the read-only index block on the `my-index-000001` index:
-
-[source,console]
---------------------------------------------------
-PUT /my-index-000001/_settings
-{
-  "index.blocks.read_only_allow_delete": null
-}
---------------------------------------------------
-// TEST[setup:my_index]
+Controls the flood stage watermark, which defaults to 95%. {es} enforces a read-only index block (<<index-block-settings,`index.blocks.read_only_allow_delete`>>) on every index that has one or more shards allocated on the node, and that has at least one disk exceeding the flood stage. This setting is a last resort to prevent nodes from running out of disk space. The index block is automatically released when the disk utilization falls below the high watermark. Similarly to the low and high watermark values, it can alternatively be set to a ratio value, e.g., `0.95`, or an absolute byte value.
 --
 // end::cluster-routing-flood-stage-tag[]
 
@@ -121,10 +127,10 @@ Defaults to 100GB when
 `cluster.routing.allocation.disk.watermark.flood_stage` is not explicitly set.
 This caps the amount of free space required.
 
-NOTE: You cannot mix the usage of percentage/ratio values and byte values across
+NOTE: You can't mix the usage of percentage/ratio values and byte values across
 the `cluster.routing.allocation.disk.watermark.low`, `cluster.routing.allocation.disk.watermark.high`,
 and `cluster.routing.allocation.disk.watermark.flood_stage` settings. Either all values
-are set to percentage/ratio values, or all are set to byte values. This enforcement is
+must be set to percentage/ratio values, or all must be set to byte values. This is required
 so that {es} can validate that the settings are internally consistent, ensuring that the
 low disk threshold is less than the high disk threshold, and the high disk threshold is
 less than the flood stage threshold. A similar comparison check is done for the max
@@ -150,44 +156,6 @@ set. This caps the amount of free space required on dedicated frozen nodes.
     cluster. Defaults to `30s`.
 
 NOTE: Percentage values refer to used disk space, while byte values refer to
-free disk space. This can be confusing, since it flips the meaning of high and
+free disk space. This can be confusing, because it flips the meaning of high and
 low. For example, it makes sense to set the low watermark to 10gb and the high
-watermark to 5gb, but not the other way around.
-
-An example of updating the low watermark to at least 100 gigabytes free, a high
-watermark of at least 50 gigabytes free, and a flood stage watermark of 10
-gigabytes free, and updating the information about the cluster every minute:
-
-[source,console]
---------------------------------------------------
-PUT _cluster/settings
-{
-  "persistent": {
-    "cluster.routing.allocation.disk.watermark.low": "100gb",
-    "cluster.routing.allocation.disk.watermark.high": "50gb",
-    "cluster.routing.allocation.disk.watermark.flood_stage": "10gb",
-    "cluster.info.update.interval": "1m"
-  }
-}
---------------------------------------------------
-
-Concerning the max headroom settings for the watermarks, please note
-that these apply only in the case that the watermark settings are percentages/ratios.
-The aim of a max headroom value is to cap the required free disk space before hitting
-the respective watermark. This is especially useful for servers with larger
-disks, where a percentage/ratio watermark could translate to a big free disk space requirement,
-and the max headroom can be used to cap the required free disk space amount.
-As an example, let us take the default settings for the flood watermark.
-It has a 95% default value, and the flood max headroom setting has a default value of 100GB.
-This means that:
-
-* For a smaller disk, e.g., of 100GB, the flood watermark will hit at 95%, meaning at 5GB
-of free space, since 5GB is smaller than the 100GB max headroom value.
-* For a larger disk, e.g., of 100TB, the flood watermark will hit at 100GB of free space.
-That is because the 95% flood watermark alone would require 5TB of free disk space, but
-that is capped by the max headroom setting to 100GB.
-
-Finally, the max headroom settings have their default values only if their respective watermark
-settings are not explicitly set (thus, they have their default percentage values).
-If watermarks are explicitly set, then the max headroom settings do not have their default values,
-and would need to be explicitly set if they are desired.
+watermark to 5gb, but not the other way around.
\ No newline at end of file
diff --git a/docs/reference/modules/cluster/misc.asciidoc b/docs/reference/modules/cluster/misc.asciidoc
index 75eaca88c66b1..b66ac1fdb0cca 100644
--- a/docs/reference/modules/cluster/misc.asciidoc
+++ b/docs/reference/modules/cluster/misc.asciidoc
@@ -1,6 +1,9 @@
 [[misc-cluster-settings]]
 === Miscellaneous cluster settings
 
+[[cluster-name]]
+include::{es-ref-dir}/setup/important-settings/cluster-name.asciidoc[]
+
 [discrete]
 [[cluster-read-only]]
 ==== Metadata
diff --git a/docs/reference/modules/discovery/bootstrapping.asciidoc b/docs/reference/modules/discovery/bootstrapping.asciidoc
index 5120c1d17e69b..a885f1633ea49 100644
--- a/docs/reference/modules/discovery/bootstrapping.asciidoc
+++ b/docs/reference/modules/discovery/bootstrapping.asciidoc
@@ -2,7 +2,7 @@
 === Bootstrapping a cluster
 
 Starting an Elasticsearch cluster for the very first time requires the initial
-set of <<master-node,master-eligible nodes>> to be explicitly defined on one or
+set of <<master-node-role,master-eligible nodes>> to be explicitly defined on one or
 more of the master-eligible nodes in the cluster. This is known as _cluster
 bootstrapping_. This is only required the first time a cluster starts up.
 Freshly-started nodes that are joining a running cluster obtain this
diff --git a/docs/reference/modules/discovery/publishing.asciidoc b/docs/reference/modules/discovery/publishing.asciidoc
index af664585085c2..2bb031e6a9860 100644
--- a/docs/reference/modules/discovery/publishing.asciidoc
+++ b/docs/reference/modules/discovery/publishing.asciidoc
@@ -1,5 +1,23 @@
+[[cluster-state-overview]]
+=== Cluster state
+
+The _cluster state_ is an internal data structure which keeps track of a
+variety of information needed by every node, including:
+
+* The identity and attributes of the other nodes in the cluster
+
+* Cluster-wide settings
+
+* Index metadata, including the mapping and settings for each index
+
+* The location and status of every shard copy in the cluster
+
+The elected master node ensures that every node in the cluster has a copy of
+the same cluster state. The <<cluster-state,cluster state API>> lets you retrieve a
+representation of this internal state for debugging or diagnostic purposes.
+
 [[cluster-state-publishing]]
-=== Publishing the cluster state
+==== Publishing the cluster state
 
 The elected master node is the only node in a cluster that can make changes to
 the cluster state. The elected master node processes one batch of cluster state
@@ -58,3 +76,16 @@ speed of the storage on each master-eligible node, as well as the reliability
 and latency of the network interconnections between all nodes in the cluster.
 You must therefore ensure that the storage and networking available to the
 nodes in your cluster are good enough to meet your performance goals.
+
+[[dangling-index]]
+==== Dangling indices
+
+When a node joins the cluster, if it finds any shards stored in its local
+data directory that do not already exist in the cluster state, it will consider
+those shards to belong to a "dangling" index. You can list, import or
+delete dangling indices using the <<dangling-indices-api,Dangling indices
+API>>.
+
+NOTE: The API cannot offer any guarantees as to whether the imported data
+truly represents the latest state of the data when the index was still part
+of the cluster.
\ No newline at end of file
diff --git a/docs/reference/modules/discovery/voting.asciidoc b/docs/reference/modules/discovery/voting.asciidoc
index 04cae9d02ab66..f4bd4756d8978 100644
--- a/docs/reference/modules/discovery/voting.asciidoc
+++ b/docs/reference/modules/discovery/voting.asciidoc
@@ -2,7 +2,7 @@
 === Voting configurations
 
 Each {es} cluster has a _voting configuration_, which is the set of
-<<master-node,master-eligible nodes>> whose responses are counted when making
+<<master-node-role,master-eligible nodes>> whose responses are counted when making
 decisions such as electing a new master or committing a new cluster state.
 Decisions are made only after a majority (more than half) of the nodes in the
 voting configuration respond.
diff --git a/docs/reference/modules/gateway.asciidoc b/docs/reference/modules/gateway.asciidoc
index bf7e6de64f093..35a5ef6ddec00 100644
--- a/docs/reference/modules/gateway.asciidoc
+++ b/docs/reference/modules/gateway.asciidoc
@@ -1,10 +1,11 @@
 [[modules-gateway]]
 === Local gateway settings
 
+[[dangling-indices]]
 The local gateway stores the cluster state and shard data across full
 cluster restarts.
 
-The following _static_ settings, which must be set on every <<master-node,master-eligible node>>,
+The following _static_ settings, which must be set on every <<master-node-role,master-eligible node>>,
 control how long a freshly elected master should wait before it tries to
 recover the <<cluster-state,cluster state>> and the cluster's data.
 
@@ -36,17 +37,4 @@ These settings can be configured in `elasticsearch.yml` as follows:
 gateway.expected_data_nodes: 3
 gateway.recover_after_time: 600s
 gateway.recover_after_data_nodes: 3
---------------------------------------------------
-
-[[dangling-indices]]
-==== Dangling indices
-
-When a node joins the cluster, if it finds any shards stored in its local
-data directory that do not already exist in the cluster, it will consider
-those shards to belong to a "dangling" index. You can list, import or
-delete dangling indices using the <<dangling-indices-api,Dangling indices
-API>>.
-
-NOTE: The API cannot offer any guarantees as to whether the imported data
-truly represents the latest state of the data when the index was still part
-of the cluster.
+--------------------------------------------------
\ No newline at end of file
diff --git a/docs/reference/modules/indices/fielddata.asciidoc b/docs/reference/modules/indices/fielddata.asciidoc
index 1383bf74d6d4c..688685c0a2247 100644
--- a/docs/reference/modules/indices/fielddata.asciidoc
+++ b/docs/reference/modules/indices/fielddata.asciidoc
@@ -5,10 +5,6 @@ The field data cache contains <<fielddata-mapping-param, field data>> and <<eage
 which are both used to support aggregations on certain field types.
 Since these are on-heap data structures, it is important to monitor the cache's use.
 
-[discrete]
-[[fielddata-sizing]]
-==== Cache size
-
 The entries in the cache are expensive to build, so the default behavior is
 to keep the cache loaded in memory. The default cache size is unlimited,
 causing the cache to grow until it reaches the limit set by the <<fielddata-circuit-breaker, field data circuit breaker>>. This behavior can be configured.
@@ -20,16 +16,12 @@ at the cost of rebuilding the cache as needed.
 If the circuit breaker limit is reached, further requests that increase the cache
 size will be prevented. In this case you should manually <<indices-clearcache, clear the cache>>.
 
+TIP: You can monitor memory usage for field data as well as the field data circuit
+breaker using
+the <<cluster-nodes-stats,nodes stats API>> or the <<cat-fielddata,cat fielddata API>>.
+
 `indices.fielddata.cache.size`::
 (<<static-cluster-setting,Static>>)
 The max size of the field data cache, eg `38%` of node heap space, or an
 absolute value, eg `12GB`. Defaults to unbounded. If you choose to set it,
-it should be smaller than <<fielddata-circuit-breaker>> limit.
-
-[discrete]
-[[fielddata-monitoring]]
-==== Monitoring field data
-
-You can monitor memory usage for field data as well as the field data circuit
-breaker using
-the <<cluster-nodes-stats,nodes stats API>> or the <<cat-fielddata,cat fielddata API>>.
+it should be smaller than <<fielddata-circuit-breaker>> limit.
\ No newline at end of file
diff --git a/docs/reference/modules/indices/request_cache.asciidoc b/docs/reference/modules/indices/request_cache.asciidoc
index 4d4d349c685a1..f6ad65245836f 100644
--- a/docs/reference/modules/indices/request_cache.asciidoc
+++ b/docs/reference/modules/indices/request_cache.asciidoc
@@ -1,4 +1,4 @@
-[[shard-request-cache]]
+[[shard-request-cache-settings]]
 === Shard request cache settings
 
 When a search request is run against an index or against many indices, each
@@ -10,139 +10,16 @@ The shard-level request cache module caches the local results on each shard.
 This allows frequently used (and potentially heavy) search requests to return
 results almost instantly. The requests cache is a very good fit for the logging
 use case, where only the most recent index is being actively updated --
-results from older indices will be served directly from the cache.
+results from older indices will be served directly from the cache. You can use shard request cache settings to control the size and expiration of the cache.
 
-[IMPORTANT]
-===================================
-
-By default, the requests cache will only cache the results of search requests
-where `size=0`, so it will not cache `hits`,
-but it will cache `hits.total`,  <<search-aggregations,aggregations>>, and
-<<search-suggesters,suggestions>>.
-
-Most queries that use `now` (see <<date-math>>) cannot be cached.
-
-Scripted queries that use the API calls which are non-deterministic, such as
-`Math.random()` or `new Date()` are not cached.
-===================================
-
-[discrete]
-==== Cache invalidation
-
-The cache is smart -- it keeps the same _near real-time_ promise as uncached
-search.
-
-Cached results are invalidated automatically whenever the shard refreshes to
-pick up changes to the documents or when you update the mapping. In other
-words you will always get the same results from the cache as you would for an
-uncached search request.
-
-The longer the refresh interval, the longer that cached entries will remain
-valid even if there are changes to the documents. If the cache is full, the
-least recently used cache keys will be evicted.
-
-The cache can be expired manually with the <<indices-clearcache,`clear-cache` API>>:
-
-[source,console]
-------------------------
-POST /my-index-000001,my-index-000002/_cache/clear?request=true
-------------------------
-// TEST[s/^/PUT my-index-000001\nPUT my-index-000002\n/]
-
-[discrete]
-==== Enabling and disabling caching
-
-The cache is enabled by default, but can be disabled when creating a new
-index as follows:
-
-[source,console]
------------------------------
-PUT /my-index-000001
-{
-  "settings": {
-    "index.requests.cache.enable": false
-  }
-}
------------------------------
-
-It can also be enabled or disabled dynamically on an existing index with the
-<<indices-update-settings,`update-settings`>> API:
-
-[source,console]
------------------------------
-PUT /my-index-000001/_settings
-{ "index.requests.cache.enable": true }
------------------------------
-// TEST[continued]
-
-
-[discrete]
-==== Enabling and disabling caching per request
-
-The `request_cache` query-string parameter can be used to enable or disable
-caching on a *per-request* basis. If set, it overrides the index-level setting:
-
-[source,console]
------------------------------
-GET /my-index-000001/_search?request_cache=true
-{
-  "size": 0,
-  "aggs": {
-    "popular_colors": {
-      "terms": {
-        "field": "colors"
-      }
-    }
-  }
-}
------------------------------
-// TEST[continued]
-
-Requests where `size` is greater than 0 will not be cached even if the request cache is
-enabled in the index settings. To cache these requests you will need to use the
-query-string parameter detailed here.
-
-[discrete]
-==== Cache key
-
-A hash of the whole JSON body is used as the cache key. This means that if the JSON
-changes -- for instance if keys are output in a different order -- then the
-cache key will not be recognised.
-
-TIP: Most JSON libraries support a _canonical_ mode which ensures that JSON
-keys are always emitted in the same order. This canonical mode can be used in
-the application to ensure that a request is always serialized in the same way.
+To learn more about the shard request cache, see <<shard-request-cache>>.
 
 [discrete]
 ==== Cache settings
 
-The cache is managed at the node level, and has a default maximum size of `1%`
-of the heap. This can be changed in the `config/elasticsearch.yml` file with:
-
-[source,yaml]
---------------------------------
-indices.requests.cache.size: 2%
---------------------------------
-
-Also, you can use the +indices.requests.cache.expire+ setting to specify a TTL
-for cached results, but there should be no reason to do so. Remember that
-stale results are automatically invalidated when the index is refreshed. This
-setting is provided for completeness' sake only.
-
-[discrete]
-==== Monitoring cache usage
-
-The size of the cache (in bytes) and the number of evictions can be viewed
-by index, with the <<indices-stats,`indices-stats`>> API:
-
-[source,console]
-------------------------
-GET /_stats/request_cache?human
-------------------------
+`indices.requests.cache.size`::
+(<<static-cluster-setting,Static>>) The maximum size of the cache, as a percentage of the heap. Default: `1%`.
 
-or by node with the <<cluster-nodes-stats,`nodes-stats`>> API:
+`indices.requests.cache.expire`::
+(<<static-cluster-setting,Static>>) The TTL for cached results. Stale results are automatically invalidated when the index is refreshed, so you shouldn't need to use this setting.
 
-[source,console]
-------------------------
-GET /_nodes/stats/indices/request_cache?human
-------------------------
diff --git a/docs/reference/modules/network.asciidoc b/docs/reference/modules/network.asciidoc
index 1e4c5a21d386c..5095d73058d75 100644
--- a/docs/reference/modules/network.asciidoc
+++ b/docs/reference/modules/network.asciidoc
@@ -286,3 +286,22 @@ include::remote-cluster-network.asciidoc[]
 include::network/tracers.asciidoc[]
 
 include::network/threading.asciidoc[]
+
+[[tcp-readiness-port]]
+==== TCP readiness port
+
+preview::[]
+
+If configured, a node can open a TCP port when the node is in a ready state. A node is deemed
+ready when it has successfully joined a cluster. In a single node configuration, the node is
+said to be ready, when it's able to accept requests.
+
+To enable the readiness TCP port, use the `readiness.port` setting. The readiness service will bind to
+all host addresses.
+
+If the node leaves the cluster, or the <<put-shutdown,Shutdown API>> is used to mark the node
+for shutdown, the readiness port is immediately closed.
+
+A successful connection to the readiness TCP port signals that the {es} node is ready. When a client
+connects to the readiness port, the server simply terminates the socket connection. No data is sent back
+to the client. If a client cannot connect to the readiness port, the node is not ready.
\ No newline at end of file
diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc
index 022e8b5d1e2fe..f0cb3eecdf390 100644
--- a/docs/reference/modules/node.asciidoc
+++ b/docs/reference/modules/node.asciidoc
@@ -1,5 +1,5 @@
 [[modules-node]]
-=== Nodes
+=== Node settings
 
 Any time that you start an instance of {es}, you are starting a _node_. A
 collection of connected nodes is called a <<modules-cluster,cluster>>. If you
@@ -18,24 +18,35 @@ TIP: The performance of an {es} node is often limited by the performance of the
 Review our recommendations for optimizing your storage for <<indexing-use-faster-hardware,indexing>> and 
 <<search-use-faster-hardware,search>>.
 
+[[node-name-settings]]
+==== Node name setting
+
+include::{es-ref-dir}/setup/important-settings/node-name.asciidoc[]
+
 [[node-roles]]
-==== Node roles
+==== Node role settings
 
 You define a node's roles by setting `node.roles` in `elasticsearch.yml`. If you
 set `node.roles`, the node is only assigned the roles you specify. If you don't
 set `node.roles`, the node is assigned the following roles:
 
-* `master`
-* `data`
+* [[master-node]]`master`
+* [[data-node]]`data`
 * `data_content`
 * `data_hot`
 * `data_warm`
 * `data_cold`
 * `data_frozen`
 * `ingest`
-* `ml`
+* [[ml-node]]`ml`
 * `remote_cluster_client`
-* `transform`
+* [[transform-node]]`transform`
+
+The following additional roles are available:
+
+* `voting_only`
+
+[[coordinating-only-node]]If you leave `node.roles` unset, then the node is considered to be a <<coordinating-only-node-role,coordinating only node>>.
 
 [IMPORTANT]
 ====
@@ -65,386 +76,7 @@ As the cluster grows and in particular if you have large {ml} jobs or
 {ctransforms}, consider separating dedicated master-eligible nodes from
 dedicated data nodes, {ml} nodes, and {transform} nodes.
 
-<<master-node,Master-eligible node>>::
-
-A node that has the `master` role, which makes it eligible to be
-<<modules-discovery,elected as the _master_ node>>, which controls the cluster.
-
-<<data-node,Data node>>::
-
-A node that has one of several data roles. Data nodes hold data and perform data
-related operations such as CRUD, search, and aggregations. A node with a generic `data` role can fill any of the specialized data node roles.
-
-<<node-ingest-node,Ingest node>>::
-
-A node that has the `ingest` role. Ingest nodes are able to apply an
-<<ingest,ingest pipeline>> to a document in order to transform and enrich the
-document before indexing. With a heavy ingest load, it makes sense to use
-dedicated ingest nodes and to not include the `ingest` role from nodes that have
-the `master` or `data` roles.
-
-<<remote-node,Remote-eligible node>>::
-
-A node that has the `remote_cluster_client` role, which makes it eligible to act
-as a remote client.
-
-<<ml-node,Machine learning node>>::
-
-A node that has the `ml` role. If you want to use {ml-features}, there must be
-at least one {ml} node in your cluster. For more information, see
-<<ml-settings>> and {ml-docs}/index.html[Machine learning in the {stack}].
-
-<<transform-node,{transform-cap} node>>::
-
-A node that has the `transform` role. If you want to use {transforms}, there
-must be at least one {transform} node in your cluster. For more information, see
-<<transform-settings>> and <<transforms>>.
-
-[NOTE]
-[[coordinating-node]]
-.Coordinating node
-===============================================
-
-Requests like search requests or bulk-indexing requests may involve data held
-on different data nodes. A search request, for example, is executed in two
-phases which are coordinated by the node which receives the client request --
-the _coordinating node_.
-
-In the _scatter_ phase, the coordinating node forwards the request to the data
-nodes which hold the data. Each data node executes the request locally and
-returns its results to the coordinating node. In the _gather_ phase, the
-coordinating node reduces each data node's results into a single global
-result set.
-
-Every node is implicitly a coordinating node. This means that a node that has
-an explicit empty list of roles via `node.roles` will only act as a coordinating
-node, which cannot be disabled. As a result, such a node needs to have enough
-memory and CPU in order to deal with the gather phase.
-
-===============================================
-
-[[master-node]]
-==== Master-eligible node
-
-The master node is responsible for lightweight cluster-wide actions such as
-creating or deleting an index, tracking which nodes are part of the cluster,
-and deciding which shards to allocate to which nodes. It is important for
-cluster health to have a stable master node.
-
-Any master-eligible node that is not a <<voting-only-node,voting-only node>> may
-be elected to become the master node by the <<modules-discovery,master election
-process>>.
-
-IMPORTANT: Master nodes must have a `path.data` directory whose contents
-persist across restarts, just like data nodes, because this is where the
-cluster metadata is stored. The cluster metadata describes how to read the data
-stored on the data nodes, so if it is lost then the data stored on the data
-nodes cannot be read.
-
-[[dedicated-master-node]]
-===== Dedicated master-eligible node
-
-It is important for the health of the cluster that the elected master node has
-the resources it needs to fulfill its responsibilities. If the elected master
-node is overloaded with other tasks then the cluster will not operate well. The
-most reliable way to avoid overloading the master with other tasks is to
-configure all the master-eligible nodes to be _dedicated master-eligible nodes_
-which only have the `master` role, allowing them to focus on managing the
-cluster. Master-eligible nodes will still also behave as
-<<coordinating-node,coordinating nodes>> that route requests from clients to
-the other nodes in the cluster, but you should _not_ use dedicated master nodes
-for this purpose.
-
-A small or lightly-loaded cluster may operate well if its master-eligible nodes
-have other roles and responsibilities, but once your cluster comprises more
-than a handful of nodes it usually makes sense to use dedicated master-eligible
-nodes.
-
-To create a dedicated master-eligible node, set:
-
-[source,yaml]
--------------------
-node.roles: [ master ]
--------------------
-
-[[voting-only-node]]
-===== Voting-only master-eligible node
-
-A voting-only master-eligible node is a node that participates in
-<<modules-discovery,master elections>> but which will not act as the cluster's
-elected master node. In particular, a voting-only node can serve as a tiebreaker
-in elections.
-
-It may seem confusing to use the term "master-eligible" to describe a
-voting-only node since such a node is not actually eligible to become the master
-at all. This terminology is an unfortunate consequence of history:
-master-eligible nodes are those nodes that participate in elections and perform
-certain tasks during cluster state publications, and voting-only nodes have the
-same responsibilities even if they can never become the elected master.
-
-To configure a master-eligible node as a voting-only node, include `master` and
-`voting_only` in the list of roles. For example to create a voting-only data
-node:
-
-[source,yaml]
--------------------
-node.roles: [ data, master, voting_only ]
--------------------
-
-IMPORTANT: Only nodes with the `master` role can be marked as having the
-`voting_only` role.
-
-High availability (HA) clusters require at least three master-eligible nodes, at
-least two of which are not voting-only nodes. Such a cluster will be able to
-elect a master node even if one of the nodes fails.
-
-Voting-only master-eligible nodes may also fill other roles in your cluster.
-For instance, a node may be both a data node and a voting-only master-eligible
-node. A _dedicated_ voting-only master-eligible nodes is a voting-only
-master-eligible node that fills no other roles in the cluster. To create a
-dedicated voting-only master-eligible node, set:
-
-[source,yaml]
--------------------
-node.roles: [ master, voting_only ]
--------------------
-
-Since dedicated voting-only nodes never act as the cluster's elected master,
-they may require less heap and a less powerful CPU than the true master nodes.
-However all master-eligible nodes, including voting-only nodes, are on the
-critical path for <<cluster-state-publishing,publishing cluster state
-updates>>. Cluster state updates are usually independent of
-performance-critical workloads such as indexing or searches, but they are
-involved in management activities such as index creation and rollover, mapping
-updates, and recovery after a failure. The performance characteristics of these
-activities are a function of the speed of the storage on each master-eligible
-node, as well as the reliability and latency of the network interconnections
-between the elected master node and the other nodes in the cluster. You must
-therefore ensure that the storage and networking available to the nodes in your
-cluster are good enough to meet your performance goals.
-
-[[data-node]]
-==== Data nodes
-
-Data nodes hold the shards that contain the documents you have indexed. Data
-nodes handle data related operations like CRUD, search, and aggregations.
-These operations are I/O-, memory-, and CPU-intensive. It is important to
-monitor these resources and to add more data nodes if they are overloaded.
-
-The main benefit of having dedicated data nodes is the separation of the master
-and data roles.
-
-In a multi-tier deployment architecture, you use specialized data roles to
-assign data nodes to specific tiers: `data_content`,`data_hot`, `data_warm`,
-`data_cold`, or `data_frozen`. A node can belong to multiple tiers. 
-
-If you want to include a node in all tiers, or if your cluster does not use multiple tiers, then you can use the generic `data` role.
-
-include::../how-to/shard-limits.asciidoc[]
-
-WARNING: If you assign a node to a specific tier using a specialized data role, then you shouldn't also assign it the generic `data` role. The generic `data` role takes precedence over specialized data roles.
-
-[[generic-data-node]]
-===== Generic data node
-
-Generic data nodes are included in all content tiers. 
-
-To create a dedicated generic data node, set:
-[source,yaml]
-----
-node.roles: [ data ]
-----
-
-[[data-content-node]]
-===== Content data node
-
-Content data nodes are part of the content tier.
-include::{es-ref-dir}/datatiers.asciidoc[tag=content-tier]
-
-To create a dedicated content node, set:
-[source,yaml]
-----
-node.roles: [ data_content ]
-----
-
-[[data-hot-node]]
-===== Hot data node
-
-Hot data nodes are part of the hot tier.
-include::{es-ref-dir}/datatiers.asciidoc[tag=hot-tier]
-
-To create a dedicated hot node, set:
-[source,yaml]
-----
-node.roles: [ data_hot ]
-----
-
-[[data-warm-node]]
-===== Warm data node
-
-Warm data nodes are part of the warm tier.
-include::{es-ref-dir}/datatiers.asciidoc[tag=warm-tier]
-
-To create a dedicated warm node, set:
-[source,yaml]
-----
-node.roles: [ data_warm ]
-----
-
-[[data-cold-node]]
-===== Cold data node
-
-Cold data nodes are part of the cold tier.
-include::{es-ref-dir}/datatiers.asciidoc[tag=cold-tier]
-
-To create a dedicated cold node, set:
-[source,yaml]
-----
-node.roles: [ data_cold ]
-----
-
-[[data-frozen-node]]
-===== Frozen data node
-
-Frozen data nodes are part of the frozen tier.
-include::{es-ref-dir}/datatiers.asciidoc[tag=frozen-tier]
-
-To create a dedicated frozen node, set:
-[source,yaml]
-----
-node.roles: [ data_frozen ]
-----
-
-[[node-ingest-node]]
-==== Ingest node
-
-Ingest nodes can execute pre-processing pipelines, composed of one or more
-ingest processors. Depending on the type of operations performed by the ingest
-processors and the required resources, it may make sense to have dedicated
-ingest nodes, that will only perform this specific task.
-
-To create a dedicated ingest node, set:
-
-[source,yaml]
-----
-node.roles: [ ingest ]
-----
-
-[[coordinating-only-node]]
-==== Coordinating only node
-
-If you take away the ability to be able to handle master duties, to hold data,
-and pre-process documents, then you are left with a _coordinating_ node that
-can only route requests, handle the search reduce phase, and distribute bulk
-indexing. Essentially, coordinating only nodes behave as smart load balancers.
-
-Coordinating only nodes can benefit large clusters by offloading the
-coordinating node role from data and master-eligible nodes. They join the
-cluster and receive the full <<cluster-state,cluster state>>, like every other
-node, and they use the cluster state to route requests directly to the
-appropriate place(s).
-
-WARNING: Adding too many coordinating only nodes to a cluster can increase the
-burden on the entire cluster because the elected master node must await
-acknowledgement of cluster state updates from every node! The benefit of
-coordinating only nodes should not be overstated -- data nodes can happily
-serve the same purpose.
-
-To create a dedicated coordinating node, set:
-
-[source,yaml]
-----
-node.roles: [ ]
-----
-
-[[remote-node]]
-==== Remote-eligible node
-
-A remote-eligible node acts as a cross-cluster client and connects to
-<<remote-clusters,remote clusters>>. Once connected, you can search
-remote clusters using <<modules-cross-cluster-search,{ccs}>>. You can also sync
-data between clusters using <<xpack-ccr,{ccr}>>.
-
-[source,yaml]
-----
-node.roles: [ remote_cluster_client ]
-----
-
-[[ml-node]]
-==== [xpack]#Machine learning node#
-
-{ml-cap} nodes run jobs and handle {ml} API requests. For more information, see
-<<ml-settings>>.
-
-To create a dedicated {ml} node, set:
-
-[source,yaml]
-----
-node.roles: [ ml, remote_cluster_client]
-----
-
-The `remote_cluster_client` role is optional but strongly recommended.
-Otherwise, {ccs} fails when used in {ml} jobs or {dfeeds}. If you use {ccs} in
-your {anomaly-jobs}, the `remote_cluster_client` role is also required on all
-master-eligible nodes. Otherwise, the {dfeed} cannot start. See <<remote-node>>.
-
-[[transform-node]]
-==== [xpack]#{transform-cap} node#
-
-{transform-cap} nodes run {transforms} and handle {transform} API requests. For
-more information, see <<transform-settings>>.
-
-To create a dedicated {transform} node, set:
-
-[source,yaml]
-----
-node.roles: [ transform, remote_cluster_client ]
-----
-
-The `remote_cluster_client` role is optional but strongly recommended.
-Otherwise, {ccs} fails when used in {transforms}. See <<remote-node>>.
-
-[[change-node-role]]
-==== Changing the role of a node
-
-Each data node maintains the following data on disk:
-
-* the shard data for every shard allocated to that node,
-* the index metadata corresponding with every shard allocated to that node, and
-* the cluster-wide metadata, such as settings and index templates.
-
-Similarly, each master-eligible node maintains the following data on disk:
-
-* the index metadata for every index in the cluster, and
-* the cluster-wide metadata, such as settings and index templates.
-
-Each node checks the contents of its data path at startup. If it discovers
-unexpected data then it will refuse to start. This is to avoid importing
-unwanted <<dangling-indices,dangling indices>> which can lead
-to a red cluster health. To be more precise, nodes without the `data` role will
-refuse to start if they find any shard data on disk at startup, and nodes
-without both the `master` and `data` roles will refuse to start if they have any
-index metadata on disk at startup.
-
-It is possible to change the roles of a node by adjusting its
-`elasticsearch.yml` file and restarting it. This is known as _repurposing_ a
-node. In order to satisfy the checks for unexpected data described above, you
-must perform some extra steps to prepare a node for repurposing when starting
-the node without the `data` or `master` roles.
-
-* If you want to repurpose a data node by removing the `data` role then you
-  should first use an <<cluster-shard-allocation-filtering,allocation filter>> to safely
-  migrate all the shard data onto other nodes in the cluster.
-
-* If you want to repurpose a node to have neither the `data` nor `master` roles
-  then it is simplest to start a brand-new node with an empty data path and the
-  desired roles. You may find it safest to use an
-  <<cluster-shard-allocation-filtering,allocation filter>> to migrate the shard data elsewhere
-  in the cluster first.
-
-If it is not possible to follow these extra steps then you may be able to use
-the <<node-tool-repurpose,`elasticsearch-node repurpose`>> tool to delete any
-excess data that prevents a node from starting.
+To learn more about the available node roles, see <<node-roles-overview>>.
 
 [discrete]
 === Node data path settings
@@ -495,6 +127,25 @@ modify the contents of the data directory. The data directory contains no
 executables so a virus scan will only find false positives.
 // end::modules-node-data-path-warning-tag[]
 
+[[custom-node-attributes]]
+==== Custom node attributes
+
+If needed, you can add custom attributes to a node. These attributes can be used to <<cluster-routing-settings,filter which nodes a shard can be allocated to>>, or to group nodes together for <<shard-allocation-awareness,shard allocation awareness>>.
+
+[TIP]
+===============================================
+You can also set a node attribute using the `-E` command line argument when you start a node:
+
+[source,sh]
+--------------------------------------------------------
+./bin/elasticsearch -Enode.attr.rack_id=rack_one
+--------------------------------------------------------
+===============================================
+
+`node.attr.<attribute-name>`::
+      (<<dynamic-cluster-setting,Dynamic>>)
+      A custom attribute that you can assign to a node. For example, you might assign a `rack_id` attribute to each node to ensure that primary and replica shards are not allocated on the same rack. You can specify multiple attributes as a comma-separated list.
+
 [discrete]
 [[other-node-settings]]
 === Other node settings
@@ -504,4 +155,4 @@ including:
 
 * <<cluster-name,`cluster.name`>>
 * <<node-name,`node.name`>>
-* <<modules-network,network settings>>
+* <<modules-network,network settings>>
\ No newline at end of file
diff --git a/docs/reference/modules/remote-clusters.asciidoc b/docs/reference/modules/remote-clusters.asciidoc
index ca1c507aa4ed9..87078c0f1956f 100644
--- a/docs/reference/modules/remote-clusters.asciidoc
+++ b/docs/reference/modules/remote-clusters.asciidoc
@@ -80,7 +80,7 @@ The _gateway nodes_ selection depends on the following criteria:
 +
 * *version*: Remote nodes must be compatible with the cluster they are
 registered to.
-* *role*: By default, any non-<<master-node,master-eligible>> node can act as a
+* *role*: By default, any non-<<master-node-role,master-eligible>> node can act as a
 gateway node. Dedicated master nodes are never selected as gateway nodes.
 * *attributes*: You can define the gateway nodes for a cluster by setting
 <<cluster-remote-node-attr,`cluster.remote.node.attr.gateway`>> to `true`.
diff --git a/docs/reference/modules/shard-ops.asciidoc b/docs/reference/modules/shard-ops.asciidoc
index c0e5ee6a220f0..93d6b6d3468f8 100644
--- a/docs/reference/modules/shard-ops.asciidoc
+++ b/docs/reference/modules/shard-ops.asciidoc
@@ -1,5 +1,5 @@
 [[shard-allocation-relocation-recovery]]
-=== Shard allocation, relocation, and recovery
+== Shard allocation, relocation, and recovery
 
 Each <<documents-indices,index>> in Elasticsearch is divided into one or more <<scalability,shards>>. 
 Each document in an index belongs to a single shard.
@@ -12,22 +12,25 @@ Over the course of normal operation, Elasticsearch allocates shard copies to nod
 
 TIP: To learn about optimizing the number and size of shards in your cluster, refer to <<size-your-shards,Size your shards>>. To learn about how read and write operations are replicated across shards and shard copies, refer to <<docs-replication,Reading and writing documents>>.
 
+[discrete]
 [[shard-allocation]]
-==== Shard allocation
+=== Shard allocation
 
 include::{es-ref-dir}/modules/shard-allocation-desc.asciidoc[]
 
 By default, the primary and replica shard copies for an index can be allocated to any node in the cluster, and may be relocated to rebalance the cluster. 
 
-===== Adjust shard allocation settings
+[discrete]
+==== Adjust shard allocation settings
 
 You can control how shard copies are allocated using the following settings:
 
-- <<modules-cluster,Cluster-level shard allocation settings>>: Use these settings to control how shard copies are allocated and balanced across the entire cluster. For example, you might want to allocate nodes availability zones, or prevent certain nodes from being used so you can perform maintenance.
+- <<modules-cluster,Cluster-level shard allocation settings>>: Use these settings to control how shard copies are allocated and balanced across the entire cluster. For example, you might want to <<shard-allocation-awareness,allocate nodes availability zones>>, or prevent certain nodes from being used so you can perform maintenance.
 
 - <<index-modules-allocation,Index-level shard allocation settings>>: Use these settings to control how the shard copies for a specific index are allocated. For example, you might want to allocate an index to a node in a specific data tier, or to an node with specific attributes.
 
-===== Monitor shard allocation
+[discrete]
+==== Monitor shard allocation
 
 If a shard copy is unassigned, it means that the shard copy is not allocated to any node in the cluster. This can happen if there are not enough nodes in the cluster to allocate the shard copy, or if the shard copy can't be allocated to any node that satisfies the shard allocation filtering rules. When a shard copy is unassigned, your cluster is considered unhealthy and returns a yellow or red cluster health status.
 
@@ -39,12 +42,14 @@ You can use the following APIs to monitor shard allocation:
 
 <<red-yellow-cluster-status,Learn more about troubleshooting unassigned shard copies and recovering your cluster health>>.
 
+[discrete]
 [[shard-recovery]]
-==== Shard recovery
+=== Shard recovery
 
 include::{es-ref-dir}/modules/shard-recovery-desc.asciidoc[]
 
-===== Adjust shard recovery settings
+[discrete]
+==== Adjust shard recovery settings
 
 To control how shards are recovered, for example the resources that can be used by recovery operations, and which indices should be prioritized for recovery, you can adjust the following settings: 
 
@@ -54,22 +59,25 @@ To control how shards are recovered, for example the resources that can be used
 
 Shard recovery operations also respect general shard allocation settings. 
 
-===== Monitor shard recovery
+[discrete]
+==== Monitor shard recovery
 
 You can use the following APIs to monitor shard allocation:
 
  - View a list of in-progress and completed recoveries using the <<cat-recovery,cat recovery API>>
  - View detailed information about a specific recovery using the <<indices-recovery,index recovery API>>
 
+[discrete]
 [[shard-relocation]]
-==== Shard relocation
+=== Shard relocation
 
 Shard relocation is the process of moving shard copies from one node to another. This can happen when a node joins or leaves the cluster, or when the cluster is rebalancing.
 
 When a shard copy is relocated, it is created as a new shard copy on the target node. When the shard copy is fully allocated and recovered, the old shard copy is deleted. If the shard copy being relocated is a primary, then the new shard copy is marked as primary before the old shard copy is deleted.
 
-===== Adjust shard relocation settings
+[discrete]
+==== Adjust shard relocation settings
 
 You can control how and when shard copies are relocated. For example, you can adjust the rebalancing settings that control when shard copies are relocated to balance the cluster, or the high watermark for disk-based shard allocation that can trigger relocation. These settings are part of the <<modules-cluster,cluster-level shard allocation settings>>.
 
-Shard relocation operations also respect shard allocation and recovery settings. 
\ No newline at end of file
+Shard relocation operations also respect shard allocation and recovery settings.
\ No newline at end of file
diff --git a/docs/reference/monitoring/index.asciidoc b/docs/reference/monitoring/index.asciidoc
index 1b83f4c11ba54..82e1447ba8a1f 100644
--- a/docs/reference/monitoring/index.asciidoc
+++ b/docs/reference/monitoring/index.asciidoc
@@ -9,6 +9,7 @@ performance of your {es} cluster.
 
 * <<monitoring-overview>>
 * <<how-monitoring-works>>
+* <<logging>>
 * <<monitoring-production>>
 * <<configuring-elastic-agent>>
 * <<configuring-metricbeat>>
@@ -23,6 +24,8 @@ include::overview.asciidoc[]
 
 include::how-monitoring-works.asciidoc[]
 
+include::{es-ref-dir}/setup/logging-config.asciidoc[]
+
 include::production.asciidoc[]
 
 include::configuring-elastic-agent.asciidoc[]
diff --git a/docs/reference/monitoring/overview.asciidoc b/docs/reference/monitoring/overview.asciidoc
index d2a0130f71bb0..5c5d016f45ea0 100644
--- a/docs/reference/monitoring/overview.asciidoc
+++ b/docs/reference/monitoring/overview.asciidoc
@@ -13,6 +13,10 @@ All of the monitoring metrics are stored in {es}, which enables you to easily
 visualize the data in {kib}. By default, the monitoring metrics are stored in
 local indices.
 
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
+
 TIP: In production, we strongly recommend using a separate monitoring cluster.
 Using a separate monitoring cluster prevents production cluster outages from
 impacting your ability to access your monitoring data. It also prevents
diff --git a/docs/reference/node-roles.asciidoc b/docs/reference/node-roles.asciidoc
new file mode 100644
index 0000000000000..296c76e6dba9b
--- /dev/null
+++ b/docs/reference/node-roles.asciidoc
@@ -0,0 +1,437 @@
+[[node-roles-overview]]
+== Node roles
+
+Any time that you start an instance of {es}, you are starting a _node_. A
+collection of connected nodes is called a <<modules-cluster,cluster>>. If you
+are running a single node of {es}, then you have a cluster of one node. All nodes know about all the other nodes in the cluster and can forward client
+requests to the appropriate node.
+
+Each node performs one or more roles. Roles control the behavior of the node in the cluster.
+
+[discrete]
+[[set-node-roles]]
+=== Set node roles
+
+You define a node's roles by setting `node.roles` in <<settings,`elasticsearch.yml`>>. If you set `node.roles`, the node is only assigned the roles you specify. If you don't set `node.roles`, the node is assigned the following roles:
+
+* `master`
+* `data`
+* `data_content`
+* `data_hot`
+* `data_warm`
+* `data_cold`
+* `data_frozen`
+* `ingest`
+* `ml`
+* `remote_cluster_client`
+* `transform`
+
+[IMPORTANT]
+====
+If you set `node.roles`, ensure you specify every node role your cluster needs.
+Every cluster requires the following node roles:
+
+* `master`
+* {blank}
++
+--
+`data_content` and `data_hot` +
+OR +
+`data`
+--
+
+Some {stack} features also require specific node roles:
+
+- {ccs-cap} and {ccr} require the `remote_cluster_client` role.
+- {stack-monitor-app} and ingest pipelines require the `ingest` role.
+- {fleet}, the {security-app}, and {transforms} require the `transform` role.
+  The `remote_cluster_client` role is also required to use {ccs} with these
+  features.
+- {ml-cap} features, such as {anomaly-detect}, require the `ml` role.
+====
+
+As the cluster grows and in particular if you have large {ml} jobs or
+{ctransforms}, consider separating dedicated master-eligible nodes from
+dedicated data nodes, {ml} nodes, and {transform} nodes.
+
+[discrete]
+[[change-node-role]]
+=== Change the role of a node
+
+Each data node maintains the following data on disk:
+
+* the shard data for every shard allocated to that node,
+* the index metadata corresponding with every shard allocated to that node, and
+* the cluster-wide metadata, such as settings and index templates.
+
+Similarly, each master-eligible node maintains the following data on disk:
+
+* the index metadata for every index in the cluster, and
+* the cluster-wide metadata, such as settings and index templates.
+
+Each node checks the contents of its data path at startup. If it discovers
+unexpected data then it will refuse to start. This is to avoid importing
+unwanted <<dangling-indices,dangling indices>> which can lead
+to a red cluster health. To be more precise, nodes without the `data` role will
+refuse to start if they find any shard data on disk at startup, and nodes
+without both the `master` and `data` roles will refuse to start if they have any
+index metadata on disk at startup.
+
+It is possible to change the roles of a node by adjusting its
+`elasticsearch.yml` file and restarting it. This is known as _repurposing_ a
+node. In order to satisfy the checks for unexpected data described above, you
+must perform some extra steps to prepare a node for repurposing when starting
+the node without the `data` or `master` roles.
+
+* If you want to repurpose a data node by removing the `data` role then you
+  should first use an <<cluster-shard-allocation-filtering,allocation filter>> to safely
+  migrate all the shard data onto other nodes in the cluster.
+
+* If you want to repurpose a node to have neither the `data` nor `master` roles
+  then it is simplest to start a brand-new node with an empty data path and the
+  desired roles. You may find it safest to use an
+  <<cluster-shard-allocation-filtering,allocation filter>> to migrate the shard data elsewhere
+  in the cluster first.
+
+If it is not possible to follow these extra steps then you may be able to use
+the <<node-tool-repurpose,`elasticsearch-node repurpose`>> tool to delete any
+excess data that prevents a node from starting.
+
+[discrete]
+[[node-roles-list]]
+=== Available node roles
+
+The following is a list of the roles that a node can perform in a cluster. A node can have one or more roles.
+
+* <<master-node-role,Master-eligible node>> (`master`): A node that is eligible to be
+<<modules-discovery,elected as the _master_ node>>, which controls the cluster.
+
+* <<data-node-role,Data node>> (`data`, `data_content`, `data_hot`, `data_warm`, `data_cold`, `data_frozen`): A node that has one of several data roles. Data nodes hold data and perform data related operations such as CRUD, search, and aggregations. You might use multiple data roles in a cluster so you can implement <<data-tiers,data tiers>>.
+
+* <<node-ingest-node,Ingest node>> (`ingest`): Ingest nodes are able to apply an <<ingest,ingest pipeline>> to a document in order to transform and enrich the document before indexing. With a heavy ingest load, it makes sense to use dedicated ingest nodes and to not include the `ingest` role from nodes that have the `master` or `data` roles.
+
+* <<remote-node,Remote-eligible node>> (`remote_cluster_client`): A node that is eligible to act as a remote client.
+
+* <<ml-node-role,Machine learning node>> (`ml`): A node that can run {ml-features}. If you want to use {ml-features}, there must be at least one {ml} node in your cluster. For more information, see <<ml-settings>> and {ml-docs}/index.html[Machine learning in the {stack}].
+
+* <<transform-node-role,{transform-cap} node>> (`transform`): A node that can perform {transforms}. If you want to use {transforms}, there must be at least one {transform} node in your cluster. For more information, see <<transform-settings>> and <<transforms>>.
+
+[NOTE]
+[[coordinating-node]]
+.Coordinating node
+===============================================
+
+Requests like search requests or bulk-indexing requests may involve data held
+on different data nodes. A search request, for example, is executed in two
+phases which are coordinated by the node which receives the client request --
+the _coordinating node_.
+
+In the _scatter_ phase, the coordinating node forwards the request to the data
+nodes which hold the data. Each data node executes the request locally and
+returns its results to the coordinating node. In the _gather_ phase, the
+coordinating node reduces each data node's results into a single global
+result set.
+
+Every node is implicitly a coordinating node. This means that a node that has
+an explicit empty list of roles in the `node.roles` setting will only act as a coordinating
+node, which cannot be disabled. As a result, such a node needs to have enough
+memory and CPU in order to deal with the gather phase.
+
+===============================================
+
+[discrete]
+
+[[master-node-role]]
+==== Master-eligible node
+
+The master node is responsible for lightweight cluster-wide actions such as
+creating or deleting an index, tracking which nodes are part of the cluster,
+and deciding which shards to allocate to which nodes. It is important for
+cluster health to have a stable master node.
+
+Any master-eligible node that is not a <<voting-only-node,voting-only node>> may
+be elected to become the master node by the <<modules-discovery,master election
+process>>.
+
+IMPORTANT: Master nodes must have a `path.data` directory whose contents
+persist across restarts, just like data nodes, because this is where the
+cluster metadata is stored. The cluster metadata describes how to read the data
+stored on the data nodes, so if it is lost then the data stored on the data
+nodes cannot be read.
+
+[discrete]
+[[dedicated-master-node]]
+===== Dedicated master-eligible node
+
+It is important for the health of the cluster that the elected master node has
+the resources it needs to fulfill its responsibilities. If the elected master
+node is overloaded with other tasks then the cluster will not operate well. The
+most reliable way to avoid overloading the master with other tasks is to
+configure all the master-eligible nodes to be _dedicated master-eligible nodes_
+which only have the `master` role, allowing them to focus on managing the
+cluster. Master-eligible nodes will still also behave as
+<<coordinating-node,coordinating nodes>> that route requests from clients to
+the other nodes in the cluster, but you should _not_ use dedicated master nodes
+for this purpose.
+
+A small or lightly-loaded cluster may operate well if its master-eligible nodes
+have other roles and responsibilities, but once your cluster comprises more
+than a handful of nodes it usually makes sense to use dedicated master-eligible
+nodes.
+
+To create a dedicated master-eligible node, set:
+
+[source,yaml]
+-------------------
+node.roles: [ master ]
+-------------------
+
+[discrete]
+[[voting-only-node]]
+===== Voting-only master-eligible node
+
+A voting-only master-eligible node is a node that participates in
+<<modules-discovery,master elections>> but which will not act as the cluster's
+elected master node. In particular, a voting-only node can serve as a tiebreaker
+in elections.
+
+It may seem confusing to use the term "master-eligible" to describe a
+voting-only node since such a node is not actually eligible to become the master
+at all. This terminology is an unfortunate consequence of history:
+master-eligible nodes are those nodes that participate in elections and perform
+certain tasks during cluster state publications, and voting-only nodes have the
+same responsibilities even if they can never become the elected master.
+
+To configure a master-eligible node as a voting-only node, include `master` and
+`voting_only` in the list of roles. For example to create a voting-only data
+node:
+
+[source,yaml]
+-------------------
+node.roles: [ data, master, voting_only ]
+-------------------
+
+IMPORTANT: Only nodes with the `master` role can be marked as having the
+`voting_only` role.
+
+High availability (HA) clusters require at least three master-eligible nodes, at
+least two of which are not voting-only nodes. Such a cluster will be able to
+elect a master node even if one of the nodes fails.
+
+Voting-only master-eligible nodes may also fill other roles in your cluster.
+For instance, a node may be both a data node and a voting-only master-eligible
+node. A _dedicated_ voting-only master-eligible nodes is a voting-only
+master-eligible node that fills no other roles in the cluster. To create a
+dedicated voting-only master-eligible node, set:
+
+[source,yaml]
+-------------------
+node.roles: [ master, voting_only ]
+-------------------
+
+Since dedicated voting-only nodes never act as the cluster's elected master,
+they may require less heap and a less powerful CPU than the true master nodes.
+However all master-eligible nodes, including voting-only nodes, are on the
+critical path for <<cluster-state-publishing,publishing cluster state
+updates>>. Cluster state updates are usually independent of
+performance-critical workloads such as indexing or searches, but they are
+involved in management activities such as index creation and rollover, mapping
+updates, and recovery after a failure. The performance characteristics of these
+activities are a function of the speed of the storage on each master-eligible
+node, as well as the reliability and latency of the network interconnections
+between the elected master node and the other nodes in the cluster. You must
+therefore ensure that the storage and networking available to the nodes in your
+cluster are good enough to meet your performance goals.
+
+[discrete]
+[[data-node-role]]
+==== Data nodes
+
+Data nodes hold the shards that contain the documents you have indexed. Data
+nodes handle data related operations like CRUD, search, and aggregations.
+These operations are I/O-, memory-, and CPU-intensive. It is important to
+monitor these resources and to add more data nodes if they are overloaded.
+
+The main benefit of having dedicated data nodes is the separation of the master
+and data roles.
+
+In a multi-tier deployment architecture, you use specialized data roles to
+assign data nodes to specific tiers: `data_content`,`data_hot`, `data_warm`,
+`data_cold`, or `data_frozen`. A node can belong to multiple tiers. 
+
+If you want to include a node in all tiers, or if your cluster does not use multiple tiers, then you can use the generic `data` role.
+
+include::{es-ref-dir}/how-to/shard-limits.asciidoc[]
+
+WARNING: If you assign a node to a specific tier using a specialized data role, then you shouldn't also assign it the generic `data` role. The generic `data` role takes precedence over specialized data roles.
+
+[discrete]
+[[generic-data-node]]
+===== Generic data node
+
+Generic data nodes are included in all content tiers. A node with a generic `data` role can fill any of the specialized data node roles.
+
+To create a dedicated generic data node, set:
+[source,yaml]
+----
+node.roles: [ data ]
+----
+
+[discrete]
+[[data-content-node]]
+===== Content data node
+
+Content data nodes are part of the content tier.
+include::{es-ref-dir}/datatiers.asciidoc[tag=content-tier]
+
+To create a dedicated content node, set:
+[source,yaml]
+----
+node.roles: [ data_content ]
+----
+
+[discrete]
+[[data-hot-node]]
+===== Hot data node
+
+Hot data nodes are part of the hot tier.
+include::{es-ref-dir}/datatiers.asciidoc[tag=hot-tier]
+
+To create a dedicated hot node, set:
+[source,yaml]
+----
+node.roles: [ data_hot ]
+----
+
+[discrete]
+[[data-warm-node]]
+===== Warm data node
+
+Warm data nodes are part of the warm tier.
+include::{es-ref-dir}/datatiers.asciidoc[tag=warm-tier]
+
+To create a dedicated warm node, set:
+[source,yaml]
+----
+node.roles: [ data_warm ]
+----
+
+[discrete]
+[[data-cold-node]]
+===== Cold data node
+
+Cold data nodes are part of the cold tier.
+include::{es-ref-dir}/datatiers.asciidoc[tag=cold-tier]
+
+To create a dedicated cold node, set:
+[source,yaml]
+----
+node.roles: [ data_cold ]
+----
+
+[discrete]
+[[data-frozen-node]]
+===== Frozen data node
+
+Frozen data nodes are part of the frozen tier.
+include::{es-ref-dir}/datatiers.asciidoc[tag=frozen-tier]
+
+To create a dedicated frozen node, set:
+[source,yaml]
+----
+node.roles: [ data_frozen ]
+----
+
+[discrete]
+[[node-ingest-node]]
+==== Ingest node
+
+Ingest nodes can execute pre-processing pipelines, composed of one or more
+ingest processors. Depending on the type of operations performed by the ingest
+processors and the required resources, it may make sense to have dedicated
+ingest nodes, that will only perform this specific task.
+
+To create a dedicated ingest node, set:
+
+[source,yaml]
+----
+node.roles: [ ingest ]
+----
+
+[discrete]
+[[coordinating-only-node-role]]
+==== Coordinating only node
+
+If you take away the ability to be able to handle master duties, to hold data,
+and pre-process documents, then you are left with a _coordinating_ node that
+can only route requests, handle the search reduce phase, and distribute bulk
+indexing. Essentially, coordinating only nodes behave as smart load balancers.
+
+Coordinating only nodes can benefit large clusters by offloading the
+coordinating node role from data and master-eligible nodes. They join the
+cluster and receive the full <<cluster-state,cluster state>>, like every other
+node, and they use the cluster state to route requests directly to the
+appropriate place(s).
+
+WARNING: Adding too many coordinating only nodes to a cluster can increase the
+burden on the entire cluster because the elected master node must await
+acknowledgement of cluster state updates from every node! The benefit of
+coordinating only nodes should not be overstated -- data nodes can happily
+serve the same purpose.
+
+To create a dedicated coordinating node, set:
+
+[source,yaml]
+----
+node.roles: [ ]
+----
+
+[discrete]
+[[remote-node]]
+==== Remote-eligible node
+
+A remote-eligible node acts as a cross-cluster client and connects to
+<<remote-clusters,remote clusters>>. Once connected, you can search
+remote clusters using <<modules-cross-cluster-search,{ccs}>>. You can also sync
+data between clusters using <<xpack-ccr,{ccr}>>.
+
+[source,yaml]
+----
+node.roles: [ remote_cluster_client ]
+----
+
+[discrete]
+[[ml-node-role]]
+==== [xpack]#Machine learning node#
+
+{ml-cap} nodes run jobs and handle {ml} API requests. For more information, see
+<<ml-settings>>.
+
+To create a dedicated {ml} node, set:
+
+[source,yaml]
+----
+node.roles: [ ml, remote_cluster_client]
+----
+
+The `remote_cluster_client` role is optional but strongly recommended.
+Otherwise, {ccs} fails when used in {ml} jobs or {dfeeds}. If you use {ccs} in
+your {anomaly-jobs}, the `remote_cluster_client` role is also required on all
+master-eligible nodes. Otherwise, the {dfeed} cannot start. See <<remote-node>>.
+
+[discrete]
+[[transform-node-role]]
+==== [xpack]#{transform-cap} node#
+
+{transform-cap} nodes run {transforms} and handle {transform} API requests. For
+more information, see <<transform-settings>>.
+
+To create a dedicated {transform} node, set:
+
+[source,yaml]
+----
+node.roles: [ transform, remote_cluster_client ]
+----
+
+The `remote_cluster_client` role is optional but strongly recommended.
+Otherwise, {ccs} fails when used in {transforms}. See <<remote-node>>.
\ No newline at end of file
diff --git a/docs/reference/nodes-shards.asciidoc b/docs/reference/nodes-shards.asciidoc
new file mode 100644
index 0000000000000..11095ed7b7eb3
--- /dev/null
+++ b/docs/reference/nodes-shards.asciidoc
@@ -0,0 +1,43 @@
+[[nodes-shards]]
+== Nodes and shards
+
+[NOTE]
+====
+Nodes and shards are what make {es} distributed and scalable.
+These concepts aren't essential if you're just getting started. How you <<elasticsearch-intro-deploy,deploy {es}>> in production determines what you need to know:
+
+* *Self-managed {es}*: You are responsible for setting up and managing nodes, clusters, shards, and replicas. This includes managing the underlying infrastructure, scaling, and ensuring high availability through failover and backup strategies.
+* *Elastic Cloud*: Elastic can autoscale resources in response to workload changes. Choose from different deployment types to apply sensible defaults for your use case. A basic understanding of nodes, shards, and replicas is still important.
+* *Elastic Cloud Serverless*: You don't need to worry about nodes, shards, or replicas. These resources are 100% automated on the serverless platform, which is designed to scale with your workload.
+====
+
+You can add servers (_nodes_) to a cluster to increase capacity, and {es} automatically distributes your data and query load across all of the available nodes.
+
+Elastic is able to distribute your data across nodes by subdividing an index into _shards_. Each index in {es} is a grouping
+of one or more physical shards, where each shard is a self-contained Lucene index containing a subset of the documents in
+the index. By distributing the documents in an index across multiple shards, and distributing those shards across multiple
+nodes, {es} increases indexing and query capacity.
+
+There are two types of shards: _primaries_ and _replicas_. Each document in an index belongs to one primary shard. A replica
+shard is a copy of a primary shard. Replicas maintain redundant copies of your data across the nodes in your cluster. 
+This protects against hardware failure and increases capacity to serve read requests like searching or retrieving a document.
+
+[TIP]
+====
+The number of primary shards in an index is fixed at the time that an index is created, but the number of replica shards can
+be changed at any time, without interrupting indexing or query operations.
+====
+
+Shard copies in your cluster are automatically balanced across nodes to provide scale and high availability. All nodes are
+aware of all the other nodes in the cluster and can forward client requests to the appropriate node. This allows {es}
+to distribute indexing and query load across the cluster.
+
+If you're exploring {es} for the first time or working in a development environment, then you can use a cluster with a single node and create indices
+with only one shard. However, in a production environment, you should build a cluster with multiple nodes and indices
+with multiple shards to increase performance and resilience.
+
+// TODO - diagram
+
+* To learn about optimizing the number and size of shards in your cluster, refer to <<size-your-shards,Size your shards>>. 
+* To learn about how read and write operations are replicated across shards and shard copies, refer to <<docs-replication,Reading and writing documents>>.
+* To adjust how shards are allocated and balanced across nodes, refer to <<shard-allocation-relocation-recovery,Shard allocation, relocation, and recovery>>.
\ No newline at end of file
diff --git a/docs/reference/path-settings-overview.asciidoc b/docs/reference/path-settings-overview.asciidoc
new file mode 100644
index 0000000000000..0740b9769c9b2
--- /dev/null
+++ b/docs/reference/path-settings-overview.asciidoc
@@ -0,0 +1,112 @@
+[[path-settings-overview]]
+=== Path settings
+
+include::{es-ref-dir}/setup/important-settings/path-settings.asciidoc[]
+
+[[multiple-data-paths]]
+==== Multiple data paths
+deprecated::[7.13.0]
+
+If needed, you can specify multiple paths in `path.data`. {es} stores the node's
+data across all provided paths but keeps each shard's data on the same path.
+
+{es} does not balance shards across a node's data paths. High disk
+usage in a single path can trigger a <<disk-based-shard-allocation,high disk
+usage watermark>> for the entire node. If triggered, {es} will not add shards to
+the node, even if the node’s other paths have available disk space. If you need
+additional disk space, we recommend you add a new node rather than additional
+data paths.
+
+include::{es-ref-dir}/tab-widgets/multi-data-path-widget.asciidoc[]
+
+[[mdp-migrate]]
+===== Migrate from multiple data paths
+
+Support for multiple data paths was deprecated in 7.13 and will be removed
+in a future release.
+
+As an alternative to multiple data paths, you can create a filesystem which
+spans multiple disks with a hardware virtualisation layer such as RAID, or a
+software virtualisation layer such as Logical Volume Manager (LVM) on Linux or
+Storage Spaces on Windows. If you wish to use multiple data paths on a single
+machine then you must run one node for each data path.
+
+If you currently use multiple data paths in a
+{ref}/high-availability-cluster-design.html[highly available cluster] then you
+can migrate to a setup that uses a single path for each node without downtime
+using a process similar to a
+{ref}/restart-cluster.html#restart-cluster-rolling[rolling restart]: shut each
+node down in turn and replace it with one or more nodes each configured to use
+a single data path. In more detail, for each node that currently has multiple
+data paths you should follow the following process. In principle you can
+perform this migration during a rolling upgrade to 8.0, but we recommend
+migrating to a single-data-path setup before starting to upgrade.
+
+1. Take a snapshot to protect your data in case of disaster.
+
+2. Optionally, migrate the data away from the target node by using an
+{ref}/modules-cluster.html#cluster-shard-allocation-filtering[allocation filter]:
++
+[source,console]
+--------------------------------------------------
+PUT _cluster/settings
+{
+  "persistent": {
+    "cluster.routing.allocation.exclude._name": "target-node-name"
+  }
+}
+--------------------------------------------------
++
+You can use the {ref}/cat-allocation.html[cat allocation API] to track progress
+of this data migration. If some shards do not migrate then the
+{ref}/cluster-allocation-explain.html[cluster allocation explain API] will help
+you to determine why.
+
+3. Follow the steps in the
+{ref}/restart-cluster.html#restart-cluster-rolling[rolling restart process]
+up to and including shutting the target node down.
+
+4. Ensure your cluster health is `yellow` or `green`, so that there is a copy
+of every shard assigned to at least one of the other nodes in your cluster.
+
+5. If applicable, remove the allocation filter applied in the earlier step.
++
+[source,console]
+--------------------------------------------------
+PUT _cluster/settings
+{
+  "persistent": {
+    "cluster.routing.allocation.exclude._name": null
+  }
+}
+--------------------------------------------------
+
+6. Discard the data held by the stopped node by deleting the contents of its
+data paths.
+
+7. Reconfigure your storage. For instance, combine your disks into a single
+filesystem using LVM or Storage Spaces. Ensure that your reconfigured storage
+has sufficient space for the data that it will hold.
+
+8. Reconfigure your node by adjusting the `path.data` setting in its
+`elasticsearch.yml` file. If needed, install more nodes each with their own
+`path.data` setting pointing at a separate data path.
+
+9. Start the new nodes and follow the rest of the
+{ref}/restart-cluster.html#restart-cluster-rolling[rolling restart process] for
+them.
+
+10. Ensure your cluster health is `green`, so that every shard has been
+assigned.
+
+You can alternatively add some number of single-data-path nodes to your
+cluster, migrate all your data over to these new nodes using
+{ref}/modules-cluster.html#cluster-shard-allocation-filtering[allocation filters],
+and then remove the old nodes from the cluster. This approach will temporarily
+double the size of your cluster so it will only work if you have the capacity to
+expand your cluster like this.
+
+If you currently use multiple data paths but your cluster is not highly
+available then you can migrate to a non-deprecated configuration by taking
+a snapshot, creating a new cluster with the desired configuration and restoring
+the snapshot into it.
diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc
index c24135a370914..27220f0d85149 100644
--- a/docs/reference/query-dsl/bool-query.asciidoc
+++ b/docs/reference/query-dsl/bool-query.asciidoc
@@ -13,21 +13,24 @@ occurrence types are:
 |=======================================================================
 |Occur |Description
 |`must` |The clause (query) must appear in matching documents and will
-contribute to the score.
+contribute to the score. Each query defined under a `must` acts as a logical "AND", returning only documents that match _all_ the specified queries.
+
+|`should` |The clause (query) should appear in the matching document. Each query defined under a `should` acts as a logical "OR", returning documents that match _any_ of the specified queries.
 
 |`filter` |The clause (query) must appear in matching documents. However unlike
 `must` the score of the query will be ignored. Filter clauses are executed
 in <<query-filter-context,filter context>>, meaning that scoring is ignored
-and clauses are considered for caching.
-
-|`should` |The clause (query) should appear in the matching document.
+and clauses are considered for caching. Each query defined under a `filter` acts as a logical "AND", returning only documents that match _all_ the specified queries.
 
 |`must_not` |The clause (query) must not appear in the matching
 documents. Clauses are executed in <<query-filter-context,filter context>> meaning
 that scoring is ignored and clauses are considered for caching. Because scoring is
-ignored, a score of `0` for all documents is returned.
+ignored, a score of `0` for all documents is returned. Each query defined under a `must_not` acts as a logical "NOT", returning only documents that do not match any of the specified queries.
+
 |=======================================================================
 
+The `must` and `should` clauses function as logical AND, OR operators, contributing to the scoring of results. However, these results will not be cached for faster retrieval. In contrast, the `filter` and `must_not` clauses are used to include or exclude results without impacting the score, unless used within a `constant_score` query.
+
 The `bool` query takes a _more-matches-is-better_ approach, so the score from
 each matching `must` or `should` clause will be added together to provide the
 final `_score` for each document.
diff --git a/docs/reference/query-dsl/intervals-query.asciidoc b/docs/reference/query-dsl/intervals-query.asciidoc
index 069021dddb69f..267e5a502efd4 100644
--- a/docs/reference/query-dsl/intervals-query.asciidoc
+++ b/docs/reference/query-dsl/intervals-query.asciidoc
@@ -303,12 +303,22 @@ matches. Defaults to `-1`.
 
 If unspecified or set to `-1`, there is no width restriction on the match. If
 set to `0`, the terms must appear next to each other.
+
+Internal intervals can have their own `max_gaps` values. In this case
+we first find internal intervals with their `max_gaps` values, and then
+combine them to see if a gap between internal intervals match
+the value of `max_gaps` of the `all_of` rule.
+
+For examples, how `max_gaps` works, see <<interval-max_gaps-all-rule>>.
 --
 
 `ordered`::
 (Optional, Boolean) If `true`, intervals produced by the rules should appear in
 the order in which they are specified. Defaults to `false`.
 
+If `ordered` is `false`, intervals can appear in any order,
+including overlapping with each other.
+
 `filter`::
 (Optional, <<interval_filter,interval filter>> rule object) Rule used to filter
 returned intervals.
@@ -468,3 +478,94 @@ This query does *not* match a document containing the phrase `hot porridge is
 salty porridge`, because the intervals returned by the match query for `hot
 porridge` only cover the initial two terms in this document, and these do not
 overlap the intervals covering `salty`.
+
+[[interval-max_gaps-all-rule]]
+===== max_gaps in `all_of` ordered and unordered rule
+
+The following `intervals` search returns documents containing `my
+favorite food` without any gap, followed by `cold porridge` that
+can have at most 4 tokens between "cold" and "porridge". These
+two inner intervals when combined in the outer `all_of` interval,
+must have at most 1 gap between each other.
+
+Because the `all_of` rule has `ordered` set to `true`, the inner
+intervals are expected to be in the provided order. Thus,
+this search would match a `my_text` value of `my favorite food is cold
+porridge` but not `when it's cold my favorite food is porridge`.
+
+[source,console]
+--------------------------------------------------
+POST _search
+{
+  "query": {
+    "intervals" : {
+      "my_text" : {
+        "all_of" : {
+          "ordered" : true,     <1>
+          "max_gaps": 1,
+          "intervals" : [
+            {
+              "match" : {
+                "query" : "my favorite food",
+                "max_gaps" : 0,
+                "ordered" : true
+              }
+            },
+            {
+              "match" : {
+                "query" : "cold porridge",
+                "max_gaps" : 4,
+                "ordered" : true
+              }
+            }
+          ]
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+<1> The `ordered` parameter is set to `true`, so intervals must appear in the order specified.
+
+
+Below is the same query, but with `ordered` set to `false`. This means that
+intervals can appear in any order, even overlap with each other.
+Thus, this search would match a `my_text` value of `my favorite food is cold
+porridge`, as well as `when it's cold my favorite food is porridge`.
+In `when it's cold my favorite food is porridge`, `cold .... porridge` interval
+overlaps with `my favorite food` interval.
+
+[source,console]
+--------------------------------------------------
+POST _search
+{
+  "query": {
+    "intervals" : {
+      "my_text" : {
+        "all_of" : {
+          "ordered" : false, <1>
+          "max_gaps": 1,
+          "intervals" : [
+            {
+              "match" : {
+                "query" : "my favorite food",
+                "max_gaps" : 0,
+                "ordered" : true
+              }
+            },
+            {
+              "match" : {
+                "query" : "cold porridge",
+                "max_gaps" : 4,
+                "ordered" : true
+              }
+            }
+          ]
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+<1> The `ordered` parameter is set to `true`, so intervals can appear in any order,
+even overlap with each other.
diff --git a/docs/reference/query-dsl/knn-query.asciidoc b/docs/reference/query-dsl/knn-query.asciidoc
index daf9e9499a189..29ccadfa70a03 100644
--- a/docs/reference/query-dsl/knn-query.asciidoc
+++ b/docs/reference/query-dsl/knn-query.asciidoc
@@ -8,7 +8,8 @@ Finds the _k_ nearest vectors to a query vector, as measured by a similarity
 metric. _knn_ query finds nearest vectors through approximate search on indexed
 dense_vectors. The preferred way to do approximate kNN search is through the
 <<knn-search,top level knn section>> of a search request. _knn_ query is reserved for
-expert cases, where there is a need to combine this query with other queries.
+expert cases, where there is a need to combine this query with other queries, or
+perform a kNN search against a <<semantic-text, semantic_text>> field.
 
 [[knn-query-ex-request]]
 ==== Example request
@@ -77,7 +78,8 @@ POST my-image-index/_search
 +
 --
 (Required, string) The name of the vector field to search against. Must be a
-<<index-vectors-knn-search, `dense_vector` field with indexing enabled>>.
+<<index-vectors-knn-search, `dense_vector` field with indexing enabled>>, or a
+<<semantic-text, `semantic_text` field>> with a compatible dense vector inference model.
 --
 
 `query_vector`::
@@ -93,6 +95,7 @@ Either this or `query_vector_builder` must be provided.
 --
 (Optional, object) Query vector builder.
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-query-vector-builder]
+If all queried fields are of type <<semantic-text, semantic_text>>, the inference ID associated with the `semantic_text` field may be inferred.
 --
 
 `k`::
@@ -100,7 +103,7 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-query-vector-builde
 --
 (Optional, integer) The number of nearest neighbors to return from each shard.
 {es} collects `k` results from each shard, then merges them to find the global top results.
-This value must be less than or equal to `num_candidates`. Defaults to `num_candidates`.
+This value must be less than or equal to `num_candidates`. Defaults to search request size.
 --
 
 `num_candidates`::
diff --git a/docs/reference/query-dsl/match-query.asciidoc b/docs/reference/query-dsl/match-query.asciidoc
index eb840508aba39..67300ffca2d26 100644
--- a/docs/reference/query-dsl/match-query.asciidoc
+++ b/docs/reference/query-dsl/match-query.asciidoc
@@ -10,6 +10,10 @@ provided text is analyzed before matching.
 The `match` query is the standard query for performing a full-text search,
 including options for fuzzy matching.
 
+`Match` will also work against <<semantic-text, semantic_text>> fields,
+however when performing `match` queries against `semantic_text` fields options
+that specifically target lexical search such as `fuzziness` or `analyzer` will be ignored.
+
 
 [[match-query-ex-request]]
 ==== Example request
@@ -296,4 +300,3 @@ The example above creates a boolean query:
 
 that matches documents with the term `ny` or the conjunction `new AND york`.
 By default the parameter `auto_generate_synonyms_phrase_query` is set to `true`.
-
diff --git a/docs/reference/query-dsl/semantic-query.asciidoc b/docs/reference/query-dsl/semantic-query.asciidoc
index 11e19d6356081..914f4429f7f9c 100644
--- a/docs/reference/query-dsl/semantic-query.asciidoc
+++ b/docs/reference/query-dsl/semantic-query.asciidoc
@@ -117,79 +117,3 @@ GET my-index/_search
 }
 ------------------------------------------------------------
 // TEST[skip: Requires inference endpoints]
-
-
-[discrete]
-[[advanced-search]]
-==== Advanced search on `semantic_text` fields
-
-The `semantic` query uses default settings for searching on `semantic_text` fields for ease of use.
-If you want to fine-tune a search on a `semantic_text` field, you need to know the task type used by the `inference_id` configured in `semantic_text`.
-You can find the task type using the <<get-inference-api>>, and check the `task_type` associated with the {infer} service.
-Depending on the `task_type`, use either the <<query-dsl-sparse-vector-query,`sparse_vector`>> or the <<query-dsl-knn-query,`knn`>> query for greater flexibility and customization.
-
-NOTE: While it is possible to use the `sparse_vector` query or the `knn` query
-on a `semantic_text` field, it is not supported to use the `semantic_query` on a
-`sparse_vector` or `dense_vector` field type.
-
-
-[discrete]
-[[search-sparse-inference]]
-===== Search with `sparse_embedding` inference
-
-When the {infer} endpoint uses a `sparse_embedding` model, you can use a <<query-dsl-sparse-vector-query,`sparse_vector` query>> on a <<semantic-text,`semantic_text`>> field in the following way:
-
-[source,console]
-------------------------------------------------------------
-GET test-index/_search
-{
-  "query": {
-    "nested": {
-      "path": "inference_field.inference.chunks",
-      "query": {
-        "sparse_vector": {
-          "field": "inference_field.inference.chunks.embeddings",
-          "inference_id": "my-inference-id",
-          "query": "mountain lake"
-        }
-      }
-    }
-  }
-}
-------------------------------------------------------------
-// TEST[skip: Requires inference endpoints]
-
-You can customize the `sparse_vector` query to include specific settings, like <<sparse-vector-query-with-pruning-config-and-rescore-example,pruning configuration>>.
-
-
-[discrete]
-[[search-text-inferece]]
-===== Search with `text_embedding` inference
-
-When the {infer} endpoint uses a `text_embedding` model, you can use a <<query-dsl-knn-query,`knn` query>> on a `semantic_text` field in the following way:
-
-[source,console]
-------------------------------------------------------------
-GET test-index/_search
-{
-  "query": {
-    "nested": {
-      "path": "inference_field.inference.chunks",
-      "query": {
-        "knn": {
-          "field": "inference_field.inference.chunks.embeddings",
-          "query_vector_builder": {
-            "text_embedding": {
-              "model_id": "my_inference_id",
-	      "model_text": "mountain lake"
-            }
-          }
-        }
-      }
-    }
-  }
-}
-------------------------------------------------------------
-// TEST[skip: Requires inference endpoints]
-
-You can customize the `knn` query to include specific settings, like `num_candidates` and `k`.
diff --git a/docs/reference/query-dsl/sparse-vector-query.asciidoc b/docs/reference/query-dsl/sparse-vector-query.asciidoc
index 399cf29d4dd12..d46d649079d70 100644
--- a/docs/reference/query-dsl/sparse-vector-query.asciidoc
+++ b/docs/reference/query-dsl/sparse-vector-query.asciidoc
@@ -11,7 +11,8 @@ This can be achieved with one of two strategies:
 - Using an {nlp} model to convert query text into a list of token-weight pairs
 - Sending in precalculated token-weight pairs as query vectors
 
-These token-weight pairs are then used in a query against a <<sparse-vector,sparse vector>>.
+These token-weight pairs are then used in a query against a <<sparse-vector,sparse vector>> 
+or a <<semantic-text, semantic_text>> field with a compatible sparse inference model.
 At query time, query vectors are calculated using the same inference model that was used to create the tokens.
 When querying, these query vectors are ORed together with their respective weights, which means scoring is effectively a <<vector-functions-dot-product,dot product>> calculation between stored dimensions and query dimensions.
 
@@ -65,6 +66,7 @@ GET _search
 It must be the same inference ID that was used to create the tokens from the input text.
 Only one of `inference_id` and `query_vector` is allowed.
 If `inference_id` is specified, `query` must also be specified.
+If all queried fields are of type <<semantic-text, semantic_text>>, the inference ID associated with the `semantic_text` field will be inferred.
 
 `query`::
 (Optional, string) The query text you want to use for search.
@@ -291,5 +293,3 @@ GET my-index/_search
 //TEST[skip: Requires inference]
 
 NOTE: When performing <<modules-cross-cluster-search, cross-cluster search>>, inference is performed on the local cluster.
-
-
diff --git a/docs/reference/query-rules/apis/delete-query-rule.asciidoc b/docs/reference/query-rules/apis/delete-query-rule.asciidoc
index 01b73033aa361..4d91092eaf8af 100644
--- a/docs/reference/query-rules/apis/delete-query-rule.asciidoc
+++ b/docs/reference/query-rules/apis/delete-query-rule.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Delete query rule</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs].
+--
+
 Removes an individual query rule within an existing query ruleset.
 This is a destructive action that is only recoverable by re-adding the same rule via the <<put-query-rule, create or update query rule>> API.
 
diff --git a/docs/reference/query-rules/apis/delete-query-ruleset.asciidoc b/docs/reference/query-rules/apis/delete-query-ruleset.asciidoc
index 31507dce3d12d..168310dcd4078 100644
--- a/docs/reference/query-rules/apis/delete-query-ruleset.asciidoc
+++ b/docs/reference/query-rules/apis/delete-query-ruleset.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Delete query ruleset</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs].
+--
+
 Removes a query ruleset and its associated data.
 This is a destructive action that is not recoverable.
 
diff --git a/docs/reference/query-rules/apis/get-query-rule.asciidoc b/docs/reference/query-rules/apis/get-query-rule.asciidoc
index 56713965d7bdc..742982e5897e1 100644
--- a/docs/reference/query-rules/apis/get-query-rule.asciidoc
+++ b/docs/reference/query-rules/apis/get-query-rule.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Get query rule</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs].
+--
+
 Retrieves information about an individual query rule within a query ruleset.
 
 [[get-query-rule-request]]
diff --git a/docs/reference/query-rules/apis/get-query-ruleset.asciidoc b/docs/reference/query-rules/apis/get-query-ruleset.asciidoc
index 6bbcd157ea9e1..55574fb7c67e9 100644
--- a/docs/reference/query-rules/apis/get-query-ruleset.asciidoc
+++ b/docs/reference/query-rules/apis/get-query-ruleset.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Get query ruleset</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs].
+--
+
 Retrieves information about a query ruleset.
 
 [[get-query-ruleset-request]]
diff --git a/docs/reference/query-rules/apis/index.asciidoc b/docs/reference/query-rules/apis/index.asciidoc
index fbeb477acacb5..7dcdf0e8f4e1c 100644
--- a/docs/reference/query-rules/apis/index.asciidoc
+++ b/docs/reference/query-rules/apis/index.asciidoc
@@ -7,6 +7,12 @@
 
 ---
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs].
+--
+
 <<search-using-query-rules,Query rules>> allow you to configure per-query rules that are applied at query time to queries that match the specific rule.
 Query rules are organized into _rulesets_, collections of query rules that are matched against incoming queries.
 Query rules are applied using the <<query-dsl-rule-query, rule query>>.
diff --git a/docs/reference/query-rules/apis/list-query-rulesets.asciidoc b/docs/reference/query-rules/apis/list-query-rulesets.asciidoc
index 304b8c7745007..5f61bcb98b085 100644
--- a/docs/reference/query-rules/apis/list-query-rulesets.asciidoc
+++ b/docs/reference/query-rules/apis/list-query-rulesets.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>List query rulesets</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs].
+--
+
 Returns information about all stored query rulesets.
 Summary information on the number of rules per ruleset will be returned, and full details can be returned with the <<get-query-ruleset>> command.
 
diff --git a/docs/reference/query-rules/apis/put-query-rule.asciidoc b/docs/reference/query-rules/apis/put-query-rule.asciidoc
index 714ed9b096d1d..df33d22b39029 100644
--- a/docs/reference/query-rules/apis/put-query-rule.asciidoc
+++ b/docs/reference/query-rules/apis/put-query-rule.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Create or update query rule</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs].
+--
+
 Creates or updates an individual query rule within a query ruleset.
 
 [[put-query-rule-request]]
diff --git a/docs/reference/query-rules/apis/put-query-ruleset.asciidoc b/docs/reference/query-rules/apis/put-query-ruleset.asciidoc
index df7ec100db076..ea689dc0bf305 100644
--- a/docs/reference/query-rules/apis/put-query-ruleset.asciidoc
+++ b/docs/reference/query-rules/apis/put-query-ruleset.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Create or update query ruleset</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs].
+--
+
 Creates or updates a query ruleset.
 
 [[put-query-ruleset-request]]
diff --git a/docs/reference/query-rules/apis/test-query-ruleset.asciidoc b/docs/reference/query-rules/apis/test-query-ruleset.asciidoc
index 4a670645cea6e..4a47754f572e5 100644
--- a/docs/reference/query-rules/apis/test-query-ruleset.asciidoc
+++ b/docs/reference/query-rules/apis/test-query-ruleset.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Tests query ruleset</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs].
+--
+
 Evaluates match criteria against a query ruleset to identify the rules that would match that criteria.
 
 preview::[]
diff --git a/docs/reference/quickstart/full-text-filtering-tutorial.asciidoc b/docs/reference/quickstart/full-text-filtering-tutorial.asciidoc
index a024305588cae..b602ee5076434 100644
--- a/docs/reference/quickstart/full-text-filtering-tutorial.asciidoc
+++ b/docs/reference/quickstart/full-text-filtering-tutorial.asciidoc
@@ -4,7 +4,7 @@
 <titleabbrev>Basics: Full-text search and filtering</titleabbrev>
 ++++
 
-This is a hands-on introduction to the basics of full-text search with {es}, also known as _lexical search_, using the <<search-search,`_search` API>> and <<query-dsl,Query DSL>>.
+This is a hands-on introduction to the basics of <<full-text-search,full-text search>> with {es}, also known as _lexical search_, using the <<search-search,`_search` API>> and <<query-dsl,Query DSL>>.
 You'll also learn how to filter data, to narrow down search results based on exact criteria.
 
 In this scenario, we're implementing a search function for a cooking blog.
@@ -632,6 +632,7 @@ This tutorial introduced the basics of full-text search and filtering in {es}.
 Building a real-world search experience requires understanding many more advanced concepts and techniques.
 Here are some resources once you're ready to dive deeper:
 
+* <<full-text-search, Full-text search>>: Learn about the core components of full-text search in {es}.
 * <<search-analyze, Elasticsearch basics — Search and analyze data>>: Understand all your options for searching and analyzing data in {es}.
 * <<analysis,Text analysis>>: Understand how text is processed for full-text search.
 * <<search-with-elasticsearch>>: Learn about more advanced search techniques using the `_search` API, including semantic search.
diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc
index c3bf84fa600d2..9c0f0092214ed 100644
--- a/docs/reference/redirects.asciidoc
+++ b/docs/reference/redirects.asciidoc
@@ -156,10 +156,16 @@ See <<restore-entire-cluster>>.
 The freeze index API was removed in 8.0.
 // tag::frozen-removal-explanation[]
 Frozen indices are no longer useful due to
-https://www.elastic.co/blog/significantly-decrease-your-elasticsearch-heap-memory-usage[recent
-improvements in heap memory usage].
+https://www.elastic.co/blog/significantly-decrease-your-elasticsearch-heap-memory-usage[improvements
+in heap memory usage].
 // end::frozen-removal-explanation[]
 
+[role="exclude",id="unfreeze-index-api"]
+=== Unfreeze index API
+
+The unfreeze index API was removed in 9.0.
+include::redirects.asciidoc[tag=frozen-removal-explanation]
+
 [role="exclude",id="ilm-freeze"]
 === Freeze {ilm-init} action
 
@@ -1749,8 +1755,10 @@ See <<search-terms-enum>>.
 === Frozen indices
 
 // tag::frozen-index-redirect[]
-
-For API documentation, see <<unfreeze-index-api>>.
+Older versions of {es} provided the option to reduce the amount of data kept in memory for an index, at the expense of
+increasing search latency. This was known as 'freezing' the index.
+include::redirects.asciidoc[tag=frozen-removal-explanation]
+The freeze index API was removed in 8.0, and the unfreeze index API was removed in 9.0.
 // end::frozen-index-redirect[]
 
 [role="exclude",id="best_practices"]
diff --git a/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc b/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc
index addcd65f0e84a..bd05d76705eb9 100644
--- a/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc
+++ b/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Clear repositories metering archive</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/operation/operation-nodes-clear-repositories-metering-archive[Clear the archived repositories metering API].
+--
+
 Removes the archived repositories metering information present in the cluster.
 
 [[clear-repositories-metering-archive-api-request]]
diff --git a/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc b/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc
index 314f85a7dba52..6b6d98a69ded2 100644
--- a/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc
+++ b/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get repositories metering information</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/operation/operation-nodes-get-repositories-metering-info[Get cluster repositories metering API].
+--
+
 Returns cluster repositories metering information.
 
 [[get-repositories-metering-api-request]]
diff --git a/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc b/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc
index b838e0fb213f4..ca9bffa32a917 100644
--- a/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc
+++ b/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc
@@ -4,6 +4,12 @@
 
 experimental[]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs].
+--
+
 You can use the following APIs to retrieve repositories metering information.
 
 This is an API used by Elastic's commercial offerings.
diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc
index 993bb8cb894f9..83c11c9256a67 100644
--- a/docs/reference/rest-api/common-parms.asciidoc
+++ b/docs/reference/rest-api/common-parms.asciidoc
@@ -668,6 +668,16 @@ node only. Defaults to `false`, which means information is retrieved from
 the master node.
 end::local[]
 
+tag::local-deprecated-9.0.0[]
+`local`::
+(Optional, Boolean) If `true`, the request retrieves information from the local
+node only. Defaults to `false`, which means information is retrieved from
+the master node.
++
+deprecated::[9.0.0, "The `?local` query parameter to this API has no effect, is now deprecated, and will be removed in a future version."]
+
+end::local-deprecated-9.0.0[]
+
 tag::mappings[]
 `mappings`::
 +
diff --git a/docs/reference/rest-api/index.asciidoc b/docs/reference/rest-api/index.asciidoc
index 50c9f96ad81b0..9d9047c93cc97 100644
--- a/docs/reference/rest-api/index.asciidoc
+++ b/docs/reference/rest-api/index.asciidoc
@@ -6,9 +6,9 @@
 {es} exposes REST APIs that are used by the UI components and can be called
 directly to configure and access {es} features.
 
-[NOTE]
-We are working on including more {es} APIs in this section. Some content might
-not be included yet.
+.New API reference
+[sidebar]
+For the most up-to-date API details, refer to {api-es}[{es} APIs].
 
 * <<api-conventions, API conventions>>
 * <<common-options, Common Options>>
diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc
index fda5b07d28205..b9547163b07ba 100644
--- a/docs/reference/rest-api/info.asciidoc
+++ b/docs/reference/rest-api/info.asciidoc
@@ -2,6 +2,12 @@
 [[info-api]]
 == Info API
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-info[Info APIs].
+--
+
 Provides general information about the installed {xpack} features.
 
 [discrete]
diff --git a/docs/reference/rest-api/logstash/delete-pipeline.asciidoc b/docs/reference/rest-api/logstash/delete-pipeline.asciidoc
index ff7494d34e615..2e56b0289a394 100644
--- a/docs/reference/rest-api/logstash/delete-pipeline.asciidoc
+++ b/docs/reference/rest-api/logstash/delete-pipeline.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete {ls} pipeline</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-logstash[{ls} APIs].
+--
+
 This API deletes a pipeline used for
 {logstash-ref}/logstash-centralized-pipeline-management.html[{ls} Central
 Management].
diff --git a/docs/reference/rest-api/logstash/get-pipeline.asciidoc b/docs/reference/rest-api/logstash/get-pipeline.asciidoc
index 8409a5128d525..8e440f218aa09 100644
--- a/docs/reference/rest-api/logstash/get-pipeline.asciidoc
+++ b/docs/reference/rest-api/logstash/get-pipeline.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get {ls} pipeline</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-logstash[{ls} APIs].
+--
+
 This API retrieves pipelines used for
 {logstash-ref}/logstash-centralized-pipeline-management.html[{ls} Central
 Management].
diff --git a/docs/reference/rest-api/logstash/index.asciidoc b/docs/reference/rest-api/logstash/index.asciidoc
index a52be19a75df4..7f28020a8b183 100644
--- a/docs/reference/rest-api/logstash/index.asciidoc
+++ b/docs/reference/rest-api/logstash/index.asciidoc
@@ -2,6 +2,12 @@
 [[logstash-apis]]
 == {ls} APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-logstash[{ls} APIs].
+--
+
 The following APIs are used to manage pipelines used by
 {logstash-ref}/logstash-centralized-pipeline-management.html[{ls} Central
 Management]:
diff --git a/docs/reference/rest-api/logstash/put-pipeline.asciidoc b/docs/reference/rest-api/logstash/put-pipeline.asciidoc
index a0d2f02f2bf6f..26af8d0f124cb 100644
--- a/docs/reference/rest-api/logstash/put-pipeline.asciidoc
+++ b/docs/reference/rest-api/logstash/put-pipeline.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Create or update {ls} pipeline</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-logstash[{ls} APIs].
+--
+
 This API creates or updates a {ls} pipeline used for
 {logstash-ref}/logstash-centralized-pipeline-management.html[{ls} Central
 Management].
diff --git a/docs/reference/rest-api/root.asciidoc b/docs/reference/rest-api/root.asciidoc
index 8821981c2afe3..aaf40b31db21c 100644
--- a/docs/reference/rest-api/root.asciidoc
+++ b/docs/reference/rest-api/root.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Root API</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-info[Info APIs].
+--
+
 The Elasticsearch API's base url returns its basic build, 
 version, and cluster information. 
 
diff --git a/docs/reference/rest-api/security.asciidoc b/docs/reference/rest-api/security.asciidoc
index 82cf38e52bd80..57726b074ac3f 100644
--- a/docs/reference/rest-api/security.asciidoc
+++ b/docs/reference/rest-api/security.asciidoc
@@ -1,6 +1,13 @@
 [role="xpack"]
 [[security-api]]
 == Security APIs
+
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 To use the security APIs, you must set `xpack.security.enabled` to `true` in
 the `elasticsearch.yml` file.
 
diff --git a/docs/reference/rest-api/security/activate-user-profile.asciidoc b/docs/reference/rest-api/security/activate-user-profile.asciidoc
index f6ce32e1bb19e..0db41937f2ff2 100644
--- a/docs/reference/rest-api/security/activate-user-profile.asciidoc
+++ b/docs/reference/rest-api/security/activate-user-profile.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Activate user profile</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 NOTE: The user profile feature is designed only for use by {kib} and
 Elastic’s {observability}, {ents}, and {elastic-sec} solutions. Individual
 users and external applications should not call this API directly. Elastic reserves
diff --git a/docs/reference/rest-api/security/authenticate.asciidoc b/docs/reference/rest-api/security/authenticate.asciidoc
index a02deb444628d..78d9cc2bcaa9f 100644
--- a/docs/reference/rest-api/security/authenticate.asciidoc
+++ b/docs/reference/rest-api/security/authenticate.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Authenticate</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Enables you to submit a request with a basic auth header to
 authenticate a user and retrieve information about the authenticated user.
 
diff --git a/docs/reference/rest-api/security/bulk-create-roles.asciidoc b/docs/reference/rest-api/security/bulk-create-roles.asciidoc
index 37f49f2445770..030e5e42bf29a 100644
--- a/docs/reference/rest-api/security/bulk-create-roles.asciidoc
+++ b/docs/reference/rest-api/security/bulk-create-roles.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Bulk create or update roles API</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Bulk adds and updates roles in the native realm.
 
 [[security-api-bulk-put-role-request]]
diff --git a/docs/reference/rest-api/security/bulk-delete-roles.asciidoc b/docs/reference/rest-api/security/bulk-delete-roles.asciidoc
index b9978c89bef3a..899591b3276db 100644
--- a/docs/reference/rest-api/security/bulk-delete-roles.asciidoc
+++ b/docs/reference/rest-api/security/bulk-delete-roles.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Bulk delete roles API</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Bulk deletes roles in the native realm.
 
 [[security-api-bulk-delete-role-request]]
diff --git a/docs/reference/rest-api/security/bulk-update-api-keys.asciidoc b/docs/reference/rest-api/security/bulk-update-api-keys.asciidoc
index faf87c67d1ccc..8206ac5d9d4f3 100644
--- a/docs/reference/rest-api/security/bulk-update-api-keys.asciidoc
+++ b/docs/reference/rest-api/security/bulk-update-api-keys.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Bulk update API keys</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 [[security-api-bulk-update-api-keys-request]]
 ==== {api-request-title}
 
diff --git a/docs/reference/rest-api/security/change-password.asciidoc b/docs/reference/rest-api/security/change-password.asciidoc
index c035661cdd707..f2dc45cad20b0 100644
--- a/docs/reference/rest-api/security/change-password.asciidoc
+++ b/docs/reference/rest-api/security/change-password.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Change passwords</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Changes the passwords of users in the native realm and built-in users.
 
 [[security-api-change-password-request]]
diff --git a/docs/reference/rest-api/security/clear-api-key-cache.asciidoc b/docs/reference/rest-api/security/clear-api-key-cache.asciidoc
index 7828026b604f0..61f05a85dfaff 100644
--- a/docs/reference/rest-api/security/clear-api-key-cache.asciidoc
+++ b/docs/reference/rest-api/security/clear-api-key-cache.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Clear API key cache</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Evicts a subset of all entries from the API key cache.
 The cache is also automatically cleared on state changes of the security index.
 
diff --git a/docs/reference/rest-api/security/clear-cache.asciidoc b/docs/reference/rest-api/security/clear-cache.asciidoc
index 3e4e5432768bd..270856ba28e6b 100644
--- a/docs/reference/rest-api/security/clear-cache.asciidoc
+++ b/docs/reference/rest-api/security/clear-cache.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Clear cache</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Evicts users from the user cache. You can completely clear
 the cache or evict specific users.
 
diff --git a/docs/reference/rest-api/security/clear-privileges-cache.asciidoc b/docs/reference/rest-api/security/clear-privileges-cache.asciidoc
index 69a5743419d2c..cf615010779b7 100644
--- a/docs/reference/rest-api/security/clear-privileges-cache.asciidoc
+++ b/docs/reference/rest-api/security/clear-privileges-cache.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Clear privileges cache</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Evicts privileges from the native application privilege cache.
 The cache is also automatically cleared for applications that have their privileges updated.
 
diff --git a/docs/reference/rest-api/security/clear-roles-cache.asciidoc b/docs/reference/rest-api/security/clear-roles-cache.asciidoc
index 63c54b51dcf2f..edcb2c2723512 100644
--- a/docs/reference/rest-api/security/clear-roles-cache.asciidoc
+++ b/docs/reference/rest-api/security/clear-roles-cache.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Clear roles cache</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Evicts roles from the native role cache. 
 
 [[security-api-clear-role-cache-request]]
diff --git a/docs/reference/rest-api/security/clear-service-token-caches.asciidoc b/docs/reference/rest-api/security/clear-service-token-caches.asciidoc
index ff4587549534d..26158220418e0 100644
--- a/docs/reference/rest-api/security/clear-service-token-caches.asciidoc
+++ b/docs/reference/rest-api/security/clear-service-token-caches.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Clear service account token caches</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Evicts a subset of all entries from the  <<service-accounts,service account>>
 token caches.
 
diff --git a/docs/reference/rest-api/security/create-api-keys.asciidoc b/docs/reference/rest-api/security/create-api-keys.asciidoc
index f740e9413e3e9..20f1c2cb155ce 100644
--- a/docs/reference/rest-api/security/create-api-keys.asciidoc
+++ b/docs/reference/rest-api/security/create-api-keys.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Create API keys</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Creates an API key for access without requiring basic authentication.
 
 [[security-api-create-api-key-request]]
diff --git a/docs/reference/rest-api/security/create-cross-cluster-api-key.asciidoc b/docs/reference/rest-api/security/create-cross-cluster-api-key.asciidoc
index d7d55bafc6d18..63b2f37063f9f 100644
--- a/docs/reference/rest-api/security/create-cross-cluster-api-key.asciidoc
+++ b/docs/reference/rest-api/security/create-cross-cluster-api-key.asciidoc
@@ -1,11 +1,16 @@
 [role="xpack"]
 [[security-api-create-cross-cluster-api-key]]
 === Create Cross-Cluster API key API
-
 ++++
 <titleabbrev>Create Cross-Cluster API key</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Creates an API key of the `cross_cluster` type for the <<remote-clusters-api-key,API key based remote cluster>> access.
 A `cross_cluster` API key cannot be used to authenticate through the REST interface.
 On the contrary, a <<security-api-create-api-key,REST API key>> is meant to be used through the REST interface
diff --git a/docs/reference/rest-api/security/create-role-mappings.asciidoc b/docs/reference/rest-api/security/create-role-mappings.asciidoc
index e78d06a5676e4..71c931260865f 100644
--- a/docs/reference/rest-api/security/create-role-mappings.asciidoc
+++ b/docs/reference/rest-api/security/create-role-mappings.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Create or update role mappings</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Creates and updates role mappings.
 
 [[security-api-put-role-mapping-request]]
diff --git a/docs/reference/rest-api/security/create-roles.asciidoc b/docs/reference/rest-api/security/create-roles.asciidoc
index d23b9f06e2d87..0b0cd828140a1 100644
--- a/docs/reference/rest-api/security/create-roles.asciidoc
+++ b/docs/reference/rest-api/security/create-roles.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Create or update roles</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Adds and updates roles in the native realm.
 
 [[security-api-put-role-request]]
diff --git a/docs/reference/rest-api/security/create-service-token.asciidoc b/docs/reference/rest-api/security/create-service-token.asciidoc
index 9a3c2b678c92e..30195d89ff47d 100644
--- a/docs/reference/rest-api/security/create-service-token.asciidoc
+++ b/docs/reference/rest-api/security/create-service-token.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Create service account tokens</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Creates a  <<service-accounts,service accounts>> token for access without requiring basic
 authentication.
 
diff --git a/docs/reference/rest-api/security/create-users.asciidoc b/docs/reference/rest-api/security/create-users.asciidoc
index 428df1102329c..6d8e1cdaa9ba0 100644
--- a/docs/reference/rest-api/security/create-users.asciidoc
+++ b/docs/reference/rest-api/security/create-users.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Create or update users</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Adds and updates users in the native realm. These users are commonly referred
 to as _native users_.
 
diff --git a/docs/reference/rest-api/security/delegate-pki-authentication.asciidoc b/docs/reference/rest-api/security/delegate-pki-authentication.asciidoc
index 860708011cd25..00a80b3fd27bc 100644
--- a/docs/reference/rest-api/security/delegate-pki-authentication.asciidoc
+++ b/docs/reference/rest-api/security/delegate-pki-authentication.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delegate PKI authentication</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Implements the exchange of an _X509Certificate_ chain into an {es} access
 token.
 
diff --git a/docs/reference/rest-api/security/delete-app-privileges.asciidoc b/docs/reference/rest-api/security/delete-app-privileges.asciidoc
index 39ac1706c6dc2..dad57bcac409c 100644
--- a/docs/reference/rest-api/security/delete-app-privileges.asciidoc
+++ b/docs/reference/rest-api/security/delete-app-privileges.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete application privileges</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Removes <<application-privileges,application privileges>>.
 
 [[security-api-delete-privilege-request]]
diff --git a/docs/reference/rest-api/security/delete-role-mappings.asciidoc b/docs/reference/rest-api/security/delete-role-mappings.asciidoc
index c5dd1aa9c909f..4ec7e3817b031 100644
--- a/docs/reference/rest-api/security/delete-role-mappings.asciidoc
+++ b/docs/reference/rest-api/security/delete-role-mappings.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete role mappings</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Removes role mappings.
 
 [[security-api-delete-role-mapping-request]]
diff --git a/docs/reference/rest-api/security/delete-roles.asciidoc b/docs/reference/rest-api/security/delete-roles.asciidoc
index 427e7c6b1860d..d30a4c5251809 100644
--- a/docs/reference/rest-api/security/delete-roles.asciidoc
+++ b/docs/reference/rest-api/security/delete-roles.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete roles</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Removes roles in the native realm.
 
 [[security-api-delete-role-request]]
diff --git a/docs/reference/rest-api/security/delete-service-token.asciidoc b/docs/reference/rest-api/security/delete-service-token.asciidoc
index b704fb9121263..f7c488e9e7139 100644
--- a/docs/reference/rest-api/security/delete-service-token.asciidoc
+++ b/docs/reference/rest-api/security/delete-service-token.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete service account token</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Deletes  <<service-accounts,service account>> tokens for a `service` in a
 specified `namespace`.
 
diff --git a/docs/reference/rest-api/security/delete-users.asciidoc b/docs/reference/rest-api/security/delete-users.asciidoc
index b08f99e809b44..ff781a7d9ef0b 100644
--- a/docs/reference/rest-api/security/delete-users.asciidoc
+++ b/docs/reference/rest-api/security/delete-users.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete users</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Deletes users from the native realm. 
 
 [[security-api-delete-user-request]]
diff --git a/docs/reference/rest-api/security/disable-user-profile.asciidoc b/docs/reference/rest-api/security/disable-user-profile.asciidoc
index 35658f071679b..f665b8955e0d0 100644
--- a/docs/reference/rest-api/security/disable-user-profile.asciidoc
+++ b/docs/reference/rest-api/security/disable-user-profile.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Disable user profile</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 NOTE: The user profile feature is designed only for use by {kib} and
 Elastic’s {observability}, {ents}, and {elastic-sec} solutions. Individual
 users and external applications should not call this API directly. Elastic reserves
diff --git a/docs/reference/rest-api/security/disable-users.asciidoc b/docs/reference/rest-api/security/disable-users.asciidoc
index 9859085cb1824..3f8bc74d7e106 100644
--- a/docs/reference/rest-api/security/disable-users.asciidoc
+++ b/docs/reference/rest-api/security/disable-users.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Disable users</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Disables users in the native realm. 
 
 
diff --git a/docs/reference/rest-api/security/enable-user-profile.asciidoc b/docs/reference/rest-api/security/enable-user-profile.asciidoc
index e27673b07f598..9f74d90f88b99 100644
--- a/docs/reference/rest-api/security/enable-user-profile.asciidoc
+++ b/docs/reference/rest-api/security/enable-user-profile.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Enable user profile</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 NOTE: The user profile feature is designed only for use by {kib} and
 Elastic’s {observability}, {ents}, and {elastic-sec} solutions. Individual
 users and external applications should not call this API directly. Elastic reserves
diff --git a/docs/reference/rest-api/security/enable-users.asciidoc b/docs/reference/rest-api/security/enable-users.asciidoc
index 04193e6c27944..db78a5c222116 100644
--- a/docs/reference/rest-api/security/enable-users.asciidoc
+++ b/docs/reference/rest-api/security/enable-users.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Enable users</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Enables users in the native realm. 
 
 
diff --git a/docs/reference/rest-api/security/enroll-kibana.asciidoc b/docs/reference/rest-api/security/enroll-kibana.asciidoc
index 55de31b5407d1..78bc0569fa193 100644
--- a/docs/reference/rest-api/security/enroll-kibana.asciidoc
+++ b/docs/reference/rest-api/security/enroll-kibana.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Enroll {kib}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Enables a {kib} instance to configure itself for communication with a secured {es} cluster.
 
 NOTE: This API is currently intended for internal use only by {kib}.
diff --git a/docs/reference/rest-api/security/enroll-node.asciidoc b/docs/reference/rest-api/security/enroll-node.asciidoc
index 81cae73fb22d6..d5c2ce2a51746 100644
--- a/docs/reference/rest-api/security/enroll-node.asciidoc
+++ b/docs/reference/rest-api/security/enroll-node.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Enroll node</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Allows a new node to join an existing cluster with security features enabled.
 
 [[security-api-node-enrollment-api-request]]
diff --git a/docs/reference/rest-api/security/get-api-keys.asciidoc b/docs/reference/rest-api/security/get-api-keys.asciidoc
index bf49297539895..6a133f136d4b3 100644
--- a/docs/reference/rest-api/security/get-api-keys.asciidoc
+++ b/docs/reference/rest-api/security/get-api-keys.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get API key information</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Retrieves information for one or more API keys.
 
 [[security-api-get-api-key-request]]
diff --git a/docs/reference/rest-api/security/get-app-privileges.asciidoc b/docs/reference/rest-api/security/get-app-privileges.asciidoc
index f0f3f1b69071c..c8bb709f96d5a 100644
--- a/docs/reference/rest-api/security/get-app-privileges.asciidoc
+++ b/docs/reference/rest-api/security/get-app-privileges.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get application privileges</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Retrieves <<application-privileges,application privileges>>.
 
 [[security-api-get-privileges-request]]
diff --git a/docs/reference/rest-api/security/get-builtin-privileges.asciidoc b/docs/reference/rest-api/security/get-builtin-privileges.asciidoc
index 7f3d75b926780..08a03a5b1e830 100644
--- a/docs/reference/rest-api/security/get-builtin-privileges.asciidoc
+++ b/docs/reference/rest-api/security/get-builtin-privileges.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Get builtin privileges</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Retrieves the list of <<privileges-list-cluster,cluster privileges>> and
 <<privileges-list-indices,index privileges>> that are
 available in this version of {es}.
diff --git a/docs/reference/rest-api/security/get-role-mappings.asciidoc b/docs/reference/rest-api/security/get-role-mappings.asciidoc
index 8272ec4d015a8..49063e775982b 100644
--- a/docs/reference/rest-api/security/get-role-mappings.asciidoc
+++ b/docs/reference/rest-api/security/get-role-mappings.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get role mappings</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Retrieves role mappings.
 
 [[security-api-get-role-mapping-request]]
diff --git a/docs/reference/rest-api/security/get-roles.asciidoc b/docs/reference/rest-api/security/get-roles.asciidoc
index 3cc2f95c6ea7e..03f083e8202f0 100644
--- a/docs/reference/rest-api/security/get-roles.asciidoc
+++ b/docs/reference/rest-api/security/get-roles.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get roles</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Retrieves roles in the native realm.
 
 [[security-api-get-role-request]]
diff --git a/docs/reference/rest-api/security/get-service-accounts.asciidoc b/docs/reference/rest-api/security/get-service-accounts.asciidoc
index 74f98f2602e34..b80136365d879 100644
--- a/docs/reference/rest-api/security/get-service-accounts.asciidoc
+++ b/docs/reference/rest-api/security/get-service-accounts.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Get service accounts</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Retrieves information about <<service-accounts,service accounts>>.
 
 NOTE: Currently, only the `elastic/fleet-server` service account is available.
@@ -270,6 +276,21 @@ GET /_security/service/elastic/fleet-server
             "view_index_metadata"
           ],
           "allow_restricted_indices": false
+        },
+        {
+          "names": [
+            "agentless-*",
+          ],
+          "privileges": [
+            "read",
+            "write",
+            "monitor",
+            "create_index",
+            "auto_configure",
+            "maintenance",
+            "view_index_metadata"
+          ],
+          "allow_restricted_indices": false
         }
       ],
       "applications": [
diff --git a/docs/reference/rest-api/security/get-service-credentials.asciidoc b/docs/reference/rest-api/security/get-service-credentials.asciidoc
index 3da6c3d860558..7a24aef059ae9 100644
--- a/docs/reference/rest-api/security/get-service-credentials.asciidoc
+++ b/docs/reference/rest-api/security/get-service-credentials.asciidoc
@@ -1,11 +1,16 @@
 [role="xpack"]
 [[security-api-get-service-credentials]]
 === Get service account credentials API
-
 ++++
 <titleabbrev>Get service account credentials</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Retrieves all service credentials for a  <<service-accounts,service account>>.
 
 [[security-api-get-service-credentials-request]]
diff --git a/docs/reference/rest-api/security/get-settings.asciidoc b/docs/reference/rest-api/security/get-settings.asciidoc
index 46e4e0cf529bb..c99b9bcedba21 100644
--- a/docs/reference/rest-api/security/get-settings.asciidoc
+++ b/docs/reference/rest-api/security/get-settings.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get Security settings</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Retrieves settings for the security internal indices.
 
 [[security-api-get-settings-prereqs]]
diff --git a/docs/reference/rest-api/security/get-tokens.asciidoc b/docs/reference/rest-api/security/get-tokens.asciidoc
index 9f5261a477bcb..eefc86528ef43 100644
--- a/docs/reference/rest-api/security/get-tokens.asciidoc
+++ b/docs/reference/rest-api/security/get-tokens.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get token</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Creates a bearer token for access without requiring basic authentication.
 
 [[security-api-get-token-request]]
diff --git a/docs/reference/rest-api/security/get-user-privileges.asciidoc b/docs/reference/rest-api/security/get-user-privileges.asciidoc
index 8115cd365c5a3..4e0d68b4b679c 100644
--- a/docs/reference/rest-api/security/get-user-privileges.asciidoc
+++ b/docs/reference/rest-api/security/get-user-privileges.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get user privileges</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Retrieves the <<security-privileges,security privileges>> for the logged in 
 user.
 
diff --git a/docs/reference/rest-api/security/get-user-profile.asciidoc b/docs/reference/rest-api/security/get-user-profile.asciidoc
index 9f0ba64d136ac..60732cf0ab395 100644
--- a/docs/reference/rest-api/security/get-user-profile.asciidoc
+++ b/docs/reference/rest-api/security/get-user-profile.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get user profiles</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 NOTE: The user profile feature is designed only for use by {kib} and
 Elastic’s {observability}, {ents}, and {elastic-sec} solutions. Individual
 users and external applications should not call this API directly. Elastic reserves
diff --git a/docs/reference/rest-api/security/get-users.asciidoc b/docs/reference/rest-api/security/get-users.asciidoc
index 59a390f6f2538..8770c90cdec0a 100644
--- a/docs/reference/rest-api/security/get-users.asciidoc
+++ b/docs/reference/rest-api/security/get-users.asciidoc
@@ -5,8 +5,13 @@
 <titleabbrev>Get users</titleabbrev>
 ++++
 
-Retrieves information about users in the native realm and built-in users.
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
 
+Retrieves information about users in the native realm and built-in users.
 
 [[security-api-get-user-request]]
 ==== {api-request-title}
diff --git a/docs/reference/rest-api/security/grant-api-keys.asciidoc b/docs/reference/rest-api/security/grant-api-keys.asciidoc
index 10c109b00bbf9..4ab599c18dd61 100644
--- a/docs/reference/rest-api/security/grant-api-keys.asciidoc
+++ b/docs/reference/rest-api/security/grant-api-keys.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Grant API keys</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Creates an API key on behalf of another user.
 
 [[security-api-grant-api-key-request]]
diff --git a/docs/reference/rest-api/security/has-privileges-user-profile.asciidoc b/docs/reference/rest-api/security/has-privileges-user-profile.asciidoc
index afadf394aa43c..3fc825b60bce8 100644
--- a/docs/reference/rest-api/security/has-privileges-user-profile.asciidoc
+++ b/docs/reference/rest-api/security/has-privileges-user-profile.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Has privileges user profile</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 NOTE: The user profile feature is designed only for use by {kib} and
 Elastic’s {observability}, {ents}, and {elastic-sec} solutions. Individual
 users and external applications should not call this API directly. Elastic reserves
diff --git a/docs/reference/rest-api/security/has-privileges.asciidoc b/docs/reference/rest-api/security/has-privileges.asciidoc
index 229ffb4997273..1e08b41d92fe0 100644
--- a/docs/reference/rest-api/security/has-privileges.asciidoc
+++ b/docs/reference/rest-api/security/has-privileges.asciidoc
@@ -6,6 +6,12 @@
 ++++
 [[security-api-has-privilege]]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Determines whether the logged in user has a specified list of privileges.
 
 [[security-api-has-privileges-request]]
diff --git a/docs/reference/rest-api/security/invalidate-api-keys.asciidoc b/docs/reference/rest-api/security/invalidate-api-keys.asciidoc
index 57a36a97634ac..27c5a8c831808 100644
--- a/docs/reference/rest-api/security/invalidate-api-keys.asciidoc
+++ b/docs/reference/rest-api/security/invalidate-api-keys.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Invalidate API key</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Invalidates one or more API keys.
 
 [[security-api-invalidate-api-key-request]]
diff --git a/docs/reference/rest-api/security/invalidate-tokens.asciidoc b/docs/reference/rest-api/security/invalidate-tokens.asciidoc
index 58f20fdcdc425..9a6fe6185b477 100644
--- a/docs/reference/rest-api/security/invalidate-tokens.asciidoc
+++ b/docs/reference/rest-api/security/invalidate-tokens.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Invalidate token</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Invalidates one or more access tokens or refresh tokens.
 
 [[security-api-invalidate-token-request]]
diff --git a/docs/reference/rest-api/security/oidc-authenticate-api.asciidoc b/docs/reference/rest-api/security/oidc-authenticate-api.asciidoc
index 282a054717e6d..7bf8a1aad3ee1 100644
--- a/docs/reference/rest-api/security/oidc-authenticate-api.asciidoc
+++ b/docs/reference/rest-api/security/oidc-authenticate-api.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>OpenID Connect authenticate</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Submits the response to an oAuth 2.0 authentication request for consumption from
 {es}. Upon successful validation, {es} will respond with an {es} internal Access
 Token and Refresh Token that can be subsequently used for authentication.
diff --git a/docs/reference/rest-api/security/oidc-logout-api.asciidoc b/docs/reference/rest-api/security/oidc-logout-api.asciidoc
index a181f4c836fbd..d8bd60b3cd85b 100644
--- a/docs/reference/rest-api/security/oidc-logout-api.asciidoc
+++ b/docs/reference/rest-api/security/oidc-logout-api.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>OpenID Connect logout</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Submits a request to invalidate a refresh token and an access token that was
 generated as a response to a call to `/_security/oidc/authenticate`.
 
diff --git a/docs/reference/rest-api/security/oidc-prepare-authentication-api.asciidoc b/docs/reference/rest-api/security/oidc-prepare-authentication-api.asciidoc
index 4452020b4547c..227f154934306 100644
--- a/docs/reference/rest-api/security/oidc-prepare-authentication-api.asciidoc
+++ b/docs/reference/rest-api/security/oidc-prepare-authentication-api.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>OpenID Connect prepare authentication</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Creates an oAuth 2.0 authentication request as a URL string based on the
 configuration of the respective OpenID Connect authentication realm in {es}.
 
diff --git a/docs/reference/rest-api/security/put-app-privileges.asciidoc b/docs/reference/rest-api/security/put-app-privileges.asciidoc
index 28be4c08c4e71..cf903d99a724d 100644
--- a/docs/reference/rest-api/security/put-app-privileges.asciidoc
+++ b/docs/reference/rest-api/security/put-app-privileges.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Create or update application privileges</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Adds or updates <<application-privileges,application privileges>>.
 
 [[security-api-put-privileges-request]]
diff --git a/docs/reference/rest-api/security/query-api-key.asciidoc b/docs/reference/rest-api/security/query-api-key.asciidoc
index 513cb99a55a4c..1dec37c166f89 100644
--- a/docs/reference/rest-api/security/query-api-key.asciidoc
+++ b/docs/reference/rest-api/security/query-api-key.asciidoc
@@ -1,11 +1,16 @@
 [role="xpack"]
 [[security-api-query-api-key]]
 === Query API key information API
-
 ++++
 <titleabbrev>Query API key information</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 ////
 [source,console]
 ----
diff --git a/docs/reference/rest-api/security/query-role.asciidoc b/docs/reference/rest-api/security/query-role.asciidoc
index 937bd263140fc..acdfbb45b84f6 100644
--- a/docs/reference/rest-api/security/query-role.asciidoc
+++ b/docs/reference/rest-api/security/query-role.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Query Role</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Retrieves roles with <<query-dsl,Query DSL>> in a <<paginate-search-results,paginated>> fashion.
 
 [[security-api-query-role-request]]
diff --git a/docs/reference/rest-api/security/saml-authenticate-api.asciidoc b/docs/reference/rest-api/security/saml-authenticate-api.asciidoc
index aa556a42d699c..4c156df6e1bb3 100644
--- a/docs/reference/rest-api/security/saml-authenticate-api.asciidoc
+++ b/docs/reference/rest-api/security/saml-authenticate-api.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>SAML authenticate</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Submits a SAML `Response` message to {es} for consumption.
 
 NOTE: This API is intended for use by custom web applications other than {kib}.
diff --git a/docs/reference/rest-api/security/saml-complete-logout-api.asciidoc b/docs/reference/rest-api/security/saml-complete-logout-api.asciidoc
index 1fb4ab1581abc..d4847fb481cdb 100644
--- a/docs/reference/rest-api/security/saml-complete-logout-api.asciidoc
+++ b/docs/reference/rest-api/security/saml-complete-logout-api.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>SAML complete logout</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Verifies the logout response sent from the SAML IdP.
 
 NOTE: This API is intended for use by custom web applications other than {kib}.
diff --git a/docs/reference/rest-api/security/saml-invalidate-api.asciidoc b/docs/reference/rest-api/security/saml-invalidate-api.asciidoc
index 21c10341c6fee..fb233f3903d1c 100644
--- a/docs/reference/rest-api/security/saml-invalidate-api.asciidoc
+++ b/docs/reference/rest-api/security/saml-invalidate-api.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>SAML invalidate</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Submits a SAML LogoutRequest message to {es} for consumption.
 
 NOTE: This API is intended for use by custom web applications other than {kib}.
diff --git a/docs/reference/rest-api/security/saml-logout-api.asciidoc b/docs/reference/rest-api/security/saml-logout-api.asciidoc
index 71729365865dc..560a713b5bf15 100644
--- a/docs/reference/rest-api/security/saml-logout-api.asciidoc
+++ b/docs/reference/rest-api/security/saml-logout-api.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>SAML logout</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Submits a request to invalidate an access token and refresh token.
 
 NOTE: This API is intended for use by custom web applications other than {kib}.
diff --git a/docs/reference/rest-api/security/saml-prepare-authentication-api.asciidoc b/docs/reference/rest-api/security/saml-prepare-authentication-api.asciidoc
index b62d3d2ac9f75..60b3451cc531d 100644
--- a/docs/reference/rest-api/security/saml-prepare-authentication-api.asciidoc
+++ b/docs/reference/rest-api/security/saml-prepare-authentication-api.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>SAML prepare authentication</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Creates a SAML authentication request (`<AuthnRequest>`) as a URL string, based on the configuration of the respective SAML realm in {es}.
 
 NOTE: This API is intended for use by custom web applications other than {kib}.
diff --git a/docs/reference/rest-api/security/saml-sp-metadata.asciidoc b/docs/reference/rest-api/security/saml-sp-metadata.asciidoc
index deecbf5f0e664..0f66b7a9f76f8 100644
--- a/docs/reference/rest-api/security/saml-sp-metadata.asciidoc
+++ b/docs/reference/rest-api/security/saml-sp-metadata.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>SAML service provider metadata</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Generate SAML metadata for a SAML 2.0 Service Provider.
 
 [[security-api-saml-sp-metadata-request]]
diff --git a/docs/reference/rest-api/security/ssl.asciidoc b/docs/reference/rest-api/security/ssl.asciidoc
index 3b8ba0eab6888..78b9aee301096 100644
--- a/docs/reference/rest-api/security/ssl.asciidoc
+++ b/docs/reference/rest-api/security/ssl.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>SSL certificate</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 The `certificates` API enables you to retrieve information about the X.509
 certificates that are used to encrypt communications in your {es} cluster.
 
diff --git a/docs/reference/rest-api/security/suggest-user-profile.asciidoc b/docs/reference/rest-api/security/suggest-user-profile.asciidoc
index ad01987a1e704..4ea04864e4a6e 100644
--- a/docs/reference/rest-api/security/suggest-user-profile.asciidoc
+++ b/docs/reference/rest-api/security/suggest-user-profile.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Suggest user profile</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 NOTE: The user profile feature is designed only for use by {kib} and
 Elastic’s {observability}, {ents}, and {elastic-sec} solutions. Individual
 users and external applications should not call this API directly. Elastic reserves
diff --git a/docs/reference/rest-api/security/update-api-key.asciidoc b/docs/reference/rest-api/security/update-api-key.asciidoc
index f297e3922a657..12dd883b5955d 100644
--- a/docs/reference/rest-api/security/update-api-key.asciidoc
+++ b/docs/reference/rest-api/security/update-api-key.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Update API key</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 [[security-api-update-api-key-request]]
 ==== {api-request-title}
 
diff --git a/docs/reference/rest-api/security/update-cross-cluster-api-key.asciidoc b/docs/reference/rest-api/security/update-cross-cluster-api-key.asciidoc
index b90cb6368eefb..5c5f55b73597a 100644
--- a/docs/reference/rest-api/security/update-cross-cluster-api-key.asciidoc
+++ b/docs/reference/rest-api/security/update-cross-cluster-api-key.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Update Cross-Cluster API key</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Update an existing cross-cluster API Key that is used for <<remote-clusters-api-key,API key based remote cluster>> access.
 
 
diff --git a/docs/reference/rest-api/security/update-settings.asciidoc b/docs/reference/rest-api/security/update-settings.asciidoc
index b227bb70b31d7..3ec2ef98153c4 100644
--- a/docs/reference/rest-api/security/update-settings.asciidoc
+++ b/docs/reference/rest-api/security/update-settings.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Update Security settings</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 Updates the settings of the security internal indices.
 
 
diff --git a/docs/reference/rest-api/security/update-user-profile-data.asciidoc b/docs/reference/rest-api/security/update-user-profile-data.asciidoc
index 01fa5e11d10e8..c461c169a517a 100644
--- a/docs/reference/rest-api/security/update-user-profile-data.asciidoc
+++ b/docs/reference/rest-api/security/update-user-profile-data.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Update user profile data</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs].
+--
+
 NOTE: The user profile feature is designed only for use by {kib} and
 Elastic’s {observability}, {ents}, and {elastic-sec} solutions. Individual
 users and external applications should not call this API directly. Elastic reserves
diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc
index b57d2aee9d190..cb9d80cc3a97b 100644
--- a/docs/reference/rest-api/usage.asciidoc
+++ b/docs/reference/rest-api/usage.asciidoc
@@ -2,6 +2,12 @@
 [[usage-api]]
 == Usage API
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-xpack[Usage APIs].
+--
+
 Provides usage information about the installed {xpack} features.
 
 [discrete]
@@ -207,16 +213,8 @@ GET /_xpack/usage
   "inference": {
     "available" : true,
     "enabled" : true,
-    "models" : [{
-        "service": "elasticsearch",
-        "task_type": "SPARSE_EMBEDDING",
-        "count": 1
-      },
-      {
-        "service": "elasticsearch",
-        "task_type": "TEXT_EMBEDDING",
-        "count": 1
-      },
+    "models" : [
+      ...
     ]
   },
   "logstash" : {
@@ -523,7 +521,10 @@ GET /_xpack/usage
     "available": true,
     "enabled": false,
     "indices_count": 0,
-    "indices_with_synthetic_source": 0
+    "indices_with_synthetic_source": 0,
+    "num_docs": 0,
+    "size_in_bytes": 0,
+    "has_custom_cutoff_date": false
   }
 }
 ------------------------------------------------------------
@@ -535,6 +536,7 @@ GET /_xpack/usage
 // TESTRESPONSE[s/"policy_stats" : \[[^\]]*\]/"policy_stats" : $body.$_path/]
 // TESTRESPONSE[s/"slm" : \{[^\}]*\},/"slm" : $body.$_path,/]
 // TESTRESPONSE[s/"health_api" : \{[^\}]*\}\s*\}/"health_api" : $body.$_path/]
+// TESTRESPONSE[s/"models" : \[[^\]]*\]/"models" : $body.$_path/]
 // TESTRESPONSE[s/"data_streams" : \{[^\}]*\},/"data_streams" : $body.$_path,/]
 // TESTRESPONSE[s/ : true/ : $body.$_path/]
 // TESTRESPONSE[s/ : false/ : $body.$_path/]
@@ -551,4 +553,5 @@ GET /_xpack/usage
 // 5. All of the numbers and strings on the right hand side of *every* field in
 //    the response are ignored. So we're really only asserting things about the
 //    the shape of this response, not the values in it.
-// 6. Ignore the contents of data streams until the failure store is tech preview.
+// 6. Ignore the contents of the `inference.models` array because the models might not yet have been initialized
+// 7. Ignore the contents of data streams until the failure store is tech preview.
diff --git a/docs/reference/rest-api/watcher.asciidoc b/docs/reference/rest-api/watcher.asciidoc
index 4c4ce1ab7ee00..227eda6605465 100644
--- a/docs/reference/rest-api/watcher.asciidoc
+++ b/docs/reference/rest-api/watcher.asciidoc
@@ -2,6 +2,12 @@
 [[watcher-api]]
 == Watcher APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs].
+--
+
 * <<watcher-api-put-watch>>
 * <<watcher-api-get-watch>>
 * <<watcher-api-query-watches>>
diff --git a/docs/reference/rest-api/watcher/ack-watch.asciidoc b/docs/reference/rest-api/watcher/ack-watch.asciidoc
index 3c45b068a34c6..b3ea9b5d6b41f 100644
--- a/docs/reference/rest-api/watcher/ack-watch.asciidoc
+++ b/docs/reference/rest-api/watcher/ack-watch.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Ack watch</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs].
+--
+
 <<actions-ack-throttle,Acknowledging a watch>> enables you
 to manually throttle execution of the watch's actions.
 
diff --git a/docs/reference/rest-api/watcher/activate-watch.asciidoc b/docs/reference/rest-api/watcher/activate-watch.asciidoc
index d8af79854c83e..c37d85cc50299 100644
--- a/docs/reference/rest-api/watcher/activate-watch.asciidoc
+++ b/docs/reference/rest-api/watcher/activate-watch.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Activate watch</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs].
+--
+
 A watch can be either <<watch-active-state,active or inactive>>. This
 API enables you to activate a currently inactive watch.
 
diff --git a/docs/reference/rest-api/watcher/deactivate-watch.asciidoc b/docs/reference/rest-api/watcher/deactivate-watch.asciidoc
index ba4170174343a..058ada195f97e 100644
--- a/docs/reference/rest-api/watcher/deactivate-watch.asciidoc
+++ b/docs/reference/rest-api/watcher/deactivate-watch.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Deactivate watch</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs].
+--
+
 A watch can be either <<watch-active-state,active or inactive>>. This
 API enables you to deactivate a currently active watch.
 
diff --git a/docs/reference/rest-api/watcher/delete-watch.asciidoc b/docs/reference/rest-api/watcher/delete-watch.asciidoc
index 3ffcb43ed65e2..536ec293ab8fd 100644
--- a/docs/reference/rest-api/watcher/delete-watch.asciidoc
+++ b/docs/reference/rest-api/watcher/delete-watch.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete watch</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs].
+--
+
 Removes a watch from {watcher}.
 
 [[watcher-api-delete-watch-request]]
diff --git a/docs/reference/rest-api/watcher/execute-watch.asciidoc b/docs/reference/rest-api/watcher/execute-watch.asciidoc
index 7acecf1709034..eab15a152b154 100644
--- a/docs/reference/rest-api/watcher/execute-watch.asciidoc
+++ b/docs/reference/rest-api/watcher/execute-watch.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Execute watch</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs].
+--
+
 Forces the execution of a stored watch.
 
 [[watcher-api-execute-watch-request]]
diff --git a/docs/reference/rest-api/watcher/get-settings.asciidoc b/docs/reference/rest-api/watcher/get-settings.asciidoc
index c5773e6ee32b0..80d86cc455daa 100644
--- a/docs/reference/rest-api/watcher/get-settings.asciidoc
+++ b/docs/reference/rest-api/watcher/get-settings.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get Watcher settings</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs].
+--
+
 This API allows a user to retrieve the user-configurable settings for the Watcher internal index (`.watches`). Only a subset of the index settings—those that are user-configurable—will be shown. This includes:
 
 - `index.auto_expand_replicas`
diff --git a/docs/reference/rest-api/watcher/get-watch.asciidoc b/docs/reference/rest-api/watcher/get-watch.asciidoc
index e80bfed88b6e5..1b5dbe2d0e47e 100644
--- a/docs/reference/rest-api/watcher/get-watch.asciidoc
+++ b/docs/reference/rest-api/watcher/get-watch.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get watch</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs].
+--
+
 Retrieves a watch by its ID.
 
 [[watcher-api-get-watch-request]]
diff --git a/docs/reference/rest-api/watcher/put-watch.asciidoc b/docs/reference/rest-api/watcher/put-watch.asciidoc
index deab44f106fb6..134e8149fde79 100644
--- a/docs/reference/rest-api/watcher/put-watch.asciidoc
+++ b/docs/reference/rest-api/watcher/put-watch.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Create or update watch</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs].
+--
+
 Either registers a new watch in {watcher} or updates an existing one.
 
 [[watcher-api-put-watch-request]]
diff --git a/docs/reference/rest-api/watcher/query-watches.asciidoc b/docs/reference/rest-api/watcher/query-watches.asciidoc
index 7a006243ed7f4..b96261d4a47ea 100644
--- a/docs/reference/rest-api/watcher/query-watches.asciidoc
+++ b/docs/reference/rest-api/watcher/query-watches.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Query watches</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs].
+--
+
 Retrieves all registered watches.
 
 [[watcher-api-query-watches-request]]
diff --git a/docs/reference/rest-api/watcher/start.asciidoc b/docs/reference/rest-api/watcher/start.asciidoc
index b153410ed2901..15606ba0a8571 100644
--- a/docs/reference/rest-api/watcher/start.asciidoc
+++ b/docs/reference/rest-api/watcher/start.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Start watch service</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs].
+--
+
 Starts the {watcher} service if it is not already running.
 
 [[watcher-api-start-request]]
diff --git a/docs/reference/rest-api/watcher/stats.asciidoc b/docs/reference/rest-api/watcher/stats.asciidoc
index 2dbca69a67616..e0bdc1b5a8892 100644
--- a/docs/reference/rest-api/watcher/stats.asciidoc
+++ b/docs/reference/rest-api/watcher/stats.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Get {watcher} stats</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs].
+--
+
 Retrieves the current {watcher} metrics.
 
 [[watcher-api-stats-request]]
diff --git a/docs/reference/rest-api/watcher/stop.asciidoc b/docs/reference/rest-api/watcher/stop.asciidoc
index 50acd6e9eb2d1..272899a2cfa3f 100644
--- a/docs/reference/rest-api/watcher/stop.asciidoc
+++ b/docs/reference/rest-api/watcher/stop.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Stop watch service</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs].
+--
+
 Stops the {watcher} service if it is running.
 
 [[watcher-api-stop-request]]
diff --git a/docs/reference/rest-api/watcher/update-settings.asciidoc b/docs/reference/rest-api/watcher/update-settings.asciidoc
index 8602c6776997d..9ad38064e34ab 100644
--- a/docs/reference/rest-api/watcher/update-settings.asciidoc
+++ b/docs/reference/rest-api/watcher/update-settings.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Update Watcher settings</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs].
+--
+
 This API allows a user to modify the settings for the Watcher internal index (`.watches`). Only a subset of settings are allowed to by modified. This includes:
 
 - `index.auto_expand_replicas`
diff --git a/docs/reference/rollup/apis/delete-job.asciidoc b/docs/reference/rollup/apis/delete-job.asciidoc
index c563e705039e2..03f5349e15d4f 100644
--- a/docs/reference/rollup/apis/delete-job.asciidoc
+++ b/docs/reference/rollup/apis/delete-job.asciidoc
@@ -8,6 +8,12 @@
 
 deprecated::[8.11.0,"Rollups will be removed in a future version. Use <<downsampling,downsampling>> instead."]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs].
+--
+
 Deletes an existing {rollup-job}.
 
 [[rollup-delete-job-request]]
diff --git a/docs/reference/rollup/apis/get-job.asciidoc b/docs/reference/rollup/apis/get-job.asciidoc
index fcafbbe95159b..9fff4d665f5fd 100644
--- a/docs/reference/rollup/apis/get-job.asciidoc
+++ b/docs/reference/rollup/apis/get-job.asciidoc
@@ -7,6 +7,12 @@
 
 deprecated::[8.11.0,"Rollups will be removed in a future version. Use <<downsampling,downsampling>> instead."]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs].
+--
+
 Retrieves the configuration, stats, and status of {rollup-jobs}.
 
 [[rollup-get-job-request]]
diff --git a/docs/reference/rollup/apis/put-job.asciidoc b/docs/reference/rollup/apis/put-job.asciidoc
index c52e7a042e0ca..a60f20a3de5bf 100644
--- a/docs/reference/rollup/apis/put-job.asciidoc
+++ b/docs/reference/rollup/apis/put-job.asciidoc
@@ -12,6 +12,12 @@ WARNING: From 8.15.0 invoking this API in a cluster with no rollup usage will fa
 deprecation and planned removal. A cluster either needs to contain a rollup job or a rollup index in order for this API
 to be allowed to execute.
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs].
+--
+
 Creates a {rollup-job}.
 
 [[rollup-put-job-api-request]]
diff --git a/docs/reference/rollup/apis/rollup-caps.asciidoc b/docs/reference/rollup/apis/rollup-caps.asciidoc
index 95f652f6d4415..be1c3ed171a23 100644
--- a/docs/reference/rollup/apis/rollup-caps.asciidoc
+++ b/docs/reference/rollup/apis/rollup-caps.asciidoc
@@ -7,6 +7,12 @@
 
 deprecated::[8.11.0,"Rollups will be removed in a future version. Use <<downsampling,downsampling>> instead."]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs].
+--
+
 Returns the capabilities of any {rollup-jobs} that have been configured for a
 specific index or index pattern.
 
diff --git a/docs/reference/rollup/apis/rollup-index-caps.asciidoc b/docs/reference/rollup/apis/rollup-index-caps.asciidoc
index c5b729f2e52e6..830cc332e8f40 100644
--- a/docs/reference/rollup/apis/rollup-index-caps.asciidoc
+++ b/docs/reference/rollup/apis/rollup-index-caps.asciidoc
@@ -7,6 +7,12 @@
 
 deprecated::[8.11.0,"Rollups will be removed in a future version. Use <<downsampling,downsampling>> instead."]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs].
+--
+
 Returns the rollup capabilities of all jobs inside of a rollup index (e.g. the
 index where rollup data is stored).
 
diff --git a/docs/reference/rollup/apis/rollup-search.asciidoc b/docs/reference/rollup/apis/rollup-search.asciidoc
index 491dcc6c38ae2..088a74973806b 100644
--- a/docs/reference/rollup/apis/rollup-search.asciidoc
+++ b/docs/reference/rollup/apis/rollup-search.asciidoc
@@ -7,6 +7,12 @@
 
 deprecated::[8.11.0,"Rollups will be removed in a future version. Use <<downsampling,downsampling>> instead."]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs].
+--
+
 Enables searching rolled-up data using the standard Query DSL.
 
 [[rollup-search-request]]
diff --git a/docs/reference/rollup/apis/start-job.asciidoc b/docs/reference/rollup/apis/start-job.asciidoc
index c102c26ea5d8e..dbeed8b09d1c8 100644
--- a/docs/reference/rollup/apis/start-job.asciidoc
+++ b/docs/reference/rollup/apis/start-job.asciidoc
@@ -8,6 +8,12 @@
 
 deprecated::[8.11.0,"Rollups will be removed in a future version. Use <<downsampling,downsampling>> instead."]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs].
+--
+
 Starts an existing, stopped {rollup-job}.
 
 [[rollup-start-job-request]]
diff --git a/docs/reference/rollup/apis/stop-job.asciidoc b/docs/reference/rollup/apis/stop-job.asciidoc
index 61e561b4ceac9..8c0fd6ab2f3af 100644
--- a/docs/reference/rollup/apis/stop-job.asciidoc
+++ b/docs/reference/rollup/apis/stop-job.asciidoc
@@ -8,6 +8,12 @@
 
 deprecated::[8.11.0,"Rollups will be removed in a future version. Use <<downsampling,downsampling>> instead."]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs].
+--
+
 Stops an existing, started {rollup-job}.
 
 [[rollup-stop-job-request]]
diff --git a/docs/reference/rollup/rollup-apis.asciidoc b/docs/reference/rollup/rollup-apis.asciidoc
index 44833a0846c2f..71922f0932a0e 100644
--- a/docs/reference/rollup/rollup-apis.asciidoc
+++ b/docs/reference/rollup/rollup-apis.asciidoc
@@ -4,6 +4,12 @@
 
 deprecated::[8.11.0,"Rollups will be removed in a future version. Please <<rollup-migrating-to-downsampling,migrate>> to <<downsampling,downsampling>> instead."]
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs].
+--
+
 [discrete]
 [[rollup-jobs-endpoint]]
 === Jobs
diff --git a/docs/reference/scripting/apis/create-stored-script-api.asciidoc b/docs/reference/scripting/apis/create-stored-script-api.asciidoc
index dab1314e65dc4..5636e212180b3 100644
--- a/docs/reference/scripting/apis/create-stored-script-api.asciidoc
+++ b/docs/reference/scripting/apis/create-stored-script-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Create or update stored script</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-script[Script APIs].
+--
+
 Creates or updates a <<script-stored-scripts,stored script>> or
 <<search-template,search template>>.
 
diff --git a/docs/reference/scripting/apis/delete-stored-script-api.asciidoc b/docs/reference/scripting/apis/delete-stored-script-api.asciidoc
index e233922c9a7d5..c6e570f2013e4 100644
--- a/docs/reference/scripting/apis/delete-stored-script-api.asciidoc
+++ b/docs/reference/scripting/apis/delete-stored-script-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Delete stored script</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-script[Script APIs].
+--
+
 Deletes a <<script-stored-scripts,stored script>> or <<search-template,search
 template>>.
 
diff --git a/docs/reference/scripting/apis/get-script-contexts-api.asciidoc b/docs/reference/scripting/apis/get-script-contexts-api.asciidoc
index ca24c97e494ee..0ef6eccf947ad 100644
--- a/docs/reference/scripting/apis/get-script-contexts-api.asciidoc
+++ b/docs/reference/scripting/apis/get-script-contexts-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get script contexts</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-script[Script APIs].
+--
+
 Retrieves a list of supported script contexts and their methods.
 
 [source,console]
diff --git a/docs/reference/scripting/apis/get-script-languages-api.asciidoc b/docs/reference/scripting/apis/get-script-languages-api.asciidoc
index dd5935bc4dcd8..a35c979b2fccb 100644
--- a/docs/reference/scripting/apis/get-script-languages-api.asciidoc
+++ b/docs/reference/scripting/apis/get-script-languages-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get script languages</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-script[Script APIs].
+--
+
 Retrieves a list of supported <<scripting-available-languages,script languages>>
 and their contexts.
 
diff --git a/docs/reference/scripting/apis/get-stored-script-api.asciidoc b/docs/reference/scripting/apis/get-stored-script-api.asciidoc
index fffeb24e0331f..d2e5a7beedad1 100644
--- a/docs/reference/scripting/apis/get-stored-script-api.asciidoc
+++ b/docs/reference/scripting/apis/get-stored-script-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get stored script</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-script[Script APIs].
+--
+
 Retrieves a <<script-stored-scripts,stored script>> or <<search-template,search
 template>>.
 
diff --git a/docs/reference/scripting/apis/script-apis.asciidoc b/docs/reference/scripting/apis/script-apis.asciidoc
index e344cb00ee6fe..f5499ade8458a 100644
--- a/docs/reference/scripting/apis/script-apis.asciidoc
+++ b/docs/reference/scripting/apis/script-apis.asciidoc
@@ -1,6 +1,12 @@
 [[script-apis]]
 == Script APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-script[Script APIs].
+--
+
 Use the following APIs to manage, store, and test your
 <<modules-scripting,scripts>>.
 
diff --git a/docs/reference/scripting/using.asciidoc b/docs/reference/scripting/using.asciidoc
index d4b4fd91e3e37..7dc1e38c62e78 100644
--- a/docs/reference/scripting/using.asciidoc
+++ b/docs/reference/scripting/using.asciidoc
@@ -201,8 +201,13 @@ when you're creating <<runtime-mapping-fields,runtime fields>>.
 [[script-stored-scripts]]
 === Store and retrieve scripts
 You can store and retrieve scripts from the cluster state using the
-<<stored-script-apis,stored script APIs>>. Stored scripts reduce compilation
-time and make searches faster.
+<<stored-script-apis,stored script APIs>>. Stored scripts allow you to reference 
+shared scripts for operations like scoring, aggregating, filtering, and 
+reindexing. Instead of embedding scripts inline in each query, you can reference 
+these shared operations.
+
+Stored scripts can also reduce request payload size. Depending on script size 
+and request frequency, this can help lower latency and data transfer costs.
 
 NOTE: Unlike regular scripts, stored scripts require that you specify a script
 language using the `lang` parameter.
diff --git a/docs/reference/search-application/apis/delete-search-application.asciidoc b/docs/reference/search-application/apis/delete-search-application.asciidoc
index 4043942b09503..52a32247a79e1 100644
--- a/docs/reference/search-application/apis/delete-search-application.asciidoc
+++ b/docs/reference/search-application/apis/delete-search-application.asciidoc
@@ -1,13 +1,18 @@
 [role="xpack"]
 [[delete-search-application]]
 === Delete Search Application
-
-beta::[]
-
 ++++
 <titleabbrev>Delete Search Application</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search_application[Search application APIs].
+--
+
+beta::[]
+
 Removes a Search Application and its associated alias.
 Indices attached to the Search Application are not removed.
 
diff --git a/docs/reference/search-application/apis/get-search-application.asciidoc b/docs/reference/search-application/apis/get-search-application.asciidoc
index f0c107011eb40..adeb84bdbe0ae 100644
--- a/docs/reference/search-application/apis/get-search-application.asciidoc
+++ b/docs/reference/search-application/apis/get-search-application.asciidoc
@@ -1,13 +1,18 @@
 [role="xpack"]
 [[get-search-application]]
 === Get Search Application
-
-beta::[]
-
 ++++
 <titleabbrev>Get Search Application</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search_application[Search application APIs].
+--
+
+beta::[]
+
 Retrieves information about a search application.
 
 [[get-search-application-request]]
diff --git a/docs/reference/search-application/apis/index.asciidoc b/docs/reference/search-application/apis/index.asciidoc
index 1df38f6a841cc..a01b93a1ee4ac 100644
--- a/docs/reference/search-application/apis/index.asciidoc
+++ b/docs/reference/search-application/apis/index.asciidoc
@@ -9,6 +9,12 @@ beta::[]
 
 ---
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search_application[Search application APIs].
+--
+
 Use Search Application APIs to manage tasks and resources related to Search Applications.
 
 * <<put-search-application>>
diff --git a/docs/reference/search-application/apis/list-search-applications.asciidoc b/docs/reference/search-application/apis/list-search-applications.asciidoc
index 3cc077bf682d6..33bd8ddee009e 100644
--- a/docs/reference/search-application/apis/list-search-applications.asciidoc
+++ b/docs/reference/search-application/apis/list-search-applications.asciidoc
@@ -1,13 +1,18 @@
 [role="xpack"]
 [[list-search-applications]]
 === List Search Applications
-
-beta::[]
-
 ++++
 <titleabbrev>List Search Applications</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search_application[Search application APIs].
+--
+
+beta::[]
+
 Returns information about Search Applications.
 
 [[list-search-applications-request]]
diff --git a/docs/reference/search-application/apis/put-search-application.asciidoc b/docs/reference/search-application/apis/put-search-application.asciidoc
index dc5e20ec40b7f..bb8edb5368045 100644
--- a/docs/reference/search-application/apis/put-search-application.asciidoc
+++ b/docs/reference/search-application/apis/put-search-application.asciidoc
@@ -1,13 +1,18 @@
 [role="xpack"]
 [[put-search-application]]
 === Put Search Application
-
-beta::[]
-
 ++++
 <titleabbrev>Put Search Application</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search_application[Search application APIs].
+--
+
+beta::[]
+
 Creates or updates a Search Application.
 
 [[put-search-application-request]]
diff --git a/docs/reference/search-application/apis/search-application-render-query.asciidoc b/docs/reference/search-application/apis/search-application-render-query.asciidoc
index 687176b4fb070..42586c7186bc8 100644
--- a/docs/reference/search-application/apis/search-application-render-query.asciidoc
+++ b/docs/reference/search-application/apis/search-application-render-query.asciidoc
@@ -1,13 +1,18 @@
 [role="xpack"]
 [[search-application-render-query]]
 === Render Search Application Query
-
-preview::[]
-
 ++++
 <titleabbrev>Render Search Application Query</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search_application[Search application APIs].
+--
+
+preview::[]
+
 Given specified query parameters, generates an {es} query using the search template associated with the search
 application or a default template if none is specified.
 Unspecified template parameters will be assigned their default values (if applicable).
diff --git a/docs/reference/search-application/apis/search-application-search.asciidoc b/docs/reference/search-application/apis/search-application-search.asciidoc
index 2d13ed5f11037..1aab4ddf84763 100644
--- a/docs/reference/search-application/apis/search-application-search.asciidoc
+++ b/docs/reference/search-application/apis/search-application-search.asciidoc
@@ -1,13 +1,18 @@
 [role="xpack"]
 [[search-application-search]]
 === Search Application Search
-
-beta::[]
-
 ++++
 <titleabbrev>Search Application Search</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search_application[Search application APIs].
+--
+
+beta::[]
+
 Given specified query parameters, generates and executes an {es} query using the search template associated
 with the search application or a default template if none is specified.
 Unspecified template parameters will be assigned their default values (if applicable).
diff --git a/docs/reference/search.asciidoc b/docs/reference/search.asciidoc
index 70ffe02e44d95..7db9b8a304be5 100644
--- a/docs/reference/search.asciidoc
+++ b/docs/reference/search.asciidoc
@@ -1,6 +1,12 @@
 [[search]]
 == Search APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 Search APIs are used to search and aggregate data stored in {es} indices and
 data streams. For an overview and related tutorials, see <<search-your-data>>.
 
diff --git a/docs/reference/search/async-search.asciidoc b/docs/reference/search/async-search.asciidoc
index 786cfaee8024c..9a9e9ca45e817 100644
--- a/docs/reference/search/async-search.asciidoc
+++ b/docs/reference/search/async-search.asciidoc
@@ -2,6 +2,12 @@
 [[async-search]]
 === Async search
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 The async search API let you asynchronously execute a search request, monitor
 its progress, and retrieve partial results as they become available.
 
diff --git a/docs/reference/search/clear-scroll-api.asciidoc b/docs/reference/search/clear-scroll-api.asciidoc
index a005babfd1bef..a6a2cd4b3cab5 100644
--- a/docs/reference/search/clear-scroll-api.asciidoc
+++ b/docs/reference/search/clear-scroll-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Clear scroll</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 Clears the search context and results for a
 <<scroll-search-results,scrolling search>>.
 
diff --git a/docs/reference/search/count.asciidoc b/docs/reference/search/count.asciidoc
index 399545adf8d1d..e0e86dd131291 100644
--- a/docs/reference/search/count.asciidoc
+++ b/docs/reference/search/count.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Count</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 Gets the number of matches for a search query.
 
 [source,console]
diff --git a/docs/reference/search/explain.asciidoc b/docs/reference/search/explain.asciidoc
index 77e2d5bd63efd..01c7bb4611a12 100644
--- a/docs/reference/search/explain.asciidoc
+++ b/docs/reference/search/explain.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Explain</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 Returns information about why a specific document matches (or doesn't match) a 
 query.
 
diff --git a/docs/reference/search/field-caps.asciidoc b/docs/reference/search/field-caps.asciidoc
index 2ff2b8d18604e..f4d9146b8ea9d 100644
--- a/docs/reference/search/field-caps.asciidoc
+++ b/docs/reference/search/field-caps.asciidoc
@@ -4,6 +4,11 @@
 <titleabbrev>Field capabilities</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
 
 Allows you to retrieve the capabilities of fields among multiple indices.
 For data streams, the API returns field capabilities among the stream's backing
diff --git a/docs/reference/search/multi-search-template-api.asciidoc b/docs/reference/search/multi-search-template-api.asciidoc
index b1c9518b1f2bc..2fdb412575eb4 100644
--- a/docs/reference/search/multi-search-template-api.asciidoc
+++ b/docs/reference/search/multi-search-template-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Multi search template</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 Runs multiple <<run-multiple-templated-searches,templated searches>> with a single
 request.
 
@@ -79,7 +85,7 @@ cross-cluster search requests. Defaults to `true`.
 
 `max_concurrent_searches`::
 (Optional, integer) Maximum number of concurrent searches the API can run.
-Defaults to +max(1, (# of <<data-node,data nodes>> *
+Defaults to +max(1, (# of <<data-node-role,data nodes>> *
 min(<<search-threadpool,search thread pool size>>, 10)))+.
 
 `rest_total_hits_as_int`::
diff --git a/docs/reference/search/multi-search.asciidoc b/docs/reference/search/multi-search.asciidoc
index 9cafa756f035e..6adcc62e5ec4f 100644
--- a/docs/reference/search/multi-search.asciidoc
+++ b/docs/reference/search/multi-search.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Multi search</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 Executes several searches with a single API request.
 
 [source,console]
@@ -91,7 +97,7 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailabl
 `max_concurrent_searches`::
 (Optional, integer)
 Maximum number of concurrent searches the multi search API can execute. Defaults
-to +max(1, (# of <<data-node,data nodes>> * min(<<search-threadpool,search thread pool size>>, 10)))+.
+to +max(1, (# of <<data-node-role,data nodes>> * min(<<search-threadpool,search thread pool size>>, 10)))+.
 
 `max_concurrent_shard_requests`::
 +
diff --git a/docs/reference/search/point-in-time-api.asciidoc b/docs/reference/search/point-in-time-api.asciidoc
index 9cd91626c7600..bc7cbd01ebb36 100644
--- a/docs/reference/search/point-in-time-api.asciidoc
+++ b/docs/reference/search/point-in-time-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Point in time</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 A search request by default executes against the most recent visible data of
 the target indices, which is called point in time. Elasticsearch pit (point in time)
 is a lightweight view into the state of the data as it existed when initiated.
diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc
index 5f1a0ccfdd6b4..4fbe5ea1bb9f8 100644
--- a/docs/reference/search/profile.asciidoc
+++ b/docs/reference/search/profile.asciidoc
@@ -24,6 +24,11 @@ The output from the Profile API is *very* verbose, especially for complicated
 requests executed across many shards. Pretty-printing the response is
 recommended to help understand the output.
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
 
 [[search-profile-api-example]]
 ==== {api-examples-title}
diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc
index 05862ebbbcca5..4a03371c4da3d 100644
--- a/docs/reference/search/rank-eval.asciidoc
+++ b/docs/reference/search/rank-eval.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Ranking evaluation</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 Allows you to evaluate the quality of ranked search results over a set of
 typical search queries.
 
diff --git a/docs/reference/search/render-search-template-api.asciidoc b/docs/reference/search/render-search-template-api.asciidoc
index 0c782f26068e6..42e82b6e352be 100644
--- a/docs/reference/search/render-search-template-api.asciidoc
+++ b/docs/reference/search/render-search-template-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Render search template</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 Renders a <<search-template,search
 template>> as a <<search-search,search request body>>.
 
diff --git a/docs/reference/search/retriever.asciidoc b/docs/reference/search/retriever.asciidoc
index f20e9148bf5e7..7e98297b780e6 100644
--- a/docs/reference/search/retriever.asciidoc
+++ b/docs/reference/search/retriever.asciidoc
@@ -14,6 +14,12 @@ Refer to <<retrievers-overview>> for a high level overview of the retrievers abs
 Refer to <<retrievers-examples, Retrievers examples>> for additional examples.
 ====
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 The following retrievers are available:
 
 `standard`::
@@ -22,6 +28,9 @@ A <<standard-retriever, retriever>> that replaces the functionality of a traditi
 `knn`::
 A <<knn-retriever, retriever>> that replaces the functionality of a <<search-api-knn, knn search>>.
 
+`rescorer`::
+A <<rescorer-retriever, retriever>> that replaces the functionality of the <<rescore, query rescorer>>.
+
 `rrf`::
 A <<rrf-retriever, retriever>> that produces top documents from <<rrf, reciprocal rank fusion (RRF)>>.
 
@@ -371,6 +380,122 @@ GET movies/_search
 ----
 // TEST[skip:uses ELSER]
 
+[[rescorer-retriever]]
+==== Rescorer Retriever
+
+The `rescorer` retriever re-scores only the results produced by its child retriever.
+For the `standard` and `knn` retrievers, the `window_size` parameter specifies the number of documents examined per shard.
+
+For compound retrievers like `rrf`, the `window_size` parameter defines the total number of documents examined globally.
+
+When using the `rescorer`, an error is returned if the following conditions are not met:
+
+* The minimum configured rescore's `window_size` is:
+** Greater than or equal to the `size` of the parent retriever for nested `rescorer` setups.
+** Greater than or equal to the `size` of the search request when used as the primary retriever in the tree.
+
+* And the maximum rescore's `window_size` is:
+** Smaller than or equal to the `size` or `rank_window_size` of the child retriever.
+
+[discrete]
+[[rescorer-retriever-parameters]]
+===== Parameters
+
+`rescore`::
+(Required. <<rescore, A rescorer definition or an array of rescorer definitions>>)
++
+Defines the <<rescore, rescorers>> applied sequentially to the top documents returned by the child retriever.
+
+`retriever`::
+(Required. <<retriever, retriever>>)
++
+Specifies the child retriever responsible for generating the initial set of top documents to be re-ranked.
+
+`filter`::
+(Optional. <<query-dsl, query object or list of query objects>>)
++
+Applies a <<query-dsl-bool-query, boolean query filter>> to the retriever, ensuring that all documents match the filter criteria without affecting their scores.
+
+[discrete]
+[[rescorer-retriever-example]]
+==== Example
+
+The `rescorer` retriever can be placed at any level within the retriever tree.
+The following example demonstrates a `rescorer` applied to the results produced by an `rrf` retriever:
+
+[source,console]
+----
+GET movies/_search
+{
+  "size": 10, <1>
+  "retriever": {
+    "rescorer": { <2>
+      "rescore": {
+        "query": { <3>
+          "window_size": 50, <4>
+          "rescore_query": {
+            "script_score": {
+              "script": {
+                "source": "cosineSimilarity(params.queryVector, 'product-vector_final_stage') + 1.0",
+                "params": {
+                  "queryVector": [-0.5, 90.0, -10, 14.8, -156.0]
+                }
+              }
+            }
+          }
+        }
+      },
+      "retriever": { <5>
+        "rrf": {
+          "rank_window_size": 100, <6>
+          "retrievers": [
+            {
+              "standard": {
+                "query": {
+                  "sparse_vector": {
+                    "field": "plot_embedding",
+                    "inference_id": "my-elser-model",
+                    "query": "films that explore psychological depths"
+                  }
+                }
+              }
+            },
+            {
+              "standard": {
+                "query": {
+                  "multi_match": {
+                    "query": "crime",
+                    "fields": [
+                      "plot",
+                      "title"
+                    ]
+                  }
+                }
+              }
+            },
+            {
+              "knn": {
+                "field": "vector",
+                "query_vector": [10, 22, 77],
+                "k": 10,
+                "num_candidates": 10
+              }
+            }
+          ]
+        }
+      }
+    }
+  }
+}
+----
+// TEST[skip:uses ELSER]
+<1> Specifies the number of top documents to return in the final response.
+<2> A `rescorer` retriever applied as the final step.
+<3> The definition of the `query` rescorer.
+<4> Defines the number of documents to rescore from the child retriever.
+<5> Specifies the child retriever definition.
+<6> Defines the number of documents returned by the `rrf` retriever, which limits the available documents to
+
 [[text-similarity-reranker-retriever]]
 ==== Text Similarity Re-ranker Retriever
 
@@ -777,4 +902,4 @@ When a retriever is specified as part of a search, the following elements are no
 * <<search-after, `search_after`>>
 * <<request-body-search-terminate-after, `terminate_after`>>
 * <<search-sort-param, `sort`>>
-* <<rescore, `rescore`>>
+* <<rescore, `rescore`>> use a <<rescorer-retriever, rescorer retriever>> instead
diff --git a/docs/reference/search/rrf.asciidoc b/docs/reference/search/rrf.asciidoc
index a942c0162a80a..842bd7049e3bf 100644
--- a/docs/reference/search/rrf.asciidoc
+++ b/docs/reference/search/rrf.asciidoc
@@ -27,6 +27,12 @@ return score
 [[rrf-api]]
 ==== Reciprocal rank fusion API
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 You can use RRF as part of a <<search-search, search>> to combine and rank documents using separate sets of top documents (result sets) from a combination of <<retriever, child retrievers>> using an
 <<rrf-retriever, RRF retriever>>.
 A minimum of *two* child retrievers is required for ranking.
diff --git a/docs/reference/search/scroll-api.asciidoc b/docs/reference/search/scroll-api.asciidoc
index e3b4123ddff68..0f89df877ba74 100644
--- a/docs/reference/search/scroll-api.asciidoc
+++ b/docs/reference/search/scroll-api.asciidoc
@@ -8,6 +8,12 @@ IMPORTANT: We no longer recommend using the scroll API for deep pagination. If
 you need to preserve the index state while paging through more than 10,000 hits,
 use the <<search-after,`search_after`>> parameter with a point in time (PIT).
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 Retrieves the next batch of results for a <<scroll-search-results,scrolling
 search>>.
 
diff --git a/docs/reference/search/search-shards.asciidoc b/docs/reference/search/search-shards.asciidoc
index 49045acf4c484..13f9ae8772858 100644
--- a/docs/reference/search/search-shards.asciidoc
+++ b/docs/reference/search/search-shards.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Search shards</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 Returns the indices and shards that a search request would be executed against.
 
 [source,console]
diff --git a/docs/reference/search/search-template-api.asciidoc b/docs/reference/search/search-template-api.asciidoc
index c60b5281c05e5..2094ee8924014 100644
--- a/docs/reference/search/search-template-api.asciidoc
+++ b/docs/reference/search/search-template-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Search template</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 Runs a search with a <<search-template,search template>>.
 
 ////
diff --git a/docs/reference/search/search-vector-tile-api.asciidoc b/docs/reference/search/search-vector-tile-api.asciidoc
index 2cdc29918a699..f63abda6fcb47 100644
--- a/docs/reference/search/search-vector-tile-api.asciidoc
+++ b/docs/reference/search/search-vector-tile-api.asciidoc
@@ -4,6 +4,11 @@
 <titleabbrev>Vector tile search</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
 
 Searches a vector tile for geospatial values. Returns results as a binary
 https://docs.mapbox.com/vector-tiles/specification[Mapbox vector tile].
diff --git a/docs/reference/search/search-your-data/full-text-search.asciidoc b/docs/reference/search/search-your-data/full-text-search.asciidoc
new file mode 100644
index 0000000000000..8641d0e45748a
--- /dev/null
+++ b/docs/reference/search/search-your-data/full-text-search.asciidoc
@@ -0,0 +1,82 @@
+[[full-text-search]]
+== Full-text search
+
+.Hands-on introduction to full-text search
+[TIP]
+====
+Would you prefer to jump straight into a hands-on tutorial?
+Refer to our quick start <<full-text-filter-tutorial,full-text search tutorial>>.
+====
+
+Full-text search, also known as lexical search, is a technique for fast, efficient searching through text fields in documents.
+Documents and search queries are transformed to enable returning https://www.elastic.co/what-is/search-relevance[relevant] results instead of simply exact term matches.
+Fields of type <<text-field-type,`text`>> are analyzed and indexed for full-text search.
+
+Built on decades of information retrieval research, full-text search delivers reliable results that scale predictably as your data grows. Because it runs efficiently on CPUs, {es}'s full-text search requires minimal computational resources compared to GPU-intensive vector operations.
+
+You can combine full-text search with <<semantic-search,semantic search using vectors>> to build modern hybrid search applications. While vector search may require additional GPU resources, the full-text component remains cost-effective by leveraging existing CPU infrastructure.
+
+[discrete]
+[[full-text-search-how-it-works]]
+=== How full-text search works
+
+The following diagram illustrates the components of full-text search.
+
+image::images/search/full-text-search-overview.svg[Components of full-text search from analysis to relevance scoring, align=center, width=500]
+
+At a high level, full-text search involves the following:
+
+* <<analysis-overview,*Text analysis*>>: Analysis consists of a pipeline of sequential transformations. Text is transformed into a format optimized for searching using techniques such as stemming, lowercasing, and stop word elimination. {es} contains a number of built-in <<analysis-analyzers,analyzers>> and tokenizers, including options to analyze specific language text. You can also create custom analyzers.
++
+[TIP]
+====
+Refer to <<test-analyzer,Test an analyzer>> to learn how to test an analyzer and inspect the tokens and metadata it generates.
+====
+* *Inverted index creation*: After analysis is complete, {es} builds an inverted index from the resulting tokens.
+An inverted index is a data structure that maps each token to the documents that contain it.
+It's made up of two key components:
+** *Dictionary*: A sorted list of all unique terms in the collection of documents in your index.
+** *Posting list*: For each term, a list of document IDs where the term appears, along with optional metadata like term frequency and position.
+* *Relevance scoring*: Results are ranked by how relevant they are to the given query. The relevance score of each document is represented by a positive floating-point number called the `_score`. The higher the `_score`, the more relevant the document.
++
+The default <<index-modules-similarity,similarity algorithm>> {es} uses for calculating relevance scores is https://en.wikipedia.org/wiki/Okapi_BM25[Okapi BM25], a variation of the https://en.wikipedia.org/wiki/Tf–idf[TF-IDF algorithm]. BM25 calculates relevance scores based on term frequency, document frequency, and document length.
+Refer to this https://www.elastic.co/blog/practical-bm25-part-2-the-bm25-algorithm-and-its-variables[technical blog post] for a deep dive into BM25.
+* *Full-text search query*: Query text is analyzed <<analysis-index-search-time,the same way as the indexed text>>, and the resulting tokens are used to search the inverted index.
++ 
+Query DSL supports a number of <<full-text-queries,full-text queries>>.
++ 
+As of 8.17, {esql} also supports <<esql-search-functions,full-text search>> functions.
+
+[discrete]
+[[full-text-search-getting-started]]
+=== Getting started
+
+For a hands-on introduction to full-text search, refer to the <<full-text-filter-tutorial,full-text search tutorial>>.
+
+[discrete]
+[[full-text-search-learn-more]]
+=== Learn more
+
+Here are some resources to help you learn more about full-text search with {es}.
+
+*Core concepts*
+
+Learn about the core components of full-text search:
+
+* <<text,Text fields>>
+* <<analysis,Text analysis>>
+** <<analysis-tokenizers,Tokenizers>>
+** <<analysis-analyzers,Analyzers>>
+
+*{es} query languages*
+
+Learn how to build full-text search queries using {es}'s query languages:
+
+* <<full-text-queries,Full-text queries using Query DSL>> 
+* <<esql-search-functions,Full-text search functions in {esql}>>
+
+*Advanced topics*
+
+For a technical deep dive into {es}'s BM25 implementation read this blog post: https://www.elastic.co/blog/practical-bm25-part-2-the-bm25-algorithm-and-its-variables[The BM25 Algorithm and its Variables].
+
+To learn how to optimize the relevance of your search results, refer to <<recipes,Search relevance optimizations>>.
\ No newline at end of file
diff --git a/docs/reference/search/search-your-data/highlighting.asciidoc b/docs/reference/search/search-your-data/highlighting.asciidoc
index 6a432e6104524..63d9c632bffcf 100644
--- a/docs/reference/search/search-your-data/highlighting.asciidoc
+++ b/docs/reference/search/search-your-data/highlighting.asciidoc
@@ -262,9 +262,11 @@ max_analyzed_offset:: By default, the maximum number of characters
 analyzed for a highlight request is bounded by the value defined in the
 <<index-max-analyzed-offset, `index.highlight.max_analyzed_offset`>> setting,
 and when the number of characters exceeds this limit an error is returned. If
-this setting is set to a non-negative value, the highlighting stops at this defined
+this setting is set to a positive value, the highlighting stops at this defined
 maximum limit, and the rest of the text is not processed, thus not highlighted and
-no error is returned. The <<max-analyzed-offset, `max_analyzed_offset`>> query setting
+no error is returned. If it is specifically set to -1 then the value of
+<<index-max-analyzed-offset, `index.highlight.max_analyzed_offset`>> is used instead.
+For values < -1 or 0, an error is returned. The <<max-analyzed-offset, `max_analyzed_offset`>> query setting
 does *not* override the <<index-max-analyzed-offset, `index.highlight.max_analyzed_offset`>>
 which prevails when it's set to lower value than the query setting.
 
diff --git a/docs/reference/search/search-your-data/retrieval-augmented-generation.asciidoc b/docs/reference/search/search-your-data/retrieval-augmented-generation.asciidoc
new file mode 100644
index 0000000000000..2958999ede91d
--- /dev/null
+++ b/docs/reference/search/search-your-data/retrieval-augmented-generation.asciidoc
@@ -0,0 +1,76 @@
+[rag-elasticsearch]
+== Retrieval augmented generation
+
+.🍿 Prefer a video introduction?
+***********************
+Check out https://www.youtube.com/watch?v=OS4ZefUPAks[this short video] from the Elastic Snackable Series.
+***********************
+
+Retrieval Augmented Generation (RAG) is a technique for improving language model responses by grounding the model with additional, verifiable sources of information. It works by first retrieving relevant context from an external datastore, which is then added to the model's context window.
+
+RAG is a form of https://arxiv.org/abs/2301.00234[in-context learning], where the model learns from information provided at inference time.
+Compared to fine-tuning or continuous pre-training, RAG can be implemented more quickly and cheaply, and offers several advantages.
+
+image::images/search/rag-venn-diagram.svg[RAG sits at the intersection of information retrieval and generative AI, align=center, width=500]
+
+RAG sits at the intersection of https://www.elastic.co/what-is/information-retrieval[information retrieval] and generative AI.
+{es} is an excellent tool for implementing RAG, because it offers various retrieval capabilities, such as full-text search, vector search, and hybrid search, as well as other tools like filtering, aggregations, and security features.
+
+[discrete]
+[[rag-elasticsearch-advantages]]
+=== Advantages of RAG
+
+Implementing RAG with {es} has several advantages:
+
+* *Improved context:* Enables grounding the language model with additional, up-to-date, and/or private data.
+* *Reduced hallucination:* Helps minimize factual errors by enabling models to cite authoritative sources.
+* *Cost efficiency:* Requires less maintenance compared to fine-tuning or continuously pre-training models.
+* *Built-in security:* Controls data access by leveraging {es}'s <<authorization, user authorization>> features, such as role-based access control and field/document-level security.
+* *Simplified response parsing:* Eliminates the need for custom parsing logic by letting the language model handle parsing {es} responses and formatting the retrieved context.
+* *Flexible implementation:* Works with basic <<full-text-search,full-text search>>, and can be gradually updated to add more advanced and computationally intensive <<semantic-search,semantic search>> capabilities.
+
+[discrete]
+[[rag-elasticsearch-components]]
+=== RAG system overview
+
+The following diagram illustrates a simple RAG system using {es}.
+
+image::images/search/rag-schema.svg[Components of a simple RAG system using Elasticsearch, align=center, role="stretch"]
+
+The workflow is as follows:
+
+. The user submits a query.
+. Elasticsearch retrieves relevant documents using full-text search, vector search, or hybrid search.
+. The language model processes the context and generates a response, using custom instructions. Examples of custom instructions include "Cite a source" or "Provide a concise summary of the `content` field in markdown format."
+. The model returns the final response to the user.
+
+[TIP]
+====
+A more advanced setup might include query rewriting between steps 1 and 2. This intermediate step could use one or more additional language models with different instructions to reformulate queries for more specific and detailed responses.
+====
+
+[discrete]
+[[rag-elasticsearch-getting-started]]
+=== Getting started
+
+Start building RAG applications quickly with Playground, which seamlessly integrates {es} with language model providers.
+The Playground UI enables you to build, test, and deploy RAG interfaces on top of your {es} indices.
+
+Playground automatically selects the best retrieval methods for your data, while providing full control over the final {es} queries and language model instructions.
+You can also download the underlying Python code to integrate with your existing applications.
+
+Learn more in the {kibana-ref}/playground.html[Playground documentation] and 
+try the https://www.elastic.co/demo-gallery/ai-playground[interactive lab] for hands-on experience.
+
+[discrete]
+[[rag-elasticsearch-learn-more]]
+=== Learn more
+
+Learn more about building RAG systems using {es} in these blog posts:
+
+* https://www.elastic.co/blog/beyond-rag-basics-semantic-search-with-elasticsearch[Beyond RAG Basics: Advanced strategies for AI applications]
+* https://www.elastic.co/search-labs/blog/building-a-rag-system-with-gemma-hugging-face-elasticsearch[Building a RAG system with Gemma, Hugging Face, and Elasticsearch]
+* https://www.elastic.co/search-labs/blog/rag-agent-tool-elasticsearch-langchain[Building an agentic RAG tool with Elasticsearch and Langchain]
+
+
+
diff --git a/docs/reference/search/search-your-data/search-across-clusters.asciidoc b/docs/reference/search/search-your-data/search-across-clusters.asciidoc
index 5f9e92c575793..8d3768817e856 100644
--- a/docs/reference/search/search-your-data/search-across-clusters.asciidoc
+++ b/docs/reference/search/search-your-data/search-across-clusters.asciidoc
@@ -22,7 +22,7 @@ The following APIs support {ccs}:
 * experimental:[] <<eql-search-api,EQL search>>
 * experimental:[] <<sql-search-api,SQL search>>
 * experimental:[] <<search-vector-tile-api,Vector tile search>>
-* experimental:[] <<esql,ES|QL>>
+* experimental:[] <<esql-cross-clusters,ES|QL>>
 
 [discrete]
 === Prerequisites
diff --git a/docs/reference/search/search-your-data/search-application-api.asciidoc b/docs/reference/search/search-your-data/search-application-api.asciidoc
index 2fe28faed546f..89d9a4318944b 100644
--- a/docs/reference/search/search-your-data/search-application-api.asciidoc
+++ b/docs/reference/search/search-your-data/search-application-api.asciidoc
@@ -206,7 +206,7 @@ will return:
 // TEST[continued]
 
 This uses the default parameters that were defined with the template.
-You can also specify one or mre parameters to the render call, for example:
+You can also specify one or more parameters to the render call, for example:
 
 [source,console]
 ----
diff --git a/docs/reference/search/search-your-data/search-your-data.asciidoc b/docs/reference/search/search-your-data/search-your-data.asciidoc
index 9ef1ae0ebc59b..0828462fd1850 100644
--- a/docs/reference/search/search-your-data/search-your-data.asciidoc
+++ b/docs/reference/search/search-your-data/search-your-data.asciidoc
@@ -18,7 +18,7 @@ Search for exact values::
 Search for <<term-level-queries,exact values or ranges>> of numbers, dates, IPs,
 or strings.
 
-Full-text search::
+<<full-text-search,Full-text search>>::
 Use <<full-text-queries,full text queries>> to query <<analysis,unstructured
 textual data>> and find documents that best match query terms.
 
@@ -43,11 +43,13 @@ DSL, with a simplified user experience. Create search applications based on your
 results directly in the Kibana Search UI.
 
 include::search-api.asciidoc[]
+include::full-text-search.asciidoc[]
 include::../../how-to/recipes.asciidoc[]
 // ☝️ search relevance recipes
 include::retrievers-overview.asciidoc[]
 include::knn-search.asciidoc[]
 include::semantic-search.asciidoc[]
+include::retrieval-augmented-generation.asciidoc[]
 include::search-across-clusters.asciidoc[]
 include::search-with-synonyms.asciidoc[]
 include::search-application-overview.asciidoc[]
diff --git a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc
index 3448940b6fad7..987a24140b6b4 100644
--- a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc
+++ b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc
@@ -157,89 +157,7 @@ GET semantic-embeddings/_search
 <2> The query text.
 
 As a result, you receive the top 10 documents that are closest in meaning to the
-query from the `semantic-embedding` index:
-
-[source,console-result]
-------------------------------------------------------------
-"hits": [
-  {
-    "_index": "semantic-embeddings",
-    "_id": "Jy5065EBBFPLbFsdh_f9",
-    "_score": 21.487484,
-    "_source": {
-      "id": 8836652,
-      "content": {
-        "text": "There are a few foods and food groups that will help to fight inflammation and delayed onset muscle soreness (both things that are inevitable after a long, hard workout) when you incorporate them into your postworkout eats, whether immediately after your run or at a meal later in the day. Advertisement. Advertisement.",
-        "inference": {
-          "inference_id": "my-elser-endpoint",
-          "model_settings": {
-            "task_type": "sparse_embedding"
-          },
-          "chunks": [
-            {
-              "text": "There are a few foods and food groups that will help to fight inflammation and delayed onset muscle soreness (both things that are inevitable after a long, hard workout) when you incorporate them into your postworkout eats, whether immediately after your run or at a meal later in the day. Advertisement. Advertisement.",
-              "embeddings": {
-                (...)
-              }
-            }
-          ]
-        }
-      }
-    }
-  },
-  {
-    "_index": "semantic-embeddings",
-    "_id": "Ji5065EBBFPLbFsdh_f9",
-    "_score": 18.211695,
-    "_source": {
-      "id": 8836651,
-      "content": {
-        "text": "During Your Workout. There are a few things you can do during your workout to help prevent muscle injury and soreness. According to personal trainer and writer for Iron Magazine, Marc David, doing warm-ups and cool-downs between sets can help keep muscle soreness to a minimum.",
-        "inference": {
-          "inference_id": "my-elser-endpoint",
-          "model_settings": {
-            "task_type": "sparse_embedding"
-          },
-          "chunks": [
-            {
-              "text": "During Your Workout. There are a few things you can do during your workout to help prevent muscle injury and soreness. According to personal trainer and writer for Iron Magazine, Marc David, doing warm-ups and cool-downs between sets can help keep muscle soreness to a minimum.",
-              "embeddings": {
-                (...)
-              }
-            }
-          ]
-        }
-      }
-    }
-  },
-  {
-    "_index": "semantic-embeddings",
-    "_id": "Wi5065EBBFPLbFsdh_b9",
-    "_score": 13.089405,
-    "_source": {
-      "id": 8800197,
-      "content": {
-        "text": "This is especially important if the soreness is due to a weightlifting routine. For this time period, do not exert more than around 50% of the level of effort (weight, distance and speed) that caused the muscle groups to be sore.",
-        "inference": {
-          "inference_id": "my-elser-endpoint",
-          "model_settings": {
-            "task_type": "sparse_embedding"
-          },
-          "chunks": [
-            {
-              "text": "This is especially important if the soreness is due to a weightlifting routine. For this time period, do not exert more than around 50% of the level of effort (weight, distance and speed) that caused the muscle groups to be sore.",
-              "embeddings": {
-                (...)
-              }
-            }
-          ]
-        }
-      }
-    }
-  }
-]
-------------------------------------------------------------
-// NOTCONSOLE
+query from the `semantic-embedding` index.
 
 [discrete]
 [[semantic-text-further-examples]]
diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc
index 2ad407b4ae1e4..d022605db22b1 100644
--- a/docs/reference/search/search.asciidoc
+++ b/docs/reference/search/search.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Search</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 Returns search hits that match the query defined in the request.
 
 [source,console]
diff --git a/docs/reference/search/suggesters.asciidoc b/docs/reference/search/suggesters.asciidoc
index c5cdbc3bd6660..22f427477b905 100644
--- a/docs/reference/search/suggesters.asciidoc
+++ b/docs/reference/search/suggesters.asciidoc
@@ -3,6 +3,12 @@
 
 Suggests similar looking terms based on a provided text by using a suggester. 
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 [source,console]
 --------------------------------------------------
 POST my-index-000001/_search
diff --git a/docs/reference/search/terms-enum.asciidoc b/docs/reference/search/terms-enum.asciidoc
index 4f34deb985abe..46f6d3560ecdb 100644
--- a/docs/reference/search/terms-enum.asciidoc
+++ b/docs/reference/search/terms-enum.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Terms enum</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs].
+--
+
 The terms enum API can be used to discover terms in the index that match
 a partial string. Supported field types are <<keyword-field-type,`keyword`>>,
 <<constant-keyword-field-type,`constant_keyword`>>, <<flattened,`flattened`>>,
diff --git a/docs/reference/search/validate.asciidoc b/docs/reference/search/validate.asciidoc
index ce682e485cd27..ab943cf72b463 100644
--- a/docs/reference/search/validate.asciidoc
+++ b/docs/reference/search/validate.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Validate</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/operation/operation-indices-validate-query[Validate a query].
+--
+
 Validates a potentially expensive query without executing it.
 
 [source,console]
diff --git a/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc b/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc
index ff67be02e6d00..ac357e29d1d17 100644
--- a/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc
+++ b/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Clear cache</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-searchable_snapshots[Searchable snapshots APIs].
+--
+
 experimental::[]
 
 Clears indices and data streams from the shared cache for
diff --git a/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc b/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc
index b47bc2370ab10..f1613ea62492d 100644
--- a/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc
+++ b/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Mount snapshot</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-searchable_snapshots[Searchable snapshots APIs].
+--
+
 Mount a snapshot as a searchable snapshot index.
 
 [[searchable-snapshots-api-mount-request]]
diff --git a/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc b/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc
index 62faceb99d4fc..d42ba02876941 100644
--- a/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc
+++ b/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Cache stats</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-searchable_snapshots[Searchable snapshots APIs].
+--
+
 Retrieves statistics about the shared cache for <<partially-mounted,partially
 mounted indices>>.
 
diff --git a/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc b/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc
index f90e27ea63224..1a1856198d286 100644
--- a/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc
+++ b/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc
@@ -2,6 +2,12 @@
 [[searchable-snapshots-apis]]
 == Searchable snapshots APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-searchable_snapshots[Searchable snapshots APIs].
+--
+
 You can use the following APIs to perform searchable snapshots operations.
 
 * <<searchable-snapshots-api-mount-snapshot,Mount snapshot>>
diff --git a/docs/reference/searchable-snapshots/apis/shard-stats.asciidoc b/docs/reference/searchable-snapshots/apis/shard-stats.asciidoc
index 369d19da9ae56..f1eb9ac8d92b2 100644
--- a/docs/reference/searchable-snapshots/apis/shard-stats.asciidoc
+++ b/docs/reference/searchable-snapshots/apis/shard-stats.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Searchable snapshot statistics</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-searchable_snapshots[Searchable snapshots APIs].
+--
+
 experimental::[]
 
 Retrieves statistics about searchable snapshots.
diff --git a/docs/reference/security/authorization/built-in-roles.asciidoc b/docs/reference/security/authorization/built-in-roles.asciidoc
index 13812b915dc5e..846ab3b6f73aa 100644
--- a/docs/reference/security/authorization/built-in-roles.asciidoc
+++ b/docs/reference/security/authorization/built-in-roles.asciidoc
@@ -33,18 +33,6 @@ suitable for writing beats output to {es}.
 
 --
 
-[[built-in-roles-data-frame-transforms-admin]] `data_frame_transforms_admin` ::
-Grants `manage_data_frame_transforms` cluster privileges, which enable you to
-manage {transforms}. This role also includes all
-{kibana-ref}/kibana-privileges.html[Kibana privileges] for the {ml-features}.
-deprecated:[7.5.0,"Replaced by <<built-in-roles-transform-admin,`transform_admin`>>"].
-
-[[built-in-roles-data-frame-transforms-user]] `data_frame_transforms_user` ::
-Grants `monitor_data_frame_transforms` cluster privileges, which enable you to
-use {transforms}. This role also includes all
-{kibana-ref}/kibana-privileges.html[Kibana privileges] for the {ml-features}.
-deprecated:[7.5.0,"Replaced by <<built-in-roles-transform-user,`transform_user`>>"].
-
 [[built-in-roles-editor]] `editor` ::
 
 Grants full access to all features in {kib} (including Solutions) and read-only access to data indices.
diff --git a/docs/reference/settings/ml-settings.asciidoc b/docs/reference/settings/ml-settings.asciidoc
index 1077a63b00249..08163b1391f2d 100644
--- a/docs/reference/settings/ml-settings.asciidoc
+++ b/docs/reference/settings/ml-settings.asciidoc
@@ -19,6 +19,8 @@ at all.
 
 // end::ml-settings-description-tag[]
 
+TIP: To control memory usage used by {ml} jobs, you can use the <<circuit-breakers-page-model-inference, machine learning circuit breaker settings>>.
+
 [discrete]
 [[general-ml-settings]]
 ==== General machine learning settings
@@ -67,7 +69,7 @@ limitations as described <<ml-settings-description,above>>.
 The inference cache exists in the JVM heap on each ingest node. The cache
 affords faster processing times for the `inference` processor. The value can be
 a static byte sized value (such as `2gb`) or a percentage of total allocated
-heap. Defaults to `40%`. See also <<model-inference-circuit-breaker>>.
+heap. Defaults to `40%`. See also <<circuit-breakers-page-model-inference>>.
 
 [[xpack-interference-model-ttl]]
 // tag::interference-model-ttl-tag[]
@@ -249,11 +251,4 @@ nodes in your cluster, you shouldn't use this setting.
 +
 If this setting is `true` it also affects the default value for
 `xpack.ml.max_model_memory_limit`. In this case `xpack.ml.max_model_memory_limit`
-defaults to the largest size that could be assigned in the current cluster.
-
-[discrete]
-[[model-inference-circuit-breaker]]
-==== {ml-cap} circuit breaker settings
-
-The relevant circuit breaker settings can be found in the <<circuit-breakers-page-model-inference, Circuit Breakers page>>.
-
+defaults to the largest size that could be assigned in the current cluster.
\ No newline at end of file
diff --git a/docs/reference/setup.asciidoc b/docs/reference/setup.asciidoc
index a284e563917c3..922f1bdba4d1f 100644
--- a/docs/reference/setup.asciidoc
+++ b/docs/reference/setup.asciidoc
@@ -27,6 +27,8 @@ the only resource-intensive application on the host or container. For example,
 you might run {metricbeat} alongside {es} for cluster statistics, but a
 resource-heavy {ls} deployment should be on its own host.
 
+// alphabetized
+
 include::run-elasticsearch-locally.asciidoc[]
 
 include::setup/install.asciidoc[]
@@ -47,30 +49,28 @@ include::settings/ccr-settings.asciidoc[]
 
 include::modules/discovery/discovery-settings.asciidoc[]
 
+include::settings/data-stream-lifecycle-settings.asciidoc[]
+
 include::modules/indices/fielddata.asciidoc[]
 
+include::modules/gateway.asciidoc[]
+
 include::settings/health-diagnostic-settings.asciidoc[]
 
 include::settings/ilm-settings.asciidoc[]
 
-include::settings/data-stream-lifecycle-settings.asciidoc[]
-
 include::modules/indices/index_management.asciidoc[]
 
 include::modules/indices/recovery.asciidoc[]
 
 include::modules/indices/indexing_buffer.asciidoc[]
 
-include::settings/license-settings.asciidoc[]
-
-include::modules/gateway.asciidoc[]
+include::settings/inference-settings.asciidoc[]
 
-include::setup/logging-config.asciidoc[]
+include::settings/license-settings.asciidoc[]
 
 include::settings/ml-settings.asciidoc[]
 
-include::settings/inference-settings.asciidoc[]
-
 include::settings/monitoring-settings.asciidoc[]
 
 include::modules/node.asciidoc[]
@@ -79,12 +79,12 @@ include::modules/network.asciidoc[]
 
 include::modules/indices/query_cache.asciidoc[]
 
+include::{es-ref-dir}/path-settings-overview.asciidoc[]
+
 include::modules/indices/search-settings.asciidoc[]
 
 include::settings/security-settings.asciidoc[]
 
-include::modules/shard-ops.asciidoc[]
-
 include::modules/indices/request_cache.asciidoc[]
 
 include::settings/snapshot-settings.asciidoc[]
diff --git a/docs/reference/setup/add-nodes.asciidoc b/docs/reference/setup/add-nodes.asciidoc
index ba749782c092f..941a3e6c40f79 100644
--- a/docs/reference/setup/add-nodes.asciidoc
+++ b/docs/reference/setup/add-nodes.asciidoc
@@ -48,7 +48,7 @@ For more information about discovery and shard allocation, refer to
 
 As nodes are added or removed Elasticsearch maintains an optimal level of fault
 tolerance by automatically updating the cluster's _voting configuration_, which
-is the set of <<master-node,master-eligible nodes>> whose responses are counted
+is the set of <<master-node-role,master-eligible nodes>> whose responses are counted
 when making decisions such as electing a new master or committing a new cluster
 state.
 
diff --git a/docs/reference/setup/advanced-configuration.asciidoc b/docs/reference/setup/advanced-configuration.asciidoc
index 2a7ccc56742de..ff80b51f0408b 100644
--- a/docs/reference/setup/advanced-configuration.asciidoc
+++ b/docs/reference/setup/advanced-configuration.asciidoc
@@ -1,13 +1,7 @@
 [[advanced-configuration]]
-=== Advanced configuration
-
-Modifying advanced settings is generally not recommended and could negatively
-impact performance and stability. Using the {es}-provided defaults
-is recommended in most circumstances.
+=== Set JVM options
 
 [[set-jvm-options]]
-==== Set JVM options
-
 If needed, you can override the default JVM options by adding custom options
 files (preferred) or setting the `ES_JAVA_OPTS` environment variable.
 
@@ -21,10 +15,15 @@ Where you put the JVM options files depends on the type of installation:
 * Docker: Bind mount custom JVM options files into
 `/usr/share/elasticsearch/config/jvm.options.d/`.
 
+CAUTION: Setting your own JVM options is generally not recommended and could negatively
+impact performance and stability. Using the {es}-provided defaults
+is recommended in most circumstances.
+
+[[readiness-tcp-port]]
 NOTE: Do not modify the root `jvm.options` file. Use files in `jvm.options.d/` instead.
 
 [[jvm-options-syntax]]
-===== JVM options syntax
+==== JVM options syntax
 
 A JVM options file contains a line-delimited list of JVM arguments.
 Arguments are preceded by a dash (`-`).
@@ -66,7 +65,7 @@ and ignored. Lines that aren't commented out and aren't recognized
 as valid JVM arguments are rejected and {es} will fail to start.
 
 [[jvm-options-env]]
-===== Use environment variables to set JVM options
+==== Use environment variables to set JVM options
 
 In production, use JVM options files to override the
 default settings. In testing and development environments,
@@ -155,23 +154,11 @@ options. We do not recommend using `ES_JAVA_OPTS` in production.
 NOTE: If you are running {es} as a Windows service, you can change the heap size
 using the service manager. See <<windows-service>>.
 
-[[readiness-tcp-port]]
-===== Enable the Elasticsearch TCP readiness port
-
-preview::[]
-
-If configured, a node can open a TCP port when the node is in a ready state. A node is deemed
-ready when it has successfully joined a cluster. In a single node configuration, the node is
-said to be ready, when it's able to accept requests.
-
-To enable the readiness TCP port, use the `readiness.port` setting. The readiness service will bind to
-all host addresses.
-
-If the node leaves the cluster, or the <<put-shutdown,Shutdown API>> is used to mark the node
-for shutdown, the readiness port is immediately closed.
-
-A successful connection to the readiness TCP port signals that the {es} node is ready. When a client
-connects to the readiness port, the server simply terminates the socket connection. No data is sent back
-to the client. If a client cannot connect to the readiness port, the node is not ready.
+[[heap-dump-path-setting]]
+include::important-settings/heap-dump-path.asciidoc[leveloffset=-1]
 
+[[gc-logging]]
+include::important-settings/gc-logging.asciidoc[leveloffset=-1]
 
+[[error-file-path]]
+include::important-settings/error-file.asciidoc[leveloffset=-1]
\ No newline at end of file
diff --git a/docs/reference/setup/important-settings.asciidoc b/docs/reference/setup/important-settings.asciidoc
index 03c891af70743..b822ee9b3f903 100644
--- a/docs/reference/setup/important-settings.asciidoc
+++ b/docs/reference/setup/important-settings.asciidoc
@@ -19,10 +19,20 @@ of items which *must* be considered before using your cluster in production:
 Our {ess-trial}[{ecloud}] service configures these items automatically, making
 your cluster production-ready by default.
 
+[[path-settings]]
+[discrete]
+==== Path settings
+
 include::important-settings/path-settings.asciidoc[]
 
+Elasticsearch offers a deprecated setting that allows you to specify multiple paths in `path.data`. 
+To learn about this setting, and how to migrate away from it, refer to <<multiple-data-paths>>.
+
 include::important-settings/cluster-name.asciidoc[]
 
+[[node-name]]
+[discrete]
+==== Node name setting
 include::important-settings/node-name.asciidoc[]
 
 include::important-settings/network-host.asciidoc[]
@@ -31,6 +41,7 @@ include::important-settings/discovery-settings.asciidoc[]
 
 include::important-settings/heap-size.asciidoc[]
 
+[[heap-dump-path]]
 include::important-settings/heap-dump-path.asciidoc[]
 
 include::important-settings/gc-logging.asciidoc[]
diff --git a/docs/reference/setup/important-settings/cluster-name.asciidoc b/docs/reference/setup/important-settings/cluster-name.asciidoc
index 3f1516f21de1e..6d489eee76cf6 100644
--- a/docs/reference/setup/important-settings/cluster-name.asciidoc
+++ b/docs/reference/setup/important-settings/cluster-name.asciidoc
@@ -1,4 +1,3 @@
-[[cluster-name]]
 [discrete]
 ==== Cluster name setting
 
diff --git a/docs/reference/setup/important-settings/error-file.asciidoc b/docs/reference/setup/important-settings/error-file.asciidoc
index ca95ded78d53f..2f654002d51f8 100644
--- a/docs/reference/setup/important-settings/error-file.asciidoc
+++ b/docs/reference/setup/important-settings/error-file.asciidoc
@@ -1,4 +1,3 @@
-[[error-file-path]]
 [discrete]
 ==== JVM fatal error log setting
 
diff --git a/docs/reference/setup/important-settings/gc-logging.asciidoc b/docs/reference/setup/important-settings/gc-logging.asciidoc
index 3534e1335c9fd..873c85d58d914 100644
--- a/docs/reference/setup/important-settings/gc-logging.asciidoc
+++ b/docs/reference/setup/important-settings/gc-logging.asciidoc
@@ -1,4 +1,3 @@
-[[gc-logging]]
 [discrete]
 ==== GC logging settings
 
@@ -20,9 +19,8 @@ To see further options not contained in the original JEP, see
 https://docs.oracle.com/en/java/javase/13/docs/specs/man/java.html#enable-logging-with-the-jvm-unified-logging-framework[Enable
 Logging with the JVM Unified Logging Framework].
 
-[[gc-logging-examples]]
 [discrete]
-==== Examples
+===== Examples
 
 Change the default GC log output location to `/opt/my-app/gc.log` by
   creating `$ES_HOME/config/jvm.options.d/gc.options` with some sample
diff --git a/docs/reference/setup/important-settings/heap-dump-path.asciidoc b/docs/reference/setup/important-settings/heap-dump-path.asciidoc
index 8f01379842a90..8b06ae752f360 100644
--- a/docs/reference/setup/important-settings/heap-dump-path.asciidoc
+++ b/docs/reference/setup/important-settings/heap-dump-path.asciidoc
@@ -1,4 +1,3 @@
-[[heap-dump-path]]
 [discrete]
 ==== JVM heap dump path setting
 
diff --git a/docs/reference/setup/important-settings/node-name.asciidoc b/docs/reference/setup/important-settings/node-name.asciidoc
index eda3052d119c9..f1260844a0549 100644
--- a/docs/reference/setup/important-settings/node-name.asciidoc
+++ b/docs/reference/setup/important-settings/node-name.asciidoc
@@ -1,7 +1,3 @@
-[[node-name]]
-[discrete]
-==== Node name setting
-
 {es} uses `node.name` as a human-readable identifier for a
 particular instance of {es}. This name is included in the response
 of many APIs. The node name defaults to the hostname of the machine when
diff --git a/docs/reference/setup/important-settings/path-settings.asciidoc b/docs/reference/setup/important-settings/path-settings.asciidoc
index a0a444ca5090a..002e08e2dc746 100644
--- a/docs/reference/setup/important-settings/path-settings.asciidoc
+++ b/docs/reference/setup/important-settings/path-settings.asciidoc
@@ -1,7 +1,3 @@
-[[path-settings]]
-[discrete]
-==== Path settings
-
 {es} writes the data you index to indices and data streams to a `data`
 directory. {es} writes its own application logs, which contain information about
 cluster health and operations, to a `logs` directory.
@@ -20,113 +16,4 @@ Supported `path.data` and `path.logs` values vary by platform:
 
 include::{es-ref-dir}/tab-widgets/customize-data-log-path-widget.asciidoc[]
 
-include::{es-ref-dir}/modules/node.asciidoc[tag=modules-node-data-path-warning-tag]
-
-[discrete]
-==== Multiple data paths
-deprecated::[7.13.0]
-
-If needed, you can specify multiple paths in `path.data`. {es} stores the node's
-data across all provided paths but keeps each shard's data on the same path.
-
-{es} does not balance shards across a node's data paths. High disk
-usage in a single path can trigger a <<disk-based-shard-allocation,high disk
-usage watermark>> for the entire node. If triggered, {es} will not add shards to
-the node, even if the node’s other paths have available disk space. If you need
-additional disk space, we recommend you add a new node rather than additional
-data paths.
-
-include::{es-ref-dir}/tab-widgets/multi-data-path-widget.asciidoc[]
-
-[discrete]
-[[mdp-migrate]]
-==== Migrate from multiple data paths
-
-Support for multiple data paths was deprecated in 7.13 and will be removed
-in a future release.
-
-As an alternative to multiple data paths, you can create a filesystem which
-spans multiple disks with a hardware virtualisation layer such as RAID, or a
-software virtualisation layer such as Logical Volume Manager (LVM) on Linux or
-Storage Spaces on Windows. If you wish to use multiple data paths on a single
-machine then you must run one node for each data path.
-
-If you currently use multiple data paths in a
-{ref}/high-availability-cluster-design.html[highly available cluster] then you
-can migrate to a setup that uses a single path for each node without downtime
-using a process similar to a
-{ref}/restart-cluster.html#restart-cluster-rolling[rolling restart]: shut each
-node down in turn and replace it with one or more nodes each configured to use
-a single data path. In more detail, for each node that currently has multiple
-data paths you should follow the following process. In principle you can
-perform this migration during a rolling upgrade to 8.0, but we recommend
-migrating to a single-data-path setup before starting to upgrade.
-
-1. Take a snapshot to protect your data in case of disaster.
-
-2. Optionally, migrate the data away from the target node by using an
-{ref}/modules-cluster.html#cluster-shard-allocation-filtering[allocation filter]:
-+
-[source,console]
---------------------------------------------------
-PUT _cluster/settings
-{
-  "persistent": {
-    "cluster.routing.allocation.exclude._name": "target-node-name"
-  }
-}
---------------------------------------------------
-+
-You can use the {ref}/cat-allocation.html[cat allocation API] to track progress
-of this data migration. If some shards do not migrate then the
-{ref}/cluster-allocation-explain.html[cluster allocation explain API] will help
-you to determine why.
-
-3. Follow the steps in the
-{ref}/restart-cluster.html#restart-cluster-rolling[rolling restart process]
-up to and including shutting the target node down.
-
-4. Ensure your cluster health is `yellow` or `green`, so that there is a copy
-of every shard assigned to at least one of the other nodes in your cluster.
-
-5. If applicable, remove the allocation filter applied in the earlier step.
-+
-[source,console]
---------------------------------------------------
-PUT _cluster/settings
-{
-  "persistent": {
-    "cluster.routing.allocation.exclude._name": null
-  }
-}
---------------------------------------------------
-
-6. Discard the data held by the stopped node by deleting the contents of its
-data paths.
-
-7. Reconfigure your storage. For instance, combine your disks into a single
-filesystem using LVM or Storage Spaces. Ensure that your reconfigured storage
-has sufficient space for the data that it will hold.
-
-8. Reconfigure your node by adjusting the `path.data` setting in its
-`elasticsearch.yml` file. If needed, install more nodes each with their own
-`path.data` setting pointing at a separate data path.
-
-9. Start the new nodes and follow the rest of the
-{ref}/restart-cluster.html#restart-cluster-rolling[rolling restart process] for
-them.
-
-10. Ensure your cluster health is `green`, so that every shard has been
-assigned.
-
-You can alternatively add some number of single-data-path nodes to your
-cluster, migrate all your data over to these new nodes using
-{ref}/modules-cluster.html#cluster-shard-allocation-filtering[allocation filters],
-and then remove the old nodes from the cluster. This approach will temporarily
-double the size of your cluster so it will only work if you have the capacity to
-expand your cluster like this.
-
-If you currently use multiple data paths but your cluster is not highly
-available then you can migrate to a non-deprecated configuration by taking
-a snapshot, creating a new cluster with the desired configuration and restoring
-the snapshot into it.
+include::{es-ref-dir}/modules/node.asciidoc[tag=modules-node-data-path-warning-tag]
\ No newline at end of file
diff --git a/docs/reference/setup/install/docker.asciidoc b/docs/reference/setup/install/docker.asciidoc
index 8694d7f5b46c6..f3576db0c786c 100644
--- a/docs/reference/setup/install/docker.asciidoc
+++ b/docs/reference/setup/install/docker.asciidoc
@@ -25,6 +25,21 @@ TIP: This setup doesn't run multiple {es} nodes or {kib} by default. To create a
 multi-node cluster with {kib}, use Docker Compose instead. See
 <<docker-compose-file>>.
 
+[[docker-wolfi-hardened-image]]
+===== Hardened Docker images
+
+You can also use the hardened https://wolfi.dev/[Wolfi] image for additional security.
+Using Wolfi images requires Docker version 20.10.10 or higher.
+
+To use the Wolfi image, append `-wolfi` to the image tag in the Docker command.
+
+For example:
+
+[source,sh,subs="attributes"]
+----
+docker pull {docker-wolfi-image}
+----
+
 ===== Start a single-node cluster
 
 . Install Docker. Visit https://docs.docker.com/get-docker/[Get Docker] to
diff --git a/docs/reference/setup/logging-config.asciidoc b/docs/reference/setup/logging-config.asciidoc
index e382bbdacb464..04e9ba3f0bef9 100644
--- a/docs/reference/setup/logging-config.asciidoc
+++ b/docs/reference/setup/logging-config.asciidoc
@@ -1,5 +1,5 @@
 [[logging]]
-=== Logging
+== Elasticsearch application logging
 
 You can use {es}'s application logs to monitor your cluster and diagnose issues.
 If you run {es} as a service, the default location of the logs varies based on
@@ -11,7 +11,7 @@ If you run {es} from the command line, {es} prints logs to the standard output
 (`stdout`).
 
 [discrete]
-[[loggin-configuration]]
+[[logging-configuration]]
 === Logging configuration
 
 IMPORTANT: Elastic strongly recommends using the Log4j 2 configuration that is shipped by default.
@@ -304,6 +304,7 @@ The user ID is included in the `X-Opaque-ID` field in deprecation JSON logs.
 Deprecation logs can be indexed into `.logs-deprecation.elasticsearch-default` data stream
 `cluster.deprecation_indexing.enabled` setting is set to true.
 
+[discrete]
 ==== Deprecation logs throttling
 :es-rate-limiting-filter-java-doc: {elasticsearch-javadoc}/org/elasticsearch/common/logging/RateLimitingFilter.html
 Deprecation logs are deduplicated based on a deprecated feature key
diff --git a/docs/reference/shard-request-cache.asciidoc b/docs/reference/shard-request-cache.asciidoc
new file mode 100644
index 0000000000000..ec79dfb531bdb
--- /dev/null
+++ b/docs/reference/shard-request-cache.asciidoc
@@ -0,0 +1,134 @@
+[[shard-request-cache]]
+=== The shard request cache
+
+When a search request is run against an index or against many indices, each
+involved shard executes the search locally and returns its local results to
+the _coordinating node_, which combines these shard-level results into a
+``global'' result set.
+
+The shard-level request cache module caches the local results on each shard.
+This allows frequently used (and potentially heavy) search requests to return
+results almost instantly. The requests cache is a very good fit for the logging
+use case, where only the most recent index is being actively updated --
+results from older indices will be served directly from the cache.
+
+You can control the size and expiration of the cache at the node level using the <<shard-request-cache-settings,shard request cache settings>>.
+
+[IMPORTANT]
+===================================
+
+By default, the requests cache will only cache the results of search requests
+where `size=0`, so it will not cache `hits`,
+but it will cache `hits.total`,  <<search-aggregations,aggregations>>, and
+<<search-suggesters,suggestions>>.
+
+Most queries that use `now` (see <<date-math>>) cannot be cached.
+
+Scripted queries that use the API calls which are non-deterministic, such as
+`Math.random()` or `new Date()` are not cached.
+===================================
+
+[discrete]
+==== Cache invalidation
+
+The cache is smart -- it keeps the same _near real-time_ promise as uncached
+search.
+
+Cached results are invalidated automatically whenever the shard refreshes to
+pick up changes to the documents or when you update the mapping. In other
+words you will always get the same results from the cache as you would for an
+uncached search request.
+
+The longer the refresh interval, the longer that cached entries will remain
+valid even if there are changes to the documents. If the cache is full, the
+least recently used cache keys will be evicted.
+
+The cache can be expired manually with the <<indices-clearcache,`clear-cache` API>>:
+
+[source,console]
+------------------------
+POST /my-index-000001,my-index-000002/_cache/clear?request=true
+------------------------
+// TEST[s/^/PUT my-index-000001\nPUT my-index-000002\n/]
+
+[discrete]
+==== Enabling and disabling caching
+
+The cache is enabled by default, but can be disabled when creating a new
+index as follows:
+
+[source,console]
+-----------------------------
+PUT /my-index-000001
+{
+  "settings": {
+    "index.requests.cache.enable": false
+  }
+}
+-----------------------------
+
+It can also be enabled or disabled dynamically on an existing index with the
+<<indices-update-settings,`update-settings`>> API:
+
+[source,console]
+-----------------------------
+PUT /my-index-000001/_settings
+{ "index.requests.cache.enable": true }
+-----------------------------
+// TEST[continued]
+
+
+[discrete]
+==== Enabling and disabling caching per request
+
+The `request_cache` query-string parameter can be used to enable or disable
+caching on a *per-request* basis. If set, it overrides the index-level setting:
+
+[source,console]
+-----------------------------
+GET /my-index-000001/_search?request_cache=true
+{
+  "size": 0,
+  "aggs": {
+    "popular_colors": {
+      "terms": {
+        "field": "colors"
+      }
+    }
+  }
+}
+-----------------------------
+// TEST[continued]
+
+Requests where `size` is greater than 0 will not be cached even if the request cache is
+enabled in the index settings. To cache these requests you will need to use the
+query-string parameter detailed here.
+
+[discrete]
+==== Cache key
+
+A hash of the whole JSON body is used as the cache key. This means that if the JSON
+changes -- for instance if keys are output in a different order -- then the
+cache key will not be recognised.
+
+TIP: Most JSON libraries support a _canonical_ mode which ensures that JSON
+keys are always emitted in the same order. This canonical mode can be used in
+the application to ensure that a request is always serialized in the same way.
+
+[discrete]
+==== Monitoring cache usage
+
+The size of the cache (in bytes) and the number of evictions can be viewed
+by index, with the <<indices-stats,`indices-stats`>> API:
+
+[source,console]
+------------------------
+GET /_stats/request_cache?human
+------------------------
+
+or by node with the <<cluster-nodes-stats,`nodes-stats`>> API:
+
+[source,console]
+------------------------
+GET /_nodes/stats/indices/request_cache?human
+------------------------
diff --git a/docs/reference/shutdown/apis/shutdown-api.asciidoc b/docs/reference/shutdown/apis/shutdown-api.asciidoc
index 24cbca720d160..b950cd3d19c56 100644
--- a/docs/reference/shutdown/apis/shutdown-api.asciidoc
+++ b/docs/reference/shutdown/apis/shutdown-api.asciidoc
@@ -4,6 +4,12 @@
 
 NOTE: {cloud-only}
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-shutdown[Node lifecycle APIs].
+--
+
 You use the shutdown APIs to prepare nodes for temporary or permanent shutdown, monitor the shutdown status, and enable a previously shut-down node to resume normal operations.
 
 [discrete]
diff --git a/docs/reference/shutdown/apis/shutdown-delete.asciidoc b/docs/reference/shutdown/apis/shutdown-delete.asciidoc
index 4d7f30c3a1e48..225d88c63a016 100644
--- a/docs/reference/shutdown/apis/shutdown-delete.asciidoc
+++ b/docs/reference/shutdown/apis/shutdown-delete.asciidoc
@@ -3,6 +3,12 @@
 
 NOTE: {cloud-only}
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-shutdown[Node lifecycle APIs].
+--
+
 Cancels shutdown preparations or clears a shutdown request
 so a node can resume normal operations.
 
diff --git a/docs/reference/shutdown/apis/shutdown-get.asciidoc b/docs/reference/shutdown/apis/shutdown-get.asciidoc
index 5feac28353ab5..b0097eb0caf7f 100644
--- a/docs/reference/shutdown/apis/shutdown-get.asciidoc
+++ b/docs/reference/shutdown/apis/shutdown-get.asciidoc
@@ -3,6 +3,12 @@
 
 NOTE: {cloud-only}
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-shutdown[Node lifecycle APIs].
+--
+
 Retrieves the status of a node that's being prepared for shutdown.
 
 [[get-shutdown-api-request]]
@@ -72,6 +78,7 @@ including the status of shard migration, task migration, and plugin cleanup:
     "nodes": [
         {
             "node_id": "USpTGYaBSIKbgSUJR2Z9lg",
+            "node_ephemeral_id": null,
             "type": "RESTART",
             "reason": "Demonstrating how the node shutdown API works",
             "shutdown_startedmillis": 1624406108685,
diff --git a/docs/reference/shutdown/apis/shutdown-put.asciidoc b/docs/reference/shutdown/apis/shutdown-put.asciidoc
index 344dd8fa36717..5eef4763a9c3a 100644
--- a/docs/reference/shutdown/apis/shutdown-put.asciidoc
+++ b/docs/reference/shutdown/apis/shutdown-put.asciidoc
@@ -3,6 +3,12 @@
 
 NOTE: {cloud-only}
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-shutdown[Node lifecycle APIs].
+--
+
 Prepares a node to be shut down.
 
 [[put-shutdown-api-request]]
diff --git a/docs/reference/slm/apis/slm-api.asciidoc b/docs/reference/slm/apis/slm-api.asciidoc
index d061ff6b0aaf7..ee624a70b00f5 100644
--- a/docs/reference/slm/apis/slm-api.asciidoc
+++ b/docs/reference/slm/apis/slm-api.asciidoc
@@ -2,6 +2,12 @@
 [[snapshot-lifecycle-management-api]]
 == {slm-cap} APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs].
+--
+
 You use the following APIs to set up policies to automatically take snapshots and 
 control how long they are retained.
 
diff --git a/docs/reference/slm/apis/slm-delete.asciidoc b/docs/reference/slm/apis/slm-delete.asciidoc
index 650ee68e24fb7..9c63d7326421f 100644
--- a/docs/reference/slm/apis/slm-delete.asciidoc
+++ b/docs/reference/slm/apis/slm-delete.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete policy</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs].
+--
+
 Deletes an existing snapshot lifecycle policy.
 
 [[slm-api-delete-lifecycle-request]]
diff --git a/docs/reference/slm/apis/slm-execute-retention.asciidoc b/docs/reference/slm/apis/slm-execute-retention.asciidoc
index 75bcdb3143668..ad8d7b15a43d7 100644
--- a/docs/reference/slm/apis/slm-execute-retention.asciidoc
+++ b/docs/reference/slm/apis/slm-execute-retention.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Execute snapshot retention policy</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs].
+--
+
 Deletes any snapshots that are expired according to the policy's retention rules.
 
 [[slm-api-execute-retention-request]]
diff --git a/docs/reference/slm/apis/slm-execute.asciidoc b/docs/reference/slm/apis/slm-execute.asciidoc
index f3977d6aed2fd..9ea35602de4e2 100644
--- a/docs/reference/slm/apis/slm-execute.asciidoc
+++ b/docs/reference/slm/apis/slm-execute.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Execute snapshot lifecycle policy</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs].
+--
+
 Immediately creates a snapshot according to the lifecycle policy, 
 without waiting for the scheduled time.
 
diff --git a/docs/reference/slm/apis/slm-get-status.asciidoc b/docs/reference/slm/apis/slm-get-status.asciidoc
index d4afbaddb1beb..181927ac35a1c 100644
--- a/docs/reference/slm/apis/slm-get-status.asciidoc
+++ b/docs/reference/slm/apis/slm-get-status.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Get {slm} status</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs].
+--
+
 Retrieves the status of {slm} ({slm-init}).
 
 [[slm-api-get-status-request]]
diff --git a/docs/reference/slm/apis/slm-get.asciidoc b/docs/reference/slm/apis/slm-get.asciidoc
index f7c847d06dc4b..723d8b374b914 100644
--- a/docs/reference/slm/apis/slm-get.asciidoc
+++ b/docs/reference/slm/apis/slm-get.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get policy</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs].
+--
+
 Retrieves one or more snapshot lifecycle policy definitions and
 information about the latest snapshot attempts.
 
diff --git a/docs/reference/slm/apis/slm-put.asciidoc b/docs/reference/slm/apis/slm-put.asciidoc
index 51ad571ee12e7..a6ee29bfd1bc9 100644
--- a/docs/reference/slm/apis/slm-put.asciidoc
+++ b/docs/reference/slm/apis/slm-put.asciidoc
@@ -4,8 +4,13 @@
 <titleabbrev>Create or update policy</titleabbrev>
 ++++
 
-Creates or updates a snapshot lifecycle policy.
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs].
+--
 
+Creates or updates a snapshot lifecycle policy.
 
 [[slm-api-put-request]]
 ==== {api-request-title}
diff --git a/docs/reference/slm/apis/slm-start.asciidoc b/docs/reference/slm/apis/slm-start.asciidoc
index 9d9b8108cb57b..87c19ec600b61 100644
--- a/docs/reference/slm/apis/slm-start.asciidoc
+++ b/docs/reference/slm/apis/slm-start.asciidoc
@@ -1,12 +1,17 @@
 [role="xpack"]
 [[slm-api-start]]
 === Start {slm} API
-
 [subs="attributes"]
 ++++
 <titleabbrev>Start {slm}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs].
+--
+
 Turns on {slm} ({slm-init}).
 
 [[slm-api-start-request]]
diff --git a/docs/reference/slm/apis/slm-stats.asciidoc b/docs/reference/slm/apis/slm-stats.asciidoc
index 340631e9c5601..d6b9cd960e45f 100644
--- a/docs/reference/slm/apis/slm-stats.asciidoc
+++ b/docs/reference/slm/apis/slm-stats.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get snapshot lifecycle stats</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs].
+--
+
 Returns global and policy-level statistics about actions taken by {slm}.
 
 [[slm-api-stats-request]]
diff --git a/docs/reference/slm/apis/slm-stop.asciidoc b/docs/reference/slm/apis/slm-stop.asciidoc
index 253abec7b4d11..2bfe9646bcffd 100644
--- a/docs/reference/slm/apis/slm-stop.asciidoc
+++ b/docs/reference/slm/apis/slm-stop.asciidoc
@@ -1,12 +1,17 @@
 [role="xpack"]
 [[slm-api-stop]]
 === Stop {slm} API
-
 [subs="attributes"]
 ++++
 <titleabbrev>Stop {slm}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs].
+--
+
 Turn off {slm} ({slm-init}).
 
 [[slm-api-stop-request]]
diff --git a/docs/reference/snapshot-restore/apis/clean-up-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/clean-up-repo-api.asciidoc
index 249e192c0c587..dbb7541693492 100644
--- a/docs/reference/snapshot-restore/apis/clean-up-repo-api.asciidoc
+++ b/docs/reference/snapshot-restore/apis/clean-up-repo-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Clean up snapshot repository</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs].
+--
+
 Triggers the review of a snapshot repository's contents and deletes any stale
 data that is not referenced by existing snapshots. See
 <<snapshots-repository-cleanup>>.
diff --git a/docs/reference/snapshot-restore/apis/clone-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/clone-snapshot-api.asciidoc
index 590bc7e7410f5..cf432d807d9c5 100644
--- a/docs/reference/snapshot-restore/apis/clone-snapshot-api.asciidoc
+++ b/docs/reference/snapshot-restore/apis/clone-snapshot-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Clone snapshot</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs].
+--
+
 Clones part or all of a snapshot into a new snapshot.
 
 [source,console]
diff --git a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc
index baa28bb7b0a53..a6c39abbda734 100644
--- a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc
+++ b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Create snapshot</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs].
+--
+
 <<snapshots-take-snapshot,Takes a snapshot>> of a cluster or specified data
 streams and indices.
 
diff --git a/docs/reference/snapshot-restore/apis/delete-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/delete-repo-api.asciidoc
index 4301fea642523..e0df427da745c 100644
--- a/docs/reference/snapshot-restore/apis/delete-repo-api.asciidoc
+++ b/docs/reference/snapshot-restore/apis/delete-repo-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Delete snapshot repository</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs].
+--
+
 Unregisters one or more <<snapshots-register-repository,snapshot repositories>>.
 
 When a repository is unregistered, {es} only removes the reference to the
diff --git a/docs/reference/snapshot-restore/apis/delete-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/delete-snapshot-api.asciidoc
index 8824977d660e4..74db60c1970c0 100644
--- a/docs/reference/snapshot-restore/apis/delete-snapshot-api.asciidoc
+++ b/docs/reference/snapshot-restore/apis/delete-snapshot-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Delete snapshot</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs].
+--
+
 Deletes a <<snapshot-restore,snapshot>>.
 
 ////
diff --git a/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc
index cf1b9813c519e..5f6b6485c5ee4 100644
--- a/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc
+++ b/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get snapshot repository</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs].
+--
+
 Gets information about one or more registered
 <<snapshots-register-repository,snapshot repositories>>.
 
diff --git a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc
index 622e1ade024b0..f9eb6a27df039 100644
--- a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc
+++ b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get snapshot</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs].
+--
+
 Retrieves information about one or more snapshots.
 
 ////
diff --git a/docs/reference/snapshot-restore/apis/get-snapshot-status-api.asciidoc b/docs/reference/snapshot-restore/apis/get-snapshot-status-api.asciidoc
index e677408da3f25..dbbf547528f46 100644
--- a/docs/reference/snapshot-restore/apis/get-snapshot-status-api.asciidoc
+++ b/docs/reference/snapshot-restore/apis/get-snapshot-status-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Get snapshot status</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs].
+--
+
 Retrieves a detailed description of the current state for each shard participating in the snapshot. Note that this API should only be used to obtain detailed shard-level information for ongoing snapshots. If this detail is not needed, or you want to obtain information about one or more existing snapshots, use the <<get-snapshot-api,get snapshot API>>.
 
 ////
diff --git a/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc
index 0d3b5586da869..55b61cc321ed3 100644
--- a/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc
+++ b/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Create or update snapshot repository</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs].
+--
+
 Registers or updates a <<snapshots-register-repository,snapshot repository>>.
 
 [source,console]
diff --git a/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc
index f18ef1ee6e826..ca46ba1fb2b57 100644
--- a/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc
+++ b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Repository analysis</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs].
+--
+
 Analyzes a repository, reporting its performance characteristics and any
 incorrect behaviour found.
 
diff --git a/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc
index 9fe06d73f1a63..89cd0f96915b9 100644
--- a/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc
+++ b/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Restore snapshot</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs].
+--
+
 Restores a <<snapshot-restore,snapshot>> of a cluster or specified data streams and indices.
 
 ////
@@ -69,7 +75,7 @@ POST /_snapshot/my_repository/my_snapshot/_restore
 
 // tag::restore-prereqs[]
 * You can only restore a snapshot to a running cluster with an elected
-<<master-node,master node>>. The snapshot's repository must be
+<<master-node-role,master node>>. The snapshot's repository must be
 <<snapshots-register-repository,registered>> and available to the cluster.
 
 * The snapshot and cluster versions must be compatible. See
diff --git a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc
index b8bb6a2cd7d13..715687f02edea 100644
--- a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc
+++ b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc
@@ -1,6 +1,12 @@
 [[snapshot-restore-apis]]
 == Snapshot and restore APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs].
+--
+
 You can use the following APIs to set up snapshot repositories, manage snapshot
 backups, and restore snapshots to a running cluster.
 
diff --git a/docs/reference/snapshot-restore/apis/verify-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/verify-repo-api.asciidoc
index dd845663be8d7..333a12ff49ac0 100644
--- a/docs/reference/snapshot-restore/apis/verify-repo-api.asciidoc
+++ b/docs/reference/snapshot-restore/apis/verify-repo-api.asciidoc
@@ -4,6 +4,12 @@
 <titleabbrev>Verify snapshot repository</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs].
+--
+
 Checks for common misconfigurations in a snapshot repository. See
 <<snapshots-repository-verification>>.
 
diff --git a/docs/reference/snapshot-restore/apis/verify-repo-integrity-api.asciidoc b/docs/reference/snapshot-restore/apis/verify-repo-integrity-api.asciidoc
index 99ae126b401f5..22a4051a546cd 100644
--- a/docs/reference/snapshot-restore/apis/verify-repo-integrity-api.asciidoc
+++ b/docs/reference/snapshot-restore/apis/verify-repo-integrity-api.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Verify repository integrity</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs].
+--
+
 Verifies the integrity of the contents of a snapshot repository.
 
 ////
diff --git a/docs/reference/snapshot-restore/repository-s3.asciidoc b/docs/reference/snapshot-restore/repository-s3.asciidoc
index 9b71fe9220385..446d3a409234d 100644
--- a/docs/reference/snapshot-restore/repository-s3.asciidoc
+++ b/docs/reference/snapshot-restore/repository-s3.asciidoc
@@ -350,7 +350,7 @@ include::repository-shared-settings.asciidoc[]
     will disable retries altogether. Note that if retries are enabled in the Azure client, each of these retries
     comprises that many client-level retries.
 
-`get_register_retry_delay`
+`get_register_retry_delay`::
 
     (<<time-units,time value>>) Sets the time to wait before trying again if an attempt to read a
     <<repository-s3-linearizable-registers,linearizable register>> fails. Defaults to `5s`.
diff --git a/docs/reference/snapshot-restore/take-snapshot.asciidoc b/docs/reference/snapshot-restore/take-snapshot.asciidoc
index 711fcfe4cc484..1ae2258c7da89 100644
--- a/docs/reference/snapshot-restore/take-snapshot.asciidoc
+++ b/docs/reference/snapshot-restore/take-snapshot.asciidoc
@@ -46,7 +46,7 @@ taking snapshots at different time intervals.
 include::register-repository.asciidoc[tag=kib-snapshot-prereqs]
 
 * You can only take a snapshot from a running cluster with an elected
-<<master-node,master node>>.
+<<master-node-role,master node>>.
 
 * A snapshot repository must be <<snapshots-register-repository,registered>> and
 available to the cluster.
diff --git a/docs/reference/sql/apis/clear-sql-cursor-api.asciidoc b/docs/reference/sql/apis/clear-sql-cursor-api.asciidoc
index 48663ca0d75aa..a16d31c4b12d7 100644
--- a/docs/reference/sql/apis/clear-sql-cursor-api.asciidoc
+++ b/docs/reference/sql/apis/clear-sql-cursor-api.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Clear SQL cursor</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-sql[SQL APIs].
+--
+
 Clears an <<sql-pagination,SQL search cursor>>.
 
 ////
diff --git a/docs/reference/sql/apis/delete-async-sql-search-api.asciidoc b/docs/reference/sql/apis/delete-async-sql-search-api.asciidoc
index 1737a39401dba..e8e4ff68cef07 100644
--- a/docs/reference/sql/apis/delete-async-sql-search-api.asciidoc
+++ b/docs/reference/sql/apis/delete-async-sql-search-api.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Delete async SQL search</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-sql[SQL APIs].
+--
+
 Deletes an <<sql-async,async SQL search>> or a <<sql-store-searches,stored
 synchronous SQL search>>. If the search is still running, the API cancels it.
 
diff --git a/docs/reference/sql/apis/get-async-sql-search-api.asciidoc b/docs/reference/sql/apis/get-async-sql-search-api.asciidoc
index 8ae575176dd3a..10ca4c648ebaa 100644
--- a/docs/reference/sql/apis/get-async-sql-search-api.asciidoc
+++ b/docs/reference/sql/apis/get-async-sql-search-api.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get async SQL search</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-sql[SQL APIs].
+--
+
 Returns results for an <<sql-async,async SQL search>> or a
 <<sql-store-searches,stored synchronous SQL search>>.
 
diff --git a/docs/reference/sql/apis/get-async-sql-search-status-api.asciidoc b/docs/reference/sql/apis/get-async-sql-search-status-api.asciidoc
index 61505bab7c454..7a0d68cd120cc 100644
--- a/docs/reference/sql/apis/get-async-sql-search-status-api.asciidoc
+++ b/docs/reference/sql/apis/get-async-sql-search-status-api.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get async SQL search status</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-sql[SQL APIs].
+--
+
 Returns the current status of an <<sql-async,async SQL search>> or a
 <<sql-store-searches,stored synchronous SQL search>>.
 
diff --git a/docs/reference/sql/apis/sql-apis.asciidoc b/docs/reference/sql/apis/sql-apis.asciidoc
index 08300522c3288..60f3c85c55d6c 100644
--- a/docs/reference/sql/apis/sql-apis.asciidoc
+++ b/docs/reference/sql/apis/sql-apis.asciidoc
@@ -2,6 +2,12 @@
 [[sql-apis]]
 == SQL APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-sql[SQL APIs].
+--
+
 {es}'s SQL APIs let you run SQL queries on {es} indices and data streams.
 For an overview of {es}'s SQL features and related tutorials, see <<xpack-sql>>.
 
diff --git a/docs/reference/sql/apis/sql-search-api.asciidoc b/docs/reference/sql/apis/sql-search-api.asciidoc
index 118d7975aefd9..b1d002d343fb8 100644
--- a/docs/reference/sql/apis/sql-search-api.asciidoc
+++ b/docs/reference/sql/apis/sql-search-api.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>SQL search</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-sql[SQL APIs].
+--
+
 Returns results for an <<sql-rest-overview,SQL search>>.
 
 [source,console]
diff --git a/docs/reference/sql/apis/sql-translate-api.asciidoc b/docs/reference/sql/apis/sql-translate-api.asciidoc
index 15e52d118800e..9183423eefd0f 100644
--- a/docs/reference/sql/apis/sql-translate-api.asciidoc
+++ b/docs/reference/sql/apis/sql-translate-api.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>SQL translate</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-sql[SQL APIs].
+--
+
 Translates an <<sql-search-api,SQL search>> into a <<search-search,search API>>
 request containing <<query-dsl,Query DSL>>. See <<sql-translate>>.
 
diff --git a/docs/reference/sql/language/indices.asciidoc b/docs/reference/sql/language/indices.asciidoc
index 1dee7f0840ade..1912a020ab0be 100644
--- a/docs/reference/sql/language/indices.asciidoc
+++ b/docs/reference/sql/language/indices.asciidoc
@@ -100,7 +100,7 @@ requires the keyword `LIKE` for SQL `LIKE` pattern.
 [[sql-index-frozen]]
 === Frozen Indices
 
-By default, {es-sql} doesn't search <<unfreeze-index-api,frozen indices>>. To
+By default, {es-sql} doesn't search <<frozen-indices,frozen indices>>. To
 search frozen indices, use one of the following features:
 
 dedicated configuration parameter::
diff --git a/docs/reference/synonyms/apis/delete-synonym-rule.asciidoc b/docs/reference/synonyms/apis/delete-synonym-rule.asciidoc
index 74cbab8c0b4a2..11f0708bafcda 100644
--- a/docs/reference/synonyms/apis/delete-synonym-rule.asciidoc
+++ b/docs/reference/synonyms/apis/delete-synonym-rule.asciidoc
@@ -1,10 +1,15 @@
 [[delete-synonym-rule]]
 === Delete synonym rule
-
 ++++
 <titleabbrev>Delete synonym rule</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-synonyms[Synonyms APIs].
+--
+
 Deletes an individual synonym rule from a synonyms set.
 
 [[delete-synonym-rule-request]]
diff --git a/docs/reference/synonyms/apis/delete-synonyms-set.asciidoc b/docs/reference/synonyms/apis/delete-synonyms-set.asciidoc
index 9ba33ff3a5c75..62162e5c45675 100644
--- a/docs/reference/synonyms/apis/delete-synonyms-set.asciidoc
+++ b/docs/reference/synonyms/apis/delete-synonyms-set.asciidoc
@@ -1,10 +1,15 @@
 [[delete-synonyms-set]]
 === Delete synonyms set
-
 ++++
 <titleabbrev>Delete synonyms set</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-synonyms[Synonyms APIs].
+--
+
 Deletes a synonyms set.
 
 [[delete-synonyms-set-request]]
diff --git a/docs/reference/synonyms/apis/get-synonym-rule.asciidoc b/docs/reference/synonyms/apis/get-synonym-rule.asciidoc
index c6c35e0efecca..3f0ee3f173245 100644
--- a/docs/reference/synonyms/apis/get-synonym-rule.asciidoc
+++ b/docs/reference/synonyms/apis/get-synonym-rule.asciidoc
@@ -1,10 +1,15 @@
 [[get-synonym-rule]]
 === Get synonym rule
-
 ++++
 <titleabbrev>Get synonym rule</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-synonyms[Synonyms APIs].
+--
+
 Retrieves a synonym rule from a synonyms set.
 
 [[get-synonym-rule-request]]
diff --git a/docs/reference/synonyms/apis/get-synonyms-set.asciidoc b/docs/reference/synonyms/apis/get-synonyms-set.asciidoc
index 70bb5fb69526d..1bb31081712e5 100644
--- a/docs/reference/synonyms/apis/get-synonyms-set.asciidoc
+++ b/docs/reference/synonyms/apis/get-synonyms-set.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Get synonyms set</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-synonyms[Synonyms APIs].
+--
+
 Retrieves a synonyms set.
 
 [[get-synonyms-set-request]]
diff --git a/docs/reference/synonyms/apis/list-synonyms-sets.asciidoc b/docs/reference/synonyms/apis/list-synonyms-sets.asciidoc
index 705a24c809e99..33ef220036b7a 100644
--- a/docs/reference/synonyms/apis/list-synonyms-sets.asciidoc
+++ b/docs/reference/synonyms/apis/list-synonyms-sets.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>List synonyms sets</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-synonyms[Synonyms APIs].
+--
+
 Retrieves a summary of all defined synonyms sets.
 
 This API allows to retrieve the total number of synonyms sets defined.
diff --git a/docs/reference/synonyms/apis/put-synonym-rule.asciidoc b/docs/reference/synonyms/apis/put-synonym-rule.asciidoc
index de2865632d55e..5bb561f0f923f 100644
--- a/docs/reference/synonyms/apis/put-synonym-rule.asciidoc
+++ b/docs/reference/synonyms/apis/put-synonym-rule.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Create or update synonym rule</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-synonyms[Synonyms APIs].
+--
+
 Creates or updates a synonym rule for a synonym set.
 
 [[put-synonym-rule-request]]
diff --git a/docs/reference/synonyms/apis/put-synonyms-set.asciidoc b/docs/reference/synonyms/apis/put-synonyms-set.asciidoc
index ca75885921456..3af85638d022d 100644
--- a/docs/reference/synonyms/apis/put-synonyms-set.asciidoc
+++ b/docs/reference/synonyms/apis/put-synonyms-set.asciidoc
@@ -5,6 +5,12 @@
 <titleabbrev>Create or update synonyms set</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-synonyms[Synonyms APIs].
+--
+
 Creates or updates a synonyms set.
 
 NOTE: Synonyms sets are limited to a maximum of 10,000 synonym rules per set.
diff --git a/docs/reference/synonyms/apis/synonyms-apis.asciidoc b/docs/reference/synonyms/apis/synonyms-apis.asciidoc
index dbbc26c36d3df..95fc0aae8c14d 100644
--- a/docs/reference/synonyms/apis/synonyms-apis.asciidoc
+++ b/docs/reference/synonyms/apis/synonyms-apis.asciidoc
@@ -7,6 +7,12 @@
 
 ---
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-synonyms[Synonyms APIs].
+--
+
 The synonyms management API provides a convenient way to define and manage synonyms in an internal system index. Related synonyms can be grouped in a "synonyms set".
 Create as many synonym sets as you need.
 
diff --git a/docs/reference/text-structure/apis/find-field-structure.asciidoc b/docs/reference/text-structure/apis/find-field-structure.asciidoc
index 4fa108e92d4cb..c4b289e6c30a9 100644
--- a/docs/reference/text-structure/apis/find-field-structure.asciidoc
+++ b/docs/reference/text-structure/apis/find-field-structure.asciidoc
@@ -2,6 +2,12 @@
 [[find-field-structure]]
 = Find field structure API
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-text_structure[Text structure APIs].
+--
+
 Finds the structure of a field in an Elasticsearch index.
 
 [discrete]
diff --git a/docs/reference/text-structure/apis/find-message-structure.asciidoc b/docs/reference/text-structure/apis/find-message-structure.asciidoc
index 6c1bf5089bed2..18b85069559e3 100644
--- a/docs/reference/text-structure/apis/find-message-structure.asciidoc
+++ b/docs/reference/text-structure/apis/find-message-structure.asciidoc
@@ -2,6 +2,12 @@
 [[find-message-structure]]
 = Find messages structure API
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-text_structure[Text structure APIs].
+--
+
 Finds the structure of a list of text messages.
 
 [discrete]
diff --git a/docs/reference/text-structure/apis/find-structure.asciidoc b/docs/reference/text-structure/apis/find-structure.asciidoc
index 361560bace4ed..fef0584222e13 100644
--- a/docs/reference/text-structure/apis/find-structure.asciidoc
+++ b/docs/reference/text-structure/apis/find-structure.asciidoc
@@ -2,6 +2,12 @@
 [[find-structure]]
 = Find text structure API
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-text_structure[Text structure APIs].
+--
+
 Finds the structure of text. The text must
 contain data that is suitable to be ingested into the
 {stack}.
diff --git a/docs/reference/text-structure/apis/index.asciidoc b/docs/reference/text-structure/apis/index.asciidoc
index 9f4af120690f7..68607ddf708be 100644
--- a/docs/reference/text-structure/apis/index.asciidoc
+++ b/docs/reference/text-structure/apis/index.asciidoc
@@ -2,6 +2,12 @@
 [[text-structure-apis]]
 == Text structure APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-text_structure[Text structure APIs].
+--
+
 You can use the following APIs to find text structures:
 
 * <<find-field-structure>>
diff --git a/docs/reference/text-structure/apis/test-grok-pattern.asciidoc b/docs/reference/text-structure/apis/test-grok-pattern.asciidoc
index 4034a24cf0a19..7c1cfb41608e9 100644
--- a/docs/reference/text-structure/apis/test-grok-pattern.asciidoc
+++ b/docs/reference/text-structure/apis/test-grok-pattern.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Test Grok pattern</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-text_structure[Text structure APIs].
+--
+
 Tests a Grok pattern on lines of text, see also <<grok,Grokking grok>>.
 
 [discrete]
diff --git a/docs/reference/transform/apis/delete-transform.asciidoc b/docs/reference/transform/apis/delete-transform.asciidoc
index 111dda23690b6..0b386f946d6c3 100644
--- a/docs/reference/transform/apis/delete-transform.asciidoc
+++ b/docs/reference/transform/apis/delete-transform.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Delete {transform}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs].
+--
+
 Deletes an existing {transform}.
 
 [[delete-transform-request]]
diff --git a/docs/reference/transform/apis/get-transform-stats.asciidoc b/docs/reference/transform/apis/get-transform-stats.asciidoc
index 273b1d094979e..e8d12a994faa3 100644
--- a/docs/reference/transform/apis/get-transform-stats.asciidoc
+++ b/docs/reference/transform/apis/get-transform-stats.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Get {transform} statistics</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs].
+--
+
 Retrieves usage information for {transforms}.
 
 
diff --git a/docs/reference/transform/apis/get-transform.asciidoc b/docs/reference/transform/apis/get-transform.asciidoc
index ece59138e2893..3eabf4ba26988 100644
--- a/docs/reference/transform/apis/get-transform.asciidoc
+++ b/docs/reference/transform/apis/get-transform.asciidoc
@@ -6,6 +6,12 @@
 <titleabbrev>Get {transforms}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs].
+--
+
 Retrieves configuration information for {transforms}.
 
 [[get-transform-request]]
diff --git a/docs/reference/transform/apis/preview-transform.asciidoc b/docs/reference/transform/apis/preview-transform.asciidoc
index fa9ad0c0fc8f9..c280bef52e0ee 100644
--- a/docs/reference/transform/apis/preview-transform.asciidoc
+++ b/docs/reference/transform/apis/preview-transform.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Preview {transform}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs].
+--
+
 Previews a {transform}.
 
 [[preview-transform-request]]
diff --git a/docs/reference/transform/apis/put-transform.asciidoc b/docs/reference/transform/apis/put-transform.asciidoc
index ed2ceba0a7a59..fc9abc8c6470c 100644
--- a/docs/reference/transform/apis/put-transform.asciidoc
+++ b/docs/reference/transform/apis/put-transform.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Create {transform}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs].
+--
+
 Instantiates a {transform}.
 
 [[put-transform-request]]
diff --git a/docs/reference/transform/apis/reset-transform.asciidoc b/docs/reference/transform/apis/reset-transform.asciidoc
index 1194d3589275d..3d9fd5db180b2 100644
--- a/docs/reference/transform/apis/reset-transform.asciidoc
+++ b/docs/reference/transform/apis/reset-transform.asciidoc
@@ -8,6 +8,12 @@
 <titleabbrev>Reset {transform}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs].
+--
+
 Resets a {transform}.
 
 [[reset-transform-request]]
diff --git a/docs/reference/transform/apis/schedule-now-transform.asciidoc b/docs/reference/transform/apis/schedule-now-transform.asciidoc
index 7a276edf08819..3349e14b02caf 100644
--- a/docs/reference/transform/apis/schedule-now-transform.asciidoc
+++ b/docs/reference/transform/apis/schedule-now-transform.asciidoc
@@ -8,6 +8,12 @@
 <titleabbrev>Schedule now {transform}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs].
+--
+
 Instantly runs a {transform} to process data.
 
 [[schedule-now-transform-request]]
diff --git a/docs/reference/transform/apis/start-transform.asciidoc b/docs/reference/transform/apis/start-transform.asciidoc
index f4f99f0f3457a..4bcb951f4c6b3 100644
--- a/docs/reference/transform/apis/start-transform.asciidoc
+++ b/docs/reference/transform/apis/start-transform.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Start {transform}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs].
+--
+
 Starts a {transform}.
 
 [[start-transform-request]]
diff --git a/docs/reference/transform/apis/stop-transform.asciidoc b/docs/reference/transform/apis/stop-transform.asciidoc
index e99fcbd413eba..d87784e036ae4 100644
--- a/docs/reference/transform/apis/stop-transform.asciidoc
+++ b/docs/reference/transform/apis/stop-transform.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Stop {transforms}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs].
+--
+
 Stops one or more {transforms}.
 
 
diff --git a/docs/reference/transform/apis/transform-apis.asciidoc b/docs/reference/transform/apis/transform-apis.asciidoc
index 20e5960e5bb18..45131f0214f9d 100644
--- a/docs/reference/transform/apis/transform-apis.asciidoc
+++ b/docs/reference/transform/apis/transform-apis.asciidoc
@@ -2,6 +2,12 @@
 [[transform-apis]]
 = {transform-cap} APIs
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs].
+--
+
 * <<put-transform>> 
 * <<delete-transform>>
 * <<get-transform>>
diff --git a/docs/reference/transform/apis/update-transform.asciidoc b/docs/reference/transform/apis/update-transform.asciidoc
index 1ac7d6d5410d2..c473ca0f83b8a 100644
--- a/docs/reference/transform/apis/update-transform.asciidoc
+++ b/docs/reference/transform/apis/update-transform.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Update {transform}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs].
+--
+
 Updates certain properties of a {transform}.
 
 [[update-transform-request]]
diff --git a/docs/reference/transform/apis/upgrade-transforms.asciidoc b/docs/reference/transform/apis/upgrade-transforms.asciidoc
index a1b01a6fd146a..826243938a9f4 100644
--- a/docs/reference/transform/apis/upgrade-transforms.asciidoc
+++ b/docs/reference/transform/apis/upgrade-transforms.asciidoc
@@ -7,6 +7,12 @@
 <titleabbrev>Upgrade {transforms}</titleabbrev>
 ++++
 
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs].
+--
+
 Upgrades all {transforms}.
 
 [[upgrade-transforms-request]]
diff --git a/docs/reference/transform/painless-examples.asciidoc b/docs/reference/transform/painless-examples.asciidoc
index 4b0802c79a340..3b4dd9bdb631d 100644
--- a/docs/reference/transform/painless-examples.asciidoc
+++ b/docs/reference/transform/painless-examples.asciidoc
@@ -8,8 +8,8 @@
 
 IMPORTANT: The examples that use the `scripted_metric` aggregation are not supported on {es} Serverless.
 
-These examples demonstrate how to use Painless in {transforms}. You can learn 
-more about the Painless scripting language in the 
+These examples demonstrate how to use Painless in {transforms}. You can learn
+more about the Painless scripting language in the
 {painless}/painless-guide.html[Painless guide].
 
 * <<painless-top-hits>>
@@ -20,24 +20,24 @@ more about the Painless scripting language in the
 * <<painless-compare>>
 * <<painless-web-session>>
 
-[NOTE] 
+[NOTE]
 --
-* While the context of the following examples is the {transform} use case, 
-the Painless scripts in the snippets below can be used in other {es} search 
+* While the context of the following examples is the {transform} use case,
+the Painless scripts in the snippets below can be used in other {es} search
 aggregations, too.
-* All the following examples use scripts, {transforms} cannot deduce mappings of 
-output fields when the fields are created by a script. {transforms-cap} don't 
-create any mappings in the destination index for these fields, which means they 
-get dynamically mapped. Create the destination index prior to starting the 
+* All the following examples use scripts, {transforms} cannot deduce mappings of
+output fields when the fields are created by a script. {transforms-cap} don't
+create any mappings in the destination index for these fields, which means they
+get dynamically mapped. Create the destination index prior to starting the
 {transform} in case you want explicit mappings.
 --
 
 [[painless-top-hits]]
 == Getting top hits by using scripted metric aggregation
 
-This snippet shows how to find the latest document, in other words the document 
-with the latest timestamp. From a technical perspective, it helps to achieve 
-the function of a <<search-aggregations-metrics-top-hits-aggregation>> by using 
+This snippet shows how to find the latest document, in other words the document
+with the latest timestamp. From a technical perspective, it helps to achieve
+the function of a <<search-aggregations-metrics-top-hits-aggregation>> by using
 scripted metric aggregation in a {transform}, which provides a metric output.
 
 IMPORTANT: This example uses a `scripted_metric` aggregation which is not supported on {es} Serverless.
@@ -45,12 +45,12 @@ IMPORTANT: This example uses a `scripted_metric` aggregation which is not suppor
 [source,js]
 --------------------------------------------------
 "aggregations": {
-  "latest_doc": { 
+  "latest_doc": {
     "scripted_metric": {
       "init_script": "state.timestamp_latest = 0L; state.last_doc = ''", <1>
       "map_script": """ <2>
-        def current_date = doc['@timestamp'].getValue().toInstant().toEpochMilli(); 
-        if (current_date > state.timestamp_latest) 
+        def current_date = doc['@timestamp'].getValue().toInstant().toEpochMilli();
+        if (current_date > state.timestamp_latest)
         {state.timestamp_latest = current_date;
         state.last_doc = new HashMap(params['_source']);}
       """,
@@ -59,7 +59,7 @@ IMPORTANT: This example uses a `scripted_metric` aggregation which is not suppor
         def last_doc = '';
         def timestamp_latest = 0L;
         for (s in states) {if (s.timestamp_latest > (timestamp_latest))
-        {timestamp_latest = s.timestamp_latest; last_doc = s.last_doc;}} 
+        {timestamp_latest = s.timestamp_latest; last_doc = s.last_doc;}}
         return last_doc
       """
     }
@@ -68,23 +68,23 @@ IMPORTANT: This example uses a `scripted_metric` aggregation which is not suppor
 --------------------------------------------------
 // NOTCONSOLE
 
-<1> The `init_script` creates a long type `timestamp_latest` and a string type 
+<1> The `init_script` creates a long type `timestamp_latest` and a string type
 `last_doc` in the `state` object.
-<2> The `map_script` defines `current_date` based on the timestamp of the 
-document, then compares `current_date` with `state.timestamp_latest`, finally 
-returns `state.last_doc` from the shard. By using `new HashMap(...)` you copy 
-the source document, this is important whenever you want to pass the full source 
+<2> The `map_script` defines `current_date` based on the timestamp of the
+document, then compares `current_date` with `state.timestamp_latest`, finally
+returns `state.last_doc` from the shard. By using `new HashMap(...)` you copy
+the source document, this is important whenever you want to pass the full source
 object from one phase to the next.
 <3> The `combine_script` returns `state` from each shard.
-<4> The `reduce_script` iterates through the value of `s.timestamp_latest` 
-returned by each shard and returns the document with the latest timestamp 
-(`last_doc`). In the response, the top hit (in other words, the `latest_doc`) is 
+<4> The `reduce_script` iterates through the value of `s.timestamp_latest`
+returned by each shard and returns the document with the latest timestamp
+(`last_doc`). In the response, the top hit (in other words, the `latest_doc`) is
 nested below the `latest_doc` field.
 
-Check the <<scripted-metric-aggregation-scope,scope of scripts>> for detailed 
+Check the <<scripted-metric-aggregation-scope,scope of scripts>> for detailed
 explanation on the respective scripts.
 
-You can retrieve the last value in a similar way: 
+You can retrieve the last value in a similar way:
 
 [source,js]
 --------------------------------------------------
@@ -93,17 +93,17 @@ You can retrieve the last value in a similar way:
     "scripted_metric": {
       "init_script": "state.timestamp_latest = 0L; state.last_value = ''",
       "map_script": """
-        def current_date = doc['@timestamp'].getValue().toInstant().toEpochMilli(); 
-        if (current_date > state.timestamp_latest) 
+        def current_date = doc['@timestamp'].getValue().toInstant().toEpochMilli();
+        if (current_date > state.timestamp_latest)
         {state.timestamp_latest = current_date;
         state.last_value = params['_source']['value'];}
       """,
       "combine_script": "return state",
       "reduce_script": """
         def last_value = '';
-        def timestamp_latest = 0L; 
-        for (s in states) {if (s.timestamp_latest > (timestamp_latest)) 
-        {timestamp_latest = s.timestamp_latest; last_value = s.last_value;}} 
+        def timestamp_latest = 0L;
+        for (s in states) {if (s.timestamp_latest > (timestamp_latest))
+        {timestamp_latest = s.timestamp_latest; last_value = s.last_value;}}
         return last_value
       """
     }
@@ -117,10 +117,10 @@ You can retrieve the last value in a similar way:
 [[top-hits-stored-scripts]]
 === Getting top hits by using stored scripts
 
-You can also use the power of 
-{ref}/create-stored-script-api.html[stored scripts] to get the latest value. 
-Stored scripts reduce compilation time,  make searches faster, and are 
-updatable. 
+You can also use the power of
+{ref}/create-stored-script-api.html[stored scripts] to get the latest value.
+Stored scripts are updatable, enable collaboration, and avoid duplication across 
+queries.
 
 1. Create the stored scripts:
 +
@@ -202,7 +202,7 @@ POST _scripts/last-value-reduce
          }
 --------------------------------------------------
 // NOTCONSOLE
-<1> The parameter `field_with_last_value` can be set any field that you want the 
+<1> The parameter `field_with_last_value` can be set any field that you want the
 latest value for.
 --
 
@@ -210,8 +210,8 @@ latest value for.
 [[painless-time-features]]
 == Getting time features by using aggregations
 
-This snippet shows how to extract time based features by using Painless in a 
-{transform}. The snippet uses an index where `@timestamp` is defined as a `date` 
+This snippet shows how to extract time based features by using Painless in a
+{transform}. The snippet uses an index where `@timestamp` is defined as a `date`
 type field.
 
 [source,js]
@@ -225,11 +225,11 @@ type field.
           return date.getHour(); <4>
         """
       }
-    }  
+    }
   },
   "avg_month_of_year": { <5>
     "avg":{
-      "script": { <6> 
+      "script": { <6>
         "source": """
           ZonedDateTime date =  doc['@timestamp'].value; <7>
           return date.getMonthValue(); <8>
@@ -255,9 +255,9 @@ type field.
 [[painless-group-by]]
 == Using Painless in `group_by`
 
-It is possible to base the `group_by` property of a {transform} on the output of 
-a script. The following example uses the {kib} sample web logs dataset. The goal 
-here is to make the {transform} output easier to understand through normalizing 
+It is possible to base the `group_by` property of a {transform} on the output of
+a script. The following example uses the {kib} sample web logs dataset. The goal
+here is to make the {transform} output easier to understand through normalizing
 the value of the fields that the data is grouped by.
 
 [source,console]
@@ -274,12 +274,12 @@ POST _transform/_preview
       "agent": {
         "terms": {
           "script": { <2>
-            "source": """String agent = doc['agent.keyword'].value; 
-            if (agent.contains("MSIE")) { 
+            "source": """String agent = doc['agent.keyword'].value;
+            if (agent.contains("MSIE")) {
               return "internet explorer";
-            } else if (agent.contains("AppleWebKit")) { 
-              return "safari"; 
-            } else if (agent.contains('Firefox')) { 
+            } else if (agent.contains("AppleWebKit")) {
+              return "safari";
+            } else if (agent.contains('Firefox')) {
               return "firefox";
             } else { return agent }""",
             "lang": "painless"
@@ -314,18 +314,18 @@ POST _transform/_preview
   "dest": { <4>
     "index": "pivot_logs"
   }
-} 
+}
 --------------------------------------------------
 // TEST[skip:setup kibana sample data]
 
 <1> Specifies the source index or indices.
-<2> The script defines an `agent` string based on the `agent` field of the 
-documents, then iterates through the values. If an `agent` field contains 
-"MSIE", than the script returns "Internet Explorer". If it contains 
-`AppleWebKit`, it returns "safari". It returns "firefox" if the field value 
-contains "Firefox". Finally, in every other case, the value of the field is 
+<2> The script defines an `agent` string based on the `agent` field of the
+documents, then iterates through the values. If an `agent` field contains
+"MSIE", than the script returns "Internet Explorer". If it contains
+`AppleWebKit`, it returns "safari". It returns "firefox" if the field value
+contains "Firefox". Finally, in every other case, the value of the field is
 returned.
-<3> The aggregations object contains filters that narrow down the results to 
+<3> The aggregations object contains filters that narrow down the results to
 documents that contains `200`, `404`, or `503` values in the `response` field.
 <4> Specifies the destination index of the {transform}.
 
@@ -374,14 +374,14 @@ The API returns the following result:
 --------------------------------------------------
 // NOTCONSOLE
 
-You can see that the `agent` values are simplified so it is easier to interpret 
-them. The table below shows how normalization modifies the output of the 
+You can see that the `agent` values are simplified so it is easier to interpret
+them. The table below shows how normalization modifies the output of the
 {transform} in our example compared to the non-normalized values.
 
 [width="50%"]
 
 |===
-| Non-normalized `agent` value                                                 | Normalized `agent` value 
+| Non-normalized `agent` value                                                 | Normalized `agent` value
 
 | "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)" | "internet explorer"
 | "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24" | "safari"
@@ -393,9 +393,9 @@ them. The table below shows how normalization modifies the output of the
 [[painless-bucket-script]]
 == Getting duration by using bucket script
 
-This example shows you how to get the duration of a session by client IP from a 
-data log by using 
-<<search-aggregations-pipeline-bucket-script-aggregation,bucket script>>. 
+This example shows you how to get the duration of a session by client IP from a
+data log by using
+<<search-aggregations-pipeline-bucket-script-aggregation,bucket script>>.
 The example uses the {kib} sample web logs dataset.
 
 [source,console]
@@ -440,22 +440,22 @@ PUT _transform/data_log
 // TEST[skip:setup kibana sample data]
 
 <1> To define the length of the sessions, we use a bucket script.
-<2> The bucket path is a map of script variables and their associated path to 
-the buckets you want to use for the variable. In this particular case, `min` and 
+<2> The bucket path is a map of script variables and their associated path to
+the buckets you want to use for the variable. In this particular case, `min` and
 `max` are variables mapped to `time_frame.gte.value` and `time_frame.lte.value`.
-<3> Finally, the script substracts the start date of the session from the end 
+<3> Finally, the script substracts the start date of the session from the end
 date which results in the duration of the session.
 
 [[painless-count-http]]
 == Counting HTTP responses by using scripted metric aggregation
 
-You can count the different HTTP response types in a web log data set by using 
-scripted metric aggregation as part of the {transform}. You can achieve a 
-similar function with filter aggregations, check the 
-{ref}/transform-examples.html#example-clientips[Finding suspicious client IPs] 
+You can count the different HTTP response types in a web log data set by using
+scripted metric aggregation as part of the {transform}. You can achieve a
+similar function with filter aggregations, check the
+{ref}/transform-examples.html#example-clientips[Finding suspicious client IPs]
 example for details.
 
-The example below assumes that the HTTP response codes are stored as keywords in 
+The example below assumes that the HTTP response codes are stored as keywords in
 the `response` field of the documents.
 
 IMPORTANT: This example uses a `scripted_metric` aggregation which is not supported on {es} Serverless.
@@ -488,32 +488,32 @@ IMPORTANT: This example uses a `scripted_metric` aggregation which is not suppor
         """
       }
     },
-  ...  
+  ...
 }
 --------------------------------------------------
 // NOTCONSOLE
 
 <1> The `aggregations` object of the {transform} that contains all aggregations.
 <2> Object of the `scripted_metric` aggregation.
-<3> This `scripted_metric` performs a distributed operation on the web log data 
+<3> This `scripted_metric` performs a distributed operation on the web log data
 to count specific types of HTTP responses (error, success, and other).
-<4> The `init_script` creates a `responses` array in the `state` object with 
+<4> The `init_script` creates a `responses` array in the `state` object with
 three properties (`error`, `success`, `other`) with long data type.
-<5> The `map_script` defines `code` based on the `response.keyword` value of the 
-document, then it counts the errors, successes, and other responses based on the 
+<5> The `map_script` defines `code` based on the `response.keyword` value of the
+document, then it counts the errors, successes, and other responses based on the
 first digit of the responses.
 <6> The `combine_script` returns `state.responses` from each shard.
-<7> The `reduce_script` creates a `counts` array with the `error`, `success`, 
-and `other` properties, then iterates through the value of `responses` returned 
-by each shard and assigns the different response types to the appropriate 
-properties of the `counts` object; error responses to the error counts, success 
-responses to the success counts, and other responses to the other counts. 
+<7> The `reduce_script` creates a `counts` array with the `error`, `success`,
+and `other` properties, then iterates through the value of `responses` returned
+by each shard and assigns the different response types to the appropriate
+properties of the `counts` object; error responses to the error counts, success
+responses to the success counts, and other responses to the other counts.
 Finally, returns the `counts` array with the response counts.
 
 [[painless-compare]]
 == Comparing indices by using scripted metric aggregations
 
-This example shows how to compare the content of two indices by a {transform} 
+This example shows how to compare the content of two indices by a {transform}
 that uses a scripted metric aggregation.
 
 IMPORTANT: This example uses a `scripted_metric` aggregation which is not supported on {es} Serverless.
@@ -570,19 +570,19 @@ POST _transform/_preview
 <2> The `dest` index contains the results of the comparison.
 <3> The `group_by` field needs to be a unique identifier for each document.
 <4> Object of the `scripted_metric` aggregation.
-<5> The `map_script` defines `doc` in the state object. By using 
-`new HashMap(...)` you copy the source document, this is important whenever you 
+<5> The `map_script` defines `doc` in the state object. By using
+`new HashMap(...)` you copy the source document, this is important whenever you
 want to pass the full source object from one phase to the next.
 <6> The `combine_script` returns `state` from each shard.
-<7> The `reduce_script` checks if the size of the indices are equal. If they are 
-not equal, than it reports back a `count_mismatch`. Then it iterates through all 
-the values of the two indices and compare them. If the values are equal, then it 
+<7> The `reduce_script` checks if the size of the indices are equal. If they are
+not equal, than it reports back a `count_mismatch`. Then it iterates through all
+the values of the two indices and compare them. If the values are equal, then it
 returns a `match`, otherwise returns a `mismatch`.
 
 [[painless-web-session]]
 == Getting web session details by using scripted metric aggregation
 
-This example shows how to derive multiple features from a single transaction. 
+This example shows how to derive multiple features from a single transaction.
 Let's take a look on the example source document from the data:
 
 .Source document
@@ -628,8 +628,8 @@ Let's take a look on the example source document from the data:
 =====
 
 
-By using the `sessionid` as a group-by field, you are able to enumerate events 
-through the session and get more details of the session by using scripted metric 
+By using the `sessionid` as a group-by field, you are able to enumerate events
+through the session and get more details of the session by using scripted metric
 aggregation.
 
 IMPORTANT: This example uses a `scripted_metric` aggregation which is not supported on {es} Serverless.
@@ -650,7 +650,7 @@ POST _transform/_preview
       }
     },
     "aggregations": { <2>
-      "distinct_paths": { 
+      "distinct_paths": {
         "cardinality": {
           "field": "apache.access.path"
         }
@@ -665,21 +665,21 @@ POST _transform/_preview
           "init_script": "state.docs = []", <3>
           "map_script": """ <4>
             Map span = [
-              '@timestamp':doc['@timestamp'].value, 
+              '@timestamp':doc['@timestamp'].value,
               'url':doc['apache.access.url'].value,
               'referrer':doc['apache.access.referrer'].value
-            ]; 
+            ];
             state.docs.add(span)
           """,
           "combine_script": "return state.docs;", <5>
           "reduce_script": """ <6>
-            def all_docs = []; 
-            for (s in states) { 
-              for (span in s) { 
-                all_docs.add(span); 
+            def all_docs = [];
+            for (s in states) {
+              for (span in s) {
+                all_docs.add(span);
               }
             }
-            all_docs.sort((HashMap o1, HashMap o2)->o1['@timestamp'].toEpochMilli().compareTo(o2['@timestamp'].toEpochMilli())); 
+            all_docs.sort((HashMap o1, HashMap o2)->o1['@timestamp'].toEpochMilli().compareTo(o2['@timestamp'].toEpochMilli()));
             def size = all_docs.size();
             def min_time = all_docs[0]['@timestamp'];
             def max_time = all_docs[size-1]['@timestamp'];
@@ -705,17 +705,17 @@ POST _transform/_preview
 // NOTCONSOLE
 
 <1> The data is grouped by `sessionid`.
-<2> The aggregations counts the number of paths and enumerate the viewed pages 
+<2> The aggregations counts the number of paths and enumerate the viewed pages
 during the session.
 <3> The `init_script` creates an array type `doc` in the `state` object.
-<4> The `map_script` defines a `span` array with a timestamp, a URL, and a 
-referrer value which are based on the corresponding values of the document, then 
+<4> The `map_script` defines a `span` array with a timestamp, a URL, and a
+referrer value which are based on the corresponding values of the document, then
 adds the value of the `span` array to the `doc` object.
 <5> The `combine_script` returns `state.docs` from each shard.
-<6> The `reduce_script` defines various objects like `min_time`, `max_time`, and 
-`duration` based on the document fields, then declares a `ret` object, and 
-copies the source document by using `new HashMap ()`. Next, the script defines 
-`first_time`, `last_time`, `duration` and other fields inside the `ret` object 
+<6> The `reduce_script` defines various objects like `min_time`, `max_time`, and
+`duration` based on the document fields, then declares a `ret` object, and
+copies the source document by using `new HashMap ()`. Next, the script defines
+`first_time`, `last_time`, `duration` and other fields inside the `ret` object
 based on the corresponding object defined earlier, finally returns `ret`.
 
 The API call results in a similar response:
diff --git a/docs/reference/transform/setup.asciidoc b/docs/reference/transform/setup.asciidoc
index dab357546d93e..3171086e43d61 100644
--- a/docs/reference/transform/setup.asciidoc
+++ b/docs/reference/transform/setup.asciidoc
@@ -11,7 +11,7 @@
 
 To use {transforms}, you must have:
 
-* at least one <<transform-node,{transform} node>>,
+* at least one <<transform-node-role,{transform} node>>,
 * management features visible in the {kib} space, and
 * security privileges that:
 +
diff --git a/docs/reference/troubleshooting.asciidoc b/docs/reference/troubleshooting.asciidoc
index ceff8619062c4..75cd0c1be49fc 100644
--- a/docs/reference/troubleshooting.asciidoc
+++ b/docs/reference/troubleshooting.asciidoc
@@ -6,6 +6,10 @@
 This section provides a series of troubleshooting solutions aimed at helping users
 fix problems that an {es} deployment might encounter.
 
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
+
 [discrete]
 [[troubleshooting-general]]
 === General
diff --git a/docs/reference/troubleshooting/common-issues/circuit-breaker-errors.asciidoc b/docs/reference/troubleshooting/common-issues/circuit-breaker-errors.asciidoc
index fbc6fe7b42a7f..8ebe962eaed80 100644
--- a/docs/reference/troubleshooting/common-issues/circuit-breaker-errors.asciidoc
+++ b/docs/reference/troubleshooting/common-issues/circuit-breaker-errors.asciidoc
@@ -9,6 +9,13 @@ By default, the <<parent-circuit-breaker,parent circuit breaker>> triggers at
 95% JVM memory usage. To prevent errors, we recommend taking steps to reduce
 memory pressure if usage consistently exceeds 85%.
 
+See https://www.youtube.com/watch?v=k3wYlRVbMSw[this video] for a walkthrough
+of diagnosing circuit breaker errors.
+
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
+
 [discrete]
 [[diagnose-circuit-breaker-errors]]
 ==== Diagnose circuit breaker errors
diff --git a/docs/reference/troubleshooting/common-issues/diagnose-unassigned-shards.asciidoc b/docs/reference/troubleshooting/common-issues/diagnose-unassigned-shards.asciidoc
index e1ceefb92bbec..c6bbcd69b0555 100644
--- a/docs/reference/troubleshooting/common-issues/diagnose-unassigned-shards.asciidoc
+++ b/docs/reference/troubleshooting/common-issues/diagnose-unassigned-shards.asciidoc
@@ -11,3 +11,7 @@ include::{es-ref-dir}/tab-widgets/troubleshooting/data/diagnose-unassigned-shard
 See https://www.youtube.com/watch?v=v2mbeSd1vTQ[this video]
 for a walkthrough of monitoring allocation health.
 
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
+
diff --git a/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc b/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc
index 2222d09c26db2..a2342c449c88c 100644
--- a/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc
+++ b/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc
@@ -21,6 +21,10 @@ usage falls below the <<cluster-routing-watermark-high,high disk watermark>>.
 To achieve this, {es} attempts to rebalance some of the affected node's shards 
 to other nodes in the same data tier.
 
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
+
 [[fix-watermark-errors-rebalance]]
 ==== Monitor rebalancing
 
diff --git a/docs/reference/troubleshooting/common-issues/high-cpu-usage.asciidoc b/docs/reference/troubleshooting/common-issues/high-cpu-usage.asciidoc
index 96a9a8f1e32b7..7d226a891251b 100644
--- a/docs/reference/troubleshooting/common-issues/high-cpu-usage.asciidoc
+++ b/docs/reference/troubleshooting/common-issues/high-cpu-usage.asciidoc
@@ -11,6 +11,10 @@ depleted, {es} will reject search requests until more threads are available.
 
 You might experience high CPU usage if a <<data-tiers,data tier>>, and therefore the nodes assigned to that tier, is experiencing more traffic than other tiers. This imbalance in resource utilization is also known as <<hotspotting,hot spotting>>.
 
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
+
 [discrete]
 [[diagnose-high-cpu-usage]]
 ==== Diagnose high CPU usage
diff --git a/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc b/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc
index 3469a0ca5bf42..842b7edda73ac 100644
--- a/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc
+++ b/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc
@@ -6,6 +6,10 @@ High JVM memory usage can degrade cluster performance and trigger
 taking steps to reduce memory pressure if a node's JVM memory usage consistently
 exceeds 85%.
 
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
+
 [discrete]
 [[diagnose-high-jvm-memory-pressure]]
 ==== Diagnose high JVM memory pressure
diff --git a/docs/reference/troubleshooting/common-issues/hotspotting.asciidoc b/docs/reference/troubleshooting/common-issues/hotspotting.asciidoc
index 344e36a999a34..53226dbfedb56 100644
--- a/docs/reference/troubleshooting/common-issues/hotspotting.asciidoc
+++ b/docs/reference/troubleshooting/common-issues/hotspotting.asciidoc
@@ -11,6 +11,10 @@ may occur in {es} when resource utilizations are unevenly distributed across
 ongoing significantly unique utilization may lead to cluster bottlenecks 
 and should be reviewed. 
 
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
+
 See link:https://www.youtube.com/watch?v=Q5ODJ5nIKAM[this video] for a walkthrough of troubleshooting a hot spotting issue.
 
 [discrete]
diff --git a/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc b/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc
index 4289242deb486..c07e92c058991 100644
--- a/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc
+++ b/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc
@@ -22,6 +22,10 @@ the remaining problems so management and cleanup activities can proceed.
 See https://www.youtube.com/watch?v=v2mbeSd1vTQ[this video]
 for a walkthrough of monitoring allocation health.
 
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
+
 [discrete]
 [[diagnose-cluster-status]]
 ==== Diagnose your cluster status
diff --git a/docs/reference/troubleshooting/common-issues/rejected-requests.asciidoc b/docs/reference/troubleshooting/common-issues/rejected-requests.asciidoc
index c863709775fcd..7b0e3eb51680e 100644
--- a/docs/reference/troubleshooting/common-issues/rejected-requests.asciidoc
+++ b/docs/reference/troubleshooting/common-issues/rejected-requests.asciidoc
@@ -12,6 +12,10 @@ thread pool returns a `TOO_MANY_REQUESTS` error message.
 * High <<index-modules-indexing-pressure,indexing pressure>> that exceeds the
 <<memory-limits,`indexing_pressure.memory.limit`>>.
 
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
+
 [discrete]
 [[check-rejected-tasks]]
 ==== Check rejected tasks
@@ -32,6 +36,9 @@ correlating log as `EsRejectedExecutionException` with either
 
 These errors are often related to <<task-queue-backlog,backlogged tasks>>.
 
+See https://www.youtube.com/watch?v=auZJRXoAVpI[this video] for a walkthrough
+of troubleshooting threadpool rejections.
+
 [discrete]
 [[check-circuit-breakers]]
 ==== Check circuit breakers
@@ -47,6 +54,9 @@ GET /_nodes/stats/breaker
 These statistics are cumulative from node startup. For more information, see
 <<circuit-breaker,circuit breaker errors>>.
 
+See https://www.youtube.com/watch?v=k3wYlRVbMSw[this video] for a walkthrough
+of diagnosing circuit breaker errors.
+
 [discrete]
 [[check-indexing-pressure]]
 ==== Check indexing pressure
@@ -63,12 +73,15 @@ These stats are cumulative from node startup.
 
 Indexing pressure rejections appear as an
 `EsRejectedExecutionException`, and indicate that they were rejected due
-to `coordinating_and_primary_bytes`, `coordinating`, `primary`, or `replica`.
+to `combined_coordinating_and_primary`, `coordinating`, `primary`, or `replica`.
 
 These errors are often related to <<task-queue-backlog,backlogged tasks>>,
 <<docs-bulk,bulk index>> sizing, or the ingest target's
 <<index-modules,`refresh_interval` setting>>.
 
+See https://www.youtube.com/watch?v=QuV8QqSfc0c[this video] for a walkthrough
+of diagnosing indexing pressure rejections.
+
 [discrete]
 [[prevent-rejected-requests]]
 ==== Prevent rejected requests
@@ -77,4 +90,4 @@ These errors are often related to <<task-queue-backlog,backlogged tasks>>,
 
 If {es} regularly rejects requests and other tasks, your cluster likely has high
 CPU usage or high JVM memory pressure. For tips, see <<high-cpu-usage>> and
-<<high-jvm-memory-pressure>>.
\ No newline at end of file
+<<high-jvm-memory-pressure>>.
diff --git a/docs/reference/troubleshooting/common-issues/task-queue-backlog.asciidoc b/docs/reference/troubleshooting/common-issues/task-queue-backlog.asciidoc
index 5aa6a0129c2d4..f233f22cb3fbe 100644
--- a/docs/reference/troubleshooting/common-issues/task-queue-backlog.asciidoc
+++ b/docs/reference/troubleshooting/common-issues/task-queue-backlog.asciidoc
@@ -1,103 +1,149 @@
 [[task-queue-backlog]]
-=== Task queue backlog
+=== Backlogged task queue
 
-A backlogged task queue can prevent tasks from completing and put the cluster
-into an unhealthy state. Resource constraints, a large number of tasks being
-triggered at once, and long running tasks can all contribute to a backlogged
-task queue.
+*******************************
+*Product:* Elasticsearch +
+*Deployment type:* Elastic Cloud Enterprise, Elastic Cloud Hosted, Elastic Cloud on Kubernetes, Elastic Self-Managed  +
+*Versions:* All
+*******************************
+
+A backlogged task queue can prevent tasks from completing and lead to an 
+unhealthy cluster state. Contributing factors include resource constraints, 
+a large number of tasks triggered at once, and long-running tasks.
 
 [discrete]
 [[diagnose-task-queue-backlog]]
-==== Diagnose a task queue backlog
+==== Diagnose a backlogged task queue
+
+To identify the cause of the backlog, try these diagnostic actions.
 
-**Check the thread pool status**
+* <<diagnose-task-queue-thread-pool>>
+* <<diagnose-task-queue-hot-thread>>
+* <<diagnose-task-queue-long-running-node-tasks>>
+* <<diagnose-task-queue-long-running-cluster-tasks>>
+
+[discrete]
+[[diagnose-task-queue-thread-pool]]
+===== Check the thread pool status
 
 A <<high-cpu-usage,depleted thread pool>> can result in
 <<rejected-requests,rejected requests>>. 
 
-Thread pool depletion might be restricted to a specific <<data-tiers,data tier>>. If <<hotspotting,hot spotting>> is occuring, one node might experience depletion faster than other nodes, leading to performance issues and a growing task backlog.
-
-You can use the <<cat-thread-pool,cat thread pool API>> to see the number of
-active threads in each thread pool and how many tasks are queued, how many
-have been rejected, and how many have completed.
+Use the <<cat-thread-pool,cat thread pool API>> to monitor
+active threads, queued tasks, rejections, and completed tasks:
 
 [source,console]
 ----
 GET /_cat/thread_pool?v&s=t,n&h=type,name,node_name,active,queue,rejected,completed
 ----
 
-The `active` and `queue` statistics are instantaneous while the `rejected` and
-`completed` statistics are cumulative from node startup.
+* Look for high `active` and `queue` metrics, which indicate potential bottlenecks
+and opportunities to <<reduce-cpu-usage,reduce CPU usage>>.
+* Determine whether thread pool issues are specific to a <<data-tiers,data tier>>.
+* Check whether a specific node's thread pool is depleting faster than others. This
+might indicate <<resolve-task-queue-backlog-hotspotting, hot spotting>>.
 
-**Inspect the hot threads on each node**
+[discrete]
+[[diagnose-task-queue-hot-thread]]
+===== Inspect hot threads on each node
 
-If a particular thread pool queue is backed up, you can periodically poll the
-<<cluster-nodes-hot-threads,Nodes hot threads>> API to determine if the thread
-has sufficient resources to progress and gauge how quickly it is progressing.
+If a particular thread pool queue is backed up, periodically poll the
+<<cluster-nodes-hot-threads,nodes hot threads API>> to gauge the thread's
+progression and ensure it has sufficient resources:
 
 [source,console]
 ----
 GET /_nodes/hot_threads
 ----
 
-**Look for long running node tasks**
+Although the hot threads API response does not list the specific tasks running on a thread, 
+it provides a summary of the thread's activities. You can correlate a hot threads response 
+with a <<tasks,task management API response>> to identify any overlap with specific tasks. For 
+example, if the hot threads response indicates the thread is `performing a search query`, you can 
+<<diagnose-task-queue-long-running-node-tasks,check for long-running search tasks>> using the task management API.
+
+[discrete]
+[[diagnose-task-queue-long-running-node-tasks]]
+===== Identify long-running node tasks
 
-Long-running tasks can also cause a backlog. You can use the <<tasks,task
-management>> API to get information about the node tasks that are running.
-Check the `running_time_in_nanos` to identify tasks that are taking an
-excessive amount of time to complete.
+Long-running tasks can also cause a backlog. Use the <<tasks,task
+management API>> to check for excessive `running_time_in_nanos` values:
 
 [source,console]
 ----
 GET /_tasks?pretty=true&human=true&detailed=true
 ----
 
-If a particular `action` is suspected, you can filter the tasks further. The most common long-running tasks are <<docs-bulk,bulk index>>- or search-related.
+You can filter on a specific `action`, such as <<docs-bulk,bulk indexing>> or search-related tasks.
+These tend to be long-running.
 
-* Filter for <<docs-bulk,bulk index>> actions:
+* Filter on <<docs-bulk,bulk index>> actions:
 +
 [source,console]
 ----
 GET /_tasks?human&detailed&actions=indices:data/write/bulk
 ----
 
-* Filter for search actions:
+* Filter on search actions:
 +
 [source,console]
 ----
 GET /_tasks?human&detailed&actions=indices:data/write/search
 ----
 
-The API response may contain additional tasks columns, including `description` and `header`, which provides the task parameters, target, and requestor. You can use this information to perform further diagnosis.
+Long-running tasks might need to be <<resolve-task-queue-backlog-stuck-tasks,canceled>>.
 
-**Look for long running cluster tasks**
+[discrete]
+[[diagnose-task-queue-long-running-cluster-tasks]]
+===== Look for long-running cluster tasks
 
-A task backlog might also appear as a delay in synchronizing the cluster state. You
-can use the <<cluster-pending,cluster pending tasks API>> to get information
-about the pending cluster state sync tasks that are running. 
+Use the <<cluster-pending,cluster pending tasks API>> to identify delays
+in cluster state synchronization: 
 
 [source,console]
 ----
 GET /_cluster/pending_tasks
 ----
 
-Check the `timeInQueue` to identify tasks that are taking an excessive amount 
-of time to complete.
+Tasks with a high `timeInQueue` value are likely contributing to the backlog and might
+need to be <<resolve-task-queue-backlog-stuck-tasks,canceled>>.
 
 [discrete]
 [[resolve-task-queue-backlog]]
-==== Resolve a task queue backlog
+==== Recommendations
+
+After identifying problematic threads and tasks, resolve the issue by increasing resources or canceling tasks.
 
-**Increase available resources** 
+[discrete]
+[[resolve-task-queue-backlog-resources]]
+===== Increase available resources
 
-If tasks are progressing slowly and the queue is backing up, 
-you might need to take steps to <<reduce-cpu-usage>>. 
+If tasks are progressing slowly, try <<reduce-cpu-usage,reducing CPU usage>>.
 
-In some cases, increasing the thread pool size might help.
-For example, the `force_merge` thread pool defaults to a single thread.
+In some cases, you might need to increase the thread pool size. For example, the `force_merge` thread pool defaults to a single thread. 
 Increasing the size to 2 might help reduce a backlog of force merge requests.
 
-**Cancel stuck tasks**
+[discrete]
+[[resolve-task-queue-backlog-stuck-tasks]]
+===== Cancel stuck tasks
+
+If an active task's <<diagnose-task-queue-hot-thread,hot thread>> shows no progress, consider <<task-cancellation,canceling the task>>.
+
+[discrete]
+[[resolve-task-queue-backlog-hotspotting]]
+===== Address hot spotting
+
+If a specific node's thread pool is depleting faster than others, try addressing
+uneven node resource utilization, also known as hot spotting.
+For details on actions you can take, such as rebalancing shards, see <<hotspotting>>.
+
+[discrete]
+==== Resources
+
+Related symptoms:
+
+* <<high-cpu-usage>>
+* <<rejected-requests>>
+* <<hotspotting>>
 
-If you find the active task's hot thread isn't progressing and there's a backlog, 
-consider canceling the task. 
\ No newline at end of file
+// TODO add link to standard Additional resources when that topic exists
diff --git a/docs/reference/troubleshooting/data/increase-cluster-shard-limit.asciidoc b/docs/reference/troubleshooting/data/increase-cluster-shard-limit.asciidoc
index 8b8703f9a9dc1..ddd0881385524 100644
--- a/docs/reference/troubleshooting/data/increase-cluster-shard-limit.asciidoc
+++ b/docs/reference/troubleshooting/data/increase-cluster-shard-limit.asciidoc
@@ -16,5 +16,8 @@ In order to fix this follow the next steps:
 
 include::{es-ref-dir}/tab-widgets/troubleshooting/data/increase-cluster-shard-limit-widget.asciidoc[]
 
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
 
 
diff --git a/docs/reference/troubleshooting/data/increase-shard-limit.asciidoc b/docs/reference/troubleshooting/data/increase-shard-limit.asciidoc
index 121b5348ab36a..51bb9fda09fef 100644
--- a/docs/reference/troubleshooting/data/increase-shard-limit.asciidoc
+++ b/docs/reference/troubleshooting/data/increase-shard-limit.asciidoc
@@ -14,5 +14,9 @@ In order to fix this follow the next steps:
 
 include::{es-ref-dir}/tab-widgets/troubleshooting/data/total-shards-per-node-widget.asciidoc[]
 
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
+
 
 
diff --git a/docs/reference/troubleshooting/data/increase-tier-capacity.asciidoc b/docs/reference/troubleshooting/data/increase-tier-capacity.asciidoc
index 362a14c3874db..09d7fedb89e6a 100644
--- a/docs/reference/troubleshooting/data/increase-tier-capacity.asciidoc
+++ b/docs/reference/troubleshooting/data/increase-tier-capacity.asciidoc
@@ -17,5 +17,8 @@ In order to fix this follow the next steps:
 
 include::{es-ref-dir}/tab-widgets/troubleshooting/data/increase-tier-capacity-widget.asciidoc[]
 
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
 
 
diff --git a/docs/reference/troubleshooting/diagnostic.asciidoc b/docs/reference/troubleshooting/diagnostic.asciidoc
index c6d46b9e94fc8..d806fa2b986c8 100644
--- a/docs/reference/troubleshooting/diagnostic.asciidoc
+++ b/docs/reference/troubleshooting/diagnostic.asciidoc
@@ -15,6 +15,10 @@ https://discuss.elastic.co[Elastic Discuss] to minimize turnaround time.
 
 See this https://www.youtube.com/watch?v=Bb6SaqhqYHw[this video] for a walkthrough of capturing an {es} diagnostic.
 
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
+
 [discrete]
 [[diagnostic-tool-requirements]]
 === Requirements
diff --git a/docs/reference/troubleshooting/fix-common-cluster-issues.asciidoc b/docs/reference/troubleshooting/fix-common-cluster-issues.asciidoc
index 47244ac17e99d..5aa90215d7ae3 100644
--- a/docs/reference/troubleshooting/fix-common-cluster-issues.asciidoc
+++ b/docs/reference/troubleshooting/fix-common-cluster-issues.asciidoc
@@ -3,6 +3,10 @@
 
 This guide describes how to fix common errors and problems with {es} clusters.
 
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
+
 <<fix-watermark-errors,Watermark errors>>::
 Fix watermark errors that occur when a data node is critically low on disk space
 and has reached the flood-stage disk usage watermark.
diff --git a/docs/reference/troubleshooting/snapshot/repeated-snapshot-failures.asciidoc b/docs/reference/troubleshooting/snapshot/repeated-snapshot-failures.asciidoc
index 2496781c0c8f4..7a7e606e2ca7d 100644
--- a/docs/reference/troubleshooting/snapshot/repeated-snapshot-failures.asciidoc
+++ b/docs/reference/troubleshooting/snapshot/repeated-snapshot-failures.asciidoc
@@ -14,5 +14,8 @@ information about the problem:
 
 include::{es-ref-dir}/tab-widgets/troubleshooting/snapshot/repeated-snapshot-failures-widget.asciidoc[]
 
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
 
 
diff --git a/docs/reference/troubleshooting/troubleshooting-shards-capacity.asciidoc b/docs/reference/troubleshooting/troubleshooting-shards-capacity.asciidoc
index bc8fb7290f1ed..d9871855faa84 100644
--- a/docs/reference/troubleshooting/troubleshooting-shards-capacity.asciidoc
+++ b/docs/reference/troubleshooting/troubleshooting-shards-capacity.asciidoc
@@ -8,3 +8,7 @@ The current shards capacity of the cluster is available in the
 <<health-api-response-details-shards-capacity, health API shards capacity section>>.
 
 include::{es-ref-dir}/tab-widgets/troubleshooting/troubleshooting-shards-capacity-widget.asciidoc[]
+
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
diff --git a/docs/reference/troubleshooting/troubleshooting-unbalanced-cluster.asciidoc b/docs/reference/troubleshooting/troubleshooting-unbalanced-cluster.asciidoc
index a1d4f5df9c4f6..4ac3155c7fa8e 100644
--- a/docs/reference/troubleshooting/troubleshooting-unbalanced-cluster.asciidoc
+++ b/docs/reference/troubleshooting/troubleshooting-unbalanced-cluster.asciidoc
@@ -7,6 +7,10 @@ Elasticsearch balances shards across data tiers to achieve a good compromise bet
 * disk usage
 * write load (for indices in data streams)
 
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
+
 Elasticsearch does not take into account the amount or complexity of search queries when rebalancing shards.
 This is indirectly achieved by balancing shard count and disk usage.
 
diff --git a/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc b/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc
index e47b85aa99547..ab0faa3b6c710 100644
--- a/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc
+++ b/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc
@@ -17,6 +17,10 @@ logs.
 
 * The master may appear busy due to frequent cluster state updates.
 
+****
+If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps].
+****
+
 To troubleshoot a cluster in this state, first ensure the cluster has a
 <<discovery-troubleshooting,stable master>>. Next, focus on the nodes
 unexpectedly leaving the cluster ahead of all other issues. It will not be
diff --git a/docs/reference/upgrade/disable-shard-alloc.asciidoc b/docs/reference/upgrade/disable-shard-alloc.asciidoc
index f69a673095257..4672ea65446b4 100644
--- a/docs/reference/upgrade/disable-shard-alloc.asciidoc
+++ b/docs/reference/upgrade/disable-shard-alloc.asciidoc
@@ -5,7 +5,7 @@ starting to replicate the shards on that node to other nodes in the cluster,
 which can involve a lot of I/O. Since the node is shortly going to be
 restarted, this I/O is unnecessary. You can avoid racing the clock by
 <<cluster-routing-allocation-enable,disabling allocation>> of replicas before
-shutting down <<data-node,data nodes>>:
+shutting down <<data-node-role,data nodes>>:
 
 [source,console]
 --------------------------------------------------
diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
index 33addef8aedd0..2c46c4642e56e 100644
--- a/gradle/verification-metadata.xml
+++ b/gradle/verification-metadata.xml
@@ -949,14 +949,9 @@
             <sha256 value="d694edd7bae3bc1a8e0dae2f5a22c479ff04d6b9bfcb0ab751a42f02e02d2100" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.netflix.nebula" name="gradle-ospackage-plugin" version="11.10.0">
-         <artifact name="gradle-ospackage-plugin-11.10.0.jar">
-            <sha256 value="93c264b05864f8db979a169a99e4bfaad84caa30b829743f67210382dff89619" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="com.netflix.nebula" name="gradle-ospackage-plugin" version="11.9.1">
-         <artifact name="gradle-ospackage-plugin-11.9.1.jar">
-            <sha256 value="63b2c387cb83481bb8d7e33e0be02e9000c0ec30843ca7730641392f3f5f28ad" origin="Generated by Gradle"/>
+      <component group="com.netflix.nebula" name="gradle-ospackage-plugin" version="11.10.1">
+         <artifact name="gradle-ospackage-plugin-11.10.1.jar">
+            <sha256 value="8f1daf5885f1a8f6bcab00bf9b52cf484d87da7839e9bbd327a94af27df5dacc" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.netflix.nebula" name="nebula-gradle-interop" version="2.0.0">
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index 22286c90de3d1..e712035eabc7b 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=89d4e70e4e84e2d2dfbb63e4daa53e21b25017cc70c37e4eea31ee51fb15098a
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-all.zip
+distributionSha256Sum=7ebdac923867a3cec0098302416d1e3c6c0c729fc4e2e05c10637a8af33a76c5
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
diff --git a/gradlew b/gradlew
index f5feea6d6b116..f3b75f3b0d4fa 100755
--- a/gradlew
+++ b/gradlew
@@ -86,8 +86,7 @@ done
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
 # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
-APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
-' "$PWD" ) || exit
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
diff --git a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java
index 9f10739486238..eaf4d0ad98ef5 100644
--- a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java
+++ b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java
@@ -66,10 +66,20 @@ public MethodVisitor visitMethod(
 
     private static final Type CLASS_TYPE = Type.getType(Class.class);
 
-    static MethodKey parseCheckerMethodSignature(String checkerMethodName, Type[] checkerMethodArgumentTypes) {
-        var classNameStartIndex = checkerMethodName.indexOf('$');
-        var classNameEndIndex = checkerMethodName.lastIndexOf('$');
+    static ParsedCheckerMethod parseCheckerMethodName(String checkerMethodName) {
+        boolean targetMethodIsStatic;
+        int classNameEndIndex = checkerMethodName.lastIndexOf("$$");
+        int methodNameStartIndex;
+        if (classNameEndIndex == -1) {
+            targetMethodIsStatic = false;
+            classNameEndIndex = checkerMethodName.lastIndexOf('$');
+            methodNameStartIndex = classNameEndIndex + 1;
+        } else {
+            targetMethodIsStatic = true;
+            methodNameStartIndex = classNameEndIndex + 2;
+        }
 
+        var classNameStartIndex = checkerMethodName.indexOf('$');
         if (classNameStartIndex == -1 || classNameStartIndex >= classNameEndIndex) {
             throw new IllegalArgumentException(
                 String.format(
@@ -82,15 +92,22 @@ static MethodKey parseCheckerMethodSignature(String checkerMethodName, Type[] ch
             );
         }
 
-        // No "className" (check$$methodName) -> method is instance, and we'll get the class from the actual typed argument
-        final boolean targetMethodIsStatic = classNameStartIndex + 1 != classNameEndIndex;
         // No "methodName" (check$package_ClassName$) -> method is ctor
         final boolean targetMethodIsCtor = classNameEndIndex + 1 == checkerMethodName.length();
-        final String targetMethodName = targetMethodIsCtor ? "<init>" : checkerMethodName.substring(classNameEndIndex + 1);
+        final String targetMethodName = targetMethodIsCtor ? "<init>" : checkerMethodName.substring(methodNameStartIndex);
+
+        final String targetClassName = checkerMethodName.substring(classNameStartIndex + 1, classNameEndIndex).replace('_', '/');
+        if (targetClassName.isBlank()) {
+            throw new IllegalArgumentException(String.format(Locale.ROOT, "Checker method %s has no class name", checkerMethodName));
+        }
+        return new ParsedCheckerMethod(targetClassName, targetMethodName, targetMethodIsStatic, targetMethodIsCtor);
+    }
+
+    static MethodKey parseCheckerMethodSignature(String checkerMethodName, Type[] checkerMethodArgumentTypes) {
+        ParsedCheckerMethod checkerMethod = parseCheckerMethodName(checkerMethodName);
 
-        final String targetClassName;
         final List<String> targetParameterTypes;
-        if (targetMethodIsStatic) {
+        if (checkerMethod.targetMethodIsStatic() || checkerMethod.targetMethodIsCtor()) {
             if (checkerMethodArgumentTypes.length < 1 || CLASS_TYPE.equals(checkerMethodArgumentTypes[0]) == false) {
                 throw new IllegalArgumentException(
                     String.format(
@@ -101,7 +118,6 @@ static MethodKey parseCheckerMethodSignature(String checkerMethodName, Type[] ch
                 );
             }
 
-            targetClassName = checkerMethodName.substring(classNameStartIndex + 1, classNameEndIndex).replace('_', '/');
             targetParameterTypes = Arrays.stream(checkerMethodArgumentTypes).skip(1).map(Type::getInternalName).toList();
         } else {
             if (checkerMethodArgumentTypes.length < 2
@@ -117,10 +133,15 @@ static MethodKey parseCheckerMethodSignature(String checkerMethodName, Type[] ch
                     )
                 );
             }
-            var targetClassType = checkerMethodArgumentTypes[1];
-            targetClassName = targetClassType.getInternalName();
             targetParameterTypes = Arrays.stream(checkerMethodArgumentTypes).skip(2).map(Type::getInternalName).toList();
         }
-        return new MethodKey(targetClassName, targetMethodName, targetParameterTypes);
+        return new MethodKey(checkerMethod.targetClassName(), checkerMethod.targetMethodName(), targetParameterTypes);
     }
+
+    private record ParsedCheckerMethod(
+        String targetClassName,
+        String targetMethodName,
+        boolean targetMethodIsStatic,
+        boolean targetMethodIsCtor
+    ) {}
 }
diff --git a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java
index 00efab829b2bb..06408941ac96e 100644
--- a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java
+++ b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java
@@ -152,6 +152,7 @@ public MethodVisitor visitMethod(int access, String name, String descriptor, Str
             if (isAnnotationPresent == false) {
                 boolean isStatic = (access & ACC_STATIC) != 0;
                 boolean isCtor = "<init>".equals(name);
+                boolean hasReceiver = (isStatic || isCtor) == false;
                 var key = new MethodKey(className, name, Stream.of(Type.getArgumentTypes(descriptor)).map(Type::getInternalName).toList());
                 var instrumentationMethod = checkMethods.get(key);
                 if (instrumentationMethod != null) {
diff --git a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java
index e3285cec8f883..ab0d96a8df96d 100644
--- a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java
+++ b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java
@@ -32,17 +32,17 @@ public class InstrumentationServiceImplTests extends ESTestCase {
     static class TestTargetClass {}
 
     interface TestChecker {
-        void check$org_example_TestTargetClass$staticMethod(Class<?> clazz, int arg0, String arg1, Object arg2);
+        void check$org_example_TestTargetClass$$staticMethod(Class<?> clazz, int arg0, String arg1, Object arg2);
 
-        void check$$instanceMethodNoArgs(Class<?> clazz, TestTargetClass that);
+        void check$org_example_TestTargetClass$instanceMethodNoArgs(Class<?> clazz, TestTargetClass that);
 
-        void check$$instanceMethodWithArgs(Class<?> clazz, TestTargetClass that, int x, int y);
+        void check$org_example_TestTargetClass$instanceMethodWithArgs(Class<?> clazz, TestTargetClass that, int x, int y);
     }
 
     interface TestCheckerOverloads {
-        void check$org_example_TestTargetClass$staticMethodWithOverload(Class<?> clazz, int x, int y);
+        void check$org_example_TestTargetClass$$staticMethodWithOverload(Class<?> clazz, int x, int y);
 
-        void check$org_example_TestTargetClass$staticMethodWithOverload(Class<?> clazz, int x, String y);
+        void check$org_example_TestTargetClass$$staticMethodWithOverload(Class<?> clazz, int x, String y);
     }
 
     interface TestCheckerCtors {
@@ -62,7 +62,7 @@ public void testInstrumentationTargetLookup() throws IOException {
                 equalTo(
                     new CheckMethod(
                         "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestChecker",
-                        "check$org_example_TestTargetClass$staticMethod",
+                        "check$org_example_TestTargetClass$$staticMethod",
                         List.of("Ljava/lang/Class;", "I", "Ljava/lang/String;", "Ljava/lang/Object;")
                     )
                 )
@@ -71,17 +71,11 @@ public void testInstrumentationTargetLookup() throws IOException {
         assertThat(
             checkMethods,
             hasEntry(
-                equalTo(
-                    new MethodKey(
-                        "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetClass",
-                        "instanceMethodNoArgs",
-                        List.of()
-                    )
-                ),
+                equalTo(new MethodKey("org/example/TestTargetClass", "instanceMethodNoArgs", List.of())),
                 equalTo(
                     new CheckMethod(
                         "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestChecker",
-                        "check$$instanceMethodNoArgs",
+                        "check$org_example_TestTargetClass$instanceMethodNoArgs",
                         List.of(
                             "Ljava/lang/Class;",
                             "Lorg/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetClass;"
@@ -93,17 +87,11 @@ public void testInstrumentationTargetLookup() throws IOException {
         assertThat(
             checkMethods,
             hasEntry(
-                equalTo(
-                    new MethodKey(
-                        "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetClass",
-                        "instanceMethodWithArgs",
-                        List.of("I", "I")
-                    )
-                ),
+                equalTo(new MethodKey("org/example/TestTargetClass", "instanceMethodWithArgs", List.of("I", "I"))),
                 equalTo(
                     new CheckMethod(
                         "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestChecker",
-                        "check$$instanceMethodWithArgs",
+                        "check$org_example_TestTargetClass$instanceMethodWithArgs",
                         List.of(
                             "Ljava/lang/Class;",
                             "Lorg/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetClass;",
@@ -127,7 +115,7 @@ public void testInstrumentationTargetLookupWithOverloads() throws IOException {
                 equalTo(
                     new CheckMethod(
                         "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestCheckerOverloads",
-                        "check$org_example_TestTargetClass$staticMethodWithOverload",
+                        "check$org_example_TestTargetClass$$staticMethodWithOverload",
                         List.of("Ljava/lang/Class;", "I", "Ljava/lang/String;")
                     )
                 )
@@ -140,7 +128,7 @@ public void testInstrumentationTargetLookupWithOverloads() throws IOException {
                 equalTo(
                     new CheckMethod(
                         "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestCheckerOverloads",
-                        "check$org_example_TestTargetClass$staticMethodWithOverload",
+                        "check$org_example_TestTargetClass$$staticMethodWithOverload",
                         List.of("Ljava/lang/Class;", "I", "I")
                     )
                 )
@@ -182,7 +170,7 @@ public void testInstrumentationTargetLookupWithCtors() throws IOException {
 
     public void testParseCheckerMethodSignatureStaticMethod() {
         var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature(
-            "check$org_example_TestClass$staticMethod",
+            "check$org_example_TestClass$$staticMethod",
             new Type[] { Type.getType(Class.class) }
         );
 
@@ -191,7 +179,7 @@ public void testParseCheckerMethodSignatureStaticMethod() {
 
     public void testParseCheckerMethodSignatureStaticMethodWithArgs() {
         var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature(
-            "check$org_example_TestClass$staticMethod",
+            "check$org_example_TestClass$$staticMethod",
             new Type[] { Type.getType(Class.class), Type.getType("I"), Type.getType(String.class) }
         );
 
@@ -200,7 +188,7 @@ public void testParseCheckerMethodSignatureStaticMethodWithArgs() {
 
     public void testParseCheckerMethodSignatureStaticMethodInnerClass() {
         var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature(
-            "check$org_example_TestClass$InnerClass$staticMethod",
+            "check$org_example_TestClass$InnerClass$$staticMethod",
             new Type[] { Type.getType(Class.class) }
         );
 
@@ -225,94 +213,80 @@ public void testParseCheckerMethodSignatureCtorWithArgs() {
         assertThat(methodKey, equalTo(new MethodKey("org/example/TestClass", "<init>", List.of("I", "java/lang/String"))));
     }
 
-    public void testParseCheckerMethodSignatureIncorrectName() {
-        var exception = assertThrows(
-            IllegalArgumentException.class,
-            () -> InstrumentationServiceImpl.parseCheckerMethodSignature("check$staticMethod", new Type[] { Type.getType(Class.class) })
-        );
+    public void testParseCheckerMethodSignatureOneDollarSign() {
+        assertParseCheckerMethodSignatureThrows("has incorrect name format", "check$method", Type.getType(Class.class));
+    }
 
-        assertThat(exception.getMessage(), containsString("has incorrect name format"));
+    public void testParseCheckerMethodSignatureMissingClass() {
+        assertParseCheckerMethodSignatureThrows("has incorrect name format", "check$$staticMethod", Type.getType(Class.class));
+    }
+
+    public void testParseCheckerMethodSignatureBlankClass() {
+        assertParseCheckerMethodSignatureThrows("no class name", "check$$$staticMethod", Type.getType(Class.class));
     }
 
     public void testParseCheckerMethodSignatureStaticMethodIncorrectArgumentCount() {
-        var exception = assertThrows(
-            IllegalArgumentException.class,
-            () -> InstrumentationServiceImpl.parseCheckerMethodSignature("check$ClassName$staticMethod", new Type[] {})
-        );
-        assertThat(exception.getMessage(), containsString("It must have a first argument of Class<?> type"));
+        assertParseCheckerMethodSignatureThrows("It must have a first argument of Class<?> type", "check$ClassName$staticMethod");
     }
 
     public void testParseCheckerMethodSignatureStaticMethodIncorrectArgumentType() {
-        var exception = assertThrows(
-            IllegalArgumentException.class,
-            () -> InstrumentationServiceImpl.parseCheckerMethodSignature(
-                "check$ClassName$staticMethod",
-                new Type[] { Type.getType(String.class) }
-            )
+        assertParseCheckerMethodSignatureThrows(
+            "It must have a first argument of Class<?> type",
+            "check$ClassName$$staticMethod",
+            Type.getType(String.class)
         );
-        assertThat(exception.getMessage(), containsString("It must have a first argument of Class<?> type"));
     }
 
     public void testParseCheckerMethodSignatureInstanceMethod() {
         var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature(
-            "check$$instanceMethod",
+            "check$org_example_TestClass$instanceMethod",
             new Type[] { Type.getType(Class.class), Type.getType(TestTargetClass.class) }
         );
 
-        assertThat(
-            methodKey,
-            equalTo(
-                new MethodKey(
-                    "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetClass",
-                    "instanceMethod",
-                    List.of()
-                )
-            )
-        );
+        assertThat(methodKey, equalTo(new MethodKey("org/example/TestClass", "instanceMethod", List.of())));
     }
 
     public void testParseCheckerMethodSignatureInstanceMethodWithArgs() {
         var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature(
-            "check$$instanceMethod",
+            "check$org_example_TestClass$instanceMethod",
             new Type[] { Type.getType(Class.class), Type.getType(TestTargetClass.class), Type.getType("I"), Type.getType(String.class) }
         );
 
-        assertThat(
-            methodKey,
-            equalTo(
-                new MethodKey(
-                    "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetClass",
-                    "instanceMethod",
-                    List.of("I", "java/lang/String")
-                )
-            )
-        );
+        assertThat(methodKey, equalTo(new MethodKey("org/example/TestClass", "instanceMethod", List.of("I", "java/lang/String"))));
     }
 
     public void testParseCheckerMethodSignatureInstanceMethodIncorrectArgumentTypes() {
-        var exception = assertThrows(
-            IllegalArgumentException.class,
-            () -> InstrumentationServiceImpl.parseCheckerMethodSignature("check$$instanceMethod", new Type[] { Type.getType(String.class) })
+        assertParseCheckerMethodSignatureThrows(
+            "It must have a first argument of Class<?> type",
+            "check$org_example_TestClass$instanceMethod",
+            Type.getType(String.class)
         );
-        assertThat(exception.getMessage(), containsString("It must have a first argument of Class<?> type"));
     }
 
     public void testParseCheckerMethodSignatureInstanceMethodIncorrectArgumentCount() {
-        var exception = assertThrows(
-            IllegalArgumentException.class,
-            () -> InstrumentationServiceImpl.parseCheckerMethodSignature("check$$instanceMethod", new Type[] { Type.getType(Class.class) })
+        assertParseCheckerMethodSignatureThrows(
+            "a second argument of the class containing the method to instrument",
+            "check$org_example_TestClass$instanceMethod",
+            Type.getType(Class.class)
         );
-        assertThat(exception.getMessage(), containsString("a second argument of the class containing the method to instrument"));
     }
 
     public void testParseCheckerMethodSignatureInstanceMethodIncorrectArgumentTypes2() {
+        assertParseCheckerMethodSignatureThrows(
+            "a second argument of the class containing the method to instrument",
+            "check$org_example_TestClass$instanceMethod",
+            Type.getType(Class.class),
+            Type.getType("I")
+        );
+    }
+
+    private static void assertParseCheckerMethodSignatureThrows(String messageText, String methodName, Type... methodArgs) {
         var exception = assertThrows(
             IllegalArgumentException.class,
-            () -> InstrumentationServiceImpl.parseCheckerMethodSignature(
-                "check$$instanceMethod",
-                new Type[] { Type.getType(Class.class), Type.getType("I") }
-            )
+            () -> InstrumentationServiceImpl.parseCheckerMethodSignature(methodName, methodArgs)
         );
-        assertThat(exception.getMessage(), containsString("a second argument of the class containing the method to instrument"));
+
+        assertThat(exception.getMessage(), containsString(messageText));
     }
+
 }
diff --git a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java
index e9af1d152dd35..5a6bf409f1ac4 100644
--- a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java
+++ b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java
@@ -85,9 +85,7 @@ public interface Testable {
      * This is a placeholder for real class library methods.
      * Without the java agent, we can't instrument the real methods, so we instrument this instead.
      * <p>
-     * Methods of this class must have the same signature and the same static/virtual condition as the corresponding real method.
-     * They should assert that the arguments came through correctly.
-     * They must not throw {@link TestException}.
+     * The instrumented copy of this class will not extend this class, but it will implement {@link Testable}.
      */
     public static class TestClassToInstrument implements Testable {
 
@@ -108,31 +106,36 @@ public static void anotherStaticMethod(int arg) {}
 
     /**
      * Interface to test specific, "synthetic" cases (e.g. overloaded methods, overloaded constructors, etc.) that
-     * may be not present/may be difficult to find or not clear in the production EntitlementChecker interface
+     * may be not present/may be difficult to find or not clear in the production EntitlementChecker interface.
+     * <p>
+     * This interface isn't subject to the {@code check$} method naming conventions because it doesn't
+     * participate in the automated scan that configures the instrumenter based on the method names;
+     * instead, we configure the instrumenter minimally as needed for each test.
      */
     public interface MockEntitlementChecker {
-        void checkSomeStaticMethod(Class<?> clazz, int arg);
+        void checkSomeStaticMethod(Class<?> callerClass, int arg);
+
+        void checkSomeStaticMethodOverload(Class<?> callerClass, int arg, String anotherArg);
 
-        void checkSomeStaticMethodOverload(Class<?> clazz, int arg, String anotherArg);
+        void checkAnotherStaticMethod(Class<?> callerClass, int arg);
 
-        void checkAnotherStaticMethod(Class<?> clazz, int arg);
+        void checkSomeInstanceMethod(Class<?> callerClass, Testable that, int arg, String anotherArg);
 
-        void checkSomeInstanceMethod(Class<?> clazz, Testable that, int arg, String anotherArg);
+        void checkCtor(Class<?> callerClass);
 
-        void checkCtor(Class<?> clazz);
+        void checkCtorOverload(Class<?> callerClass, int arg);
 
-        void checkCtorOverload(Class<?> clazz, int arg);
     }
 
     public static class TestEntitlementCheckerHolder {
-        static TestEntitlementChecker checkerInstance = new TestEntitlementChecker();
+        static MockEntitlementCheckerImpl checkerInstance = new MockEntitlementCheckerImpl();
 
         public static MockEntitlementChecker instance() {
             return checkerInstance;
         }
     }
 
-    public static class TestEntitlementChecker implements MockEntitlementChecker {
+    public static class MockEntitlementCheckerImpl implements MockEntitlementChecker {
         /**
          * This allows us to test that the instrumentation is correct in both cases:
          * if the check throws, and if it doesn't.
@@ -206,7 +209,7 @@ public void checkCtorOverload(Class<?> callerClass, int arg) {
 
     @Before
     public void resetInstance() {
-        TestEntitlementCheckerHolder.checkerInstance = new TestEntitlementChecker();
+        TestEntitlementCheckerHolder.checkerInstance = new MockEntitlementCheckerImpl();
     }
 
     public void testStaticMethod() throws Exception {
@@ -285,18 +288,16 @@ public void testConstructors() throws Exception {
         assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkCtorIntCallCount);
     }
 
-    /** This test doesn't replace classToInstrument in-place but instead loads a separate
-     * class with the same class name plus a "_NEW" suffix (classToInstrument.class.getName() + "_NEW")
-     * that contains the instrumentation. Because of this, we need to configure the Transformer to use a
-     * MethodKey and instrumentationMethod with slightly different signatures (using the common interface
-     * Testable) which is not what would happen when it's run by the agent.
+    /**
+     * These tests don't replace classToInstrument in-place but instead load a separate class with the same class name.
+     * This requires a configuration slightly different from what we'd use in production.
      */
     private static InstrumenterImpl createInstrumenter(Map<String, Executable> methods) throws NoSuchMethodException {
         Map<MethodKey, CheckMethod> checkMethods = new HashMap<>();
         for (var entry : methods.entrySet()) {
             checkMethods.put(getMethodKey(entry.getValue()), getCheckMethod(entry.getKey(), entry.getValue()));
         }
-        String checkerClass = Type.getInternalName(InstrumenterTests.MockEntitlementChecker.class);
+        String checkerClass = Type.getInternalName(MockEntitlementChecker.class);
         String handleClass = Type.getInternalName(InstrumenterTests.TestEntitlementCheckerHolder.class);
         String getCheckerClassMethodDescriptor = Type.getMethodDescriptor(Type.getObjectType(checkerClass));
 
diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java
index a6b8a31fc3894..69fc57973f68a 100644
--- a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java
+++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java
@@ -9,17 +9,73 @@
 
 package org.elasticsearch.entitlement.bridge;
 
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.net.ContentHandlerFactory;
+import java.net.DatagramPacket;
+import java.net.DatagramSocket;
+import java.net.DatagramSocketImplFactory;
+import java.net.FileNameMap;
+import java.net.InetAddress;
+import java.net.MulticastSocket;
+import java.net.NetworkInterface;
+import java.net.ProxySelector;
+import java.net.ResponseCache;
+import java.net.SocketAddress;
+import java.net.SocketImplFactory;
 import java.net.URL;
+import java.net.URLStreamHandler;
 import java.net.URLStreamHandlerFactory;
+import java.util.List;
 
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.SSLSocketFactory;
+
+@SuppressWarnings("unused") // Called from instrumentation code inserted by the Entitlements agent
 public interface EntitlementChecker {
 
+    ////////////////////
+    //
     // Exit the JVM process
-    void check$$exit(Class<?> callerClass, Runtime runtime, int status);
+    //
+
+    void check$java_lang_Runtime$exit(Class<?> callerClass, Runtime runtime, int status);
+
+    void check$java_lang_Runtime$halt(Class<?> callerClass, Runtime runtime, int status);
+
+    void check$java_lang_System$$exit(Class<?> callerClass, int status);
+
+    ////////////////////
+    //
+    // ClassLoader ctor
+    //
+
+    void check$java_lang_ClassLoader$(Class<?> callerClass);
+
+    void check$java_lang_ClassLoader$(Class<?> callerClass, ClassLoader parent);
+
+    void check$java_lang_ClassLoader$(Class<?> callerClass, String name, ClassLoader parent);
+
+    ////////////////////
+    //
+    // SecureClassLoader ctor
+    //
 
-    void check$$halt(Class<?> callerClass, Runtime runtime, int status);
+    void check$java_security_SecureClassLoader$(Class<?> callerClass);
+
+    void check$java_security_SecureClassLoader$(Class<?> callerClass, ClassLoader parent);
+
+    void check$java_security_SecureClassLoader$(Class<?> callerClass, String name, ClassLoader parent);
+
+    ////////////////////
+    //
+    // URLClassLoader constructors
+    //
 
-    // URLClassLoader ctor
     void check$java_net_URLClassLoader$(Class<?> callerClass, URL[] urls);
 
     void check$java_net_URLClassLoader$(Class<?> callerClass, URL[] urls, ClassLoader parent);
@@ -29,4 +85,138 @@ public interface EntitlementChecker {
     void check$java_net_URLClassLoader$(Class<?> callerClass, String name, URL[] urls, ClassLoader parent);
 
     void check$java_net_URLClassLoader$(Class<?> callerClass, String name, URL[] urls, ClassLoader parent, URLStreamHandlerFactory factory);
+
+    ////////////////////
+    //
+    // "setFactory" methods
+    //
+
+    void check$javax_net_ssl_HttpsURLConnection$setSSLSocketFactory(Class<?> callerClass, HttpsURLConnection conn, SSLSocketFactory sf);
+
+    void check$javax_net_ssl_HttpsURLConnection$$setDefaultSSLSocketFactory(Class<?> callerClass, SSLSocketFactory sf);
+
+    void check$javax_net_ssl_HttpsURLConnection$$setDefaultHostnameVerifier(Class<?> callerClass, HostnameVerifier hv);
+
+    void check$javax_net_ssl_SSLContext$$setDefault(Class<?> callerClass, SSLContext context);
+
+    ////////////////////
+    //
+    // Process creation
+    //
+
+    void check$java_lang_ProcessBuilder$start(Class<?> callerClass, ProcessBuilder that);
+
+    void check$java_lang_ProcessBuilder$$startPipeline(Class<?> callerClass, List<ProcessBuilder> builders);
+
+    ////////////////////
+    //
+    // JVM-wide state changes
+    //
+
+    void check$java_lang_System$$setIn(Class<?> callerClass, InputStream in);
+
+    void check$java_lang_System$$setOut(Class<?> callerClass, PrintStream out);
+
+    void check$java_lang_System$$setErr(Class<?> callerClass, PrintStream err);
+
+    void check$java_lang_Runtime$addShutdownHook(Class<?> callerClass, Runtime runtime, Thread hook);
+
+    void check$java_lang_Runtime$removeShutdownHook(Class<?> callerClass, Runtime runtime, Thread hook);
+
+    void check$jdk_tools_jlink_internal_Jlink$(Class<?> callerClass);
+
+    void check$jdk_tools_jlink_internal_Main$$run(Class<?> callerClass, PrintWriter out, PrintWriter err, String... args);
+
+    void check$jdk_vm_ci_services_JVMCIServiceLocator$$getProviders(Class<?> callerClass, Class<?> service);
+
+    void check$jdk_vm_ci_services_Services$$load(Class<?> callerClass, Class<?> service);
+
+    void check$jdk_vm_ci_services_Services$$loadSingle(Class<?> callerClass, Class<?> service, boolean required);
+
+    void check$com_sun_tools_jdi_VirtualMachineManagerImpl$$virtualMachineManager(Class<?> callerClass);
+
+    void check$java_lang_Thread$$setDefaultUncaughtExceptionHandler(Class<?> callerClass, Thread.UncaughtExceptionHandler ueh);
+
+    void check$java_util_spi_LocaleServiceProvider$(Class<?> callerClass);
+
+    void check$java_text_spi_BreakIteratorProvider$(Class<?> callerClass);
+
+    void check$java_text_spi_CollatorProvider$(Class<?> callerClass);
+
+    void check$java_text_spi_DateFormatProvider$(Class<?> callerClass);
+
+    void check$java_text_spi_DateFormatSymbolsProvider$(Class<?> callerClass);
+
+    void check$java_text_spi_DecimalFormatSymbolsProvider$(Class<?> callerClass);
+
+    void check$java_text_spi_NumberFormatProvider$(Class<?> callerClass);
+
+    void check$java_util_spi_CalendarDataProvider$(Class<?> callerClass);
+
+    void check$java_util_spi_CalendarNameProvider$(Class<?> callerClass);
+
+    void check$java_util_spi_CurrencyNameProvider$(Class<?> callerClass);
+
+    void check$java_util_spi_LocaleNameProvider$(Class<?> callerClass);
+
+    void check$java_util_spi_TimeZoneNameProvider$(Class<?> callerClass);
+
+    void check$java_util_logging_LogManager$(Class<?> callerClass);
+
+    void check$java_net_DatagramSocket$$setDatagramSocketImplFactory(Class<?> callerClass, DatagramSocketImplFactory fac);
+
+    void check$java_net_HttpURLConnection$$setFollowRedirects(Class<?> callerClass, boolean set);
+
+    void check$java_net_ServerSocket$$setSocketFactory(Class<?> callerClass, SocketImplFactory fac);
+
+    void check$java_net_Socket$$setSocketImplFactory(Class<?> callerClass, SocketImplFactory fac);
+
+    void check$java_net_URL$$setURLStreamHandlerFactory(Class<?> callerClass, URLStreamHandlerFactory fac);
+
+    void check$java_net_URLConnection$$setFileNameMap(Class<?> callerClass, FileNameMap map);
+
+    void check$java_net_URLConnection$$setContentHandlerFactory(Class<?> callerClass, ContentHandlerFactory fac);
+
+    ////////////////////
+    //
+    // Network access
+    //
+    void check$java_net_ProxySelector$$setDefault(Class<?> callerClass, ProxySelector ps);
+
+    void check$java_net_ResponseCache$$setDefault(Class<?> callerClass, ResponseCache rc);
+
+    void check$java_net_spi_InetAddressResolverProvider$(Class<?> callerClass);
+
+    void check$java_net_spi_URLStreamHandlerProvider$(Class<?> callerClass);
+
+    void check$java_net_URL$(Class<?> callerClass, String protocol, String host, int port, String file, URLStreamHandler handler);
+
+    void check$java_net_URL$(Class<?> callerClass, URL context, String spec, URLStreamHandler handler);
+
+    // The only implementation of SSLSession#getSessionContext(); unfortunately it's an interface, so we need to check the implementation
+    void check$sun_security_ssl_SSLSessionImpl$getSessionContext(Class<?> callerClass, SSLSession sslSession);
+
+    void check$java_net_DatagramSocket$bind(Class<?> callerClass, DatagramSocket that, SocketAddress addr);
+
+    void check$java_net_DatagramSocket$connect(Class<?> callerClass, DatagramSocket that, InetAddress addr);
+
+    void check$java_net_DatagramSocket$connect(Class<?> callerClass, DatagramSocket that, SocketAddress addr);
+
+    void check$java_net_DatagramSocket$send(Class<?> callerClass, DatagramSocket that, DatagramPacket p);
+
+    void check$java_net_DatagramSocket$receive(Class<?> callerClass, DatagramSocket that, DatagramPacket p);
+
+    void check$java_net_DatagramSocket$joinGroup(Class<?> callerClass, DatagramSocket that, SocketAddress addr, NetworkInterface ni);
+
+    void check$java_net_DatagramSocket$leaveGroup(Class<?> callerClass, DatagramSocket that, SocketAddress addr, NetworkInterface ni);
+
+    void check$java_net_MulticastSocket$joinGroup(Class<?> callerClass, MulticastSocket that, InetAddress addr);
+
+    void check$java_net_MulticastSocket$joinGroup(Class<?> callerClass, MulticastSocket that, SocketAddress addr, NetworkInterface ni);
+
+    void check$java_net_MulticastSocket$leaveGroup(Class<?> callerClass, MulticastSocket that, InetAddress addr);
+
+    void check$java_net_MulticastSocket$leaveGroup(Class<?> callerClass, MulticastSocket that, SocketAddress addr, NetworkInterface ni);
+
+    void check$java_net_MulticastSocket$send(Class<?> callerClass, MulticastSocket that, DatagramPacket p, byte ttl);
 }
diff --git a/libs/entitlement/qa/common/src/main/java/module-info.java b/libs/entitlement/qa/common/src/main/java/module-info.java
index 2dd37e3174e08..211b7041e97ea 100644
--- a/libs/entitlement/qa/common/src/main/java/module-info.java
+++ b/libs/entitlement/qa/common/src/main/java/module-info.java
@@ -12,5 +12,8 @@
     requires org.elasticsearch.base;
     requires org.elasticsearch.logging;
 
+    // Modules we'll attempt to use in order to exercise entitlements
+    requires java.logging;
+
     exports org.elasticsearch.entitlement.qa.common;
 }
diff --git a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/DummyImplementations.java b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/DummyImplementations.java
new file mode 100644
index 0000000000000..fae873123528d
--- /dev/null
+++ b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/DummyImplementations.java
@@ -0,0 +1,410 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.qa.common;
+
+import java.io.IOException;
+import java.net.DatagramPacket;
+import java.net.DatagramSocket;
+import java.net.DatagramSocketImpl;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.Socket;
+import java.net.SocketAddress;
+import java.net.SocketException;
+import java.security.cert.Certificate;
+import java.text.BreakIterator;
+import java.text.Collator;
+import java.text.DateFormat;
+import java.text.DateFormatSymbols;
+import java.text.DecimalFormatSymbols;
+import java.text.NumberFormat;
+import java.text.spi.BreakIteratorProvider;
+import java.text.spi.CollatorProvider;
+import java.text.spi.DateFormatProvider;
+import java.text.spi.DateFormatSymbolsProvider;
+import java.text.spi.DecimalFormatSymbolsProvider;
+import java.text.spi.NumberFormatProvider;
+import java.util.Locale;
+import java.util.Map;
+import java.util.spi.CalendarDataProvider;
+import java.util.spi.CalendarNameProvider;
+import java.util.spi.CurrencyNameProvider;
+import java.util.spi.LocaleNameProvider;
+import java.util.spi.LocaleServiceProvider;
+import java.util.spi.TimeZoneNameProvider;
+
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
+
+/**
+ * A collection of concrete subclasses that we can instantiate but that don't actually work.
+ * <p>
+ * A bit like Mockito but way more painful.
+ */
+class DummyImplementations {
+
+    static class DummyLocaleServiceProvider extends LocaleServiceProvider {
+
+        @Override
+        public Locale[] getAvailableLocales() {
+            throw unexpected();
+        }
+    }
+
+    static class DummyBreakIteratorProvider extends BreakIteratorProvider {
+
+        @Override
+        public BreakIterator getWordInstance(Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public BreakIterator getLineInstance(Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public BreakIterator getCharacterInstance(Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public BreakIterator getSentenceInstance(Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public Locale[] getAvailableLocales() {
+            throw unexpected();
+        }
+    }
+
+    static class DummyCollatorProvider extends CollatorProvider {
+
+        @Override
+        public Collator getInstance(Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public Locale[] getAvailableLocales() {
+            throw unexpected();
+        }
+    }
+
+    static class DummyDateFormatProvider extends DateFormatProvider {
+
+        @Override
+        public DateFormat getTimeInstance(int style, Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public DateFormat getDateInstance(int style, Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public DateFormat getDateTimeInstance(int dateStyle, int timeStyle, Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public Locale[] getAvailableLocales() {
+            throw unexpected();
+        }
+    }
+
+    static class DummyDateFormatSymbolsProvider extends DateFormatSymbolsProvider {
+
+        @Override
+        public DateFormatSymbols getInstance(Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public Locale[] getAvailableLocales() {
+            throw unexpected();
+        }
+    }
+
+    static class DummyDecimalFormatSymbolsProvider extends DecimalFormatSymbolsProvider {
+
+        @Override
+        public DecimalFormatSymbols getInstance(Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public Locale[] getAvailableLocales() {
+            throw unexpected();
+        }
+    }
+
+    static class DummyNumberFormatProvider extends NumberFormatProvider {
+
+        @Override
+        public NumberFormat getCurrencyInstance(Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public NumberFormat getIntegerInstance(Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public NumberFormat getNumberInstance(Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public NumberFormat getPercentInstance(Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public Locale[] getAvailableLocales() {
+            throw unexpected();
+        }
+    }
+
+    static class DummyCalendarDataProvider extends CalendarDataProvider {
+
+        @Override
+        public int getFirstDayOfWeek(Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public int getMinimalDaysInFirstWeek(Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public Locale[] getAvailableLocales() {
+            throw unexpected();
+        }
+    }
+
+    static class DummyCalendarNameProvider extends CalendarNameProvider {
+
+        @Override
+        public String getDisplayName(String calendarType, int field, int value, int style, Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public Map<String, Integer> getDisplayNames(String calendarType, int field, int style, Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public Locale[] getAvailableLocales() {
+            throw unexpected();
+        }
+    }
+
+    static class DummyCurrencyNameProvider extends CurrencyNameProvider {
+
+        @Override
+        public String getSymbol(String currencyCode, Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public Locale[] getAvailableLocales() {
+            throw unexpected();
+        }
+    }
+
+    static class DummyLocaleNameProvider extends LocaleNameProvider {
+
+        @Override
+        public String getDisplayLanguage(String languageCode, Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public String getDisplayCountry(String countryCode, Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public String getDisplayVariant(String variant, Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public Locale[] getAvailableLocales() {
+            throw unexpected();
+        }
+    }
+
+    static class DummyTimeZoneNameProvider extends TimeZoneNameProvider {
+
+        @Override
+        public String getDisplayName(String ID, boolean daylight, int style, Locale locale) {
+            throw unexpected();
+        }
+
+        @Override
+        public Locale[] getAvailableLocales() {
+            throw unexpected();
+        }
+    }
+
+    static class DummyHttpsURLConnection extends HttpsURLConnection {
+        DummyHttpsURLConnection() {
+            super(null);
+        }
+
+        @Override
+        public void connect() {
+            throw unexpected();
+        }
+
+        @Override
+        public void disconnect() {
+            throw unexpected();
+        }
+
+        @Override
+        public boolean usingProxy() {
+            throw unexpected();
+        }
+
+        @Override
+        public String getCipherSuite() {
+            throw unexpected();
+        }
+
+        @Override
+        public Certificate[] getLocalCertificates() {
+            throw unexpected();
+        }
+
+        @Override
+        public Certificate[] getServerCertificates() {
+            throw unexpected();
+        }
+    }
+
+    static class DummySSLSocketFactory extends SSLSocketFactory {
+        @Override
+        public Socket createSocket(String host, int port) {
+            throw unexpected();
+        }
+
+        @Override
+        public Socket createSocket(String host, int port, InetAddress localHost, int localPort) {
+            throw unexpected();
+        }
+
+        @Override
+        public Socket createSocket(InetAddress host, int port) {
+            throw unexpected();
+        }
+
+        @Override
+        public Socket createSocket(InetAddress address, int port, InetAddress localAddress, int localPort) {
+            throw unexpected();
+        }
+
+        @Override
+        public String[] getDefaultCipherSuites() {
+            throw unexpected();
+        }
+
+        @Override
+        public String[] getSupportedCipherSuites() {
+            throw unexpected();
+        }
+
+        @Override
+        public Socket createSocket(Socket s, String host, int port, boolean autoClose) {
+            throw unexpected();
+        }
+    }
+
+    static class DummyDatagramSocket extends DatagramSocket {
+        DummyDatagramSocket() throws SocketException {
+            super(new DatagramSocketImpl() {
+                @Override
+                protected void create() throws SocketException {}
+
+                @Override
+                protected void bind(int lport, InetAddress laddr) throws SocketException {}
+
+                @Override
+                protected void send(DatagramPacket p) throws IOException {}
+
+                @Override
+                protected int peek(InetAddress i) throws IOException {
+                    return 0;
+                }
+
+                @Override
+                protected int peekData(DatagramPacket p) throws IOException {
+                    return 0;
+                }
+
+                @Override
+                protected void receive(DatagramPacket p) throws IOException {}
+
+                @Override
+                protected void setTTL(byte ttl) throws IOException {}
+
+                @Override
+                protected byte getTTL() throws IOException {
+                    return 0;
+                }
+
+                @Override
+                protected void setTimeToLive(int ttl) throws IOException {}
+
+                @Override
+                protected int getTimeToLive() throws IOException {
+                    return 0;
+                }
+
+                @Override
+                protected void join(InetAddress inetaddr) throws IOException {}
+
+                @Override
+                protected void leave(InetAddress inetaddr) throws IOException {}
+
+                @Override
+                protected void joinGroup(SocketAddress mcastaddr, NetworkInterface netIf) throws IOException {}
+
+                @Override
+                protected void leaveGroup(SocketAddress mcastaddr, NetworkInterface netIf) throws IOException {}
+
+                @Override
+                protected void close() {}
+
+                @Override
+                public void setOption(int optID, Object value) throws SocketException {}
+
+                @Override
+                public Object getOption(int optID) throws SocketException {
+                    return null;
+                }
+
+                @Override
+                protected void connect(InetAddress address, int port) throws SocketException {}
+            });
+        }
+    }
+
+    private static RuntimeException unexpected() {
+        return new IllegalStateException("This method isn't supposed to be called");
+    }
+}
diff --git a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java
index 1ac4a7506eacb..3a5480f468528 100644
--- a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java
+++ b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java
@@ -11,7 +11,20 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyBreakIteratorProvider;
+import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyCalendarDataProvider;
+import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyCalendarNameProvider;
+import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyCollatorProvider;
+import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyCurrencyNameProvider;
+import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyDateFormatProvider;
+import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyDateFormatSymbolsProvider;
+import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyDecimalFormatSymbolsProvider;
+import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyLocaleNameProvider;
+import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyLocaleServiceProvider;
+import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyNumberFormatProvider;
+import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyTimeZoneNameProvider;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -20,56 +33,389 @@
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
-import java.io.UncheckedIOException;
+import java.net.DatagramPacket;
+import java.net.DatagramSocket;
+import java.net.HttpURLConnection;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.MalformedURLException;
+import java.net.NetworkInterface;
+import java.net.ProxySelector;
+import java.net.ResponseCache;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketException;
 import java.net.URL;
 import java.net.URLClassLoader;
+import java.net.URLConnection;
+import java.net.URLStreamHandler;
+import java.net.spi.InetAddressResolver;
+import java.net.spi.InetAddressResolverProvider;
+import java.net.spi.URLStreamHandlerProvider;
+import java.security.NoSuchAlgorithmException;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.SSLSocket;
+import javax.net.ssl.SSLSocketFactory;
+
 import static java.util.Map.entry;
+import static org.elasticsearch.entitlement.qa.common.RestEntitlementsCheckAction.CheckAction.alwaysDenied;
+import static org.elasticsearch.entitlement.qa.common.RestEntitlementsCheckAction.CheckAction.deniedToPlugins;
+import static org.elasticsearch.entitlement.qa.common.RestEntitlementsCheckAction.CheckAction.forPlugins;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
+@SuppressWarnings("unused")
 public class RestEntitlementsCheckAction extends BaseRestHandler {
     private static final Logger logger = LogManager.getLogger(RestEntitlementsCheckAction.class);
+    public static final Thread NO_OP_SHUTDOWN_HOOK = new Thread(() -> {}, "Shutdown hook for testing");
     private final String prefix;
 
-    private record CheckAction(Runnable action, boolean isServerOnly) {
-
-        static CheckAction serverOnly(Runnable action) {
+    record CheckAction(CheckedRunnable<Exception> action, boolean isAlwaysDeniedToPlugins) {
+        /**
+         * These cannot be granted to plugins, so our test plugins cannot test the "allowed" case.
+         * Used both for always-denied entitlements as well as those granted only to the server itself.
+         */
+        static CheckAction deniedToPlugins(CheckedRunnable<Exception> action) {
             return new CheckAction(action, true);
         }
 
-        static CheckAction serverAndPlugin(Runnable action) {
+        static CheckAction forPlugins(CheckedRunnable<Exception> action) {
             return new CheckAction(action, false);
         }
+
+        static CheckAction alwaysDenied(CheckedRunnable<Exception> action) {
+            return new CheckAction(action, true);
+        }
     }
 
     private static final Map<String, CheckAction> checkActions = Map.ofEntries(
-        entry("runtime_exit", CheckAction.serverOnly(RestEntitlementsCheckAction::runtimeExit)),
-        entry("runtime_halt", CheckAction.serverOnly(RestEntitlementsCheckAction::runtimeHalt)),
-        entry("create_classloader", CheckAction.serverAndPlugin(RestEntitlementsCheckAction::createClassLoader))
+        entry("runtime_exit", deniedToPlugins(RestEntitlementsCheckAction::runtimeExit)),
+        entry("runtime_halt", deniedToPlugins(RestEntitlementsCheckAction::runtimeHalt)),
+        entry("system_exit", deniedToPlugins(RestEntitlementsCheckAction::systemExit)),
+        entry("create_classloader", forPlugins(RestEntitlementsCheckAction::createClassLoader)),
+        entry("processBuilder_start", deniedToPlugins(RestEntitlementsCheckAction::processBuilder_start)),
+        entry("processBuilder_startPipeline", deniedToPlugins(RestEntitlementsCheckAction::processBuilder_startPipeline)),
+        entry("set_https_connection_properties", forPlugins(RestEntitlementsCheckAction::setHttpsConnectionProperties)),
+        entry("set_default_ssl_socket_factory", alwaysDenied(RestEntitlementsCheckAction::setDefaultSSLSocketFactory)),
+        entry("set_default_hostname_verifier", alwaysDenied(RestEntitlementsCheckAction::setDefaultHostnameVerifier)),
+        entry("set_default_ssl_context", alwaysDenied(RestEntitlementsCheckAction::setDefaultSSLContext)),
+        entry("system_setIn", alwaysDenied(RestEntitlementsCheckAction::system$$setIn)),
+        entry("system_setOut", alwaysDenied(RestEntitlementsCheckAction::system$$setOut)),
+        entry("system_setErr", alwaysDenied(RestEntitlementsCheckAction::system$$setErr)),
+        entry("runtime_addShutdownHook", alwaysDenied(RestEntitlementsCheckAction::runtime$addShutdownHook)),
+        entry("runtime_removeShutdownHook", alwaysDenied(RestEntitlementsCheckAction::runtime$$removeShutdownHook)),
+        entry(
+            "thread_setDefaultUncaughtExceptionHandler",
+            alwaysDenied(RestEntitlementsCheckAction::thread$$setDefaultUncaughtExceptionHandler)
+        ),
+        entry("localeServiceProvider", alwaysDenied(RestEntitlementsCheckAction::localeServiceProvider$)),
+        entry("breakIteratorProvider", alwaysDenied(RestEntitlementsCheckAction::breakIteratorProvider$)),
+        entry("collatorProvider", alwaysDenied(RestEntitlementsCheckAction::collatorProvider$)),
+        entry("dateFormatProvider", alwaysDenied(RestEntitlementsCheckAction::dateFormatProvider$)),
+        entry("dateFormatSymbolsProvider", alwaysDenied(RestEntitlementsCheckAction::dateFormatSymbolsProvider$)),
+        entry("decimalFormatSymbolsProvider", alwaysDenied(RestEntitlementsCheckAction::decimalFormatSymbolsProvider$)),
+        entry("numberFormatProvider", alwaysDenied(RestEntitlementsCheckAction::numberFormatProvider$)),
+        entry("calendarDataProvider", alwaysDenied(RestEntitlementsCheckAction::calendarDataProvider$)),
+        entry("calendarNameProvider", alwaysDenied(RestEntitlementsCheckAction::calendarNameProvider$)),
+        entry("currencyNameProvider", alwaysDenied(RestEntitlementsCheckAction::currencyNameProvider$)),
+        entry("localeNameProvider", alwaysDenied(RestEntitlementsCheckAction::localeNameProvider$)),
+        entry("timeZoneNameProvider", alwaysDenied(RestEntitlementsCheckAction::timeZoneNameProvider$)),
+        entry("logManager", alwaysDenied(RestEntitlementsCheckAction::logManager$)),
+
+        // This group is a bit nasty: if entitlements don't prevent these, then networking is
+        // irreparably borked for the remainder of the test run.
+        entry(
+            "datagramSocket_setDatagramSocketImplFactory",
+            alwaysDenied(RestEntitlementsCheckAction::datagramSocket$$setDatagramSocketImplFactory)
+        ),
+        entry("httpURLConnection_setFollowRedirects", alwaysDenied(RestEntitlementsCheckAction::httpURLConnection$$setFollowRedirects)),
+        entry("serverSocket_setSocketFactory", alwaysDenied(RestEntitlementsCheckAction::serverSocket$$setSocketFactory)),
+        entry("socket_setSocketImplFactory", alwaysDenied(RestEntitlementsCheckAction::socket$$setSocketImplFactory)),
+        entry("url_setURLStreamHandlerFactory", alwaysDenied(RestEntitlementsCheckAction::url$$setURLStreamHandlerFactory)),
+        entry("urlConnection_setFileNameMap", alwaysDenied(RestEntitlementsCheckAction::urlConnection$$setFileNameMap)),
+        entry("urlConnection_setContentHandlerFactory", alwaysDenied(RestEntitlementsCheckAction::urlConnection$$setContentHandlerFactory)),
+
+        entry("proxySelector_setDefault", alwaysDenied(RestEntitlementsCheckAction::setDefaultProxySelector)),
+        entry("responseCache_setDefault", alwaysDenied(RestEntitlementsCheckAction::setDefaultResponseCache)),
+        entry("createInetAddressResolverProvider", alwaysDenied(RestEntitlementsCheckAction::createInetAddressResolverProvider)),
+        entry("createURLStreamHandlerProvider", alwaysDenied(RestEntitlementsCheckAction::createURLStreamHandlerProvider)),
+        entry("createURLWithURLStreamHandler", alwaysDenied(RestEntitlementsCheckAction::createURLWithURLStreamHandler)),
+        entry("createURLWithURLStreamHandler2", alwaysDenied(RestEntitlementsCheckAction::createURLWithURLStreamHandler2)),
+        entry("sslSessionImpl_getSessionContext", alwaysDenied(RestEntitlementsCheckAction::sslSessionImplGetSessionContext)),
+        entry("datagram_socket_bind", forPlugins(RestEntitlementsCheckAction::bindDatagramSocket)),
+        entry("datagram_socket_connect", forPlugins(RestEntitlementsCheckAction::connectDatagramSocket)),
+        entry("datagram_socket_send", forPlugins(RestEntitlementsCheckAction::sendDatagramSocket)),
+        entry("datagram_socket_receive", forPlugins(RestEntitlementsCheckAction::receiveDatagramSocket)),
+        entry("datagram_socket_join_group", forPlugins(RestEntitlementsCheckAction::joinGroupDatagramSocket)),
+        entry("datagram_socket_leave_group", forPlugins(RestEntitlementsCheckAction::leaveGroupDatagramSocket))
     );
 
+    private static void createURLStreamHandlerProvider() {
+        var x = new URLStreamHandlerProvider() {
+            @Override
+            public URLStreamHandler createURLStreamHandler(String protocol) {
+                return null;
+            }
+        };
+    }
+
+    private static void sslSessionImplGetSessionContext() throws IOException {
+        SSLSocketFactory factory = HttpsURLConnection.getDefaultSSLSocketFactory();
+        try (SSLSocket socket = (SSLSocket) factory.createSocket()) {
+            SSLSession session = socket.getSession();
+
+            session.getSessionContext();
+        }
+    }
+
+    @SuppressWarnings("deprecation")
+    private static void createURLWithURLStreamHandler() throws MalformedURLException {
+        var x = new URL("http", "host", 1234, "file", new URLStreamHandler() {
+            @Override
+            protected URLConnection openConnection(URL u) {
+                return null;
+            }
+        });
+    }
+
+    @SuppressWarnings("deprecation")
+    private static void createURLWithURLStreamHandler2() throws MalformedURLException {
+        var x = new URL(null, "spec", new URLStreamHandler() {
+            @Override
+            protected URLConnection openConnection(URL u) {
+                return null;
+            }
+        });
+    }
+
+    private static void createInetAddressResolverProvider() {
+        var x = new InetAddressResolverProvider() {
+            @Override
+            public InetAddressResolver get(Configuration configuration) {
+                return null;
+            }
+
+            @Override
+            public String name() {
+                return "TEST";
+            }
+        };
+    }
+
+    private static void setDefaultResponseCache() {
+        ResponseCache.setDefault(null);
+    }
+
+    private static void setDefaultProxySelector() {
+        ProxySelector.setDefault(null);
+    }
+
+    private static void setDefaultSSLContext() throws NoSuchAlgorithmException {
+        SSLContext.setDefault(SSLContext.getDefault());
+    }
+
+    private static void setDefaultHostnameVerifier() {
+        HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> false);
+    }
+
+    private static void setDefaultSSLSocketFactory() {
+        HttpsURLConnection.setDefaultSSLSocketFactory(new DummyImplementations.DummySSLSocketFactory());
+    }
+
     @SuppressForbidden(reason = "Specifically testing Runtime.exit")
     private static void runtimeExit() {
-        logger.info("Calling Runtime.exit;");
         Runtime.getRuntime().exit(123);
     }
 
     @SuppressForbidden(reason = "Specifically testing Runtime.halt")
     private static void runtimeHalt() {
-        logger.info("Calling Runtime.halt;");
         Runtime.getRuntime().halt(123);
     }
 
-    private static void createClassLoader() {
-        logger.info("Calling new URLClassLoader");
+    @SuppressForbidden(reason = "Specifically testing System.exit")
+    private static void systemExit() {
+        System.exit(123);
+    }
+
+    private static void createClassLoader() throws IOException {
         try (var classLoader = new URLClassLoader("test", new URL[0], RestEntitlementsCheckAction.class.getClassLoader())) {
             logger.info("Created URLClassLoader [{}]", classLoader.getName());
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
+        }
+    }
+
+    private static void processBuilder_start() throws IOException {
+        new ProcessBuilder("").start();
+    }
+
+    private static void processBuilder_startPipeline() throws IOException {
+        ProcessBuilder.startPipeline(List.of());
+    }
+
+    private static void setHttpsConnectionProperties() {
+        new DummyImplementations.DummyHttpsURLConnection().setSSLSocketFactory(new DummyImplementations.DummySSLSocketFactory());
+    }
+
+    private static void system$$setIn() {
+        System.setIn(System.in);
+    }
+
+    @SuppressForbidden(reason = "This should be a no-op so we don't interfere with system streams")
+    private static void system$$setOut() {
+        System.setOut(System.out);
+    }
+
+    @SuppressForbidden(reason = "This should be a no-op so we don't interfere with system streams")
+    private static void system$$setErr() {
+        System.setErr(System.err);
+    }
+
+    private static void runtime$addShutdownHook() {
+        Runtime.getRuntime().addShutdownHook(NO_OP_SHUTDOWN_HOOK);
+    }
+
+    private static void runtime$$removeShutdownHook() {
+        Runtime.getRuntime().removeShutdownHook(NO_OP_SHUTDOWN_HOOK);
+    }
+
+    private static void thread$$setDefaultUncaughtExceptionHandler() {
+        Thread.setDefaultUncaughtExceptionHandler(Thread.getDefaultUncaughtExceptionHandler());
+    }
+
+    private static void localeServiceProvider$() {
+        new DummyLocaleServiceProvider();
+    }
+
+    private static void breakIteratorProvider$() {
+        new DummyBreakIteratorProvider();
+    }
+
+    private static void collatorProvider$() {
+        new DummyCollatorProvider();
+    }
+
+    private static void dateFormatProvider$() {
+        new DummyDateFormatProvider();
+    }
+
+    private static void dateFormatSymbolsProvider$() {
+        new DummyDateFormatSymbolsProvider();
+    }
+
+    private static void decimalFormatSymbolsProvider$() {
+        new DummyDecimalFormatSymbolsProvider();
+    }
+
+    private static void numberFormatProvider$() {
+        new DummyNumberFormatProvider();
+    }
+
+    private static void calendarDataProvider$() {
+        new DummyCalendarDataProvider();
+    }
+
+    private static void calendarNameProvider$() {
+        new DummyCalendarNameProvider();
+    }
+
+    private static void currencyNameProvider$() {
+        new DummyCurrencyNameProvider();
+    }
+
+    private static void localeNameProvider$() {
+        new DummyLocaleNameProvider();
+    }
+
+    private static void timeZoneNameProvider$() {
+        new DummyTimeZoneNameProvider();
+    }
+
+    private static void logManager$() {
+        new java.util.logging.LogManager() {
+        };
+    }
+
+    @SuppressWarnings("deprecation")
+    @SuppressForbidden(reason = "We're required to prevent calls to this forbidden API")
+    private static void datagramSocket$$setDatagramSocketImplFactory() throws IOException {
+        DatagramSocket.setDatagramSocketImplFactory(() -> { throw new IllegalStateException(); });
+    }
+
+    private static void httpURLConnection$$setFollowRedirects() {
+        HttpURLConnection.setFollowRedirects(HttpURLConnection.getFollowRedirects());
+    }
+
+    @SuppressWarnings("deprecation")
+    @SuppressForbidden(reason = "We're required to prevent calls to this forbidden API")
+    private static void serverSocket$$setSocketFactory() throws IOException {
+        ServerSocket.setSocketFactory(() -> { throw new IllegalStateException(); });
+    }
+
+    @SuppressWarnings("deprecation")
+    @SuppressForbidden(reason = "We're required to prevent calls to this forbidden API")
+    private static void socket$$setSocketImplFactory() throws IOException {
+        Socket.setSocketImplFactory(() -> { throw new IllegalStateException(); });
+    }
+
+    private static void url$$setURLStreamHandlerFactory() {
+        URL.setURLStreamHandlerFactory(__ -> { throw new IllegalStateException(); });
+    }
+
+    private static void urlConnection$$setFileNameMap() {
+        URLConnection.setFileNameMap(__ -> { throw new IllegalStateException(); });
+    }
+
+    private static void urlConnection$$setContentHandlerFactory() {
+        URLConnection.setContentHandlerFactory(__ -> { throw new IllegalStateException(); });
+    }
+
+    private static void bindDatagramSocket() throws SocketException {
+        try (var socket = new DatagramSocket(null)) {
+            socket.bind(null);
+        }
+    }
+
+    @SuppressForbidden(reason = "testing entitlements")
+    private static void connectDatagramSocket() throws SocketException {
+        try (var socket = new DummyImplementations.DummyDatagramSocket()) {
+            socket.connect(new InetSocketAddress(1234));
+        }
+    }
+
+    private static void joinGroupDatagramSocket() throws IOException {
+        try (var socket = new DummyImplementations.DummyDatagramSocket()) {
+            socket.joinGroup(
+                new InetSocketAddress(InetAddress.getByAddress(new byte[] { (byte) 230, 0, 0, 1 }), 1234),
+                NetworkInterface.getByIndex(0)
+            );
+        }
+    }
+
+    private static void leaveGroupDatagramSocket() throws IOException {
+        try (var socket = new DummyImplementations.DummyDatagramSocket()) {
+            socket.leaveGroup(
+                new InetSocketAddress(InetAddress.getByAddress(new byte[] { (byte) 230, 0, 0, 1 }), 1234),
+                NetworkInterface.getByIndex(0)
+            );
+        }
+    }
+
+    @SuppressForbidden(reason = "testing entitlements")
+    private static void sendDatagramSocket() throws IOException {
+        try (var socket = new DummyImplementations.DummyDatagramSocket()) {
+            socket.send(new DatagramPacket(new byte[] { 0 }, 1, InetAddress.getLocalHost(), 1234));
+        }
+    }
+
+    @SuppressForbidden(reason = "testing entitlements")
+    private static void receiveDatagramSocket() throws IOException {
+        try (var socket = new DummyImplementations.DummyDatagramSocket()) {
+            socket.receive(new DatagramPacket(new byte[1], 1, InetAddress.getLocalHost(), 1234));
         }
     }
 
@@ -77,10 +423,10 @@ public RestEntitlementsCheckAction(String prefix) {
         this.prefix = prefix;
     }
 
-    public static Set<String> getServerAndPluginsCheckActions() {
+    public static Set<String> getCheckActionsAllowedInPlugins() {
         return checkActions.entrySet()
             .stream()
-            .filter(kv -> kv.getValue().isServerOnly() == false)
+            .filter(kv -> kv.getValue().isAlwaysDeniedToPlugins() == false)
             .map(Map.Entry::getKey)
             .collect(Collectors.toSet());
     }
@@ -112,6 +458,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
         }
 
         return channel -> {
+            logger.info("Calling check action [{}]", actionName);
             checkAction.action().run();
             channel.sendResponse(new RestResponse(RestStatus.OK, Strings.format("Succesfully executed action [%s]", actionName)));
         };
diff --git a/libs/entitlement/qa/entitlement-allowed-nonmodular/build.gradle b/libs/entitlement/qa/entitlement-allowed-nonmodular/build.gradle
index 7b3015a5ab831..316b61e15707e 100644
--- a/libs/entitlement/qa/entitlement-allowed-nonmodular/build.gradle
+++ b/libs/entitlement/qa/entitlement-allowed-nonmodular/build.gradle
@@ -10,9 +10,9 @@
 apply plugin: 'elasticsearch.base-internal-es-plugin'
 
 esplugin {
-  name 'entitlement-allowed-nonmodular'
-  description 'A non-modular test module that invokes entitlement checks that are supposed to be granted'
-  classname 'org.elasticsearch.entitlement.qa.nonmodular.EntitlementAllowedNonModularPlugin'
+  name = 'entitlement-allowed-nonmodular'
+  description = 'A non-modular test module that invokes entitlement checks that are supposed to be granted'
+  classname = 'org.elasticsearch.entitlement.qa.nonmodular.EntitlementAllowedNonModularPlugin'
 }
 
 dependencies {
diff --git a/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/java/org/elasticsearch/entitlement/qa/nonmodular/EntitlementAllowedNonModularPlugin.java b/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/java/org/elasticsearch/entitlement/qa/nonmodular/EntitlementAllowedNonModularPlugin.java
index d65981c30f0be..82146e6a87759 100644
--- a/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/java/org/elasticsearch/entitlement/qa/nonmodular/EntitlementAllowedNonModularPlugin.java
+++ b/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/java/org/elasticsearch/entitlement/qa/nonmodular/EntitlementAllowedNonModularPlugin.java
@@ -27,7 +27,6 @@
 import java.util.function.Supplier;
 
 public class EntitlementAllowedNonModularPlugin extends Plugin implements ActionPlugin {
-
     @Override
     public List<RestHandler> getRestHandlers(
         final Settings settings,
diff --git a/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml b/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml
index 45d4e57f66521..05a94f09264a8 100644
--- a/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml
+++ b/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml
@@ -1,2 +1,8 @@
 ALL-UNNAMED:
   - create_class_loader
+  - set_https_connection_properties
+  - network:
+      actions:
+        - listen
+        - accept
+        - connect
diff --git a/libs/entitlement/qa/entitlement-allowed/build.gradle b/libs/entitlement/qa/entitlement-allowed/build.gradle
index 6090d658d2081..b9518f8d65fb5 100644
--- a/libs/entitlement/qa/entitlement-allowed/build.gradle
+++ b/libs/entitlement/qa/entitlement-allowed/build.gradle
@@ -10,9 +10,9 @@
 apply plugin: 'elasticsearch.base-internal-es-plugin'
 
 esplugin {
-  name 'entitlement-allowed'
-  description 'A test module that invokes entitlement checks that are supposed to be granted'
-  classname 'org.elasticsearch.entitlement.qa.EntitlementAllowedPlugin'
+  name = 'entitlement-allowed'
+  description = 'A test module that invokes entitlement checks that are supposed to be granted'
+  classname = 'org.elasticsearch.entitlement.qa.EntitlementAllowedPlugin'
 }
 
 dependencies {
diff --git a/libs/entitlement/qa/entitlement-allowed/src/main/java/org/elasticsearch/entitlement/qa/EntitlementAllowedPlugin.java b/libs/entitlement/qa/entitlement-allowed/src/main/java/org/elasticsearch/entitlement/qa/EntitlementAllowedPlugin.java
index d81e23e311be1..8649daf272e70 100644
--- a/libs/entitlement/qa/entitlement-allowed/src/main/java/org/elasticsearch/entitlement/qa/EntitlementAllowedPlugin.java
+++ b/libs/entitlement/qa/entitlement-allowed/src/main/java/org/elasticsearch/entitlement/qa/EntitlementAllowedPlugin.java
@@ -27,7 +27,6 @@
 import java.util.function.Supplier;
 
 public class EntitlementAllowedPlugin extends Plugin implements ActionPlugin {
-
     @Override
     public List<RestHandler> getRestHandlers(
         final Settings settings,
diff --git a/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml b/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml
index 7b5e848f414b2..0d2c66c2daa2c 100644
--- a/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml
+++ b/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml
@@ -1,2 +1,8 @@
 org.elasticsearch.entitlement.qa.common:
   - create_class_loader
+  - set_https_connection_properties
+  - network:
+      actions:
+        - listen
+        - accept
+        - connect
diff --git a/libs/entitlement/qa/entitlement-denied-nonmodular/build.gradle b/libs/entitlement/qa/entitlement-denied-nonmodular/build.gradle
index bddd6c83c7cc4..6a88dd66eaf75 100644
--- a/libs/entitlement/qa/entitlement-denied-nonmodular/build.gradle
+++ b/libs/entitlement/qa/entitlement-denied-nonmodular/build.gradle
@@ -10,9 +10,9 @@
 apply plugin: 'elasticsearch.base-internal-es-plugin'
 
 esplugin {
-  name 'entitlement-denied-nonmodular'
-  description 'A non-modular test module that invokes non-granted entitlement and triggers exceptions'
-  classname 'org.elasticsearch.entitlement.qa.nonmodular.EntitlementDeniedNonModularPlugin'
+  name = 'entitlement-denied-nonmodular'
+  description = 'A non-modular test module that invokes non-granted entitlement and triggers exceptions'
+  classname = 'org.elasticsearch.entitlement.qa.nonmodular.EntitlementDeniedNonModularPlugin'
 }
 
 dependencies {
diff --git a/libs/entitlement/qa/entitlement-denied-nonmodular/src/main/java/org/elasticsearch/entitlement/qa/nonmodular/EntitlementDeniedNonModularPlugin.java b/libs/entitlement/qa/entitlement-denied-nonmodular/src/main/java/org/elasticsearch/entitlement/qa/nonmodular/EntitlementDeniedNonModularPlugin.java
index 0f908d84260fb..7ca89c735a602 100644
--- a/libs/entitlement/qa/entitlement-denied-nonmodular/src/main/java/org/elasticsearch/entitlement/qa/nonmodular/EntitlementDeniedNonModularPlugin.java
+++ b/libs/entitlement/qa/entitlement-denied-nonmodular/src/main/java/org/elasticsearch/entitlement/qa/nonmodular/EntitlementDeniedNonModularPlugin.java
@@ -27,7 +27,6 @@
 import java.util.function.Supplier;
 
 public class EntitlementDeniedNonModularPlugin extends Plugin implements ActionPlugin {
-
     @Override
     public List<RestHandler> getRestHandlers(
         final Settings settings,
diff --git a/libs/entitlement/qa/entitlement-denied/build.gradle b/libs/entitlement/qa/entitlement-denied/build.gradle
index cc269135c5bf5..9d1872563b8c5 100644
--- a/libs/entitlement/qa/entitlement-denied/build.gradle
+++ b/libs/entitlement/qa/entitlement-denied/build.gradle
@@ -10,9 +10,9 @@
 apply plugin: 'elasticsearch.base-internal-es-plugin'
 
 esplugin {
-  name 'entitlement-denied'
-  description 'A test module that invokes non-granted entitlement and triggers exceptions'
-  classname 'org.elasticsearch.entitlement.qa.EntitlementDeniedPlugin'
+  name = 'entitlement-denied'
+  description = 'A test module that invokes non-granted entitlement and triggers exceptions'
+  classname = 'org.elasticsearch.entitlement.qa.EntitlementDeniedPlugin'
 }
 
 dependencies {
diff --git a/libs/entitlement/qa/entitlement-denied/src/main/java/org/elasticsearch/entitlement/qa/EntitlementDeniedPlugin.java b/libs/entitlement/qa/entitlement-denied/src/main/java/org/elasticsearch/entitlement/qa/EntitlementDeniedPlugin.java
index 0ed27e2e576e7..2a2fd35d47cf3 100644
--- a/libs/entitlement/qa/entitlement-denied/src/main/java/org/elasticsearch/entitlement/qa/EntitlementDeniedPlugin.java
+++ b/libs/entitlement/qa/entitlement-denied/src/main/java/org/elasticsearch/entitlement/qa/EntitlementDeniedPlugin.java
@@ -27,7 +27,6 @@
 import java.util.function.Supplier;
 
 public class EntitlementDeniedPlugin extends Plugin implements ActionPlugin {
-
     @Override
     public List<RestHandler> getRestHandlers(
         final Settings settings,
diff --git a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedIT.java b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedIT.java
index 2fd4472f5cc65..c38e8b3f35efb 100644
--- a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedIT.java
+++ b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedIT.java
@@ -46,7 +46,7 @@ public EntitlementsAllowedIT(@Name("pathPrefix") String pathPrefix, @Name("actio
     public static Iterable<Object[]> data() {
         return Stream.of("allowed", "allowed_nonmodular")
             .flatMap(
-                path -> RestEntitlementsCheckAction.getServerAndPluginsCheckActions().stream().map(action -> new Object[] { path, action })
+                path -> RestEntitlementsCheckAction.getCheckActionsAllowedInPlugins().stream().map(action -> new Object[] { path, action })
             )
             .toList();
     }
diff --git a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsDeniedIT.java b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsDeniedIT.java
index 9f55a7c9e894d..e2e5a3c4c61e6 100644
--- a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsDeniedIT.java
+++ b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsDeniedIT.java
@@ -31,6 +31,8 @@ public class EntitlementsDeniedIT extends ESRestTestCase {
         .plugin("entitlement-denied-nonmodular")
         .systemProperty("es.entitlements.enabled", "true")
         .setting("xpack.security.enabled", "false")
+        // Logs in libs/entitlement/qa/build/test-results/javaRestTest/TEST-org.elasticsearch.entitlement.qa.EntitlementsDeniedIT.xml
+        // .setting("logger.org.elasticsearch.entitlement", "DEBUG")
         .build();
 
     @Override
diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java
index 2abfb11964a93..257d130302580 100644
--- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java
+++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java
@@ -16,6 +16,7 @@
 
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.entitlement.initialization.EntitlementInitialization;
+import org.elasticsearch.entitlement.runtime.api.NotEntitledException;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 
@@ -23,14 +24,24 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collection;
-import java.util.Objects;
 import java.util.function.Function;
 
+import static java.util.Objects.requireNonNull;
+
 public class EntitlementBootstrap {
 
-    public record PluginData(Path pluginPath, boolean isModular, boolean isExternalPlugin) {}
+    public record BootstrapArgs(Collection<PluginData> pluginData, Function<Class<?>, String> pluginResolver) {
+        public BootstrapArgs {
+            requireNonNull(pluginData);
+            requireNonNull(pluginResolver);
+        }
+    }
 
-    public record BootstrapArgs(Collection<PluginData> pluginData, Function<Class<?>, String> pluginResolver) {}
+    public record PluginData(Path pluginPath, boolean isModular, boolean isExternalPlugin) {
+        public PluginData {
+            requireNonNull(pluginPath);
+        }
+    }
 
     private static BootstrapArgs bootstrapArgs;
 
@@ -50,9 +61,10 @@ public static void bootstrap(Collection<PluginData> pluginData, Function<Class<?
         if (EntitlementBootstrap.bootstrapArgs != null) {
             throw new IllegalStateException("plugin data is already set");
         }
-        EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(Objects.requireNonNull(pluginData), Objects.requireNonNull(pluginResolver));
+        EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(pluginData, pluginResolver);
         exportInitializationToAgent();
         loadAgent(findAgentJar());
+        selfTest();
     }
 
     @SuppressForbidden(reason = "The VirtualMachine API is the only way to attach a java agent dynamically")
@@ -98,5 +110,63 @@ private static String findAgentJar() {
         }
     }
 
+    /**
+     * Attempt a few sensitive operations to ensure that some are permitted and some are forbidden.
+     * <p>
+     *
+     * This serves two purposes:
+     *
+     * <ol>
+     *     <li>
+     *         a smoke test to make sure the entitlements system is not completely broken, and
+     *     </li>
+     *     <li>
+     *         an early test of certain important operations so they don't fail later on at an awkward time.
+     *     </li>
+     * </ol>
+     *
+     * @throws IllegalStateException if the entitlements system can't prevent an unauthorized action of our choosing
+     */
+    private static void selfTest() {
+        ensureCannotStartProcess();
+        ensureCanCreateTempFile();
+    }
+
+    private static void ensureCannotStartProcess() {
+        try {
+            // The command doesn't matter; it doesn't even need to exist
+            new ProcessBuilder("").start();
+        } catch (NotEntitledException e) {
+            logger.debug("Success: Entitlement protection correctly prevented process creation");
+            return;
+        } catch (IOException e) {
+            throw new IllegalStateException("Failed entitlement protection self-test", e);
+        }
+        throw new IllegalStateException("Entitlement protection self-test was incorrectly permitted");
+    }
+
+    /**
+     * Originally {@code Security.selfTest}.
+     */
+    @SuppressForbidden(reason = "accesses jvm default tempdir as a self-test")
+    private static void ensureCanCreateTempFile() {
+        try {
+            Path p = Files.createTempFile(null, null);
+            p.toFile().deleteOnExit();
+
+            // Make an effort to clean up the file immediately; also, deleteOnExit leaves the file if the JVM exits abnormally.
+            try {
+                Files.delete(p);
+            } catch (IOException ignored) {
+                // Can be caused by virus scanner
+            }
+        } catch (NotEntitledException e) {
+            throw new IllegalStateException("Entitlement protection self-test was incorrectly forbidden", e);
+        } catch (Exception e) {
+            throw new IllegalStateException("Unable to perform entitlement protection self-test", e);
+        }
+        logger.debug("Success: Entitlement protection correctly permitted temp file creation");
+    }
+
     private static final Logger logger = LogManager.getLogger(EntitlementBootstrap.class);
 }
diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java
index 9118f67cdc145..ba5ccbafa70ae 100644
--- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java
+++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java
@@ -9,6 +9,7 @@
 
 package org.elasticsearch.entitlement.initialization;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.internal.provider.ProviderLocator;
 import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap;
 import org.elasticsearch.entitlement.bridge.EntitlementChecker;
@@ -19,6 +20,7 @@
 import org.elasticsearch.entitlement.instrumentation.Transformer;
 import org.elasticsearch.entitlement.runtime.api.ElasticsearchEntitlementChecker;
 import org.elasticsearch.entitlement.runtime.policy.CreateClassLoaderEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.Entitlement;
 import org.elasticsearch.entitlement.runtime.policy.ExitVMEntitlement;
 import org.elasticsearch.entitlement.runtime.policy.Policy;
 import org.elasticsearch.entitlement.runtime.policy.PolicyManager;
@@ -34,6 +36,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
@@ -53,6 +56,7 @@
 public class EntitlementInitialization {
 
     private static final String POLICY_FILE_NAME = "entitlement-policy.yaml";
+    private static final Module ENTITLEMENTS_MODULE = PolicyManager.class.getModule();
 
     private static ElasticsearchEntitlementChecker manager;
 
@@ -71,17 +75,17 @@ public static void initialize(Instrumentation inst) throws Exception {
 
         Instrumenter instrumenter = INSTRUMENTER_FACTORY.newInstrumenter(EntitlementChecker.class, checkMethods);
         inst.addTransformer(new Transformer(instrumenter, classesToTransform), true);
-        // TODO: should we limit this array somehow?
-        var classesToRetransform = classesToTransform.stream().map(EntitlementInitialization::internalNameToClass).toArray(Class[]::new);
-        inst.retransformClasses(classesToRetransform);
+        inst.retransformClasses(findClassesToRetransform(inst.getAllLoadedClasses(), classesToTransform));
     }
 
-    private static Class<?> internalNameToClass(String internalName) {
-        try {
-            return Class.forName(internalName.replace('/', '.'), false, ClassLoader.getPlatformClassLoader());
-        } catch (ClassNotFoundException e) {
-            throw new RuntimeException(e);
+    private static Class<?>[] findClassesToRetransform(Class<?>[] loadedClasses, Set<String> classesToTransform) {
+        List<Class<?>> retransform = new ArrayList<>();
+        for (Class<?> loadedClass : loadedClasses) {
+            if (classesToTransform.contains(loadedClass.getName().replace(".", "/"))) {
+                retransform.add(loadedClass);
+            }
         }
+        return retransform.toArray(new Class<?>[0]);
     }
 
     private static PolicyManager createPolicyManager() throws IOException {
@@ -90,9 +94,17 @@ private static PolicyManager createPolicyManager() throws IOException {
         // TODO(ES-10031): Decide what goes in the elasticsearch default policy and extend it
         var serverPolicy = new Policy(
             "server",
-            List.of(new Scope("org.elasticsearch.server", List.of(new ExitVMEntitlement(), new CreateClassLoaderEntitlement())))
+            List.of(
+                new Scope("org.elasticsearch.base", List.of(new CreateClassLoaderEntitlement())),
+                new Scope("org.elasticsearch.xcontent", List.of(new CreateClassLoaderEntitlement())),
+                new Scope("org.elasticsearch.server", List.of(new ExitVMEntitlement(), new CreateClassLoaderEntitlement()))
+            )
         );
-        return new PolicyManager(serverPolicy, pluginPolicies, EntitlementBootstrap.bootstrapArgs().pluginResolver());
+        // agents run without a module, so this is a special hack for the apm agent
+        // this should be removed once https://github.com/elastic/elasticsearch/issues/109335 is completed
+        List<Entitlement> agentEntitlements = List.of(new CreateClassLoaderEntitlement());
+        var resolver = EntitlementBootstrap.bootstrapArgs().pluginResolver();
+        return new PolicyManager(serverPolicy, agentEntitlements, pluginPolicies, resolver, ENTITLEMENTS_MODULE);
     }
 
     private static Map<String, Policy> createPluginPolicies(Collection<EntitlementBootstrap.PluginData> pluginData) throws IOException {
@@ -116,9 +128,17 @@ private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, Strin
         final Policy policy = parsePolicyIfExists(pluginName, policyFile, isExternalPlugin);
 
         // TODO: should this check actually be part of the parser?
-        for (Scope scope : policy.scopes) {
-            if (moduleNames.contains(scope.name) == false) {
-                throw new IllegalStateException("policy [" + policyFile + "] contains invalid module [" + scope.name + "]");
+        for (Scope scope : policy.scopes()) {
+            if (moduleNames.contains(scope.moduleName()) == false) {
+                throw new IllegalStateException(
+                    Strings.format(
+                        "Invalid module name in policy: plugin [%s] does not have module [%s]; available modules [%s]; policy file [%s]",
+                        pluginName,
+                        scope.moduleName(),
+                        String.join(", ", moduleNames),
+                        policyFile
+                    )
+                );
             }
         }
         return policy;
diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/MethodKey.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/MethodKey.java
index 256a4d709d9dc..caaa10bec52dd 100644
--- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/MethodKey.java
+++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/MethodKey.java
@@ -16,6 +16,6 @@
  *
  * @param className      the "internal name" of the class: includes the package info, but with periods replaced by slashes
  * @param methodName     the method name
- * @param parameterTypes a list of "internal names" for the parameter types
+ * @param parameterTypes a list of "internal names" for the parameter types that appear in the method's descriptor (not the receiver)
  */
 public record MethodKey(String className, String methodName, List<String> parameterTypes) {}
diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java
index a5ca0543ad15a..dd39ec3c5fe43 100644
--- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java
+++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java
@@ -10,10 +10,34 @@
 package org.elasticsearch.entitlement.runtime.api;
 
 import org.elasticsearch.entitlement.bridge.EntitlementChecker;
+import org.elasticsearch.entitlement.runtime.policy.NetworkEntitlement;
 import org.elasticsearch.entitlement.runtime.policy.PolicyManager;
 
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.net.ContentHandlerFactory;
+import java.net.DatagramPacket;
+import java.net.DatagramSocket;
+import java.net.DatagramSocketImplFactory;
+import java.net.FileNameMap;
+import java.net.InetAddress;
+import java.net.MulticastSocket;
+import java.net.NetworkInterface;
+import java.net.ProxySelector;
+import java.net.ResponseCache;
+import java.net.SocketAddress;
+import java.net.SocketImplFactory;
 import java.net.URL;
+import java.net.URLStreamHandler;
 import java.net.URLStreamHandlerFactory;
+import java.util.List;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.SSLSocketFactory;
 
 /**
  * Implementation of the {@link EntitlementChecker} interface, providing additional
@@ -21,6 +45,7 @@
  * The trampoline module loads this object via SPI.
  */
 public class ElasticsearchEntitlementChecker implements EntitlementChecker {
+
     private final PolicyManager policyManager;
 
     public ElasticsearchEntitlementChecker(PolicyManager policyManager) {
@@ -28,15 +53,50 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) {
     }
 
     @Override
-    public void check$$exit(Class<?> callerClass, Runtime runtime, int status) {
+    public void check$java_lang_Runtime$exit(Class<?> callerClass, Runtime runtime, int status) {
         policyManager.checkExitVM(callerClass);
     }
 
     @Override
-    public void check$$halt(Class<?> callerClass, Runtime runtime, int status) {
+    public void check$java_lang_Runtime$halt(Class<?> callerClass, Runtime runtime, int status) {
         policyManager.checkExitVM(callerClass);
     }
 
+    @Override
+    public void check$java_lang_System$$exit(Class<?> callerClass, int status) {
+        policyManager.checkExitVM(callerClass);
+    }
+
+    @Override
+    public void check$java_lang_ClassLoader$(Class<?> callerClass) {
+        policyManager.checkCreateClassLoader(callerClass);
+    }
+
+    @Override
+    public void check$java_lang_ClassLoader$(Class<?> callerClass, ClassLoader parent) {
+        policyManager.checkCreateClassLoader(callerClass);
+    }
+
+    @Override
+    public void check$java_lang_ClassLoader$(Class<?> callerClass, String name, ClassLoader parent) {
+        policyManager.checkCreateClassLoader(callerClass);
+    }
+
+    @Override
+    public void check$java_security_SecureClassLoader$(Class<?> callerClass) {
+        policyManager.checkCreateClassLoader(callerClass);
+    }
+
+    @Override
+    public void check$java_security_SecureClassLoader$(Class<?> callerClass, ClassLoader parent) {
+        policyManager.checkCreateClassLoader(callerClass);
+    }
+
+    @Override
+    public void check$java_security_SecureClassLoader$(Class<?> callerClass, String name, ClassLoader parent) {
+        policyManager.checkCreateClassLoader(callerClass);
+    }
+
     @Override
     public void check$java_net_URLClassLoader$(Class<?> callerClass, URL[] urls) {
         policyManager.checkCreateClassLoader(callerClass);
@@ -67,4 +127,297 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) {
     ) {
         policyManager.checkCreateClassLoader(callerClass);
     }
+
+    @Override
+    public void check$java_lang_ProcessBuilder$start(Class<?> callerClass, ProcessBuilder processBuilder) {
+        policyManager.checkStartProcess(callerClass);
+    }
+
+    @Override
+    public void check$java_lang_ProcessBuilder$$startPipeline(Class<?> callerClass, List<ProcessBuilder> builders) {
+        policyManager.checkStartProcess(callerClass);
+    }
+
+    @Override
+    public void check$java_lang_System$$setIn(Class<?> callerClass, InputStream in) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_lang_System$$setOut(Class<?> callerClass, PrintStream out) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_lang_System$$setErr(Class<?> callerClass, PrintStream err) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_lang_Runtime$addShutdownHook(Class<?> callerClass, Runtime runtime, Thread hook) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_lang_Runtime$removeShutdownHook(Class<?> callerClass, Runtime runtime, Thread hook) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$jdk_tools_jlink_internal_Jlink$(Class<?> callerClass) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$jdk_tools_jlink_internal_Main$$run(Class<?> callerClass, PrintWriter out, PrintWriter err, String... args) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$jdk_vm_ci_services_JVMCIServiceLocator$$getProviders(Class<?> callerClass, Class<?> service) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$jdk_vm_ci_services_Services$$load(Class<?> callerClass, Class<?> service) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$jdk_vm_ci_services_Services$$loadSingle(Class<?> callerClass, Class<?> service, boolean required) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$com_sun_tools_jdi_VirtualMachineManagerImpl$$virtualMachineManager(Class<?> callerClass) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_lang_Thread$$setDefaultUncaughtExceptionHandler(Class<?> callerClass, Thread.UncaughtExceptionHandler ueh) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_util_spi_LocaleServiceProvider$(Class<?> callerClass) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_text_spi_BreakIteratorProvider$(Class<?> callerClass) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_text_spi_CollatorProvider$(Class<?> callerClass) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_text_spi_DateFormatProvider$(Class<?> callerClass) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_text_spi_DateFormatSymbolsProvider$(Class<?> callerClass) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_text_spi_DecimalFormatSymbolsProvider$(Class<?> callerClass) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_text_spi_NumberFormatProvider$(Class<?> callerClass) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_util_spi_CalendarDataProvider$(Class<?> callerClass) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_util_spi_CalendarNameProvider$(Class<?> callerClass) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_util_spi_CurrencyNameProvider$(Class<?> callerClass) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_util_spi_LocaleNameProvider$(Class<?> callerClass) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_util_spi_TimeZoneNameProvider$(Class<?> callerClass) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_util_logging_LogManager$(Class<?> callerClass) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_net_DatagramSocket$$setDatagramSocketImplFactory(Class<?> callerClass, DatagramSocketImplFactory fac) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_net_HttpURLConnection$$setFollowRedirects(Class<?> callerClass, boolean set) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_net_ServerSocket$$setSocketFactory(Class<?> callerClass, SocketImplFactory fac) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_net_Socket$$setSocketImplFactory(Class<?> callerClass, SocketImplFactory fac) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_net_URL$$setURLStreamHandlerFactory(Class<?> callerClass, URLStreamHandlerFactory fac) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_net_URLConnection$$setFileNameMap(Class<?> callerClass, FileNameMap map) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_net_URLConnection$$setContentHandlerFactory(Class<?> callerClass, ContentHandlerFactory fac) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$javax_net_ssl_HttpsURLConnection$setSSLSocketFactory(
+        Class<?> callerClass,
+        HttpsURLConnection connection,
+        SSLSocketFactory sf
+    ) {
+        policyManager.checkSetHttpsConnectionProperties(callerClass);
+    }
+
+    @Override
+    public void check$javax_net_ssl_HttpsURLConnection$$setDefaultSSLSocketFactory(Class<?> callerClass, SSLSocketFactory sf) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$javax_net_ssl_HttpsURLConnection$$setDefaultHostnameVerifier(Class<?> callerClass, HostnameVerifier hv) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$javax_net_ssl_SSLContext$$setDefault(Class<?> callerClass, SSLContext context) {
+        policyManager.checkChangeJVMGlobalState(callerClass);
+    }
+
+    @Override
+    public void check$java_net_ProxySelector$$setDefault(Class<?> callerClass, ProxySelector ps) {
+        policyManager.checkChangeNetworkHandling(callerClass);
+    }
+
+    @Override
+    public void check$java_net_ResponseCache$$setDefault(Class<?> callerClass, ResponseCache rc) {
+        policyManager.checkChangeNetworkHandling(callerClass);
+    }
+
+    @Override
+    public void check$java_net_spi_InetAddressResolverProvider$(Class<?> callerClass) {
+        policyManager.checkChangeNetworkHandling(callerClass);
+    }
+
+    @Override
+    public void check$java_net_spi_URLStreamHandlerProvider$(Class<?> callerClass) {
+        policyManager.checkChangeNetworkHandling(callerClass);
+    }
+
+    @Override
+    public void check$java_net_URL$(Class<?> callerClass, String protocol, String host, int port, String file, URLStreamHandler handler) {
+        policyManager.checkChangeNetworkHandling(callerClass);
+    }
+
+    @Override
+    public void check$java_net_URL$(Class<?> callerClass, URL context, String spec, URLStreamHandler handler) {
+        policyManager.checkChangeNetworkHandling(callerClass);
+    }
+
+    @Override
+    public void check$sun_security_ssl_SSLSessionImpl$getSessionContext(Class<?> callerClass, SSLSession sslSession) {
+        policyManager.checkReadSensitiveNetworkInformation(callerClass);
+    }
+
+    @Override
+    public void check$java_net_DatagramSocket$bind(Class<?> callerClass, DatagramSocket that, SocketAddress addr) {
+        policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.LISTEN_ACTION);
+    }
+
+    @Override
+    public void check$java_net_DatagramSocket$connect(Class<?> callerClass, DatagramSocket that, InetAddress addr) {
+        policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION);
+    }
+
+    @Override
+    public void check$java_net_DatagramSocket$connect(Class<?> callerClass, DatagramSocket that, SocketAddress addr) {
+        policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION);
+    }
+
+    @Override
+    public void check$java_net_DatagramSocket$send(Class<?> callerClass, DatagramSocket that, DatagramPacket p) {
+        var actions = NetworkEntitlement.CONNECT_ACTION;
+        if (p.getAddress().isMulticastAddress()) {
+            actions |= NetworkEntitlement.ACCEPT_ACTION;
+        }
+        policyManager.checkNetworkAccess(callerClass, actions);
+    }
+
+    @Override
+    public void check$java_net_DatagramSocket$receive(Class<?> callerClass, DatagramSocket that, DatagramPacket p) {
+        policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.ACCEPT_ACTION);
+    }
+
+    @Override
+    public void check$java_net_DatagramSocket$joinGroup(Class<?> caller, DatagramSocket that, SocketAddress addr, NetworkInterface ni) {
+        policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION);
+    }
+
+    @Override
+    public void check$java_net_DatagramSocket$leaveGroup(Class<?> caller, DatagramSocket that, SocketAddress addr, NetworkInterface ni) {
+        policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION);
+    }
+
+    @Override
+    public void check$java_net_MulticastSocket$joinGroup(Class<?> callerClass, MulticastSocket that, InetAddress addr) {
+        policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION);
+    }
+
+    @Override
+    public void check$java_net_MulticastSocket$joinGroup(Class<?> caller, MulticastSocket that, SocketAddress addr, NetworkInterface ni) {
+        policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION);
+    }
+
+    @Override
+    public void check$java_net_MulticastSocket$leaveGroup(Class<?> caller, MulticastSocket that, InetAddress addr) {
+        policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION);
+    }
+
+    @Override
+    public void check$java_net_MulticastSocket$leaveGroup(Class<?> caller, MulticastSocket that, SocketAddress addr, NetworkInterface ni) {
+        policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION);
+    }
+
+    @Override
+    public void check$java_net_MulticastSocket$send(Class<?> callerClass, MulticastSocket that, DatagramPacket p, byte ttl) {
+        policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION);
+    }
 }
diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java
index 138515be9ffcb..55e4b66595642 100644
--- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java
+++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java
@@ -9,7 +9,7 @@
 
 package org.elasticsearch.entitlement.runtime.policy;
 
-public class CreateClassLoaderEntitlement implements Entitlement {
+public record CreateClassLoaderEntitlement() implements Entitlement {
     @ExternalEntitlement
-    public CreateClassLoaderEntitlement() {}
+    public CreateClassLoaderEntitlement {}
 }
diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java
index c4a8fc6833581..e5c836ea22b20 100644
--- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java
+++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java
@@ -12,4 +12,4 @@
 /**
  * Internal policy type (not-parseable -- not available to plugins).
  */
-public class ExitVMEntitlement implements Entitlement {}
+public record ExitVMEntitlement() implements Entitlement {}
diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlement.java
new file mode 100644
index 0000000000000..b6c6a41d5be7f
--- /dev/null
+++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlement.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.runtime.policy;
+
+import org.elasticsearch.core.Strings;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.StringJoiner;
+
+import static java.util.Map.entry;
+
+/**
+ * Describes a network entitlement (sockets) with actions.
+ */
+public class NetworkEntitlement implements Entitlement {
+
+    public static final int LISTEN_ACTION = 0x1;
+    public static final int CONNECT_ACTION = 0x2;
+    public static final int ACCEPT_ACTION = 0x4;
+
+    static final String LISTEN = "listen";
+    static final String CONNECT = "connect";
+    static final String ACCEPT = "accept";
+
+    private static final Map<String, Integer> ACTION_MAP = Map.ofEntries(
+        entry(LISTEN, LISTEN_ACTION),
+        entry(CONNECT, CONNECT_ACTION),
+        entry(ACCEPT, ACCEPT_ACTION)
+    );
+
+    private final int actions;
+
+    @ExternalEntitlement(parameterNames = { "actions" }, esModulesOnly = false)
+    public NetworkEntitlement(List<String> actionsList) {
+
+        int actionsInt = 0;
+
+        for (String actionString : actionsList) {
+            var action = ACTION_MAP.get(actionString);
+            if (action == null) {
+                throw new IllegalArgumentException("unknown network action [" + actionString + "]");
+            }
+            if ((actionsInt & action) == action) {
+                throw new IllegalArgumentException(Strings.format("network action [%s] specified multiple times", actionString));
+            }
+            actionsInt |= action;
+        }
+
+        this.actions = actionsInt;
+    }
+
+    public static Object printActions(int actions) {
+        var joiner = new StringJoiner(",");
+        for (var entry : ACTION_MAP.entrySet()) {
+            var action = entry.getValue();
+            if ((actions & action) == action) {
+                joiner.add(entry.getKey());
+            }
+        }
+        return joiner.toString();
+    }
+
+    /**
+     * For the actions to match, the actions present in this entitlement must be a superset
+     * of the actions required by a check.
+     * There is only one "negative" case (action required by the check but not present in the entitlement),
+     * and it can be expressed efficiently via this truth table:
+     * this.actions | requiredActions |
+     * 0            | 0               | 0
+     * 0            | 1               | 1 --> NOT this.action AND requiredActions
+     * 1            | 0               | 0
+     * 1            | 1               | 0
+     *
+     * @param requiredActions the actions required to be present for a check to pass
+     * @return true if requiredActions are present, false otherwise
+     */
+    public boolean matchActions(int requiredActions) {
+        return (~this.actions & requiredActions) == 0;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        NetworkEntitlement that = (NetworkEntitlement) o;
+        return actions == that.actions;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(actions);
+    }
+
+    @Override
+    public String toString() {
+        return "NetworkEntitlement{actions=" + actions + '}';
+    }
+}
diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java
index e8bd7a3fff357..3546472f485fb 100644
--- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java
+++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java
@@ -9,38 +9,15 @@
 
 package org.elasticsearch.entitlement.runtime.policy;
 
-import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
 /**
  * A holder for scoped entitlements.
  */
-public class Policy {
-
-    public final String name;
-    public final List<Scope> scopes;
-
+public record Policy(String name, List<Scope> scopes) {
     public Policy(String name, List<Scope> scopes) {
         this.name = Objects.requireNonNull(name);
-        this.scopes = Collections.unmodifiableList(Objects.requireNonNull(scopes));
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        Policy policy = (Policy) o;
-        return Objects.equals(name, policy.name) && Objects.equals(scopes, policy.scopes);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(name, scopes);
-    }
-
-    @Override
-    public String toString() {
-        return "Policy{" + "name='" + name + '\'' + ", scopes=" + scopes + '}';
+        this.scopes = List.copyOf(scopes);
     }
 }
diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java
index 8d3efe4eb98e6..f039fbda3dfbd 100644
--- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java
+++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java
@@ -10,38 +10,41 @@
 package org.elasticsearch.entitlement.runtime.policy;
 
 import org.elasticsearch.core.Strings;
-import org.elasticsearch.entitlement.runtime.api.ElasticsearchEntitlementChecker;
 import org.elasticsearch.entitlement.runtime.api.NotEntitledException;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 
+import java.lang.StackWalker.StackFrame;
 import java.lang.module.ModuleFinder;
 import java.lang.module.ModuleReference;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Function;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static java.lang.StackWalker.Option.RETAIN_CLASS_REFERENCE;
+import static java.util.Objects.requireNonNull;
+import static java.util.function.Predicate.not;
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.toUnmodifiableMap;
+
 public class PolicyManager {
-    private static final Logger logger = LogManager.getLogger(ElasticsearchEntitlementChecker.class);
+    private static final Logger logger = LogManager.getLogger(PolicyManager.class);
+
+    record ModuleEntitlements(Map<Class<? extends Entitlement>, List<Entitlement>> entitlementsByType) {
+        public static final ModuleEntitlements NONE = new ModuleEntitlements(Map.of());
 
-    static class ModuleEntitlements {
-        public static final ModuleEntitlements NONE = new ModuleEntitlements(List.of());
-        private final IdentityHashMap<Class<? extends Entitlement>, List<Entitlement>> entitlementsByType;
+        ModuleEntitlements {
+            entitlementsByType = Map.copyOf(entitlementsByType);
+        }
 
-        ModuleEntitlements(List<Entitlement> entitlements) {
-            this.entitlementsByType = entitlements.stream()
-                .collect(Collectors.toMap(Entitlement::getClass, e -> new ArrayList<>(List.of(e)), (a, b) -> {
-                    a.addAll(b);
-                    return a;
-                }, IdentityHashMap::new));
+        public static ModuleEntitlements from(List<Entitlement> entitlements) {
+            return new ModuleEntitlements(entitlements.stream().collect(groupingBy(Entitlement::getClass)));
         }
 
         public boolean hasEntitlement(Class<? extends Entitlement> entitlementClass) {
@@ -49,13 +52,18 @@ public boolean hasEntitlement(Class<? extends Entitlement> entitlementClass) {
         }
 
         public <E extends Entitlement> Stream<E> getEntitlements(Class<E> entitlementClass) {
-            return entitlementsByType.get(entitlementClass).stream().map(entitlementClass::cast);
+            var entitlements = entitlementsByType.get(entitlementClass);
+            if (entitlements == null) {
+                return Stream.empty();
+            }
+            return entitlements.stream().map(entitlementClass::cast);
         }
     }
 
-    final Map<Module, ModuleEntitlements> moduleEntitlementsMap = new HashMap<>();
+    final Map<Module, ModuleEntitlements> moduleEntitlementsMap = new ConcurrentHashMap<>();
 
     protected final Map<String, List<Entitlement>> serverEntitlements;
+    protected final List<Entitlement> agentEntitlements;
     protected final Map<String, Map<String, List<Entitlement>>> pluginsEntitlements;
     private final Function<Class<?>, String> pluginResolver;
 
@@ -69,7 +77,6 @@ private static Set<Module> findSystemModules() {
             .stream()
             .map(ModuleReference::descriptor)
             .collect(Collectors.toUnmodifiableSet());
-
         return ModuleLayer.boot()
             .modules()
             .stream()
@@ -77,17 +84,69 @@ private static Set<Module> findSystemModules() {
             .collect(Collectors.toUnmodifiableSet());
     }
 
-    public PolicyManager(Policy defaultPolicy, Map<String, Policy> pluginPolicies, Function<Class<?>, String> pluginResolver) {
-        this.serverEntitlements = buildScopeEntitlementsMap(Objects.requireNonNull(defaultPolicy));
-        this.pluginsEntitlements = Objects.requireNonNull(pluginPolicies)
-            .entrySet()
+    /**
+     * Frames originating from this module are ignored in the permission logic.
+     */
+    private final Module entitlementsModule;
+
+    public PolicyManager(
+        Policy serverPolicy,
+        List<Entitlement> agentEntitlements,
+        Map<String, Policy> pluginPolicies,
+        Function<Class<?>, String> pluginResolver,
+        Module entitlementsModule
+    ) {
+        this.serverEntitlements = buildScopeEntitlementsMap(requireNonNull(serverPolicy));
+        this.agentEntitlements = agentEntitlements;
+        this.pluginsEntitlements = requireNonNull(pluginPolicies).entrySet()
             .stream()
-            .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> buildScopeEntitlementsMap(e.getValue())));
+            .collect(toUnmodifiableMap(Map.Entry::getKey, e -> buildScopeEntitlementsMap(e.getValue())));
         this.pluginResolver = pluginResolver;
+        this.entitlementsModule = entitlementsModule;
     }
 
     private static Map<String, List<Entitlement>> buildScopeEntitlementsMap(Policy policy) {
-        return policy.scopes.stream().collect(Collectors.toUnmodifiableMap(scope -> scope.name, scope -> scope.entitlements));
+        return policy.scopes().stream().collect(toUnmodifiableMap(Scope::moduleName, Scope::entitlements));
+    }
+
+    public void checkStartProcess(Class<?> callerClass) {
+        neverEntitled(callerClass, "start process");
+    }
+
+    private void neverEntitled(Class<?> callerClass, String operationDescription) {
+        var requestingModule = requestingClass(callerClass);
+        if (isTriviallyAllowed(requestingModule)) {
+            return;
+        }
+
+        throw new NotEntitledException(
+            Strings.format(
+                "Not entitled: caller [%s], module [%s], operation [%s]",
+                callerClass,
+                requestingModule.getName(),
+                operationDescription
+            )
+        );
+    }
+
+    /**
+     * @param operationDescription is only called when the operation is not trivially allowed, meaning the check is about to fail;
+     *                            therefore, its performance is not a major concern.
+     */
+    private void neverEntitled(Class<?> callerClass, Supplier<String> operationDescription) {
+        var requestingModule = requestingClass(callerClass);
+        if (isTriviallyAllowed(requestingModule)) {
+            return;
+        }
+
+        throw new NotEntitledException(
+            Strings.format(
+                "Not entitled: caller [%s], module [%s], operation [%s]",
+                callerClass,
+                requestingModule.getName(),
+                operationDescription.get()
+            )
+        );
     }
 
     public void checkExitVM(Class<?> callerClass) {
@@ -98,19 +157,84 @@ public void checkCreateClassLoader(Class<?> callerClass) {
         checkEntitlementPresent(callerClass, CreateClassLoaderEntitlement.class);
     }
 
+    public void checkSetHttpsConnectionProperties(Class<?> callerClass) {
+        checkEntitlementPresent(callerClass, SetHttpsConnectionPropertiesEntitlement.class);
+    }
+
+    public void checkChangeJVMGlobalState(Class<?> callerClass) {
+        neverEntitled(callerClass, () -> {
+            // Look up the check$ method to compose an informative error message.
+            // This way, we don't need to painstakingly describe every individual global-state change.
+            Optional<String> checkMethodName = StackWalker.getInstance()
+                .walk(
+                    frames -> frames.map(StackFrame::getMethodName)
+                        .dropWhile(not(methodName -> methodName.startsWith("check$")))
+                        .findFirst()
+                );
+            return checkMethodName.map(this::operationDescription).orElse("change JVM global state");
+        });
+    }
+
+    /**
+     * Check for operations that can modify the way network operations are handled
+     */
+    public void checkChangeNetworkHandling(Class<?> callerClass) {
+        checkChangeJVMGlobalState(callerClass);
+    }
+
+    /**
+     * Check for operations that can access sensitive network information, e.g. secrets, tokens or SSL sessions
+     */
+    public void checkReadSensitiveNetworkInformation(Class<?> callerClass) {
+        neverEntitled(callerClass, "access sensitive network information");
+    }
+
+    private String operationDescription(String methodName) {
+        // TODO: Use a more human-readable description. Perhaps share code with InstrumentationServiceImpl.parseCheckerMethodName
+        return methodName.substring(methodName.indexOf('$'));
+    }
+
+    public void checkNetworkAccess(Class<?> callerClass, int actions) {
+        var requestingClass = requestingClass(callerClass);
+        if (isTriviallyAllowed(requestingClass)) {
+            return;
+        }
+
+        ModuleEntitlements entitlements = getEntitlements(requestingClass);
+        if (entitlements.getEntitlements(NetworkEntitlement.class).anyMatch(n -> n.matchActions(actions))) {
+            logger.debug(
+                () -> Strings.format(
+                    "Entitled: class [%s], module [%s], entitlement [Network], actions [Ox%X]",
+                    requestingClass,
+                    requestingClass.getModule().getName(),
+                    actions
+                )
+            );
+            return;
+        }
+        throw new NotEntitledException(
+            Strings.format(
+                "Missing entitlement: class [%s], module [%s], entitlement [Network], actions [%s]",
+                requestingClass,
+                requestingClass.getModule().getName(),
+                NetworkEntitlement.printActions(actions)
+            )
+        );
+    }
+
     private void checkEntitlementPresent(Class<?> callerClass, Class<? extends Entitlement> entitlementClass) {
-        var requestingModule = requestingModule(callerClass);
-        if (isTriviallyAllowed(requestingModule)) {
+        var requestingClass = requestingClass(callerClass);
+        if (isTriviallyAllowed(requestingClass)) {
             return;
         }
 
-        ModuleEntitlements entitlements = getEntitlementsOrThrow(callerClass, requestingModule);
+        ModuleEntitlements entitlements = getEntitlements(requestingClass);
         if (entitlements.hasEntitlement(entitlementClass)) {
             logger.debug(
                 () -> Strings.format(
-                    "Entitled: caller [%s], module [%s], type [%s]",
-                    callerClass,
-                    requestingModule.getName(),
+                    "Entitled: class [%s], module [%s], entitlement [%s]",
+                    requestingClass,
+                    requestingClass.getModule().getName(),
                     entitlementClass.getSimpleName()
                 )
             );
@@ -118,30 +242,26 @@ private void checkEntitlementPresent(Class<?> callerClass, Class<? extends Entit
         }
         throw new NotEntitledException(
             Strings.format(
-                "Missing entitlement: caller [%s], module [%s], type [%s]",
-                callerClass,
-                requestingModule.getName(),
+                "Missing entitlement: class [%s], module [%s], entitlement [%s]",
+                requestingClass,
+                requestingClass.getModule().getName(),
                 entitlementClass.getSimpleName()
             )
         );
     }
 
-    ModuleEntitlements getEntitlementsOrThrow(Class<?> callerClass, Module requestingModule) {
-        ModuleEntitlements cachedEntitlement = moduleEntitlementsMap.get(requestingModule);
-        if (cachedEntitlement != null) {
-            if (cachedEntitlement == ModuleEntitlements.NONE) {
-                throw new NotEntitledException(buildModuleNoPolicyMessage(callerClass, requestingModule) + "[CACHED]");
-            }
-            return cachedEntitlement;
-        }
+    ModuleEntitlements getEntitlements(Class<?> requestingClass) {
+        return moduleEntitlementsMap.computeIfAbsent(requestingClass.getModule(), m -> computeEntitlements(requestingClass));
+    }
 
+    private ModuleEntitlements computeEntitlements(Class<?> requestingClass) {
+        Module requestingModule = requestingClass.getModule();
         if (isServerModule(requestingModule)) {
-            var scopeName = requestingModule.getName();
-            return getModuleEntitlementsOrThrow(callerClass, requestingModule, serverEntitlements, scopeName);
+            return getModuleScopeEntitlements(requestingClass, serverEntitlements, requestingModule.getName());
         }
 
         // plugins
-        var pluginName = pluginResolver.apply(callerClass);
+        var pluginName = pluginResolver.apply(requestingClass);
         if (pluginName != null) {
             var pluginEntitlements = pluginsEntitlements.get(pluginName);
             if (pluginEntitlements != null) {
@@ -151,66 +271,80 @@ ModuleEntitlements getEntitlementsOrThrow(Class<?> callerClass, Module requestin
                 } else {
                     scopeName = requestingModule.getName();
                 }
-                return getModuleEntitlementsOrThrow(callerClass, requestingModule, pluginEntitlements, scopeName);
+                return getModuleScopeEntitlements(requestingClass, pluginEntitlements, scopeName);
             }
         }
 
-        moduleEntitlementsMap.put(requestingModule, ModuleEntitlements.NONE);
-        throw new NotEntitledException(buildModuleNoPolicyMessage(callerClass, requestingModule));
-    }
+        if (requestingModule.isNamed() == false) {
+            // agents are the only thing running non-modular
+            return ModuleEntitlements.from(agentEntitlements);
+        }
 
-    private static String buildModuleNoPolicyMessage(Class<?> callerClass, Module requestingModule) {
-        return Strings.format("Missing entitlement policy: caller [%s], module [%s]", callerClass, requestingModule.getName());
+        logger.warn("No applicable entitlement policy for class [{}]", requestingClass.getName());
+        return ModuleEntitlements.NONE;
     }
 
-    private ModuleEntitlements getModuleEntitlementsOrThrow(
+    private ModuleEntitlements getModuleScopeEntitlements(
         Class<?> callerClass,
-        Module module,
         Map<String, List<Entitlement>> scopeEntitlements,
         String moduleName
     ) {
         var entitlements = scopeEntitlements.get(moduleName);
         if (entitlements == null) {
-            // Module without entitlements - remember we don't have any
-            moduleEntitlementsMap.put(module, ModuleEntitlements.NONE);
-            throw new NotEntitledException(buildModuleNoPolicyMessage(callerClass, module));
+            logger.warn("No applicable entitlement policy for module [{}], class [{}]", moduleName, callerClass);
+            return ModuleEntitlements.NONE;
         }
-        // We have a policy for this module
-        var classEntitlements = new ModuleEntitlements(entitlements);
-        moduleEntitlementsMap.put(module, classEntitlements);
-        return classEntitlements;
+        return ModuleEntitlements.from(entitlements);
     }
 
     private static boolean isServerModule(Module requestingModule) {
         return requestingModule.isNamed() && requestingModule.getLayer() == ModuleLayer.boot();
     }
 
-    private static Module requestingModule(Class<?> callerClass) {
+    /**
+     * Walks the stack to determine which class should be checked for entitlements.
+     *
+     * @param callerClass when non-null will be returned;
+     *                    this is a fast-path check that can avoid the stack walk
+     *                    in cases where the caller class is available.
+     * @return the requesting class, or {@code null} if the entire call stack
+     * comes from the entitlement library itself.
+     */
+    Class<?> requestingClass(Class<?> callerClass) {
         if (callerClass != null) {
-            Module callerModule = callerClass.getModule();
-            if (systemModules.contains(callerModule) == false) {
-                // fast path
-                return callerModule;
-            }
+            // fast path
+            return callerClass;
         }
-        int framesToSkip = 1  // getCallingClass (this method)
-            + 1  // the checkXxx method
-            + 1  // the runtime config method
-            + 1  // the instrumented method
-        ;
-        Optional<Module> module = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE)
-            .walk(
-                s -> s.skip(framesToSkip)
-                    .map(f -> f.getDeclaringClass().getModule())
-                    .filter(m -> systemModules.contains(m) == false)
-                    .findFirst()
-            );
-        return module.orElse(null);
+        Optional<Class<?>> result = StackWalker.getInstance(RETAIN_CLASS_REFERENCE)
+            .walk(frames -> findRequestingClass(frames.map(StackFrame::getDeclaringClass)));
+        return result.orElse(null);
     }
 
-    private static boolean isTriviallyAllowed(Module requestingModule) {
-        if (requestingModule == null) {
-            logger.debug("Entitlement trivially allowed: entire call stack is in composed of classes in system modules");
+    /**
+     * Given a stream of classes corresponding to the frames from a {@link StackWalker},
+     * returns the module whose entitlements should be checked.
+     *
+     * @throws NullPointerException if the requesting module is {@code null}
+     */
+    Optional<Class<?>> findRequestingClass(Stream<Class<?>> classes) {
+        return classes.filter(c -> c.getModule() != entitlementsModule)  // Ignore the entitlements library
+            .skip(1)                                           // Skip the sensitive caller method
+            .findFirst();
+    }
+
+    /**
+     * @return true if permission is granted regardless of the entitlement
+     */
+    private static boolean isTriviallyAllowed(Class<?> requestingClass) {
+        if (logger.isTraceEnabled()) {
+            logger.trace("Stack trace for upcoming trivially-allowed check", new Exception());
+        }
+        if (requestingClass == null) {
+            logger.debug("Entitlement trivially allowed: no caller frames outside the entitlement library");
+            return true;
+        }
+        if (systemModules.contains(requestingClass.getModule())) {
+            logger.debug("Entitlement trivially allowed from system module [{}]", requestingClass.getModule().getName());
             return true;
         }
         logger.trace("Entitlement not trivially allowed");
diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java
index fb63d5ffbeb48..ac4d4afdd97f8 100644
--- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java
+++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java
@@ -34,8 +34,12 @@
  */
 public class PolicyParser {
 
-    private static final Map<String, Class<?>> EXTERNAL_ENTITLEMENTS = Stream.of(FileEntitlement.class, CreateClassLoaderEntitlement.class)
-        .collect(Collectors.toUnmodifiableMap(PolicyParser::getEntitlementTypeName, Function.identity()));
+    private static final Map<String, Class<?>> EXTERNAL_ENTITLEMENTS = Stream.of(
+        FileEntitlement.class,
+        CreateClassLoaderEntitlement.class,
+        SetHttpsConnectionPropertiesEntitlement.class,
+        NetworkEntitlement.class
+    ).collect(Collectors.toUnmodifiableMap(PolicyParser::getEntitlementTypeName, Function.identity()));
 
     protected final XContentParser policyParser;
     protected final String policyName;
diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java
index 0fe63eb8da1b7..55e257797d603 100644
--- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java
+++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java
@@ -9,38 +9,17 @@
 
 package org.elasticsearch.entitlement.runtime.policy;
 
-import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
 /**
  * A holder for entitlements within a single scope.
  */
-public class Scope {
+public record Scope(String moduleName, List<Entitlement> entitlements) {
 
-    public final String name;
-    public final List<Entitlement> entitlements;
-
-    public Scope(String name, List<Entitlement> entitlements) {
-        this.name = Objects.requireNonNull(name);
-        this.entitlements = Collections.unmodifiableList(Objects.requireNonNull(entitlements));
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        Scope scope = (Scope) o;
-        return Objects.equals(name, scope.name) && Objects.equals(entitlements, scope.entitlements);
+    public Scope(String moduleName, List<Entitlement> entitlements) {
+        this.moduleName = Objects.requireNonNull(moduleName);
+        this.entitlements = List.copyOf(entitlements);
     }
 
-    @Override
-    public int hashCode() {
-        return Objects.hash(name, entitlements);
-    }
-
-    @Override
-    public String toString() {
-        return "Scope{" + "name='" + name + '\'' + ", entitlements=" + entitlements + '}';
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataFeatures.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/SetHttpsConnectionPropertiesEntitlement.java
similarity index 56%
rename from server/src/main/java/org/elasticsearch/cluster/metadata/MetadataFeatures.java
rename to libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/SetHttpsConnectionPropertiesEntitlement.java
index 49bd38330e3af..bb2f65def9e18 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataFeatures.java
+++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/SetHttpsConnectionPropertiesEntitlement.java
@@ -7,16 +7,12 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.cluster.metadata;
+package org.elasticsearch.entitlement.runtime.policy;
 
-import org.elasticsearch.features.FeatureSpecification;
-import org.elasticsearch.features.NodeFeature;
-
-import java.util.Set;
-
-public class MetadataFeatures implements FeatureSpecification {
-    @Override
-    public Set<NodeFeature> getFeatures() {
-        return Set.of(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED);
-    }
+/**
+ * An Entitlement to allow setting properties to a single Https connection after this has been created
+ */
+public record SetHttpsConnectionPropertiesEntitlement() implements Entitlement {
+    @ExternalEntitlement(esModulesOnly = false)
+    public SetHttpsConnectionPropertiesEntitlement {}
 }
diff --git a/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java
index 912d76ecfc01a..706a6649de329 100644
--- a/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java
+++ b/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java
@@ -19,8 +19,8 @@ public Java23ElasticsearchEntitlementChecker(PolicyManager policyManager) {
     }
 
     @Override
-    public void check$$exit(Class<?> callerClass, Runtime runtime, int status) {
+    public void check$java_lang_Runtime$exit(Class<?> callerClass, Runtime runtime, int status) {
         // TODO: this is just an example, we shouldn't really override a method implemented in the superclass
-        super.check$$exit(callerClass, runtime, status);
+        super.check$java_lang_Runtime$exit(callerClass, runtime, status);
     }
 }
diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlementTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlementTests.java
new file mode 100644
index 0000000000000..91051d48c365f
--- /dev/null
+++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlementTests.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.runtime.policy;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.List;
+
+import static org.hamcrest.Matchers.is;
+
+public class NetworkEntitlementTests extends ESTestCase {
+
+    public void testMatchesActions() {
+        var listenEntitlement = new NetworkEntitlement(List.of(NetworkEntitlement.LISTEN));
+        var emptyEntitlement = new NetworkEntitlement(List.of());
+        var connectAcceptEntitlement = new NetworkEntitlement(List.of(NetworkEntitlement.CONNECT, NetworkEntitlement.ACCEPT));
+
+        assertThat(listenEntitlement.matchActions(0), is(true));
+        assertThat(listenEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION), is(true));
+        assertThat(listenEntitlement.matchActions(NetworkEntitlement.ACCEPT_ACTION), is(false));
+        assertThat(listenEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION), is(false));
+        assertThat(listenEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false));
+        assertThat(listenEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.CONNECT_ACTION), is(false));
+        assertThat(listenEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false));
+
+        assertThat(connectAcceptEntitlement.matchActions(0), is(true));
+        assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION), is(false));
+        assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.ACCEPT_ACTION), is(true));
+        assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION), is(true));
+        assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false));
+        assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.CONNECT_ACTION), is(false));
+        assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(true));
+
+        assertThat(emptyEntitlement.matchActions(0), is(true));
+        assertThat(emptyEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION), is(false));
+        assertThat(emptyEntitlement.matchActions(NetworkEntitlement.ACCEPT_ACTION), is(false));
+        assertThat(emptyEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION), is(false));
+        assertThat(emptyEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false));
+        assertThat(emptyEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.CONNECT_ACTION), is(false));
+        assertThat(emptyEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false));
+    }
+}
diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java
index 45bdf2e457824..d22c2f598e344 100644
--- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java
+++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java
@@ -9,10 +9,11 @@
 
 package org.elasticsearch.entitlement.runtime.policy;
 
-import org.elasticsearch.entitlement.runtime.api.NotEntitledException;
+import org.elasticsearch.entitlement.runtime.policy.PolicyManager.ModuleEntitlements;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.compiler.InMemoryJavaCompiler;
 import org.elasticsearch.test.jar.JarUtils;
+import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.lang.module.Configuration;
@@ -22,6 +23,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Stream;
 
 import static java.util.Map.entry;
 import static org.elasticsearch.entitlement.runtime.policy.PolicyManager.ALL_UNNAMED;
@@ -29,93 +31,93 @@
 import static org.hamcrest.Matchers.aMapWithSize;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.endsWith;
-import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.sameInstance;
 
 @ESTestCase.WithoutSecurityManager
 public class PolicyManagerTests extends ESTestCase {
+    /**
+     * A module you can use for test cases that don't actually care about the
+     * entitlements module.
+     */
+    private static Module NO_ENTITLEMENTS_MODULE;
+
+    @BeforeClass
+    public static void beforeClass() {
+        try {
+            // Any old module will do for tests using NO_ENTITLEMENTS_MODULE
+            NO_ENTITLEMENTS_MODULE = makeClassInItsOwnModule().getModule();
+        } catch (Exception e) {
+            throw new IllegalStateException(e);
+        }
+
+    }
 
     public void testGetEntitlementsThrowsOnMissingPluginUnnamedModule() {
         var policyManager = new PolicyManager(
             createEmptyTestServerPolicy(),
+            List.of(),
             Map.of("plugin1", createPluginPolicy("plugin.module")),
-            c -> "plugin1"
+            c -> "plugin1",
+            NO_ENTITLEMENTS_MODULE
         );
 
         // Any class from the current module (unnamed) will do
         var callerClass = this.getClass();
         var requestingModule = callerClass.getModule();
 
-        var ex = assertThrows(
-            "No policy for the unnamed module",
-            NotEntitledException.class,
-            () -> policyManager.getEntitlementsOrThrow(callerClass, requestingModule)
-        );
+        assertEquals("No policy for the unnamed module", ModuleEntitlements.NONE, policyManager.getEntitlements(callerClass));
 
-        assertEquals(
-            "Missing entitlement policy: caller [class org.elasticsearch.entitlement.runtime.policy.PolicyManagerTests], module [null]",
-            ex.getMessage()
-        );
-        assertThat(policyManager.moduleEntitlementsMap, hasEntry(requestingModule, PolicyManager.ModuleEntitlements.NONE));
+        assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap);
     }
 
     public void testGetEntitlementsThrowsOnMissingPolicyForPlugin() {
-        var policyManager = new PolicyManager(createEmptyTestServerPolicy(), Map.of(), c -> "plugin1");
+        var policyManager = new PolicyManager(createEmptyTestServerPolicy(), List.of(), Map.of(), c -> "plugin1", NO_ENTITLEMENTS_MODULE);
 
         // Any class from the current module (unnamed) will do
         var callerClass = this.getClass();
         var requestingModule = callerClass.getModule();
 
-        var ex = assertThrows(
-            "No policy for this plugin",
-            NotEntitledException.class,
-            () -> policyManager.getEntitlementsOrThrow(callerClass, requestingModule)
-        );
+        assertEquals("No policy for this plugin", ModuleEntitlements.NONE, policyManager.getEntitlements(callerClass));
 
-        assertEquals(
-            "Missing entitlement policy: caller [class org.elasticsearch.entitlement.runtime.policy.PolicyManagerTests], module [null]",
-            ex.getMessage()
-        );
-        assertThat(policyManager.moduleEntitlementsMap, hasEntry(requestingModule, PolicyManager.ModuleEntitlements.NONE));
+        assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap);
     }
 
     public void testGetEntitlementsFailureIsCached() {
-        var policyManager = new PolicyManager(createEmptyTestServerPolicy(), Map.of(), c -> "plugin1");
+        var policyManager = new PolicyManager(createEmptyTestServerPolicy(), List.of(), Map.of(), c -> "plugin1", NO_ENTITLEMENTS_MODULE);
 
         // Any class from the current module (unnamed) will do
         var callerClass = this.getClass();
         var requestingModule = callerClass.getModule();
 
-        assertThrows(NotEntitledException.class, () -> policyManager.getEntitlementsOrThrow(callerClass, requestingModule));
-        assertThat(policyManager.moduleEntitlementsMap, hasEntry(requestingModule, PolicyManager.ModuleEntitlements.NONE));
+        assertEquals(ModuleEntitlements.NONE, policyManager.getEntitlements(callerClass));
+        assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap);
 
         // A second time
-        var ex = assertThrows(NotEntitledException.class, () -> policyManager.getEntitlementsOrThrow(callerClass, requestingModule));
+        assertEquals(ModuleEntitlements.NONE, policyManager.getEntitlements(callerClass));
 
-        assertThat(ex.getMessage(), endsWith("[CACHED]"));
         // Nothing new in the map
-        assertThat(policyManager.moduleEntitlementsMap, aMapWithSize(1));
+        assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap);
     }
 
     public void testGetEntitlementsReturnsEntitlementsForPluginUnnamedModule() {
         var policyManager = new PolicyManager(
             createEmptyTestServerPolicy(),
+            List.of(),
             Map.ofEntries(entry("plugin2", createPluginPolicy(ALL_UNNAMED))),
-            c -> "plugin2"
+            c -> "plugin2",
+            NO_ENTITLEMENTS_MODULE
         );
 
         // Any class from the current module (unnamed) will do
         var callerClass = this.getClass();
-        var requestingModule = callerClass.getModule();
 
-        var entitlements = policyManager.getEntitlementsOrThrow(callerClass, requestingModule);
+        var entitlements = policyManager.getEntitlements(callerClass);
         assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true));
     }
 
     public void testGetEntitlementsThrowsOnMissingPolicyForServer() throws ClassNotFoundException {
-        var policyManager = new PolicyManager(createTestServerPolicy("example"), Map.of(), c -> null);
+        var policyManager = new PolicyManager(createTestServerPolicy("example"), List.of(), Map.of(), c -> null, NO_ENTITLEMENTS_MODULE);
 
         // Tests do not run modular, so we cannot use a server class.
         // But we know that in production code the server module and its classes are in the boot layer.
@@ -124,21 +126,19 @@ public void testGetEntitlementsThrowsOnMissingPolicyForServer() throws ClassNotF
         var mockServerClass = ModuleLayer.boot().findLoader("jdk.httpserver").loadClass("com.sun.net.httpserver.HttpServer");
         var requestingModule = mockServerClass.getModule();
 
-        var ex = assertThrows(
-            "No policy for this module in server",
-            NotEntitledException.class,
-            () -> policyManager.getEntitlementsOrThrow(mockServerClass, requestingModule)
-        );
+        assertEquals("No policy for this module in server", ModuleEntitlements.NONE, policyManager.getEntitlements(mockServerClass));
 
-        assertEquals(
-            "Missing entitlement policy: caller [class com.sun.net.httpserver.HttpServer], module [jdk.httpserver]",
-            ex.getMessage()
-        );
-        assertThat(policyManager.moduleEntitlementsMap, hasEntry(requestingModule, PolicyManager.ModuleEntitlements.NONE));
+        assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap);
     }
 
     public void testGetEntitlementsReturnsEntitlementsForServerModule() throws ClassNotFoundException {
-        var policyManager = new PolicyManager(createTestServerPolicy("jdk.httpserver"), Map.of(), c -> null);
+        var policyManager = new PolicyManager(
+            createTestServerPolicy("jdk.httpserver"),
+            List.of(),
+            Map.of(),
+            c -> null,
+            NO_ENTITLEMENTS_MODULE
+        );
 
         // Tests do not run modular, so we cannot use a server class.
         // But we know that in production code the server module and its classes are in the boot layer.
@@ -147,7 +147,7 @@ public void testGetEntitlementsReturnsEntitlementsForServerModule() throws Class
         var mockServerClass = ModuleLayer.boot().findLoader("jdk.httpserver").loadClass("com.sun.net.httpserver.HttpServer");
         var requestingModule = mockServerClass.getModule();
 
-        var entitlements = policyManager.getEntitlementsOrThrow(mockServerClass, requestingModule);
+        var entitlements = policyManager.getEntitlements(mockServerClass);
         assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true));
         assertThat(entitlements.hasEntitlement(ExitVMEntitlement.class), is(true));
     }
@@ -155,19 +155,21 @@ public void testGetEntitlementsReturnsEntitlementsForServerModule() throws Class
     public void testGetEntitlementsReturnsEntitlementsForPluginModule() throws IOException, ClassNotFoundException {
         final Path home = createTempDir();
 
-        Path jar = creteMockPluginJar(home);
+        Path jar = createMockPluginJar(home);
 
         var policyManager = new PolicyManager(
             createEmptyTestServerPolicy(),
+            List.of(),
             Map.of("mock-plugin", createPluginPolicy("org.example.plugin")),
-            c -> "mock-plugin"
+            c -> "mock-plugin",
+            NO_ENTITLEMENTS_MODULE
         );
 
         var layer = createLayerForJar(jar, "org.example.plugin");
         var mockPluginClass = layer.findLoader("org.example.plugin").loadClass("q.B");
         var requestingModule = mockPluginClass.getModule();
 
-        var entitlements = policyManager.getEntitlementsOrThrow(mockPluginClass, requestingModule);
+        var entitlements = policyManager.getEntitlements(mockPluginClass);
         assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true));
         assertThat(
             entitlements.getEntitlements(FileEntitlement.class).toList(),
@@ -178,25 +180,68 @@ public void testGetEntitlementsReturnsEntitlementsForPluginModule() throws IOExc
     public void testGetEntitlementsResultIsCached() {
         var policyManager = new PolicyManager(
             createEmptyTestServerPolicy(),
+            List.of(),
             Map.ofEntries(entry("plugin2", createPluginPolicy(ALL_UNNAMED))),
-            c -> "plugin2"
+            c -> "plugin2",
+            NO_ENTITLEMENTS_MODULE
         );
 
         // Any class from the current module (unnamed) will do
         var callerClass = this.getClass();
-        var requestingModule = callerClass.getModule();
 
-        var entitlements = policyManager.getEntitlementsOrThrow(callerClass, requestingModule);
+        var entitlements = policyManager.getEntitlements(callerClass);
         assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true));
         assertThat(policyManager.moduleEntitlementsMap, aMapWithSize(1));
         var cachedResult = policyManager.moduleEntitlementsMap.values().stream().findFirst().get();
-        var entitlementsAgain = policyManager.getEntitlementsOrThrow(callerClass, requestingModule);
+        var entitlementsAgain = policyManager.getEntitlements(callerClass);
 
         // Nothing new in the map
         assertThat(policyManager.moduleEntitlementsMap, aMapWithSize(1));
         assertThat(entitlementsAgain, sameInstance(cachedResult));
     }
 
+    public void testRequestingClassFastPath() throws IOException, ClassNotFoundException {
+        var callerClass = makeClassInItsOwnModule();
+        assertEquals(callerClass, policyManagerWithEntitlementsModule(NO_ENTITLEMENTS_MODULE).requestingClass(callerClass));
+    }
+
+    public void testRequestingModuleWithStackWalk() throws IOException, ClassNotFoundException {
+        var entitlementsClass = makeClassInItsOwnModule();    // A class in the entitlements library itself
+        var requestingClass = makeClassInItsOwnModule();      // This guy is always the right answer
+        var instrumentedClass = makeClassInItsOwnModule();    // The class that called the check method
+        var ignorableClass = makeClassInItsOwnModule();
+
+        var policyManager = policyManagerWithEntitlementsModule(entitlementsClass.getModule());
+
+        assertEquals(
+            "Skip entitlement library and the instrumented method",
+            requestingClass,
+            policyManager.findRequestingClass(Stream.of(entitlementsClass, instrumentedClass, requestingClass, ignorableClass)).orElse(null)
+        );
+        assertEquals(
+            "Skip multiple library frames",
+            requestingClass,
+            policyManager.findRequestingClass(Stream.of(entitlementsClass, entitlementsClass, instrumentedClass, requestingClass))
+                .orElse(null)
+        );
+        assertThrows(
+            "Non-modular caller frames are not supported",
+            NullPointerException.class,
+            () -> policyManager.findRequestingClass(Stream.of(entitlementsClass, null))
+        );
+    }
+
+    private static Class<?> makeClassInItsOwnModule() throws IOException, ClassNotFoundException {
+        final Path home = createTempDir();
+        Path jar = createMockPluginJar(home);
+        var layer = createLayerForJar(jar, "org.example.plugin");
+        return layer.findLoader("org.example.plugin").loadClass("q.B");
+    }
+
+    private static PolicyManager policyManagerWithEntitlementsModule(Module entitlementsModule) {
+        return new PolicyManager(createEmptyTestServerPolicy(), List.of(), Map.of(), c -> "test", entitlementsModule);
+    }
+
     private static Policy createEmptyTestServerPolicy() {
         return new Policy("server", List.of());
     }
@@ -219,7 +264,7 @@ private static Policy createPluginPolicy(String... pluginModules) {
         );
     }
 
-    private static Path creteMockPluginJar(Path home) throws IOException {
+    private static Path createMockPluginJar(Path home) throws IOException {
         Path jar = home.resolve("mock-plugin.jar");
 
         Map<String, CharSequence> sources = Map.ofEntries(
diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java
index 633c76cb8c04f..1e0c31d2280b8 100644
--- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java
+++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java
@@ -16,11 +16,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 
-import static org.elasticsearch.test.LambdaMatchers.transformedMatch;
-import static org.hamcrest.Matchers.both;
-import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
 
 public class PolicyParserTests extends ESTestCase {
 
@@ -39,21 +35,37 @@ public void testGetEntitlementTypeName() {
     public void testPolicyBuilder() throws IOException {
         Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", false)
             .parsePolicy();
-        Policy builtPolicy = new Policy(
+        Policy expected = new Policy(
             "test-policy.yaml",
             List.of(new Scope("entitlement-module-name", List.of(new FileEntitlement("test/path/to/file", List.of("read", "write")))))
         );
-        assertEquals(parsedPolicy, builtPolicy);
+        assertEquals(expected, parsedPolicy);
     }
 
     public void testPolicyBuilderOnExternalPlugin() throws IOException {
         Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", true)
             .parsePolicy();
-        Policy builtPolicy = new Policy(
+        Policy expected = new Policy(
             "test-policy.yaml",
             List.of(new Scope("entitlement-module-name", List.of(new FileEntitlement("test/path/to/file", List.of("read", "write")))))
         );
-        assertEquals(parsedPolicy, builtPolicy);
+        assertEquals(expected, parsedPolicy);
+    }
+
+    public void testParseNetwork() throws IOException {
+        Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream("""
+            entitlement-module-name:
+              - network:
+                  actions:
+                    - listen
+                    - accept
+                    - connect
+            """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", false).parsePolicy();
+        Policy expected = new Policy(
+            "test-policy.yaml",
+            List.of(new Scope("entitlement-module-name", List.of(new NetworkEntitlement(List.of("listen", "accept", "connect")))))
+        );
+        assertEquals(expected, parsedPolicy);
     }
 
     public void testParseCreateClassloader() throws IOException {
@@ -61,17 +73,22 @@ public void testParseCreateClassloader() throws IOException {
             entitlement-module-name:
               - create_class_loader
             """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", false).parsePolicy();
-        Policy builtPolicy = new Policy(
+        Policy expected = new Policy(
             "test-policy.yaml",
             List.of(new Scope("entitlement-module-name", List.of(new CreateClassLoaderEntitlement())))
         );
-        assertThat(
-            parsedPolicy.scopes,
-            contains(
-                both(transformedMatch((Scope scope) -> scope.name, equalTo("entitlement-module-name"))).and(
-                    transformedMatch(scope -> scope.entitlements, contains(instanceOf(CreateClassLoaderEntitlement.class)))
-                )
-            )
+        assertEquals(expected, parsedPolicy);
+    }
+
+    public void testParseSetHttpsConnectionProperties() throws IOException {
+        Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream("""
+            entitlement-module-name:
+              - set_https_connection_properties
+            """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", true).parsePolicy();
+        Policy expected = new Policy(
+            "test-policy.yaml",
+            List.of(new Scope("entitlement-module-name", List.of(new SetHttpsConnectionPropertiesEntitlement())))
         );
+        assertEquals(expected, parsedPolicy);
     }
 }
diff --git a/libs/grok/src/main/java/org/elasticsearch/grok/PatternBank.java b/libs/grok/src/main/java/org/elasticsearch/grok/PatternBank.java
index e6f8af9b566d5..cf8edf07f280a 100644
--- a/libs/grok/src/main/java/org/elasticsearch/grok/PatternBank.java
+++ b/libs/grok/src/main/java/org/elasticsearch/grok/PatternBank.java
@@ -23,7 +23,7 @@
 
 public class PatternBank {
 
-    public static PatternBank EMPTY = new PatternBank(Map.of());
+    public static final PatternBank EMPTY = new PatternBank(Map.of());
 
     private final Map<String, String> bank;
 
diff --git a/libs/h3/build.gradle b/libs/h3/build.gradle
index 81a0d56ed4606..6036323e160fc 100644
--- a/libs/h3/build.gradle
+++ b/libs/h3/build.gradle
@@ -35,7 +35,7 @@ tasks.named('forbiddenApisMain').configure {
   replaceSignatureFiles 'jdk-signatures'
 }
 
-ext.projectLicenses.set(['The Apache Software License, Version 2.0': 'http://www.apache.org/licenses/LICENSE-2.0'])
+ext.projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')])
 licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt'))
 
 tasks.withType(LicenseHeadersTask.class).configureEach {
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java b/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java
index 3b3f760c0534f..fa52ff6ee56cf 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java
@@ -29,7 +29,7 @@ final class Constants {
     /**
      * sqrt(3) / 2.0
      */
-    public static double M_SQRT3_2 = 0.8660254037844386467637231707529361834714;
+    public static final double M_SQRT3_2 = 0.8660254037844386467637231707529361834714;
     /**
      * 2.0 * PI
      */
@@ -37,19 +37,19 @@ final class Constants {
     /**
      * The number of H3 base cells
      */
-    public static int NUM_BASE_CELLS = 122;
+    public static final int NUM_BASE_CELLS = 122;
     /**
      * The number of vertices in a hexagon
      */
-    public static int NUM_HEX_VERTS = 6;
+    public static final int NUM_HEX_VERTS = 6;
     /**
      * The number of vertices in a pentagon
      */
-    public static int NUM_PENT_VERTS = 5;
+    public static final int NUM_PENT_VERTS = 5;
     /**
      * H3 index modes
      */
-    public static int H3_CELL_MODE = 1;
+    public static final int H3_CELL_MODE = 1;
     /**
      * square root of 7
      */
@@ -64,14 +64,14 @@ final class Constants {
      * (or distance between adjacent cell center points
      * on the plane) to gnomonic unit length.
      */
-    public static double RES0_U_GNOMONIC = 0.38196601125010500003;
+    public static final double RES0_U_GNOMONIC = 0.38196601125010500003;
     /**
      * rotation angle between Class II and Class III resolution axes
      * (asin(sqrt(3.0 / 28.0)))
      */
-    public static double M_AP7_ROT_RADS = 0.333473172251832115336090755351601070065900389;
+    public static final double M_AP7_ROT_RADS = 0.333473172251832115336090755351601070065900389;
     /**
      * threshold epsilon
      */
-    public static double EPSILON = 0.0000000000000001;
+    public static final double EPSILON = 0.0000000000000001;
 }
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/H3.java b/libs/h3/src/main/java/org/elasticsearch/h3/H3.java
index 08031088728ba..ac35fe6670c1b 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/H3.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/H3.java
@@ -33,7 +33,7 @@ public final class H3 {
     /**
      * max H3 resolution; H3 version 1 has 16 resolutions, numbered 0 through 15
      */
-    public static int MAX_H3_RES = 15;
+    public static final int MAX_H3_RES = 15;
 
     private static final long[] NORTH = new long[MAX_H3_RES + 1];
     private static final long[] SOUTH = new long[MAX_H3_RES + 1];
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java b/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java
index 2b1b9cade21a4..88261dd458feb 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java
@@ -41,22 +41,22 @@ public static boolean H3_is_pentagon(long h3) {
         return BaseCells.isBaseCellPentagon(H3Index.H3_get_base_cell(h3)) && H3Index.h3LeadingNonZeroDigit(h3) == 0;
     }
 
-    public static long H3_INIT = 35184372088831L;
+    public static final long H3_INIT = 35184372088831L;
 
     /**
      * The bit offset of the mode in an H3 index.
      */
-    public static int H3_MODE_OFFSET = 59;
+    public static final int H3_MODE_OFFSET = 59;
 
     /**
      * 1's in the 4 mode bits, 0's everywhere else.
      */
-    public static long H3_MODE_MASK = 15L << H3_MODE_OFFSET;
+    public static final long H3_MODE_MASK = 15L << H3_MODE_OFFSET;
 
     /**
      * 0's in the 4 mode bits, 1's everywhere else.
      */
-    public static long H3_MODE_MASK_NEGATIVE = ~H3_MODE_MASK;
+    public static final long H3_MODE_MASK_NEGATIVE = ~H3_MODE_MASK;
 
     public static long H3_set_mode(long h3, long mode) {
         return (h3 & H3_MODE_MASK_NEGATIVE) | (mode << H3_MODE_OFFSET);
@@ -65,16 +65,16 @@ public static long H3_set_mode(long h3, long mode) {
     /**
      * The bit offset of the base cell in an H3 index.
      */
-    public static int H3_BC_OFFSET = 45;
+    public static final int H3_BC_OFFSET = 45;
     /**
      * 1's in the 7 base cell bits, 0's everywhere else.
      */
-    public static long H3_BC_MASK = 127L << H3_BC_OFFSET;
+    public static final long H3_BC_MASK = 127L << H3_BC_OFFSET;
 
     /**
      * 0's in the 7 base cell bits, 1's everywhere else.
      */
-    public static long H3_BC_MASK_NEGATIVE = ~H3_BC_MASK;
+    public static final long H3_BC_MASK_NEGATIVE = ~H3_BC_MASK;
 
     /**
      * Sets the integer base cell of h3 to bc.
@@ -83,26 +83,26 @@ public static long H3_set_base_cell(long h3, long bc) {
         return (h3 & H3_BC_MASK_NEGATIVE) | (bc << H3_BC_OFFSET);
     }
 
-    public static int H3_RES_OFFSET = 52;
+    public static final int H3_RES_OFFSET = 52;
     /**
      * 1's in the 4 resolution bits, 0's everywhere else.
      */
-    public static long H3_RES_MASK = 15L << H3_RES_OFFSET;
+    public static final long H3_RES_MASK = 15L << H3_RES_OFFSET;
 
     /**
      * 0's in the 4 resolution bits, 1's everywhere else.
      */
-    public static long H3_RES_MASK_NEGATIVE = ~H3_RES_MASK;
+    public static final long H3_RES_MASK_NEGATIVE = ~H3_RES_MASK;
 
     /**
      * The bit offset of the max resolution digit in an H3 index.
      */
-    public static int H3_MAX_OFFSET = 63;
+    public static final int H3_MAX_OFFSET = 63;
 
     /**
      * 1 in the highest bit, 0's everywhere else.
      */
-    public static long H3_HIGH_BIT_MASK = (1L << H3_MAX_OFFSET);
+    public static final long H3_HIGH_BIT_MASK = (1L << H3_MAX_OFFSET);
 
     /**
      * Gets the highest bit of the H3 index.
@@ -121,12 +121,12 @@ public static long H3_set_resolution(long h3, long res) {
     /**
      * The bit offset of the reserved bits in an H3 index.
      */
-    public static int H3_RESERVED_OFFSET = 56;
+    public static final int H3_RESERVED_OFFSET = 56;
 
     /**
      * 1's in the 3 reserved bits, 0's everywhere else.
      */
-    public static long H3_RESERVED_MASK = (7L << H3_RESERVED_OFFSET);
+    public static final long H3_RESERVED_MASK = (7L << H3_RESERVED_OFFSET);
 
     /**
      * Gets a value in the reserved space. Should always be zero for valid indexes.
@@ -149,12 +149,12 @@ public static int H3_get_resolution(long h3) {
     /**
      * The number of bits in a single H3 resolution digit.
      */
-    public static int H3_PER_DIGIT_OFFSET = 3;
+    public static final int H3_PER_DIGIT_OFFSET = 3;
 
     /**
      * 1's in the 3 bits of res 15 digit bits, 0's everywhere else.
      */
-    public static long H3_DIGIT_MASK = 7L;
+    public static final long H3_DIGIT_MASK = 7L;
 
     /**
      * Gets the resolution res integer digit (0-7) of h3.
diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/IngestDocumentBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/IngestDocumentBridge.java
index da97be39d97c5..1df089c0427dc 100644
--- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/IngestDocumentBridge.java
+++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/IngestDocumentBridge.java
@@ -20,8 +20,6 @@
 
 public class IngestDocumentBridge extends StableBridgeAPI.Proxy<IngestDocument> {
 
-    public static String INGEST_KEY = IngestDocument.INGEST_KEY;
-
     public static IngestDocumentBridge wrap(final IngestDocument ingestDocument) {
         if (ingestDocument == null) {
             return null;
diff --git a/libs/native/libraries/build.gradle b/libs/native/libraries/build.gradle
index c6684052334c5..ed37d4a70931f 100644
--- a/libs/native/libraries/build.gradle
+++ b/libs/native/libraries/build.gradle
@@ -25,7 +25,7 @@ repositories {
   exclusiveContent {
     forRepository {
       maven {
-        url "https://artifactory.elastic.dev/artifactory/elasticsearch-native"
+        url = "https://artifactory.elastic.dev/artifactory/elasticsearch-native"
         metadataSources {
           artifact()
         }
diff --git a/libs/simdvec/output.txt b/libs/simdvec/output.txt
new file mode 100644
index 0000000000000..acccaea19b765
--- /dev/null
+++ b/libs/simdvec/output.txt
@@ -0,0 +1,3809 @@
+Using gradle at '/Users/rene/dev/elastic/elasticsearch/gradlew' to run buildfile '/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build.gradle':
+
+Initialized native services in: /Users/rene/.gradle/native
+Initialized jansi services in: /Users/rene/.gradle/native
+Found daemon DaemonInfo{pid=51106, address=[aba1969e-035e-4d64-b357-459afffcd02b port:60032, addresses:[/127.0.0.1]], state=Idle, lastBusy=1734939945982, context=DefaultDaemonContext[uid=2770ac86-e90f-46e0-aca4-eb484b46539b,javaHome=/Users/rene/.gradle-idea2/jdks/oracle_corporation-22-aarch64-os_x/jdk-22.jdk/Contents/Home,javaVersion=22,javaVendor=Oracle Corporation,daemonRegistryDir=/Users/rene/.gradle/daemon,pid=51106,idleTimeout=10800000,priority=NORMAL,applyInstrumentationAgent=true,nativeServicesMode=ENABLED,daemonOpts=-XX:+HeapDumpOnOutOfMemoryError,-Xss2m,--add-exports,jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED,--add-exports,jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED,--add-exports,jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED,--add-exports,jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED,--add-exports,jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED,--add-opens,java.base/java.time=ALL-UNNAMED,-Dfile.encoding=UTF-8,-Duser.country=DE,-Duser.language=en,-Duser.variant]} however its context does not match the desired criteria.
+JVM is incompatible.
+Wanted: DaemonRequestContext{jvmCriteria=/Users/rene/.sdkman/candidates/java/21.0.5-oracle (no JDK specified, using current Java home), daemonOpts=[-XX:+HeapDumpOnOutOfMemoryError, -Xss2m, --add-exports, jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED, --add-exports, jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED, --add-exports, jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED, --add-exports, jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED, --add-exports, jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED, --add-opens, java.base/java.time=ALL-UNNAMED, -Dfile.encoding=UTF-8, -Duser.country=DE, -Duser.language=en, -Duser.variant], applyInstrumentationAgent=true, nativeServicesMode=ENABLED, priority=NORMAL}
+Actual: DefaultDaemonContext[uid=2770ac86-e90f-46e0-aca4-eb484b46539b,javaHome=/Users/rene/.gradle-idea2/jdks/oracle_corporation-22-aarch64-os_x/jdk-22.jdk/Contents/Home,javaVersion=22,javaVendor=Oracle Corporation,daemonRegistryDir=/Users/rene/.gradle/daemon,pid=51106,idleTimeout=10800000,priority=NORMAL,applyInstrumentationAgent=true,nativeServicesMode=ENABLED,daemonOpts=-XX:+HeapDumpOnOutOfMemoryError,-Xss2m,--add-exports,jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED,--add-exports,jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED,--add-exports,jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED,--add-exports,jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED,--add-exports,jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED,--add-opens,java.base/java.time=ALL-UNNAMED,-Dfile.encoding=UTF-8,-Duser.country=DE,-Duser.language=en,-Duser.variant]
+
+  Looking for a different daemon...
+The client will now receive all logging from the daemon (pid: 49343). The daemon log file: /Users/rene/.gradle/daemon/8.11.1/daemon-49343.out.log
+Starting 7th build in daemon [uptime: 28 mins 11.835 secs, performance: 98%, GC rate: 0.00/s, heap usage: 0% of 8 GiB]
+Using 12 worker leases.
+Now considering [/Users/rene/dev/elastic/elasticsearch, /Users/rene/dev/elastic/elasticsearch/build-tools-internal, /Users/rene/dev/elastic/elasticsearch/build-tools, /Users/rene/dev/elastic/elasticsearch/build-conventions] as hierarchies to watch
+Now considering [/Users/rene/dev/elastic/elasticsearch/build-conventions, /Users/rene/dev/elastic/elasticsearch, /Users/rene/dev/elastic/elasticsearch/build-tools-internal, /Users/rene/dev/elastic/elasticsearch/build-tools] as hierarchies to watch
+Now considering [/Users/rene/dev/elastic/elasticsearch/build-tools, /Users/rene/dev/elastic/elasticsearch/build-conventions, /Users/rene/dev/elastic/elasticsearch, /Users/rene/dev/elastic/elasticsearch/build-tools-internal] as hierarchies to watch
+Now considering [/Users/rene/dev/elastic/elasticsearch/build-tools-internal, /Users/rene/dev/elastic/elasticsearch/build-tools, /Users/rene/dev/elastic/elasticsearch/build-conventions, /Users/rene/dev/elastic/elasticsearch] as hierarchies to watch
+Calculating task graph as configuration cache cannot be reused because directory 'libs/simdvec' has changed.
+Now considering [/Users/rene/dev/elastic/elasticsearch, /Users/rene/dev/elastic/elasticsearch/build-tools-internal, /Users/rene/dev/elastic/elasticsearch/build-tools, /Users/rene/dev/elastic/elasticsearch/build-conventions] as hierarchies to watch
+Watching the file system is configured to be enabled if available
+File system watching is active
+Starting Build
+Now considering [/Users/rene/dev/elastic/elasticsearch/build-conventions, /Users/rene/dev/elastic/elasticsearch, /Users/rene/dev/elastic/elasticsearch/build-tools-internal, /Users/rene/dev/elastic/elasticsearch/build-tools] as hierarchies to watch
+Now considering [/Users/rene/dev/elastic/elasticsearch/build-tools, /Users/rene/dev/elastic/elasticsearch/build-conventions, /Users/rene/dev/elastic/elasticsearch, /Users/rene/dev/elastic/elasticsearch/build-tools-internal] as hierarchies to watch
+Now considering [/Users/rene/dev/elastic/elasticsearch/build-tools-internal, /Users/rene/dev/elastic/elasticsearch/build-tools, /Users/rene/dev/elastic/elasticsearch/build-conventions, /Users/rene/dev/elastic/elasticsearch] as hierarchies to watch
+
+> Configure project :build-conventions
+Evaluating project ':build-conventions' using build file '/Users/rene/dev/elastic/elasticsearch/build-conventions/build.gradle'.
+Registering project ':build-conventions' in composite build. Will substitute for module 'org.elasticsearch:build-conventions'.
+Resolve mutations for :build-conventions:compileJava (Thread[#1322,Execution worker,5,main]) started.
+:build-conventions:compileJava (Thread[#1329,Execution worker Thread 8,5,main]) started.
+
+> Task :build-conventions:compileJava UP-TO-DATE
+Caching disabled for task ':build-conventions:compileJava' because:
+  Build cache is disabled
+Skipping task ':build-conventions:compileJava' as it is up-to-date.
+Resolve mutations for :build-conventions:pluginDescriptors (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:build-conventions:pluginDescriptors (Thread[#1329,Execution worker Thread 8,5,main]) started.
+
+> Task :build-conventions:pluginDescriptors UP-TO-DATE
+Caching disabled for task ':build-conventions:pluginDescriptors' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':build-conventions:pluginDescriptors' as it is up-to-date.
+Resolve mutations for :build-conventions:processResources (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:build-conventions:processResources (Thread[#1329,Execution worker Thread 8,5,main]) started.
+
+> Task :build-conventions:processResources UP-TO-DATE
+Caching disabled for task ':build-conventions:processResources' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':build-conventions:processResources' as it is up-to-date.
+Resolve mutations for :build-conventions:classes (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:build-conventions:classes (Thread[#1329,Execution worker Thread 8,5,main]) started.
+
+> Task :build-conventions:classes UP-TO-DATE
+Skipping task ':build-conventions:classes' as it has no actions.
+Resolve mutations for :build-conventions:jar (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:build-conventions:jar (Thread[#1329,Execution worker Thread 8,5,main]) started.
+
+> Task :build-conventions:jar UP-TO-DATE
+Caching disabled for task ':build-conventions:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':build-conventions:jar' as it is up-to-date.
+
+> Configure project :build-tools
+Evaluating project ':build-tools' using build file '/Users/rene/dev/elastic/elasticsearch/build-tools/build.gradle'.
+Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with InstrumentationAnalysisTransform
+Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with InstrumentationAnalysisTransform
+Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with InstrumentationAnalysisTransform
+Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with InstrumentationAnalysisTransform
+Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with InstrumentationAnalysisTransform
+Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with InstrumentationAnalysisTransform
+Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with InstrumentationAnalysisTransform
+Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with InstrumentationAnalysisTransform
+Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with InstrumentationAnalysisTransform
+Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with InstrumentationAnalysisTransform
+Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with InstrumentationAnalysisTransform
+Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with InstrumentationAnalysisTransform
+Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with InstrumentationAnalysisTransform
+Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with InstrumentationAnalysisTransform
+Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with InstrumentationAnalysisTransform
+Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with InstrumentationAnalysisTransform
+Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with InstrumentationAnalysisTransform
+Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with InstrumentationAnalysisTransform
+Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with InstrumentationAnalysisTransform
+Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with InstrumentationAnalysisTransform
+Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with InstrumentationAnalysisTransform
+Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with InstrumentationAnalysisTransform
+Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with InstrumentationAnalysisTransform
+Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with InstrumentationAnalysisTransform
+Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with InstrumentationAnalysisTransform
+Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with InstrumentationAnalysisTransform
+Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with InstrumentationAnalysisTransform
+Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with InstrumentationAnalysisTransform
+Transforming commons-compress-1.5.jar (org.apache.commons:commons-compress:1.5) with InstrumentationAnalysisTransform
+Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with InstrumentationAnalysisTransform
+Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with InstrumentationAnalysisTransform
+Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with InstrumentationAnalysisTransform
+Transforming commons-codec-1.16.0.jar (commons-codec:commons-codec:1.16.0) with InstrumentationAnalysisTransform
+Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with InstrumentationAnalysisTransform
+Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with InstrumentationAnalysisTransform
+Transforming xz-1.9.jar (org.tukaani:xz:1.9) with InstrumentationAnalysisTransform
+Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with InstrumentationAnalysisTransform
+Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with InstrumentationAnalysisTransform
+Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with InstrumentationAnalysisTransform
+Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with InstrumentationAnalysisTransform
+Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with InstrumentationAnalysisTransform
+Transforming main (project :build-conventions) with InstrumentationAnalysisTransform
+Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with InstrumentationAnalysisTransform
+Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with MergeInstrumentationAnalysisTransform
+Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with InstrumentationAnalysisTransform
+Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with MergeInstrumentationAnalysisTransform
+Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with InstrumentationAnalysisTransform
+Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with MergeInstrumentationAnalysisTransform
+Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with InstrumentationAnalysisTransform
+Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with ExternalDependencyInstrumentingArtifactTransform
+Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with MergeInstrumentationAnalysisTransform
+Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with InstrumentationAnalysisTransform
+Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with MergeInstrumentationAnalysisTransform
+Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with ExternalDependencyInstrumentingArtifactTransform
+Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with InstrumentationAnalysisTransform
+Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with MergeInstrumentationAnalysisTransform
+Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with InstrumentationAnalysisTransform
+Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with MergeInstrumentationAnalysisTransform
+Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with InstrumentationAnalysisTransform
+Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with MergeInstrumentationAnalysisTransform
+Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with ExternalDependencyInstrumentingArtifactTransform
+Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with InstrumentationAnalysisTransform
+Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with MergeInstrumentationAnalysisTransform
+Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with InstrumentationAnalysisTransform
+Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with MergeInstrumentationAnalysisTransform
+Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with ExternalDependencyInstrumentingArtifactTransform
+Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with InstrumentationAnalysisTransform
+Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with MergeInstrumentationAnalysisTransform
+Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with InstrumentationAnalysisTransform
+Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with MergeInstrumentationAnalysisTransform
+Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with InstrumentationAnalysisTransform
+Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with MergeInstrumentationAnalysisTransform
+Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with InstrumentationAnalysisTransform
+Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with MergeInstrumentationAnalysisTransform
+Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with InstrumentationAnalysisTransform
+Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with MergeInstrumentationAnalysisTransform
+Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with InstrumentationAnalysisTransform
+Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with MergeInstrumentationAnalysisTransform
+Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with ExternalDependencyInstrumentingArtifactTransform
+Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with InstrumentationAnalysisTransform
+Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with MergeInstrumentationAnalysisTransform
+Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with InstrumentationAnalysisTransform
+Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with MergeInstrumentationAnalysisTransform
+Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with ExternalDependencyInstrumentingArtifactTransform
+Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with InstrumentationAnalysisTransform
+Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with MergeInstrumentationAnalysisTransform
+Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with InstrumentationAnalysisTransform
+Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with MergeInstrumentationAnalysisTransform
+Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with InstrumentationAnalysisTransform
+Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with MergeInstrumentationAnalysisTransform
+Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with InstrumentationAnalysisTransform
+Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with MergeInstrumentationAnalysisTransform
+Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with ExternalDependencyInstrumentingArtifactTransform
+Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with InstrumentationAnalysisTransform
+Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with MergeInstrumentationAnalysisTransform
+Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with InstrumentationAnalysisTransform
+Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with MergeInstrumentationAnalysisTransform
+Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with InstrumentationAnalysisTransform
+Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with MergeInstrumentationAnalysisTransform
+Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform
+Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with InstrumentationAnalysisTransform
+Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform
+Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with MergeInstrumentationAnalysisTransform
+Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with InstrumentationAnalysisTransform
+Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with MergeInstrumentationAnalysisTransform
+Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with InstrumentationAnalysisTransform
+Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with MergeInstrumentationAnalysisTransform
+Transforming commons-compress-1.5.jar (org.apache.commons:commons-compress:1.5) with InstrumentationAnalysisTransform
+Transforming commons-compress-1.5.jar (org.apache.commons:commons-compress:1.5) with MergeInstrumentationAnalysisTransform
+Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with InstrumentationAnalysisTransform
+Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with MergeInstrumentationAnalysisTransform
+Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with InstrumentationAnalysisTransform
+Transforming commons-compress-1.5.jar (org.apache.commons:commons-compress:1.5) with ExternalDependencyInstrumentingArtifactTransform
+Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with MergeInstrumentationAnalysisTransform
+Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with ExternalDependencyInstrumentingArtifactTransform
+Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with InstrumentationAnalysisTransform
+Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with ExternalDependencyInstrumentingArtifactTransform
+Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with MergeInstrumentationAnalysisTransform
+Transforming commons-codec-1.16.0.jar (commons-codec:commons-codec:1.16.0) with InstrumentationAnalysisTransform
+Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-codec-1.16.0.jar (commons-codec:commons-codec:1.16.0) with MergeInstrumentationAnalysisTransform
+Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with InstrumentationAnalysisTransform
+Transforming commons-codec-1.16.0.jar (commons-codec:commons-codec:1.16.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with MergeInstrumentationAnalysisTransform
+Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform
+Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with InstrumentationAnalysisTransform
+Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform
+Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with MergeInstrumentationAnalysisTransform
+Transforming xz-1.9.jar (org.tukaani:xz:1.9) with InstrumentationAnalysisTransform
+Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming xz-1.9.jar (org.tukaani:xz:1.9) with MergeInstrumentationAnalysisTransform
+Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with InstrumentationAnalysisTransform
+Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with MergeInstrumentationAnalysisTransform
+Transforming xz-1.9.jar (org.tukaani:xz:1.9) with ExternalDependencyInstrumentingArtifactTransform
+Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with InstrumentationAnalysisTransform
+Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with MergeInstrumentationAnalysisTransform
+Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform
+Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with InstrumentationAnalysisTransform
+Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with MergeInstrumentationAnalysisTransform
+Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform
+Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with InstrumentationAnalysisTransform
+Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with MergeInstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with MergeInstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with InstrumentationAnalysisTransform
+Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with MergeInstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with ExternalDependencyInstrumentingArtifactTransform
+Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with MergeInstrumentationAnalysisTransform
+Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with ExternalDependencyInstrumentingArtifactTransform
+Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with ExternalDependencyInstrumentingArtifactTransform
+Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with MergeInstrumentationAnalysisTransform
+Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with InstrumentationAnalysisTransform
+Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with MergeInstrumentationAnalysisTransform
+Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with ExternalDependencyInstrumentingArtifactTransform
+Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming build-conventions.jar (project :build-conventions) with ProjectDependencyInstrumentingArtifactTransform
+
+> Configure project :build-tools:reaper
+Evaluating project ':build-tools:reaper' using build file '/Users/rene/dev/elastic/elasticsearch/build-tools/reaper/build.gradle'.
+Registering project ':build-tools' in composite build. Will substitute for module 'org.elasticsearch.gradle:build-tools'.
+Registering project ':build-tools:reaper' in composite build. Will substitute for module 'org.elasticsearch.gradle:reaper'.
+
+> Configure project :build-tools-internal
+Evaluating project ':build-tools-internal' using build file '/Users/rene/dev/elastic/elasticsearch/build-tools-internal/build.gradle'.
+Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with InstrumentationAnalysisTransform
+Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with InstrumentationAnalysisTransform
+Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with InstrumentationAnalysisTransform
+Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with InstrumentationAnalysisTransform
+Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with InstrumentationAnalysisTransform
+Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with InstrumentationAnalysisTransform
+Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with InstrumentationAnalysisTransform
+Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with InstrumentationAnalysisTransform
+Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with InstrumentationAnalysisTransform
+Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with InstrumentationAnalysisTransform
+Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with InstrumentationAnalysisTransform
+Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with InstrumentationAnalysisTransform
+Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with InstrumentationAnalysisTransform
+Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with InstrumentationAnalysisTransform
+Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with InstrumentationAnalysisTransform
+Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with InstrumentationAnalysisTransform
+Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with InstrumentationAnalysisTransform
+Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with InstrumentationAnalysisTransform
+Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with InstrumentationAnalysisTransform
+Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with InstrumentationAnalysisTransform
+Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with InstrumentationAnalysisTransform
+Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with InstrumentationAnalysisTransform
+Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with InstrumentationAnalysisTransform
+Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with InstrumentationAnalysisTransform
+Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with InstrumentationAnalysisTransform
+Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with InstrumentationAnalysisTransform
+Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with InstrumentationAnalysisTransform
+Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with InstrumentationAnalysisTransform
+Transforming commons-compress-1.5.jar (org.apache.commons:commons-compress:1.5) with InstrumentationAnalysisTransform
+Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with InstrumentationAnalysisTransform
+Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with InstrumentationAnalysisTransform
+Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with InstrumentationAnalysisTransform
+Transforming commons-codec-1.16.0.jar (commons-codec:commons-codec:1.16.0) with InstrumentationAnalysisTransform
+Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with InstrumentationAnalysisTransform
+Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with InstrumentationAnalysisTransform
+Transforming xz-1.9.jar (org.tukaani:xz:1.9) with InstrumentationAnalysisTransform
+Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with InstrumentationAnalysisTransform
+Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with InstrumentationAnalysisTransform
+Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with InstrumentationAnalysisTransform
+Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with InstrumentationAnalysisTransform
+Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with InstrumentationAnalysisTransform
+Transforming main (project :build-conventions) with InstrumentationAnalysisTransform
+Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with InstrumentationAnalysisTransform
+Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with MergeInstrumentationAnalysisTransform
+Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with InstrumentationAnalysisTransform
+Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with MergeInstrumentationAnalysisTransform
+Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with InstrumentationAnalysisTransform
+Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with MergeInstrumentationAnalysisTransform
+Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with InstrumentationAnalysisTransform
+Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with MergeInstrumentationAnalysisTransform
+Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with ExternalDependencyInstrumentingArtifactTransform
+Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with InstrumentationAnalysisTransform
+Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with MergeInstrumentationAnalysisTransform
+Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with ExternalDependencyInstrumentingArtifactTransform
+Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with InstrumentationAnalysisTransform
+Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with MergeInstrumentationAnalysisTransform
+Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with InstrumentationAnalysisTransform
+Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with MergeInstrumentationAnalysisTransform
+Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with InstrumentationAnalysisTransform
+Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with MergeInstrumentationAnalysisTransform
+Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with ExternalDependencyInstrumentingArtifactTransform
+Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with InstrumentationAnalysisTransform
+Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with MergeInstrumentationAnalysisTransform
+Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with InstrumentationAnalysisTransform
+Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with MergeInstrumentationAnalysisTransform
+Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with ExternalDependencyInstrumentingArtifactTransform
+Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with InstrumentationAnalysisTransform
+Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with MergeInstrumentationAnalysisTransform
+Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with InstrumentationAnalysisTransform
+Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with MergeInstrumentationAnalysisTransform
+Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with InstrumentationAnalysisTransform
+Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with MergeInstrumentationAnalysisTransform
+Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with InstrumentationAnalysisTransform
+Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with MergeInstrumentationAnalysisTransform
+Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with InstrumentationAnalysisTransform
+Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with MergeInstrumentationAnalysisTransform
+Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with InstrumentationAnalysisTransform
+Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with MergeInstrumentationAnalysisTransform
+Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with InstrumentationAnalysisTransform
+Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with MergeInstrumentationAnalysisTransform
+Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with ExternalDependencyInstrumentingArtifactTransform
+Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with InstrumentationAnalysisTransform
+Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with MergeInstrumentationAnalysisTransform
+Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with ExternalDependencyInstrumentingArtifactTransform
+Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with InstrumentationAnalysisTransform
+Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with MergeInstrumentationAnalysisTransform
+Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with InstrumentationAnalysisTransform
+Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with MergeInstrumentationAnalysisTransform
+Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with InstrumentationAnalysisTransform
+Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with MergeInstrumentationAnalysisTransform
+Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with InstrumentationAnalysisTransform
+Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with MergeInstrumentationAnalysisTransform
+Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with ExternalDependencyInstrumentingArtifactTransform
+Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with InstrumentationAnalysisTransform
+Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with MergeInstrumentationAnalysisTransform
+Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with InstrumentationAnalysisTransform
+Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with MergeInstrumentationAnalysisTransform
+Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with InstrumentationAnalysisTransform
+Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with MergeInstrumentationAnalysisTransform
+Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform
+Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with ExternalDependencyInstrumentingArtifactTransform
+Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with InstrumentationAnalysisTransform
+Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform
+Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with MergeInstrumentationAnalysisTransform
+Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with InstrumentationAnalysisTransform
+Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with MergeInstrumentationAnalysisTransform
+Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with InstrumentationAnalysisTransform
+Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with MergeInstrumentationAnalysisTransform
+Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-compress-1.5.jar (org.apache.commons:commons-compress:1.5) with InstrumentationAnalysisTransform
+Transforming commons-compress-1.5.jar (org.apache.commons:commons-compress:1.5) with MergeInstrumentationAnalysisTransform
+Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with InstrumentationAnalysisTransform
+Transforming commons-compress-1.5.jar (org.apache.commons:commons-compress:1.5) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with MergeInstrumentationAnalysisTransform
+Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with InstrumentationAnalysisTransform
+Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with MergeInstrumentationAnalysisTransform
+Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with InstrumentationAnalysisTransform
+Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with ExternalDependencyInstrumentingArtifactTransform
+Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with ExternalDependencyInstrumentingArtifactTransform
+Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with MergeInstrumentationAnalysisTransform
+Transforming commons-codec-1.16.0.jar (commons-codec:commons-codec:1.16.0) with InstrumentationAnalysisTransform
+Transforming commons-codec-1.16.0.jar (commons-codec:commons-codec:1.16.0) with MergeInstrumentationAnalysisTransform
+Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with InstrumentationAnalysisTransform
+Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with MergeInstrumentationAnalysisTransform
+Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-codec-1.16.0.jar (commons-codec:commons-codec:1.16.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform
+Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with InstrumentationAnalysisTransform
+Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform
+Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with MergeInstrumentationAnalysisTransform
+Transforming xz-1.9.jar (org.tukaani:xz:1.9) with InstrumentationAnalysisTransform
+Transforming xz-1.9.jar (org.tukaani:xz:1.9) with MergeInstrumentationAnalysisTransform
+Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming xz-1.9.jar (org.tukaani:xz:1.9) with ExternalDependencyInstrumentingArtifactTransform
+Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with InstrumentationAnalysisTransform
+Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with MergeInstrumentationAnalysisTransform
+Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with InstrumentationAnalysisTransform
+Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with MergeInstrumentationAnalysisTransform
+Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform
+Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with InstrumentationAnalysisTransform
+Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with MergeInstrumentationAnalysisTransform
+Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform
+Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with InstrumentationAnalysisTransform
+Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with MergeInstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with InstrumentationAnalysisTransform
+Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with MergeInstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with MergeInstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with ExternalDependencyInstrumentingArtifactTransform
+Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with ExternalDependencyInstrumentingArtifactTransform
+Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with MergeInstrumentationAnalysisTransform
+Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with MergeInstrumentationAnalysisTransform
+Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with ExternalDependencyInstrumentingArtifactTransform
+Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with InstrumentationAnalysisTransform
+Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with MergeInstrumentationAnalysisTransform
+Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with ExternalDependencyInstrumentingArtifactTransform
+Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming build-conventions.jar (project :build-conventions) with ProjectDependencyInstrumentingArtifactTransform
+Found project 'project :build-conventions' as substitute for module 'org.elasticsearch:build-conventions'.
+Found project 'project :build-tools' as substitute for module 'org.elasticsearch.gradle:build-tools'.
+Found project 'project :build-tools:reaper' as substitute for module 'org.elasticsearch.gradle:reaper'.
+Found project 'project :build-conventions' as substitute for module 'org.elasticsearch:build-conventions'.
+Found project 'project :build-tools' as substitute for module 'org.elasticsearch.gradle:build-tools'.
+Resolve mutations for :build-tools:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started.
+Resolve mutations for :build-tools:reaper:compileJava (Thread[#1329,Execution worker Thread 8,5,main]) started.
+Resolve mutations for :build-tools-internal:extractPluginRequests (Thread[#1331,Execution worker Thread 10,5,main]) started.
+work action resolve build-conventions.jar (project :build-conventions) (Thread[#1330,Execution worker Thread 9,5,main]) started.
+:build-tools-internal:extractPluginRequests (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:build-tools:reaper:compileJava (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:build-tools:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :build-tools-internal:extractPluginRequests UP-TO-DATE
+Caching disabled for task ':build-tools-internal:extractPluginRequests' because:
+  Build cache is disabled
+Skipping task ':build-tools-internal:extractPluginRequests' as it is up-to-date.
+Resolve mutations for :build-tools-internal:generatePluginAdapters (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :build-tools:reaper:compileJava UP-TO-DATE
+Caching disabled for task ':build-tools:reaper:compileJava' because:
+  Build cache is disabled
+Skipping task ':build-tools:reaper:compileJava' as it is up-to-date.
+No compile result for :build-tools:reaper:compileJava
+:build-tools-internal:generatePluginAdapters (Thread[#1325,Execution worker Thread 4,5,main]) started.
+No compile result for :build-tools:reaper:compileJava
+Resolve mutations for :build-tools:reaper:processResources (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:build-tools:reaper:processResources (Thread[#1329,Execution worker Thread 8,5,main]) started.
+
+> Task :build-tools-internal:generatePluginAdapters UP-TO-DATE
+Caching disabled for task ':build-tools-internal:generatePluginAdapters' because:
+  Build cache is disabled
+Skipping task ':build-tools-internal:generatePluginAdapters' as it is up-to-date.
+
+> Task :build-tools:reaper:processResources NO-SOURCE
+Skipping task ':build-tools:reaper:processResources' as it has no source files and no previous output files.
+Resolve mutations for :build-tools:reaper:classes (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:build-tools:reaper:classes (Thread[#1329,Execution worker Thread 8,5,main]) started.
+
+> Task :build-tools:reaper:classes UP-TO-DATE
+Skipping task ':build-tools:reaper:classes' as it has no actions.
+Resolve mutations for :build-tools-internal:pluginDescriptors (Thread[#1325,Execution worker Thread 4,5,main]) started.
+Resolve mutations for :build-tools:reaper:jar (Thread[#1330,Execution worker Thread 9,5,main]) started.
+:build-tools-internal:pluginDescriptors (Thread[#1325,Execution worker Thread 4,5,main]) started.
+:build-tools:reaper:jar (Thread[#1330,Execution worker Thread 9,5,main]) started.
+
+> Task :build-tools-internal:pluginDescriptors UP-TO-DATE
+Caching disabled for task ':build-tools-internal:pluginDescriptors' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':build-tools-internal:pluginDescriptors' as it is up-to-date.
+Resolve mutations for :build-tools-internal:processResources (Thread[#1325,Execution worker Thread 4,5,main]) started.
+:build-tools-internal:processResources (Thread[#1325,Execution worker Thread 4,5,main]) started.
+
+> Task :build-tools:reaper:jar UP-TO-DATE
+Caching disabled for task ':build-tools:reaper:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':build-tools:reaper:jar' as it is up-to-date.
+work action resolve reaper.jar (project :build-tools:reaper) (Thread[#1330,Execution worker Thread 9,5,main]) started.
+
+> Task :build-tools-internal:processResources UP-TO-DATE
+Caching disabled for task ':build-tools-internal:processResources' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':build-tools-internal:processResources' as it is up-to-date.
+
+> Task :build-tools:compileJava UP-TO-DATE
+Caching disabled for task ':build-tools:compileJava' because:
+  Build cache is disabled
+Skipping task ':build-tools:compileJava' as it is up-to-date.
+No compile result for :build-tools:compileJava
+No compile result for :build-tools:compileJava
+Resolve mutations for :build-tools:compileGroovy (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:build-tools:compileGroovy (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :build-tools:compileGroovy NO-SOURCE
+Skipping task ':build-tools:compileGroovy' as it has no source files and no previous output files.
+Resolve mutations for :build-tools:generateVersionProperties (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:build-tools:generateVersionProperties (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :build-tools:generateVersionProperties UP-TO-DATE
+Caching disabled for task ':build-tools:generateVersionProperties' because:
+  Build cache is disabled
+Skipping task ':build-tools:generateVersionProperties' as it is up-to-date.
+Resolve mutations for :build-tools:pluginDescriptors (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:build-tools:pluginDescriptors (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :build-tools:pluginDescriptors UP-TO-DATE
+Caching disabled for task ':build-tools:pluginDescriptors' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':build-tools:pluginDescriptors' as it is up-to-date.
+Resolve mutations for :build-tools:processResources (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:build-tools:processResources (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :build-tools:processResources UP-TO-DATE
+Caching disabled for task ':build-tools:processResources' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':build-tools:processResources' as it is up-to-date.
+Resolve mutations for :build-tools:classes (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:build-tools:classes (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :build-tools:classes UP-TO-DATE
+Skipping task ':build-tools:classes' as it has no actions.
+Resolve mutations for :build-tools:jar (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:build-tools:jar (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :build-tools:jar UP-TO-DATE
+Caching disabled for task ':build-tools:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':build-tools:jar' as it is up-to-date.
+other build task :build-tools:jar (Thread[#1328,Execution worker Thread 7,5,main]) started.
+work action resolve build-tools-9.0.0-SNAPSHOT.jar (project :build-tools) (Thread[#1328,Execution worker Thread 7,5,main]) started.
+Resolve mutations for :build-tools-internal:compileJava (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:build-tools-internal:compileJava (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :build-tools-internal:compileJava UP-TO-DATE
+Caching disabled for task ':build-tools-internal:compileJava' because:
+  Build cache is disabled
+Skipping task ':build-tools-internal:compileJava' as it is up-to-date.
+No compile result for :build-tools-internal:compileJava
+No compile result for :build-tools-internal:compileJava
+Resolve mutations for :build-tools-internal:compileGroovy (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:build-tools-internal:compileGroovy (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :build-tools-internal:compileGroovy UP-TO-DATE
+Caching disabled for task ':build-tools-internal:compileGroovy' because:
+  Build cache is disabled
+Skipping task ':build-tools-internal:compileGroovy' as it is up-to-date.
+Resolve mutations for :build-tools-internal:compileGroovyPlugins (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:build-tools-internal:compileGroovyPlugins (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :build-tools-internal:compileGroovyPlugins UP-TO-DATE
+Caching disabled for task ':build-tools-internal:compileGroovyPlugins' because:
+  Build cache is disabled
+Skipping task ':build-tools-internal:compileGroovyPlugins' as it is up-to-date.
+Resolve mutations for :build-tools-internal:classes (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:build-tools-internal:classes (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :build-tools-internal:classes UP-TO-DATE
+Skipping task ':build-tools-internal:classes' as it has no actions.
+Resolve mutations for :build-tools-internal:jar (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:build-tools-internal:jar (Thread[#1330,Execution worker Thread 9,5,main]) started.
+
+> Task :build-tools-internal:jar UP-TO-DATE
+Caching disabled for task ':build-tools-internal:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':build-tools-internal:jar' as it is up-to-date.
+Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with InstrumentationAnalysisTransform
+Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with InstrumentationAnalysisTransform
+Transforming develocity-gradle-plugin-3.18.1.jar (com.gradle:develocity-gradle-plugin:3.18.1) with InstrumentationAnalysisTransform
+Transforming jackson-dataformat-yaml-2.15.0.jar (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.0) with InstrumentationAnalysisTransform
+Transforming json-schema-validator-1.0.72.jar (com.networknt:json-schema-validator:1.0.72) with InstrumentationAnalysisTransform
+Transforming jackson-databind-2.15.0.jar (com.fasterxml.jackson.core:jackson-databind:2.15.0) with InstrumentationAnalysisTransform
+Transforming jackson-core-2.15.0.jar (com.fasterxml.jackson.core:jackson-core:2.15.0) with InstrumentationAnalysisTransform
+Transforming jackson-annotations-2.15.0.jar (com.fasterxml.jackson.core:jackson-annotations:2.15.0) with InstrumentationAnalysisTransform
+Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with InstrumentationAnalysisTransform
+Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with InstrumentationAnalysisTransform
+Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with InstrumentationAnalysisTransform
+Transforming commons-compress-1.26.1.jar (org.apache.commons:commons-compress:1.26.1) with InstrumentationAnalysisTransform
+Transforming httpclient-4.5.14.jar (org.apache.httpcomponents:httpclient:4.5.14) with InstrumentationAnalysisTransform
+Transforming gradle-info-plugin-11.3.3.jar (com.netflix.nebula:gradle-info-plugin:11.3.3) with InstrumentationAnalysisTransform
+Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with InstrumentationAnalysisTransform
+Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with InstrumentationAnalysisTransform
+Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with InstrumentationAnalysisTransform
+Transforming commons-codec-1.16.1.jar (commons-codec:commons-codec:1.16.1) with InstrumentationAnalysisTransform
+Transforming svnkit-1.8.12.jar (org.tmatesoft.svnkit:svnkit:1.8.12) with InstrumentationAnalysisTransform
+Transforming jna-platform-5.7.0.jar (net.java.dev.jna:jna-platform:5.7.0) with InstrumentationAnalysisTransform
+Transforming jsch.agentproxy.connector-factory-0.0.7.jar (com.jcraft:jsch.agentproxy.connector-factory:0.0.7) with InstrumentationAnalysisTransform
+Transforming jsch.agentproxy.usocket-jna-0.0.7.jar (com.jcraft:jsch.agentproxy.usocket-jna:0.0.7) with InstrumentationAnalysisTransform
+Transforming jsch.agentproxy.pageant-0.0.7.jar (com.jcraft:jsch.agentproxy.pageant:0.0.7) with InstrumentationAnalysisTransform
+Transforming jna-5.10.0.jar (net.java.dev.jna:jna:5.10.0) with InstrumentationAnalysisTransform
+Transforming gradle-idea-ext-1.1.4.jar (gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext:1.1.4) with InstrumentationAnalysisTransform
+Transforming forbiddenapis-3.8.jar (de.thetaphi:forbiddenapis:3.8) with InstrumentationAnalysisTransform
+Transforming gradle-docker-compose-plugin-0.17.5.jar (com.avast.gradle:gradle-docker-compose-plugin:0.17.5) with InstrumentationAnalysisTransform
+Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with InstrumentationAnalysisTransform
+Transforming ST4-4.3.4.jar (org.antlr:ST4:4.3.4) with InstrumentationAnalysisTransform
+Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with InstrumentationAnalysisTransform
+Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with InstrumentationAnalysisTransform
+Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with InstrumentationAnalysisTransform
+Transforming httpcore-4.4.16.jar (org.apache.httpcomponents:httpcore:4.4.16) with InstrumentationAnalysisTransform
+Transforming snakeyaml-2.0.jar (org.yaml:snakeyaml:2.0) with InstrumentationAnalysisTransform
+Transforming javaparser-core-3.18.0.jar (com.github.javaparser:javaparser-core:3.18.0) with InstrumentationAnalysisTransform
+Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with InstrumentationAnalysisTransform
+Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with InstrumentationAnalysisTransform
+Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with InstrumentationAnalysisTransform
+Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with InstrumentationAnalysisTransform
+Transforming commons-lang3-3.14.0.jar (org.apache.commons:commons-lang3:3.14.0) with InstrumentationAnalysisTransform
+Transforming p4java-2015.2.1365273.jar (com.perforce:p4java:2015.2.1365273) with InstrumentationAnalysisTransform
+Transforming nebula-gradle-interop-2.0.0.jar (com.netflix.nebula:nebula-gradle-interop:2.0.0) with InstrumentationAnalysisTransform
+Transforming gradle-contacts-plugin-6.0.0.jar (com.netflix.nebula:gradle-contacts-plugin:6.0.0) with InstrumentationAnalysisTransform
+Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with InstrumentationAnalysisTransform
+Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with InstrumentationAnalysisTransform
+Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with InstrumentationAnalysisTransform
+Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with InstrumentationAnalysisTransform
+Transforming gson-2.8.6.jar (com.google.code.gson:gson:2.8.6) with InstrumentationAnalysisTransform
+Transforming guava-28.2-jre.jar (com.google.guava:guava:28.2-jre) with InstrumentationAnalysisTransform
+Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with InstrumentationAnalysisTransform
+Transforming itu-1.7.0.jar (com.ethlo.time:itu:1.7.0) with InstrumentationAnalysisTransform
+Transforming sqljet-1.1.10.jar (org.tmatesoft.sqljet:sqljet:1.1.10) with InstrumentationAnalysisTransform
+Transforming antlr-runtime-3.5.3.jar (org.antlr:antlr-runtime:3.5.3) with InstrumentationAnalysisTransform
+Transforming commons-logging-1.2.jar (commons-logging:commons-logging:1.2) with InstrumentationAnalysisTransform
+Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with InstrumentationAnalysisTransform
+Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with InstrumentationAnalysisTransform
+Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with InstrumentationAnalysisTransform
+Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with InstrumentationAnalysisTransform
+Transforming fastdoubleparser-0.8.0.jar (ch.randelshofer:fastdoubleparser:0.8.0) with InstrumentationAnalysisTransform
+Transforming jzlib-1.1.2.jar (com.jcraft:jzlib:1.1.2) with InstrumentationAnalysisTransform
+Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with InstrumentationAnalysisTransform
+Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with InstrumentationAnalysisTransform
+Transforming jsch.agentproxy.svnkit-trilead-ssh2-0.0.7.jar (com.jcraft:jsch.agentproxy.svnkit-trilead-ssh2:0.0.7) with InstrumentationAnalysisTransform
+Transforming trilead-ssh2-1.0.0-build220.jar (com.trilead:trilead-ssh2:1.0.0-build220) with InstrumentationAnalysisTransform
+Transforming sequence-library-1.0.3.jar (de.regnis.q.sequence:sequence-library:1.0.3) with InstrumentationAnalysisTransform
+Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with InstrumentationAnalysisTransform
+Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with InstrumentationAnalysisTransform
+Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with InstrumentationAnalysisTransform
+Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with InstrumentationAnalysisTransform
+Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with InstrumentationAnalysisTransform
+Transforming failureaccess-1.0.1.jar (com.google.guava:failureaccess:1.0.1) with InstrumentationAnalysisTransform
+Transforming listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava) with InstrumentationAnalysisTransform
+Transforming jsr305-3.0.2.jar (com.google.code.findbugs:jsr305:3.0.2) with InstrumentationAnalysisTransform
+Transforming checker-qual-2.10.0.jar (org.checkerframework:checker-qual:2.10.0) with InstrumentationAnalysisTransform
+Transforming error_prone_annotations-2.3.4.jar (com.google.errorprone:error_prone_annotations:2.3.4) with InstrumentationAnalysisTransform
+Transforming j2objc-annotations-1.3.jar (com.google.j2objc:j2objc-annotations:1.3) with InstrumentationAnalysisTransform
+Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with InstrumentationAnalysisTransform
+Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with InstrumentationAnalysisTransform
+Transforming jsch.agentproxy.usocket-nc-0.0.7.jar (com.jcraft:jsch.agentproxy.usocket-nc:0.0.7) with InstrumentationAnalysisTransform
+Transforming jsch.agentproxy.sshagent-0.0.7.jar (com.jcraft:jsch.agentproxy.sshagent:0.0.7) with InstrumentationAnalysisTransform
+Transforming jsch.agentproxy.core-0.0.7.jar (com.jcraft:jsch.agentproxy.core:0.0.7) with InstrumentationAnalysisTransform
+Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with InstrumentationAnalysisTransform
+Transforming xz-1.9.jar (org.tukaani:xz:1.9) with InstrumentationAnalysisTransform
+Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with InstrumentationAnalysisTransform
+Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with InstrumentationAnalysisTransform
+Transforming platform-3.4.0.jar (net.java.dev.jna:platform:3.4.0) with InstrumentationAnalysisTransform
+Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with InstrumentationAnalysisTransform
+Transforming main (project :build-tools-internal) with InstrumentationAnalysisTransform
+Transforming main (project :build-tools-internal) with InstrumentationAnalysisTransform
+Transforming main (project :build-conventions) with InstrumentationAnalysisTransform
+Transforming main (project :build-tools) with InstrumentationAnalysisTransform
+Transforming main (project :build-tools) with InstrumentationAnalysisTransform
+Transforming main (project :build-tools:reaper) with InstrumentationAnalysisTransform
+Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with InstrumentationAnalysisTransform
+Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with MergeInstrumentationAnalysisTransform
+Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with InstrumentationAnalysisTransform
+Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with MergeInstrumentationAnalysisTransform
+Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with ExternalDependencyInstrumentingArtifactTransform
+Transforming develocity-gradle-plugin-3.18.1.jar (com.gradle:develocity-gradle-plugin:3.18.1) with InstrumentationAnalysisTransform
+Transforming develocity-gradle-plugin-3.18.1.jar (com.gradle:develocity-gradle-plugin:3.18.1) with MergeInstrumentationAnalysisTransform
+Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jackson-dataformat-yaml-2.15.0.jar (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.0) with InstrumentationAnalysisTransform
+Transforming jackson-dataformat-yaml-2.15.0.jar (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.0) with MergeInstrumentationAnalysisTransform
+Transforming develocity-gradle-plugin-3.18.1.jar (com.gradle:develocity-gradle-plugin:3.18.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming json-schema-validator-1.0.72.jar (com.networknt:json-schema-validator:1.0.72) with InstrumentationAnalysisTransform
+Transforming json-schema-validator-1.0.72.jar (com.networknt:json-schema-validator:1.0.72) with MergeInstrumentationAnalysisTransform
+Transforming jackson-dataformat-yaml-2.15.0.jar (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jackson-databind-2.15.0.jar (com.fasterxml.jackson.core:jackson-databind:2.15.0) with InstrumentationAnalysisTransform
+Transforming jackson-databind-2.15.0.jar (com.fasterxml.jackson.core:jackson-databind:2.15.0) with MergeInstrumentationAnalysisTransform
+Transforming json-schema-validator-1.0.72.jar (com.networknt:json-schema-validator:1.0.72) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jackson-core-2.15.0.jar (com.fasterxml.jackson.core:jackson-core:2.15.0) with InstrumentationAnalysisTransform
+Transforming jackson-databind-2.15.0.jar (com.fasterxml.jackson.core:jackson-databind:2.15.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jackson-core-2.15.0.jar (com.fasterxml.jackson.core:jackson-core:2.15.0) with MergeInstrumentationAnalysisTransform
+Transforming jackson-annotations-2.15.0.jar (com.fasterxml.jackson.core:jackson-annotations:2.15.0) with InstrumentationAnalysisTransform
+Transforming jackson-annotations-2.15.0.jar (com.fasterxml.jackson.core:jackson-annotations:2.15.0) with MergeInstrumentationAnalysisTransform
+Transforming jackson-core-2.15.0.jar (com.fasterxml.jackson.core:jackson-core:2.15.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with InstrumentationAnalysisTransform
+Transforming jackson-annotations-2.15.0.jar (com.fasterxml.jackson.core:jackson-annotations:2.15.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with MergeInstrumentationAnalysisTransform
+Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with InstrumentationAnalysisTransform
+Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with MergeInstrumentationAnalysisTransform
+Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with ExternalDependencyInstrumentingArtifactTransform
+Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with InstrumentationAnalysisTransform
+Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with MergeInstrumentationAnalysisTransform
+Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-compress-1.26.1.jar (org.apache.commons:commons-compress:1.26.1) with InstrumentationAnalysisTransform
+Transforming commons-compress-1.26.1.jar (org.apache.commons:commons-compress:1.26.1) with MergeInstrumentationAnalysisTransform
+Transforming httpclient-4.5.14.jar (org.apache.httpcomponents:httpclient:4.5.14) with InstrumentationAnalysisTransform
+Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with ExternalDependencyInstrumentingArtifactTransform
+Transforming httpclient-4.5.14.jar (org.apache.httpcomponents:httpclient:4.5.14) with MergeInstrumentationAnalysisTransform
+Transforming gradle-info-plugin-11.3.3.jar (com.netflix.nebula:gradle-info-plugin:11.3.3) with InstrumentationAnalysisTransform
+Transforming commons-compress-1.26.1.jar (org.apache.commons:commons-compress:1.26.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming httpclient-4.5.14.jar (org.apache.httpcomponents:httpclient:4.5.14) with ExternalDependencyInstrumentingArtifactTransform
+Transforming gradle-info-plugin-11.3.3.jar (com.netflix.nebula:gradle-info-plugin:11.3.3) with MergeInstrumentationAnalysisTransform
+Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with InstrumentationAnalysisTransform
+Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with MergeInstrumentationAnalysisTransform
+Transforming gradle-info-plugin-11.3.3.jar (com.netflix.nebula:gradle-info-plugin:11.3.3) with ExternalDependencyInstrumentingArtifactTransform
+Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with InstrumentationAnalysisTransform
+Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with MergeInstrumentationAnalysisTransform
+Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with InstrumentationAnalysisTransform
+Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with MergeInstrumentationAnalysisTransform
+Transforming commons-codec-1.16.1.jar (commons-codec:commons-codec:1.16.1) with InstrumentationAnalysisTransform
+Transforming commons-codec-1.16.1.jar (commons-codec:commons-codec:1.16.1) with MergeInstrumentationAnalysisTransform
+Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with ExternalDependencyInstrumentingArtifactTransform
+Transforming svnkit-1.8.12.jar (org.tmatesoft.svnkit:svnkit:1.8.12) with InstrumentationAnalysisTransform
+Transforming commons-codec-1.16.1.jar (commons-codec:commons-codec:1.16.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming svnkit-1.8.12.jar (org.tmatesoft.svnkit:svnkit:1.8.12) with MergeInstrumentationAnalysisTransform
+Transforming jna-platform-5.7.0.jar (net.java.dev.jna:jna-platform:5.7.0) with InstrumentationAnalysisTransform
+Transforming svnkit-1.8.12.jar (org.tmatesoft.svnkit:svnkit:1.8.12) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jna-platform-5.7.0.jar (net.java.dev.jna:jna-platform:5.7.0) with MergeInstrumentationAnalysisTransform
+Transforming jsch.agentproxy.connector-factory-0.0.7.jar (com.jcraft:jsch.agentproxy.connector-factory:0.0.7) with InstrumentationAnalysisTransform
+Transforming jna-platform-5.7.0.jar (net.java.dev.jna:jna-platform:5.7.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jsch.agentproxy.connector-factory-0.0.7.jar (com.jcraft:jsch.agentproxy.connector-factory:0.0.7) with MergeInstrumentationAnalysisTransform
+Transforming jsch.agentproxy.usocket-jna-0.0.7.jar (com.jcraft:jsch.agentproxy.usocket-jna:0.0.7) with InstrumentationAnalysisTransform
+Transforming jsch.agentproxy.connector-factory-0.0.7.jar (com.jcraft:jsch.agentproxy.connector-factory:0.0.7) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jsch.agentproxy.usocket-jna-0.0.7.jar (com.jcraft:jsch.agentproxy.usocket-jna:0.0.7) with MergeInstrumentationAnalysisTransform
+Transforming jsch.agentproxy.pageant-0.0.7.jar (com.jcraft:jsch.agentproxy.pageant:0.0.7) with InstrumentationAnalysisTransform
+Transforming jsch.agentproxy.usocket-jna-0.0.7.jar (com.jcraft:jsch.agentproxy.usocket-jna:0.0.7) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jsch.agentproxy.pageant-0.0.7.jar (com.jcraft:jsch.agentproxy.pageant:0.0.7) with MergeInstrumentationAnalysisTransform
+Transforming jna-5.10.0.jar (net.java.dev.jna:jna:5.10.0) with InstrumentationAnalysisTransform
+Transforming jsch.agentproxy.pageant-0.0.7.jar (com.jcraft:jsch.agentproxy.pageant:0.0.7) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jna-5.10.0.jar (net.java.dev.jna:jna:5.10.0) with MergeInstrumentationAnalysisTransform
+Transforming gradle-idea-ext-1.1.4.jar (gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext:1.1.4) with InstrumentationAnalysisTransform
+Transforming jna-5.10.0.jar (net.java.dev.jna:jna:5.10.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming gradle-idea-ext-1.1.4.jar (gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext:1.1.4) with MergeInstrumentationAnalysisTransform
+Transforming forbiddenapis-3.8.jar (de.thetaphi:forbiddenapis:3.8) with InstrumentationAnalysisTransform
+Transforming gradle-idea-ext-1.1.4.jar (gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext:1.1.4) with ExternalDependencyInstrumentingArtifactTransform
+Transforming forbiddenapis-3.8.jar (de.thetaphi:forbiddenapis:3.8) with MergeInstrumentationAnalysisTransform
+Transforming gradle-docker-compose-plugin-0.17.5.jar (com.avast.gradle:gradle-docker-compose-plugin:0.17.5) with InstrumentationAnalysisTransform
+Transforming gradle-docker-compose-plugin-0.17.5.jar (com.avast.gradle:gradle-docker-compose-plugin:0.17.5) with MergeInstrumentationAnalysisTransform
+Transforming forbiddenapis-3.8.jar (de.thetaphi:forbiddenapis:3.8) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with InstrumentationAnalysisTransform
+Transforming gradle-docker-compose-plugin-0.17.5.jar (com.avast.gradle:gradle-docker-compose-plugin:0.17.5) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with MergeInstrumentationAnalysisTransform
+Transforming ST4-4.3.4.jar (org.antlr:ST4:4.3.4) with InstrumentationAnalysisTransform
+Transforming ST4-4.3.4.jar (org.antlr:ST4:4.3.4) with MergeInstrumentationAnalysisTransform
+Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with ExternalDependencyInstrumentingArtifactTransform
+Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with InstrumentationAnalysisTransform
+Transforming ST4-4.3.4.jar (org.antlr:ST4:4.3.4) with ExternalDependencyInstrumentingArtifactTransform
+Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with MergeInstrumentationAnalysisTransform
+Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with InstrumentationAnalysisTransform
+Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with MergeInstrumentationAnalysisTransform
+Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with InstrumentationAnalysisTransform
+Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with MergeInstrumentationAnalysisTransform
+Transforming httpcore-4.4.16.jar (org.apache.httpcomponents:httpcore:4.4.16) with InstrumentationAnalysisTransform
+Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming httpcore-4.4.16.jar (org.apache.httpcomponents:httpcore:4.4.16) with MergeInstrumentationAnalysisTransform
+Transforming snakeyaml-2.0.jar (org.yaml:snakeyaml:2.0) with InstrumentationAnalysisTransform
+Transforming snakeyaml-2.0.jar (org.yaml:snakeyaml:2.0) with MergeInstrumentationAnalysisTransform
+Transforming javaparser-core-3.18.0.jar (com.github.javaparser:javaparser-core:3.18.0) with InstrumentationAnalysisTransform
+Transforming httpcore-4.4.16.jar (org.apache.httpcomponents:httpcore:4.4.16) with ExternalDependencyInstrumentingArtifactTransform
+Transforming javaparser-core-3.18.0.jar (com.github.javaparser:javaparser-core:3.18.0) with MergeInstrumentationAnalysisTransform
+Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with InstrumentationAnalysisTransform
+Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with MergeInstrumentationAnalysisTransform
+Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with InstrumentationAnalysisTransform
+Transforming snakeyaml-2.0.jar (org.yaml:snakeyaml:2.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with ExternalDependencyInstrumentingArtifactTransform
+Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with MergeInstrumentationAnalysisTransform
+Transforming javaparser-core-3.18.0.jar (com.github.javaparser:javaparser-core:3.18.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with InstrumentationAnalysisTransform
+Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with ExternalDependencyInstrumentingArtifactTransform
+Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with MergeInstrumentationAnalysisTransform
+Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with InstrumentationAnalysisTransform
+Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with MergeInstrumentationAnalysisTransform
+Transforming commons-lang3-3.14.0.jar (org.apache.commons:commons-lang3:3.14.0) with InstrumentationAnalysisTransform
+Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-lang3-3.14.0.jar (org.apache.commons:commons-lang3:3.14.0) with MergeInstrumentationAnalysisTransform
+Transforming p4java-2015.2.1365273.jar (com.perforce:p4java:2015.2.1365273) with InstrumentationAnalysisTransform
+Transforming p4java-2015.2.1365273.jar (com.perforce:p4java:2015.2.1365273) with MergeInstrumentationAnalysisTransform
+Transforming nebula-gradle-interop-2.0.0.jar (com.netflix.nebula:nebula-gradle-interop:2.0.0) with InstrumentationAnalysisTransform
+Transforming commons-lang3-3.14.0.jar (org.apache.commons:commons-lang3:3.14.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming p4java-2015.2.1365273.jar (com.perforce:p4java:2015.2.1365273) with ExternalDependencyInstrumentingArtifactTransform
+Transforming nebula-gradle-interop-2.0.0.jar (com.netflix.nebula:nebula-gradle-interop:2.0.0) with MergeInstrumentationAnalysisTransform
+Transforming gradle-contacts-plugin-6.0.0.jar (com.netflix.nebula:gradle-contacts-plugin:6.0.0) with InstrumentationAnalysisTransform
+Transforming gradle-contacts-plugin-6.0.0.jar (com.netflix.nebula:gradle-contacts-plugin:6.0.0) with MergeInstrumentationAnalysisTransform
+Transforming nebula-gradle-interop-2.0.0.jar (com.netflix.nebula:nebula-gradle-interop:2.0.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with InstrumentationAnalysisTransform
+Transforming gradle-contacts-plugin-6.0.0.jar (com.netflix.nebula:gradle-contacts-plugin:6.0.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with MergeInstrumentationAnalysisTransform
+Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with InstrumentationAnalysisTransform
+Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with MergeInstrumentationAnalysisTransform
+Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with InstrumentationAnalysisTransform
+Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with MergeInstrumentationAnalysisTransform
+Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with InstrumentationAnalysisTransform
+Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with MergeInstrumentationAnalysisTransform
+Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with ExternalDependencyInstrumentingArtifactTransform
+Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with ExternalDependencyInstrumentingArtifactTransform
+Transforming gson-2.8.6.jar (com.google.code.gson:gson:2.8.6) with InstrumentationAnalysisTransform
+Transforming gson-2.8.6.jar (com.google.code.gson:gson:2.8.6) with MergeInstrumentationAnalysisTransform
+Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with ExternalDependencyInstrumentingArtifactTransform
+Transforming guava-28.2-jre.jar (com.google.guava:guava:28.2-jre) with InstrumentationAnalysisTransform
+Transforming gson-2.8.6.jar (com.google.code.gson:gson:2.8.6) with ExternalDependencyInstrumentingArtifactTransform
+Transforming guava-28.2-jre.jar (com.google.guava:guava:28.2-jre) with MergeInstrumentationAnalysisTransform
+Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with InstrumentationAnalysisTransform
+Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with MergeInstrumentationAnalysisTransform
+Transforming itu-1.7.0.jar (com.ethlo.time:itu:1.7.0) with InstrumentationAnalysisTransform
+Transforming guava-28.2-jre.jar (com.google.guava:guava:28.2-jre) with ExternalDependencyInstrumentingArtifactTransform
+Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with ExternalDependencyInstrumentingArtifactTransform
+Transforming itu-1.7.0.jar (com.ethlo.time:itu:1.7.0) with MergeInstrumentationAnalysisTransform
+Transforming sqljet-1.1.10.jar (org.tmatesoft.sqljet:sqljet:1.1.10) with InstrumentationAnalysisTransform
+Transforming sqljet-1.1.10.jar (org.tmatesoft.sqljet:sqljet:1.1.10) with MergeInstrumentationAnalysisTransform
+Transforming itu-1.7.0.jar (com.ethlo.time:itu:1.7.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming antlr-runtime-3.5.3.jar (org.antlr:antlr-runtime:3.5.3) with InstrumentationAnalysisTransform
+Transforming sqljet-1.1.10.jar (org.tmatesoft.sqljet:sqljet:1.1.10) with ExternalDependencyInstrumentingArtifactTransform
+Transforming antlr-runtime-3.5.3.jar (org.antlr:antlr-runtime:3.5.3) with MergeInstrumentationAnalysisTransform
+Transforming commons-logging-1.2.jar (commons-logging:commons-logging:1.2) with InstrumentationAnalysisTransform
+Transforming commons-logging-1.2.jar (commons-logging:commons-logging:1.2) with MergeInstrumentationAnalysisTransform
+Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with InstrumentationAnalysisTransform
+Transforming antlr-runtime-3.5.3.jar (org.antlr:antlr-runtime:3.5.3) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-logging-1.2.jar (commons-logging:commons-logging:1.2) with ExternalDependencyInstrumentingArtifactTransform
+Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with MergeInstrumentationAnalysisTransform
+Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with InstrumentationAnalysisTransform
+Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with MergeInstrumentationAnalysisTransform
+Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with InstrumentationAnalysisTransform
+Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with MergeInstrumentationAnalysisTransform
+Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with InstrumentationAnalysisTransform
+Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with MergeInstrumentationAnalysisTransform
+Transforming fastdoubleparser-0.8.0.jar (ch.randelshofer:fastdoubleparser:0.8.0) with InstrumentationAnalysisTransform
+Transforming fastdoubleparser-0.8.0.jar (ch.randelshofer:fastdoubleparser:0.8.0) with MergeInstrumentationAnalysisTransform
+Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jzlib-1.1.2.jar (com.jcraft:jzlib:1.1.2) with InstrumentationAnalysisTransform
+Transforming jzlib-1.1.2.jar (com.jcraft:jzlib:1.1.2) with MergeInstrumentationAnalysisTransform
+Transforming fastdoubleparser-0.8.0.jar (ch.randelshofer:fastdoubleparser:0.8.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with InstrumentationAnalysisTransform
+Transforming jzlib-1.1.2.jar (com.jcraft:jzlib:1.1.2) with ExternalDependencyInstrumentingArtifactTransform
+Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with MergeInstrumentationAnalysisTransform
+Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with InstrumentationAnalysisTransform
+Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with MergeInstrumentationAnalysisTransform
+Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with InstrumentationAnalysisTransform
+Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with MergeInstrumentationAnalysisTransform
+Transforming jsch.agentproxy.svnkit-trilead-ssh2-0.0.7.jar (com.jcraft:jsch.agentproxy.svnkit-trilead-ssh2:0.0.7) with InstrumentationAnalysisTransform
+Transforming jsch.agentproxy.svnkit-trilead-ssh2-0.0.7.jar (com.jcraft:jsch.agentproxy.svnkit-trilead-ssh2:0.0.7) with MergeInstrumentationAnalysisTransform
+Transforming trilead-ssh2-1.0.0-build220.jar (com.trilead:trilead-ssh2:1.0.0-build220) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jsch.agentproxy.svnkit-trilead-ssh2-0.0.7.jar (com.jcraft:jsch.agentproxy.svnkit-trilead-ssh2:0.0.7) with ExternalDependencyInstrumentingArtifactTransform
+Transforming trilead-ssh2-1.0.0-build220.jar (com.trilead:trilead-ssh2:1.0.0-build220) with MergeInstrumentationAnalysisTransform
+Transforming sequence-library-1.0.3.jar (de.regnis.q.sequence:sequence-library:1.0.3) with InstrumentationAnalysisTransform
+Transforming sequence-library-1.0.3.jar (de.regnis.q.sequence:sequence-library:1.0.3) with MergeInstrumentationAnalysisTransform
+Transforming trilead-ssh2-1.0.0-build220.jar (com.trilead:trilead-ssh2:1.0.0-build220) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with InstrumentationAnalysisTransform
+Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with MergeInstrumentationAnalysisTransform
+Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with InstrumentationAnalysisTransform
+Transforming sequence-library-1.0.3.jar (de.regnis.q.sequence:sequence-library:1.0.3) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with MergeInstrumentationAnalysisTransform
+Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with InstrumentationAnalysisTransform
+Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with MergeInstrumentationAnalysisTransform
+Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with InstrumentationAnalysisTransform
+Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with ExternalDependencyInstrumentingArtifactTransform
+Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with MergeInstrumentationAnalysisTransform
+Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform
+Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with InstrumentationAnalysisTransform
+Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with MergeInstrumentationAnalysisTransform
+Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform
+Transforming failureaccess-1.0.1.jar (com.google.guava:failureaccess:1.0.1) with InstrumentationAnalysisTransform
+Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming failureaccess-1.0.1.jar (com.google.guava:failureaccess:1.0.1) with MergeInstrumentationAnalysisTransform
+Transforming listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava) with InstrumentationAnalysisTransform
+Transforming failureaccess-1.0.1.jar (com.google.guava:failureaccess:1.0.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava) with MergeInstrumentationAnalysisTransform
+Transforming jsr305-3.0.2.jar (com.google.code.findbugs:jsr305:3.0.2) with InstrumentationAnalysisTransform
+Transforming listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jsr305-3.0.2.jar (com.google.code.findbugs:jsr305:3.0.2) with MergeInstrumentationAnalysisTransform
+Transforming checker-qual-2.10.0.jar (org.checkerframework:checker-qual:2.10.0) with InstrumentationAnalysisTransform
+Transforming checker-qual-2.10.0.jar (org.checkerframework:checker-qual:2.10.0) with MergeInstrumentationAnalysisTransform
+Transforming error_prone_annotations-2.3.4.jar (com.google.errorprone:error_prone_annotations:2.3.4) with InstrumentationAnalysisTransform
+Transforming jsr305-3.0.2.jar (com.google.code.findbugs:jsr305:3.0.2) with ExternalDependencyInstrumentingArtifactTransform
+Transforming error_prone_annotations-2.3.4.jar (com.google.errorprone:error_prone_annotations:2.3.4) with MergeInstrumentationAnalysisTransform
+Transforming j2objc-annotations-1.3.jar (com.google.j2objc:j2objc-annotations:1.3) with InstrumentationAnalysisTransform
+Transforming checker-qual-2.10.0.jar (org.checkerframework:checker-qual:2.10.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming error_prone_annotations-2.3.4.jar (com.google.errorprone:error_prone_annotations:2.3.4) with ExternalDependencyInstrumentingArtifactTransform
+Transforming j2objc-annotations-1.3.jar (com.google.j2objc:j2objc-annotations:1.3) with MergeInstrumentationAnalysisTransform
+Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with InstrumentationAnalysisTransform
+Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with MergeInstrumentationAnalysisTransform
+Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with InstrumentationAnalysisTransform
+Transforming j2objc-annotations-1.3.jar (com.google.j2objc:j2objc-annotations:1.3) with ExternalDependencyInstrumentingArtifactTransform
+Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with MergeInstrumentationAnalysisTransform
+Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with ExternalDependencyInstrumentingArtifactTransform
+Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with InstrumentationAnalysisTransform
+Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with MergeInstrumentationAnalysisTransform
+Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with MergeInstrumentationAnalysisTransform
+Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with ExternalDependencyInstrumentingArtifactTransform
+Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jsch.agentproxy.usocket-nc-0.0.7.jar (com.jcraft:jsch.agentproxy.usocket-nc:0.0.7) with InstrumentationAnalysisTransform
+Transforming jsch.agentproxy.usocket-nc-0.0.7.jar (com.jcraft:jsch.agentproxy.usocket-nc:0.0.7) with MergeInstrumentationAnalysisTransform
+Transforming jsch.agentproxy.sshagent-0.0.7.jar (com.jcraft:jsch.agentproxy.sshagent:0.0.7) with InstrumentationAnalysisTransform
+Transforming jsch.agentproxy.sshagent-0.0.7.jar (com.jcraft:jsch.agentproxy.sshagent:0.0.7) with MergeInstrumentationAnalysisTransform
+Transforming jsch.agentproxy.core-0.0.7.jar (com.jcraft:jsch.agentproxy.core:0.0.7) with InstrumentationAnalysisTransform
+Transforming jsch.agentproxy.usocket-nc-0.0.7.jar (com.jcraft:jsch.agentproxy.usocket-nc:0.0.7) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jsch.agentproxy.core-0.0.7.jar (com.jcraft:jsch.agentproxy.core:0.0.7) with MergeInstrumentationAnalysisTransform
+Transforming jsch.agentproxy.sshagent-0.0.7.jar (com.jcraft:jsch.agentproxy.sshagent:0.0.7) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jsch.agentproxy.core-0.0.7.jar (com.jcraft:jsch.agentproxy.core:0.0.7) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform
+Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with InstrumentationAnalysisTransform
+Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with MergeInstrumentationAnalysisTransform
+Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform
+Transforming xz-1.9.jar (org.tukaani:xz:1.9) with InstrumentationAnalysisTransform
+Transforming xz-1.9.jar (org.tukaani:xz:1.9) with MergeInstrumentationAnalysisTransform
+Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming xz-1.9.jar (org.tukaani:xz:1.9) with ExternalDependencyInstrumentingArtifactTransform
+Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with InstrumentationAnalysisTransform
+Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with MergeInstrumentationAnalysisTransform
+Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with MergeInstrumentationAnalysisTransform
+Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with ExternalDependencyInstrumentingArtifactTransform
+Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with InstrumentationAnalysisTransform
+Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with MergeInstrumentationAnalysisTransform
+Transforming platform-3.4.0.jar (net.java.dev.jna:platform:3.4.0) with InstrumentationAnalysisTransform
+Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with ExternalDependencyInstrumentingArtifactTransform
+Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming platform-3.4.0.jar (net.java.dev.jna:platform:3.4.0) with MergeInstrumentationAnalysisTransform
+Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with InstrumentationAnalysisTransform
+Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform
+Transforming platform-3.4.0.jar (net.java.dev.jna:platform:3.4.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with InstrumentationAnalysisTransform
+Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with MergeInstrumentationAnalysisTransform
+Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform
+Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming build-tools-internal-9.0.0-SNAPSHOT.jar (project :build-tools-internal) with ProjectDependencyInstrumentingArtifactTransform
+Transforming build-conventions.jar (project :build-conventions) with ProjectDependencyInstrumentingArtifactTransform
+Transforming build-tools-9.0.0-SNAPSHOT.jar (project :build-tools) with ProjectDependencyInstrumentingArtifactTransform
+Transforming reaper.jar (project :build-tools:reaper) with ProjectDependencyInstrumentingArtifactTransform
+Settings evaluated using settings file '/Users/rene/dev/elastic/elasticsearch/settings.gradle'.
+Projects loaded. Root project using build file '/Users/rene/dev/elastic/elasticsearch/build.gradle'.
+Included projects: [root project 'elasticsearch', project ':benchmarks', project ':client', project ':distribution', project ':docs', project ':libs', project ':modules', project ':plugins', project ':qa', project ':rest-api-spec', project ':server', project ':test', project ':x-pack', project ':client:benchmark', project ':client:client-benchmark-noop-api-plugin', project ':client:rest', project ':client:sniffer', project ':client:test', project ':distribution:archives', project ':distribution:bwc', project ':distribution:docker', project ':distribution:packages', project ':distribution:tools', project ':libs:cli', project ':libs:core', project ':libs:dissect', project ':libs:entitlement', project ':libs:geo', project ':libs:grok', project ':libs:h3', project ':libs:log4j', project ':libs:logging', project ':libs:logstash-bridge', project ':libs:lz4', project ':libs:native', project ':libs:plugin-analysis-api', project ':libs:plugin-api', project ':libs:plugin-scanner', project ':libs:secure-sm', project ':libs:simdvec', project ':libs:ssl-config', project ':libs:tdigest', project ':libs:x-content', project ':modules:aggregations', project ':modules:analysis-common', project ':modules:apm', project ':modules:data-streams', project ':modules:dot-prefix-validation', project ':modules:health-shards-availability', project ':modules:ingest-attachment', project ':modules:ingest-common', project ':modules:ingest-geoip', project ':modules:ingest-user-agent', project ':modules:kibana', project ':modules:lang-expression', project ':modules:lang-mustache', project ':modules:lang-painless', project ':modules:legacy-geo', project ':modules:mapper-extras', project ':modules:parent-join', project ':modules:percolator', project ':modules:rank-eval', project ':modules:reindex', project ':modules:repository-azure', project ':modules:repository-gcs', project ':modules:repository-s3', project ':modules:repository-url', project ':modules:rest-root', project ':modules:runtime-fields-common', project ':modules:systemd', project ':modules:transport-netty4', project ':plugins:analysis-icu', project ':plugins:analysis-kuromoji', project ':plugins:analysis-nori', project ':plugins:analysis-phonetic', project ':plugins:analysis-smartcn', project ':plugins:analysis-stempel', project ':plugins:analysis-ukrainian', project ':plugins:discovery-azure-classic', project ':plugins:discovery-ec2', project ':plugins:discovery-gce', project ':plugins:mapper-annotated-text', project ':plugins:mapper-murmur3', project ':plugins:mapper-size', project ':plugins:repository-hdfs', project ':plugins:store-smb', project ':qa:ccs-common-rest', project ':qa:ccs-rolling-upgrade-remote-cluster', project ':qa:ccs-unavailable-clusters', project ':qa:custom-rest-controller', project ':qa:evil-tests', project ':qa:full-cluster-restart', project ':qa:logging-config', project ':qa:logging-spi', project ':qa:lucene-index-compatibility', project ':qa:mixed-cluster', project ':qa:multi-cluster-search', project ':qa:no-bootstrap-tests', project ':qa:packaging', project ':qa:remote-clusters', project ':qa:repository-multi-version', project ':qa:restricted-loggers', project ':qa:rolling-upgrade', project ':qa:rolling-upgrade-legacy', project ':qa:smoke-test-http', project ':qa:smoke-test-ingest-disabled', project ':qa:smoke-test-ingest-with-all-dependencies', project ':qa:smoke-test-multinode', project ':qa:smoke-test-plugins', project ':qa:stable-api', project ':qa:system-indices', project ':qa:unconfigured-node-name', project ':qa:verify-version-constants', project ':test:external-modules', project ':test:fixtures', project ':test:framework', project ':test:immutable-collections-patch', project ':test:logger-usage', project ':test:metadata-extractor', project ':test:test-clusters', project ':test:x-content', project ':test:yaml-rest-runner', project ':x-pack:libs', project ':x-pack:license-tools', project ':x-pack:plugin', project ':x-pack:qa', project ':x-pack:rest-resources-zip', project ':x-pack:test', project ':distribution:archives:darwin-aarch64-tar', project ':distribution:archives:darwin-tar', project ':distribution:archives:integ-test-zip', project ':distribution:archives:linux-aarch64-tar', project ':distribution:archives:linux-tar', project ':distribution:archives:windows-zip', project ':distribution:bwc:bugfix', project ':distribution:bwc:bugfix2', project ':distribution:bwc:maintenance', project ':distribution:bwc:minor', project ':distribution:bwc:staged', project ':distribution:docker:cloud-ess-docker-aarch64-export', project ':distribution:docker:cloud-ess-docker-export', project ':distribution:docker:docker-aarch64-export', project ':distribution:docker:docker-export', project ':distribution:docker:ironbank-docker-aarch64-export', project ':distribution:docker:ironbank-docker-export', project ':distribution:docker:wolfi-docker-aarch64-export', project ':distribution:docker:wolfi-docker-export', project ':distribution:packages:aarch64-deb', project ':distribution:packages:aarch64-rpm', project ':distribution:packages:deb', project ':distribution:packages:rpm', project ':distribution:tools:ansi-console', project ':distribution:tools:cli-launcher', project ':distribution:tools:geoip-cli', project ':distribution:tools:java-version-checker', project ':distribution:tools:keystore-cli', project ':distribution:tools:plugin-cli', project ':distribution:tools:server-cli', project ':distribution:tools:windows-service-cli', project ':libs:entitlement:agent', project ':libs:entitlement:asm-provider', project ':libs:entitlement:bridge', project ':libs:entitlement:qa', project ':libs:entitlement:tools', project ':libs:native:native-libraries', project ':libs:x-content:impl', project ':modules:ingest-geoip:qa', project ':modules:lang-painless:spi', project ':plugins:discovery-ec2:qa', project ':plugins:discovery-gce:qa', project ':plugins:repository-hdfs:hadoop-client-api', project ':qa:stable-api:logging', project ':qa:stable-api:plugin-analysis-api', project ':qa:stable-api:plugin-api', project ':test:external-modules:test-apm-integration', project ':test:external-modules:test-delayed-aggs', project ':test:external-modules:test-die-with-dignity', project ':test:external-modules:test-error-query', project ':test:external-modules:test-esql-heap-attack', project ':test:external-modules:test-jvm-crash', project ':test:external-modules:test-latency-simulating-directory', project ':test:fixtures:aws-sts-fixture', project ':test:fixtures:azure-fixture', project ':test:fixtures:ec2-imds-fixture', project ':test:fixtures:gcs-fixture', project ':test:fixtures:geoip-fixture', project ':test:fixtures:hdfs-fixture', project ':test:fixtures:krb5kdc-fixture', project ':test:fixtures:minio-fixture', project ':test:fixtures:old-elasticsearch', project ':test:fixtures:s3-fixture', project ':test:fixtures:testcontainer-utils', project ':test:fixtures:url-fixture', project ':x-pack:libs:es-opensaml-security-api', project ':x-pack:plugin:analytics', project ':x-pack:plugin:apm-data', project ':x-pack:plugin:async', project ':x-pack:plugin:async-search', project ':x-pack:plugin:autoscaling', project ':x-pack:plugin:blob-cache', project ':x-pack:plugin:ccr', project ':x-pack:plugin:core', project ':x-pack:plugin:deprecation', project ':x-pack:plugin:downsample', project ':x-pack:plugin:enrich', project ':x-pack:plugin:ent-search', project ':x-pack:plugin:eql', project ':x-pack:plugin:esql', project ':x-pack:plugin:esql-core', project ':x-pack:plugin:fleet', project ':x-pack:plugin:frozen-indices', project ':x-pack:plugin:geoip-enterprise-downloader', project ':x-pack:plugin:graph', project ':x-pack:plugin:identity-provider', project ':x-pack:plugin:ilm', project ':x-pack:plugin:inference', project ':x-pack:plugin:kql', project ':x-pack:plugin:logsdb', project ':x-pack:plugin:logstash', project ':x-pack:plugin:mapper-aggregate-metric', project ':x-pack:plugin:mapper-constant-keyword', project ':x-pack:plugin:mapper-counted-keyword', project ':x-pack:plugin:mapper-unsigned-long', project ':x-pack:plugin:mapper-version', project ':x-pack:plugin:migrate', project ':x-pack:plugin:ml', project ':x-pack:plugin:ml-package-loader', project ':x-pack:plugin:monitoring', project ':x-pack:plugin:old-lucene-versions', project ':x-pack:plugin:otel-data', project ':x-pack:plugin:profiling', project ':x-pack:plugin:ql', project ':x-pack:plugin:rank-rrf', project ':x-pack:plugin:redact', project ':x-pack:plugin:repositories-metering-api', project ':x-pack:plugin:rollup', project ':x-pack:plugin:search-business-rules', project ':x-pack:plugin:searchable-snapshots', project ':x-pack:plugin:security', project ':x-pack:plugin:shutdown', project ':x-pack:plugin:slm', project ':x-pack:plugin:snapshot-based-recoveries', project ':x-pack:plugin:snapshot-repo-test-kit', project ':x-pack:plugin:spatial', project ':x-pack:plugin:sql', project ':x-pack:plugin:stack', project ':x-pack:plugin:text-structure', project ':x-pack:plugin:transform', project ':x-pack:plugin:vector-tile', project ':x-pack:plugin:voting-only-node', project ':x-pack:plugin:watcher', project ':x-pack:plugin:wildcard', project ':x-pack:plugin:write-load-forecaster', project ':x-pack:qa:core-rest-tests-with-security', project ':x-pack:qa:evil-tests', project ':x-pack:qa:freeze-plugin', project ':x-pack:qa:full-cluster-restart', project ':x-pack:qa:kerberos-tests', project ':x-pack:qa:mixed-tier-cluster', project ':x-pack:qa:multi-cluster-search-security', project ':x-pack:qa:multi-node', project ':x-pack:qa:oidc-op-tests', project ':x-pack:qa:openldap-tests', project ':x-pack:qa:password-protected-keystore', project ':x-pack:qa:reindex-tests-with-security', project ':x-pack:qa:repository-old-versions', project ':x-pack:qa:rolling-upgrade', project ':x-pack:qa:rolling-upgrade-basic', project ':x-pack:qa:rolling-upgrade-multi-cluster', project ':x-pack:qa:runtime-fields', project ':x-pack:qa:saml-idp-tests', project ':x-pack:qa:security-example-spi-extension', project ':x-pack:qa:security-setup-password-tests', project ':x-pack:qa:security-tools-tests', project ':x-pack:qa:smoke-test-plugins', project ':x-pack:qa:smoke-test-plugins-ssl', project ':x-pack:qa:smoke-test-security-with-mustache', project ':x-pack:qa:third-party', project ':x-pack:test:idp-fixture', project ':x-pack:test:smb-fixture', project ':libs:entitlement:qa:common', project ':libs:entitlement:qa:entitlement-allowed', project ':libs:entitlement:qa:entitlement-allowed-nonmodular', project ':libs:entitlement:qa:entitlement-denied', project ':libs:entitlement:qa:entitlement-denied-nonmodular', project ':libs:entitlement:tools:common', project ':libs:entitlement:tools:public-callers-finder', project ':libs:entitlement:tools:securitymanager-scanner', project ':modules:ingest-geoip:qa:file-based-update', project ':modules:ingest-geoip:qa:full-cluster-restart', project ':plugins:discovery-ec2:qa:amazon-ec2', project ':plugins:discovery-gce:qa:gce', project ':x-pack:plugin:async-search:qa', project ':x-pack:plugin:autoscaling:qa', project ':x-pack:plugin:ccr:qa', project ':x-pack:plugin:core:template-resources', project ':x-pack:plugin:deprecation:qa', project ':x-pack:plugin:downsample:qa', project ':x-pack:plugin:enrich:qa', project ':x-pack:plugin:ent-search:qa', project ':x-pack:plugin:eql:qa', project ':x-pack:plugin:esql:arrow', project ':x-pack:plugin:esql:compute', project ':x-pack:plugin:esql:qa', project ':x-pack:plugin:esql-core:test-fixtures', project ':x-pack:plugin:fleet:qa', project ':x-pack:plugin:graph:qa', project ':x-pack:plugin:identity-provider:qa', project ':x-pack:plugin:ilm:qa', project ':x-pack:plugin:inference:qa', project ':x-pack:plugin:logsdb:qa', project ':x-pack:plugin:ml:qa', project ':x-pack:plugin:ql:test-fixtures', project ':x-pack:plugin:repositories-metering-api:qa', project ':x-pack:plugin:searchable-snapshots:qa', project ':x-pack:plugin:security:cli', project ':x-pack:plugin:security:lib', project ':x-pack:plugin:security:qa', project ':x-pack:plugin:shutdown:qa', project ':x-pack:plugin:slm:qa', project ':x-pack:plugin:snapshot-based-recoveries:qa', project ':x-pack:plugin:snapshot-repo-test-kit:qa', project ':x-pack:plugin:sql:jdbc', project ':x-pack:plugin:sql:qa', project ':x-pack:plugin:sql:sql-action', project ':x-pack:plugin:sql:sql-cli', project ':x-pack:plugin:sql:sql-client', project ':x-pack:plugin:sql:sql-proto', project ':x-pack:plugin:stack:qa', project ':x-pack:plugin:text-structure:qa', project ':x-pack:plugin:transform:qa', project ':x-pack:plugin:vector-tile:qa', project ':x-pack:plugin:watcher:qa', project ':x-pack:qa:multi-cluster-search-security:legacy-with-basic-license', project ':x-pack:qa:multi-cluster-search-security:legacy-with-full-license', project ':x-pack:qa:multi-cluster-search-security:legacy-with-restricted-trust', project ':x-pack:qa:runtime-fields:core-with-mapped', project ':x-pack:qa:runtime-fields:core-with-search', project ':x-pack:qa:runtime-fields:with-security', project ':x-pack:qa:third-party:active-directory', project ':x-pack:qa:third-party:jira', project ':x-pack:qa:third-party:pagerduty', project ':x-pack:qa:third-party:slack', project ':x-pack:plugin:async-search:qa:rest', project ':x-pack:plugin:async-search:qa:security', project ':x-pack:plugin:autoscaling:qa:rest', project ':x-pack:plugin:ccr:qa:downgrade-to-basic-license', project ':x-pack:plugin:ccr:qa:multi-cluster', project ':x-pack:plugin:ccr:qa:non-compliant-license', project ':x-pack:plugin:ccr:qa:rest', project ':x-pack:plugin:ccr:qa:restart', project ':x-pack:plugin:ccr:qa:security', project ':x-pack:plugin:deprecation:qa:common', project ':x-pack:plugin:deprecation:qa:early-deprecation-rest', project ':x-pack:plugin:deprecation:qa:rest', project ':x-pack:plugin:downsample:qa:mixed-cluster', project ':x-pack:plugin:downsample:qa:rest', project ':x-pack:plugin:downsample:qa:with-security', project ':x-pack:plugin:enrich:qa:common', project ':x-pack:plugin:enrich:qa:rest', project ':x-pack:plugin:enrich:qa:rest-with-advanced-security', project ':x-pack:plugin:enrich:qa:rest-with-security', project ':x-pack:plugin:ent-search:qa:full-cluster-restart', project ':x-pack:plugin:ent-search:qa:rest', project ':x-pack:plugin:eql:qa:ccs-rolling-upgrade', project ':x-pack:plugin:eql:qa:common', project ':x-pack:plugin:eql:qa:correctness', project ':x-pack:plugin:eql:qa:mixed-node', project ':x-pack:plugin:eql:qa:multi-cluster-with-security', project ':x-pack:plugin:eql:qa:rest', project ':x-pack:plugin:eql:qa:security', project ':x-pack:plugin:esql:compute:ann', project ':x-pack:plugin:esql:compute:gen', project ':x-pack:plugin:esql:qa:action', project ':x-pack:plugin:esql:qa:security', project ':x-pack:plugin:esql:qa:server', project ':x-pack:plugin:esql:qa:testFixtures', project ':x-pack:plugin:fleet:qa:rest', project ':x-pack:plugin:graph:qa:with-security', project ':x-pack:plugin:identity-provider:qa:idp-rest-tests', project ':x-pack:plugin:ilm:qa:multi-cluster', project ':x-pack:plugin:ilm:qa:multi-node', project ':x-pack:plugin:ilm:qa:rest', project ':x-pack:plugin:ilm:qa:with-security', project ':x-pack:plugin:inference:qa:inference-service-tests', project ':x-pack:plugin:inference:qa:mixed-cluster', project ':x-pack:plugin:inference:qa:rolling-upgrade', project ':x-pack:plugin:inference:qa:test-service-plugin', project ':x-pack:plugin:logsdb:qa:with-basic', project ':x-pack:plugin:logsdb:qa:with-custom-cutoff', project ':x-pack:plugin:ml:qa:basic-multi-node', project ':x-pack:plugin:ml:qa:disabled', project ':x-pack:plugin:ml:qa:ml-inference-service-tests', project ':x-pack:plugin:ml:qa:ml-with-security', project ':x-pack:plugin:ml:qa:multi-cluster-tests-with-security', project ':x-pack:plugin:ml:qa:native-multi-node-tests', project ':x-pack:plugin:ml:qa:no-bootstrap-tests', project ':x-pack:plugin:ml:qa:single-node-tests', project ':x-pack:plugin:repositories-metering-api:qa:azure', project ':x-pack:plugin:repositories-metering-api:qa:gcs', project ':x-pack:plugin:repositories-metering-api:qa:s3', project ':x-pack:plugin:searchable-snapshots:qa:azure', project ':x-pack:plugin:searchable-snapshots:qa:gcs', project ':x-pack:plugin:searchable-snapshots:qa:hdfs', project ':x-pack:plugin:searchable-snapshots:qa:minio', project ':x-pack:plugin:searchable-snapshots:qa:rest', project ':x-pack:plugin:searchable-snapshots:qa:s3', project ':x-pack:plugin:searchable-snapshots:qa:url', project ':x-pack:plugin:security:lib:nimbus-jose-jwt-modified', project ':x-pack:plugin:security:lib:nimbus-jose-jwt-modified-part1', project ':x-pack:plugin:security:lib:nimbus-jose-jwt-modified-part2', project ':x-pack:plugin:security:qa:audit', project ':x-pack:plugin:security:qa:basic-enable-security', project ':x-pack:plugin:security:qa:consistency-checks', project ':x-pack:plugin:security:qa:jwt-realm', project ':x-pack:plugin:security:qa:multi-cluster', project ':x-pack:plugin:security:qa:operator-privileges-tests', project ':x-pack:plugin:security:qa:profile', project ':x-pack:plugin:security:qa:saml-rest-tests', project ':x-pack:plugin:security:qa:secondary-auth-actions', project ':x-pack:plugin:security:qa:security-basic', project ':x-pack:plugin:security:qa:security-disabled', project ':x-pack:plugin:security:qa:security-trial', project ':x-pack:plugin:security:qa:service-account', project ':x-pack:plugin:security:qa:smoke-test-all-realms', project ':x-pack:plugin:security:qa:tls-basic', project ':x-pack:plugin:shutdown:qa:full-cluster-restart', project ':x-pack:plugin:shutdown:qa:multi-node', project ':x-pack:plugin:shutdown:qa:rolling-upgrade', project ':x-pack:plugin:slm:qa:multi-node', project ':x-pack:plugin:slm:qa:rest', project ':x-pack:plugin:slm:qa:with-security', project ':x-pack:plugin:snapshot-based-recoveries:qa:azure', project ':x-pack:plugin:snapshot-based-recoveries:qa:fs', project ':x-pack:plugin:snapshot-based-recoveries:qa:gcs', project ':x-pack:plugin:snapshot-based-recoveries:qa:license-enforcing', project ':x-pack:plugin:snapshot-based-recoveries:qa:s3', project ':x-pack:plugin:snapshot-repo-test-kit:qa:azure', project ':x-pack:plugin:snapshot-repo-test-kit:qa:gcs', project ':x-pack:plugin:snapshot-repo-test-kit:qa:hdfs', project ':x-pack:plugin:snapshot-repo-test-kit:qa:minio', project ':x-pack:plugin:snapshot-repo-test-kit:qa:rest', project ':x-pack:plugin:snapshot-repo-test-kit:qa:s3', project ':x-pack:plugin:sql:qa:jdbc', project ':x-pack:plugin:sql:qa:mixed-node', project ':x-pack:plugin:sql:qa:server', project ':x-pack:plugin:stack:qa:rest', project ':x-pack:plugin:text-structure:qa:text-structure-with-security', project ':x-pack:plugin:transform:qa:common', project ':x-pack:plugin:transform:qa:multi-cluster-tests-with-security', project ':x-pack:plugin:transform:qa:multi-node-tests', project ':x-pack:plugin:transform:qa:single-node-tests', project ':x-pack:plugin:vector-tile:qa:multi-cluster', project ':x-pack:plugin:watcher:qa:common', project ':x-pack:plugin:watcher:qa:rest', project ':x-pack:plugin:watcher:qa:with-security', project ':x-pack:plugin:esql:qa:server:mixed-cluster', project ':x-pack:plugin:esql:qa:server:multi-clusters', project ':x-pack:plugin:esql:qa:server:multi-node', project ':x-pack:plugin:esql:qa:server:single-node', project ':x-pack:plugin:sql:qa:jdbc:multi-node', project ':x-pack:plugin:sql:qa:jdbc:no-sql', project ':x-pack:plugin:sql:qa:jdbc:security', project ':x-pack:plugin:sql:qa:jdbc:single-node', project ':x-pack:plugin:sql:qa:server:multi-cluster-with-security', project ':x-pack:plugin:sql:qa:server:multi-node', project ':x-pack:plugin:sql:qa:server:security', project ':x-pack:plugin:sql:qa:server:single-node', project ':x-pack:plugin:sql:qa:jdbc:security:with-ssl', project ':x-pack:plugin:sql:qa:jdbc:security:without-ssl', project ':x-pack:plugin:sql:qa:server:security:with-ssl', project ':x-pack:plugin:sql:qa:server:security:without-ssl']
+
+> Configure project :
+Evaluating root project 'elasticsearch' using build file '/Users/rene/dev/elastic/elasticsearch/build.gradle'.
+
+> Configure project :benchmarks
+Evaluating project ':benchmarks' using build file '/Users/rene/dev/elastic/elasticsearch/benchmarks/build.gradle'.
+
+> Configure project :client
+Evaluating project ':client' using build file '/Users/rene/dev/elastic/elasticsearch/client/build.gradle'.
+
+> Configure project :distribution
+Evaluating project ':distribution' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/build.gradle'.
+
+> Configure project :docs
+Evaluating project ':docs' using build file '/Users/rene/dev/elastic/elasticsearch/docs/build.gradle'.
+
+> Configure project :libs
+Evaluating project ':libs' using build file '/Users/rene/dev/elastic/elasticsearch/libs/build.gradle'.
+
+> Configure project :modules
+Evaluating project ':modules' using build file '/Users/rene/dev/elastic/elasticsearch/modules/build.gradle'.
+
+> Configure project :plugins
+Evaluating project ':plugins' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/build.gradle'.
+
+> Configure project :qa
+Evaluating project ':qa' using build file '/Users/rene/dev/elastic/elasticsearch/qa/build.gradle'.
+
+> Configure project :rest-api-spec
+Evaluating project ':rest-api-spec' using build file '/Users/rene/dev/elastic/elasticsearch/rest-api-spec/build.gradle'.
+
+> Configure project :server
+Evaluating project ':server' using build file '/Users/rene/dev/elastic/elasticsearch/server/build.gradle'.
+
+> Configure project :test
+Evaluating project ':test' using build file '/Users/rene/dev/elastic/elasticsearch/test/build.gradle'.
+
+> Configure project :x-pack
+Evaluating project ':x-pack' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/build.gradle'.
+
+> Configure project :client:benchmark
+Evaluating project ':client:benchmark' using build file '/Users/rene/dev/elastic/elasticsearch/client/benchmark/build.gradle'.
+
+> Configure project :client:client-benchmark-noop-api-plugin
+Evaluating project ':client:client-benchmark-noop-api-plugin' using build file '/Users/rene/dev/elastic/elasticsearch/client/client-benchmark-noop-api-plugin/build.gradle'.
+
+> Configure project :client:rest
+Evaluating project ':client:rest' using build file '/Users/rene/dev/elastic/elasticsearch/client/rest/build.gradle'.
+
+> Configure project :client:sniffer
+Evaluating project ':client:sniffer' using build file '/Users/rene/dev/elastic/elasticsearch/client/sniffer/build.gradle'.
+
+> Configure project :client:test
+Evaluating project ':client:test' using build file '/Users/rene/dev/elastic/elasticsearch/client/test/build.gradle'.
+
+> Configure project :distribution:archives
+Evaluating project ':distribution:archives' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/archives/build.gradle'.
+
+> Configure project :distribution:bwc
+Evaluating project ':distribution:bwc' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/bwc/build.gradle'.
+
+> Configure project :distribution:docker
+Evaluating project ':distribution:docker' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/build.gradle'.
+
+> Configure project :distribution:packages
+Evaluating project ':distribution:packages' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/packages/build.gradle'.
+Transforming gradle-ospackage-plugin-11.10.0.jar (com.netflix.nebula:gradle-ospackage-plugin:11.10.0) with InstrumentationAnalysisTransform
+Transforming commons-lang3-3.9.jar (org.apache.commons:commons-lang3:3.9) with InstrumentationAnalysisTransform
+Transforming gradle-docker-plugin-3.2.1.jar (com.bmuschko:gradle-docker-plugin:3.2.1) with InstrumentationAnalysisTransform
+Transforming redline-1.2.10.jar (org.redline-rpm:redline:1.2.10) with InstrumentationAnalysisTransform
+Transforming jdeb-1.10.jar (org.vafer:jdeb:1.10) with InstrumentationAnalysisTransform
+Transforming ant-1.10.12.jar (org.apache.ant:ant:1.10.12) with InstrumentationAnalysisTransform
+Transforming slf4j-api-1.7.5.jar (org.slf4j:slf4j-api:1.7.5) with InstrumentationAnalysisTransform
+Transforming maven-archiver-3.5.1.jar (org.apache.maven:maven-archiver:3.5.1) with InstrumentationAnalysisTransform
+Transforming plexus-archiver-4.2.3.jar (org.codehaus.plexus:plexus-archiver:4.2.3) with InstrumentationAnalysisTransform
+Transforming commons-compress-1.21.jar (org.apache.commons:commons-compress:1.21) with InstrumentationAnalysisTransform
+Transforming xz-1.8.jar (org.tukaani:xz:1.8) with InstrumentationAnalysisTransform
+Transforming bcpg-jdk15on-1.69.jar (org.bouncycastle:bcpg-jdk15on:1.69) with InstrumentationAnalysisTransform
+Transforming bcprov-jdk15on-1.69.jar (org.bouncycastle:bcprov-jdk15on:1.69) with InstrumentationAnalysisTransform
+Transforming ant-launcher-1.10.12.jar (org.apache.ant:ant-launcher:1.10.12) with InstrumentationAnalysisTransform
+Transforming maven-model-3.1.1.jar (org.apache.maven:maven-model:3.1.1) with InstrumentationAnalysisTransform
+Transforming maven-shared-utils-3.3.3.jar (org.apache.maven.shared:maven-shared-utils:3.3.3) with InstrumentationAnalysisTransform
+Transforming plexus-io-3.2.0.jar (org.codehaus.plexus:plexus-io:3.2.0) with InstrumentationAnalysisTransform
+Transforming commons-io-2.6.jar (commons-io:commons-io:2.6) with InstrumentationAnalysisTransform
+Transforming plexus-interpolation-1.26.jar (org.codehaus.plexus:plexus-interpolation:1.26) with InstrumentationAnalysisTransform
+Transforming snappy-0.4.jar (org.iq80.snappy:snappy:0.4) with InstrumentationAnalysisTransform
+Transforming gradle-ospackage-plugin-11.10.0.jar (com.netflix.nebula:gradle-ospackage-plugin:11.10.0) with InstrumentationAnalysisTransform
+Transforming gradle-ospackage-plugin-11.10.0.jar (com.netflix.nebula:gradle-ospackage-plugin:11.10.0) with MergeInstrumentationAnalysisTransform
+Transforming commons-lang3-3.9.jar (org.apache.commons:commons-lang3:3.9) with InstrumentationAnalysisTransform
+Transforming commons-lang3-3.9.jar (org.apache.commons:commons-lang3:3.9) with MergeInstrumentationAnalysisTransform
+Transforming gradle-ospackage-plugin-11.10.0.jar (com.netflix.nebula:gradle-ospackage-plugin:11.10.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming gradle-docker-plugin-3.2.1.jar (com.bmuschko:gradle-docker-plugin:3.2.1) with InstrumentationAnalysisTransform
+Transforming commons-lang3-3.9.jar (org.apache.commons:commons-lang3:3.9) with ExternalDependencyInstrumentingArtifactTransform
+Transforming gradle-docker-plugin-3.2.1.jar (com.bmuschko:gradle-docker-plugin:3.2.1) with MergeInstrumentationAnalysisTransform
+Transforming redline-1.2.10.jar (org.redline-rpm:redline:1.2.10) with InstrumentationAnalysisTransform
+Transforming redline-1.2.10.jar (org.redline-rpm:redline:1.2.10) with MergeInstrumentationAnalysisTransform
+Transforming gradle-docker-plugin-3.2.1.jar (com.bmuschko:gradle-docker-plugin:3.2.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming jdeb-1.10.jar (org.vafer:jdeb:1.10) with InstrumentationAnalysisTransform
+Transforming jdeb-1.10.jar (org.vafer:jdeb:1.10) with MergeInstrumentationAnalysisTransform
+Transforming redline-1.2.10.jar (org.redline-rpm:redline:1.2.10) with ExternalDependencyInstrumentingArtifactTransform
+Transforming ant-1.10.12.jar (org.apache.ant:ant:1.10.12) with InstrumentationAnalysisTransform
+Transforming jdeb-1.10.jar (org.vafer:jdeb:1.10) with ExternalDependencyInstrumentingArtifactTransform
+Transforming ant-1.10.12.jar (org.apache.ant:ant:1.10.12) with MergeInstrumentationAnalysisTransform
+Transforming slf4j-api-1.7.5.jar (org.slf4j:slf4j-api:1.7.5) with InstrumentationAnalysisTransform
+Transforming slf4j-api-1.7.5.jar (org.slf4j:slf4j-api:1.7.5) with MergeInstrumentationAnalysisTransform
+Transforming ant-1.10.12.jar (org.apache.ant:ant:1.10.12) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-archiver-3.5.1.jar (org.apache.maven:maven-archiver:3.5.1) with InstrumentationAnalysisTransform
+Transforming slf4j-api-1.7.5.jar (org.slf4j:slf4j-api:1.7.5) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-archiver-3.5.1.jar (org.apache.maven:maven-archiver:3.5.1) with MergeInstrumentationAnalysisTransform
+Transforming plexus-archiver-4.2.3.jar (org.codehaus.plexus:plexus-archiver:4.2.3) with InstrumentationAnalysisTransform
+Transforming plexus-archiver-4.2.3.jar (org.codehaus.plexus:plexus-archiver:4.2.3) with MergeInstrumentationAnalysisTransform
+Transforming maven-archiver-3.5.1.jar (org.apache.maven:maven-archiver:3.5.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-compress-1.21.jar (org.apache.commons:commons-compress:1.21) with InstrumentationAnalysisTransform
+Transforming commons-compress-1.21.jar (org.apache.commons:commons-compress:1.21) with MergeInstrumentationAnalysisTransform
+Transforming plexus-archiver-4.2.3.jar (org.codehaus.plexus:plexus-archiver:4.2.3) with ExternalDependencyInstrumentingArtifactTransform
+Transforming xz-1.8.jar (org.tukaani:xz:1.8) with InstrumentationAnalysisTransform
+Transforming xz-1.8.jar (org.tukaani:xz:1.8) with MergeInstrumentationAnalysisTransform
+Transforming commons-compress-1.21.jar (org.apache.commons:commons-compress:1.21) with ExternalDependencyInstrumentingArtifactTransform
+Transforming bcpg-jdk15on-1.69.jar (org.bouncycastle:bcpg-jdk15on:1.69) with InstrumentationAnalysisTransform
+Transforming bcpg-jdk15on-1.69.jar (org.bouncycastle:bcpg-jdk15on:1.69) with MergeInstrumentationAnalysisTransform
+Transforming xz-1.8.jar (org.tukaani:xz:1.8) with ExternalDependencyInstrumentingArtifactTransform
+Transforming bcprov-jdk15on-1.69.jar (org.bouncycastle:bcprov-jdk15on:1.69) with InstrumentationAnalysisTransform
+Transforming bcprov-jdk15on-1.69.jar (org.bouncycastle:bcprov-jdk15on:1.69) with MergeInstrumentationAnalysisTransform
+Transforming ant-launcher-1.10.12.jar (org.apache.ant:ant-launcher:1.10.12) with InstrumentationAnalysisTransform
+Transforming bcpg-jdk15on-1.69.jar (org.bouncycastle:bcpg-jdk15on:1.69) with ExternalDependencyInstrumentingArtifactTransform
+Transforming bcprov-jdk15on-1.69.jar (org.bouncycastle:bcprov-jdk15on:1.69) with ExternalDependencyInstrumentingArtifactTransform
+Transforming ant-launcher-1.10.12.jar (org.apache.ant:ant-launcher:1.10.12) with MergeInstrumentationAnalysisTransform
+Transforming maven-model-3.1.1.jar (org.apache.maven:maven-model:3.1.1) with InstrumentationAnalysisTransform
+Transforming maven-model-3.1.1.jar (org.apache.maven:maven-model:3.1.1) with MergeInstrumentationAnalysisTransform
+Transforming maven-shared-utils-3.3.3.jar (org.apache.maven.shared:maven-shared-utils:3.3.3) with InstrumentationAnalysisTransform
+Transforming ant-launcher-1.10.12.jar (org.apache.ant:ant-launcher:1.10.12) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-model-3.1.1.jar (org.apache.maven:maven-model:3.1.1) with ExternalDependencyInstrumentingArtifactTransform
+Transforming maven-shared-utils-3.3.3.jar (org.apache.maven.shared:maven-shared-utils:3.3.3) with MergeInstrumentationAnalysisTransform
+Transforming plexus-io-3.2.0.jar (org.codehaus.plexus:plexus-io:3.2.0) with InstrumentationAnalysisTransform
+Transforming plexus-io-3.2.0.jar (org.codehaus.plexus:plexus-io:3.2.0) with MergeInstrumentationAnalysisTransform
+Transforming commons-io-2.6.jar (commons-io:commons-io:2.6) with InstrumentationAnalysisTransform
+Transforming maven-shared-utils-3.3.3.jar (org.apache.maven.shared:maven-shared-utils:3.3.3) with ExternalDependencyInstrumentingArtifactTransform
+Transforming plexus-io-3.2.0.jar (org.codehaus.plexus:plexus-io:3.2.0) with ExternalDependencyInstrumentingArtifactTransform
+Transforming commons-io-2.6.jar (commons-io:commons-io:2.6) with MergeInstrumentationAnalysisTransform
+Transforming plexus-interpolation-1.26.jar (org.codehaus.plexus:plexus-interpolation:1.26) with InstrumentationAnalysisTransform
+Transforming plexus-interpolation-1.26.jar (org.codehaus.plexus:plexus-interpolation:1.26) with MergeInstrumentationAnalysisTransform
+Transforming snappy-0.4.jar (org.iq80.snappy:snappy:0.4) with InstrumentationAnalysisTransform
+Transforming commons-io-2.6.jar (commons-io:commons-io:2.6) with ExternalDependencyInstrumentingArtifactTransform
+Transforming plexus-interpolation-1.26.jar (org.codehaus.plexus:plexus-interpolation:1.26) with ExternalDependencyInstrumentingArtifactTransform
+Transforming snappy-0.4.jar (org.iq80.snappy:snappy:0.4) with MergeInstrumentationAnalysisTransform
+Transforming snappy-0.4.jar (org.iq80.snappy:snappy:0.4) with ExternalDependencyInstrumentingArtifactTransform
+
+> Configure project :distribution:tools
+Evaluating project ':distribution:tools' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/build.gradle'.
+
+> Configure project :libs:cli
+Evaluating project ':libs:cli' using build file '/Users/rene/dev/elastic/elasticsearch/libs/cli/build.gradle'.
+
+> Configure project :libs:core
+Evaluating project ':libs:core' using build file '/Users/rene/dev/elastic/elasticsearch/libs/core/build.gradle'.
+
+> Configure project :libs:dissect
+Evaluating project ':libs:dissect' using build file '/Users/rene/dev/elastic/elasticsearch/libs/dissect/build.gradle'.
+
+> Configure project :libs:entitlement
+Evaluating project ':libs:entitlement' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/build.gradle'.
+
+> Configure project :libs:geo
+Evaluating project ':libs:geo' using build file '/Users/rene/dev/elastic/elasticsearch/libs/geo/build.gradle'.
+
+> Configure project :libs:grok
+Evaluating project ':libs:grok' using build file '/Users/rene/dev/elastic/elasticsearch/libs/grok/build.gradle'.
+
+> Configure project :libs:h3
+Evaluating project ':libs:h3' using build file '/Users/rene/dev/elastic/elasticsearch/libs/h3/build.gradle'.
+
+> Configure project :libs:log4j
+Evaluating project ':libs:log4j' using build file '/Users/rene/dev/elastic/elasticsearch/libs/log4j/build.gradle'.
+
+> Configure project :libs:logging
+Evaluating project ':libs:logging' using build file '/Users/rene/dev/elastic/elasticsearch/libs/logging/build.gradle'.
+
+> Configure project :libs:logstash-bridge
+Evaluating project ':libs:logstash-bridge' using build file '/Users/rene/dev/elastic/elasticsearch/libs/logstash-bridge/build.gradle'.
+
+> Configure project :libs:lz4
+Evaluating project ':libs:lz4' using build file '/Users/rene/dev/elastic/elasticsearch/libs/lz4/build.gradle'.
+
+> Configure project :libs:native
+Evaluating project ':libs:native' using build file '/Users/rene/dev/elastic/elasticsearch/libs/native/build.gradle'.
+
+> Configure project :libs:plugin-analysis-api
+Evaluating project ':libs:plugin-analysis-api' using build file '/Users/rene/dev/elastic/elasticsearch/libs/plugin-analysis-api/build.gradle'.
+
+> Configure project :libs:plugin-api
+Evaluating project ':libs:plugin-api' using build file '/Users/rene/dev/elastic/elasticsearch/libs/plugin-api/build.gradle'.
+
+> Configure project :libs:plugin-scanner
+Evaluating project ':libs:plugin-scanner' using build file '/Users/rene/dev/elastic/elasticsearch/libs/plugin-scanner/build.gradle'.
+
+> Configure project :libs:secure-sm
+Evaluating project ':libs:secure-sm' using build file '/Users/rene/dev/elastic/elasticsearch/libs/secure-sm/build.gradle'.
+
+> Configure project :libs:simdvec
+Evaluating project ':libs:simdvec' using build file '/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build.gradle'.
+
+> Configure project :libs:ssl-config
+Evaluating project ':libs:ssl-config' using build file '/Users/rene/dev/elastic/elasticsearch/libs/ssl-config/build.gradle'.
+
+> Configure project :libs:tdigest
+Evaluating project ':libs:tdigest' using build file '/Users/rene/dev/elastic/elasticsearch/libs/tdigest/build.gradle'.
+
+> Configure project :libs:x-content
+Evaluating project ':libs:x-content' using build file '/Users/rene/dev/elastic/elasticsearch/libs/x-content/build.gradle'.
+
+> Configure project :modules:aggregations
+Evaluating project ':modules:aggregations' using build file '/Users/rene/dev/elastic/elasticsearch/modules/aggregations/build.gradle'.
+
+> Configure project :modules:analysis-common
+Evaluating project ':modules:analysis-common' using build file '/Users/rene/dev/elastic/elasticsearch/modules/analysis-common/build.gradle'.
+
+> Configure project :modules:apm
+Evaluating project ':modules:apm' using build file '/Users/rene/dev/elastic/elasticsearch/modules/apm/build.gradle'.
+
+> Configure project :modules:data-streams
+Evaluating project ':modules:data-streams' using build file '/Users/rene/dev/elastic/elasticsearch/modules/data-streams/build.gradle'.
+
+> Configure project :modules:dot-prefix-validation
+Evaluating project ':modules:dot-prefix-validation' using build file '/Users/rene/dev/elastic/elasticsearch/modules/dot-prefix-validation/build.gradle'.
+
+> Configure project :modules:health-shards-availability
+Evaluating project ':modules:health-shards-availability' using build file '/Users/rene/dev/elastic/elasticsearch/modules/health-shards-availability/build.gradle'.
+
+> Configure project :modules:ingest-attachment
+Evaluating project ':modules:ingest-attachment' using build file '/Users/rene/dev/elastic/elasticsearch/modules/ingest-attachment/build.gradle'.
+
+> Configure project :modules:ingest-common
+Evaluating project ':modules:ingest-common' using build file '/Users/rene/dev/elastic/elasticsearch/modules/ingest-common/build.gradle'.
+
+> Configure project :modules:ingest-geoip
+Evaluating project ':modules:ingest-geoip' using build file '/Users/rene/dev/elastic/elasticsearch/modules/ingest-geoip/build.gradle'.
+
+> Configure project :modules:ingest-user-agent
+Evaluating project ':modules:ingest-user-agent' using build file '/Users/rene/dev/elastic/elasticsearch/modules/ingest-user-agent/build.gradle'.
+
+> Configure project :modules:kibana
+Evaluating project ':modules:kibana' using build file '/Users/rene/dev/elastic/elasticsearch/modules/kibana/build.gradle'.
+
+> Configure project :modules:lang-expression
+Evaluating project ':modules:lang-expression' using build file '/Users/rene/dev/elastic/elasticsearch/modules/lang-expression/build.gradle'.
+
+> Configure project :modules:lang-mustache
+Evaluating project ':modules:lang-mustache' using build file '/Users/rene/dev/elastic/elasticsearch/modules/lang-mustache/build.gradle'.
+
+> Configure project :modules:lang-painless
+Evaluating project ':modules:lang-painless' using build file '/Users/rene/dev/elastic/elasticsearch/modules/lang-painless/build.gradle'.
+
+> Configure project :modules:legacy-geo
+Evaluating project ':modules:legacy-geo' using build file '/Users/rene/dev/elastic/elasticsearch/modules/legacy-geo/build.gradle'.
+
+> Configure project :modules:mapper-extras
+Evaluating project ':modules:mapper-extras' using build file '/Users/rene/dev/elastic/elasticsearch/modules/mapper-extras/build.gradle'.
+
+> Configure project :modules:parent-join
+Evaluating project ':modules:parent-join' using build file '/Users/rene/dev/elastic/elasticsearch/modules/parent-join/build.gradle'.
+
+> Configure project :modules:percolator
+Evaluating project ':modules:percolator' using build file '/Users/rene/dev/elastic/elasticsearch/modules/percolator/build.gradle'.
+
+> Configure project :modules:rank-eval
+Evaluating project ':modules:rank-eval' using build file '/Users/rene/dev/elastic/elasticsearch/modules/rank-eval/build.gradle'.
+
+> Configure project :modules:reindex
+Evaluating project ':modules:reindex' using build file '/Users/rene/dev/elastic/elasticsearch/modules/reindex/build.gradle'.
+
+> Configure project :modules:repository-azure
+Evaluating project ':modules:repository-azure' using build file '/Users/rene/dev/elastic/elasticsearch/modules/repository-azure/build.gradle'.
+
+> Configure project :modules:repository-gcs
+Evaluating project ':modules:repository-gcs' using build file '/Users/rene/dev/elastic/elasticsearch/modules/repository-gcs/build.gradle'.
+
+> Configure project :modules:repository-s3
+Evaluating project ':modules:repository-s3' using build file '/Users/rene/dev/elastic/elasticsearch/modules/repository-s3/build.gradle'.
+
+> Configure project :modules:repository-url
+Evaluating project ':modules:repository-url' using build file '/Users/rene/dev/elastic/elasticsearch/modules/repository-url/build.gradle'.
+
+> Configure project :modules:rest-root
+Evaluating project ':modules:rest-root' using build file '/Users/rene/dev/elastic/elasticsearch/modules/rest-root/build.gradle'.
+
+> Configure project :modules:runtime-fields-common
+Evaluating project ':modules:runtime-fields-common' using build file '/Users/rene/dev/elastic/elasticsearch/modules/runtime-fields-common/build.gradle'.
+
+> Configure project :modules:systemd
+Evaluating project ':modules:systemd' using build file '/Users/rene/dev/elastic/elasticsearch/modules/systemd/build.gradle'.
+
+> Configure project :modules:transport-netty4
+Evaluating project ':modules:transport-netty4' using build file '/Users/rene/dev/elastic/elasticsearch/modules/transport-netty4/build.gradle'.
+
+> Configure project :plugins:analysis-icu
+Evaluating project ':plugins:analysis-icu' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/analysis-icu/build.gradle'.
+
+> Configure project :plugins:analysis-kuromoji
+Evaluating project ':plugins:analysis-kuromoji' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/analysis-kuromoji/build.gradle'.
+
+> Configure project :plugins:analysis-nori
+Evaluating project ':plugins:analysis-nori' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/analysis-nori/build.gradle'.
+
+> Configure project :plugins:analysis-phonetic
+Evaluating project ':plugins:analysis-phonetic' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/analysis-phonetic/build.gradle'.
+
+> Configure project :plugins:analysis-smartcn
+Evaluating project ':plugins:analysis-smartcn' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/analysis-smartcn/build.gradle'.
+
+> Configure project :plugins:analysis-stempel
+Evaluating project ':plugins:analysis-stempel' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/analysis-stempel/build.gradle'.
+
+> Configure project :plugins:analysis-ukrainian
+Evaluating project ':plugins:analysis-ukrainian' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/analysis-ukrainian/build.gradle'.
+
+> Configure project :plugins:discovery-azure-classic
+Evaluating project ':plugins:discovery-azure-classic' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/discovery-azure-classic/build.gradle'.
+
+> Configure project :plugins:discovery-ec2
+Evaluating project ':plugins:discovery-ec2' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/discovery-ec2/build.gradle'.
+
+> Configure project :plugins:discovery-gce
+Evaluating project ':plugins:discovery-gce' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/discovery-gce/build.gradle'.
+
+> Configure project :plugins:mapper-annotated-text
+Evaluating project ':plugins:mapper-annotated-text' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/mapper-annotated-text/build.gradle'.
+
+> Configure project :plugins:mapper-murmur3
+Evaluating project ':plugins:mapper-murmur3' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/mapper-murmur3/build.gradle'.
+
+> Configure project :plugins:mapper-size
+Evaluating project ':plugins:mapper-size' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/mapper-size/build.gradle'.
+
+> Configure project :plugins:repository-hdfs
+Evaluating project ':plugins:repository-hdfs' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/repository-hdfs/build.gradle'.
+
+> Configure project :plugins:store-smb
+Evaluating project ':plugins:store-smb' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/store-smb/build.gradle'.
+
+> Configure project :qa:ccs-common-rest
+Evaluating project ':qa:ccs-common-rest' using build file '/Users/rene/dev/elastic/elasticsearch/qa/ccs-common-rest/build.gradle'.
+
+> Configure project :qa:ccs-rolling-upgrade-remote-cluster
+Evaluating project ':qa:ccs-rolling-upgrade-remote-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/qa/ccs-rolling-upgrade-remote-cluster/build.gradle'.
+
+> Configure project :qa:ccs-unavailable-clusters
+Evaluating project ':qa:ccs-unavailable-clusters' using build file '/Users/rene/dev/elastic/elasticsearch/qa/ccs-unavailable-clusters/build.gradle'.
+
+> Configure project :qa:custom-rest-controller
+Evaluating project ':qa:custom-rest-controller' using build file '/Users/rene/dev/elastic/elasticsearch/qa/custom-rest-controller/build.gradle'.
+
+> Configure project :qa:evil-tests
+Evaluating project ':qa:evil-tests' using build file '/Users/rene/dev/elastic/elasticsearch/qa/evil-tests/build.gradle'.
+
+> Configure project :qa:full-cluster-restart
+Evaluating project ':qa:full-cluster-restart' using build file '/Users/rene/dev/elastic/elasticsearch/qa/full-cluster-restart/build.gradle'.
+
+> Configure project :qa:logging-config
+Evaluating project ':qa:logging-config' using build file '/Users/rene/dev/elastic/elasticsearch/qa/logging-config/build.gradle'.
+
+> Configure project :qa:logging-spi
+Evaluating project ':qa:logging-spi' using build file '/Users/rene/dev/elastic/elasticsearch/qa/logging-spi/build.gradle'.
+
+> Configure project :qa:lucene-index-compatibility
+Evaluating project ':qa:lucene-index-compatibility' using build file '/Users/rene/dev/elastic/elasticsearch/qa/lucene-index-compatibility/build.gradle'.
+
+> Configure project :qa:mixed-cluster
+Evaluating project ':qa:mixed-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/qa/mixed-cluster/build.gradle'.
+
+> Configure project :qa:multi-cluster-search
+Evaluating project ':qa:multi-cluster-search' using build file '/Users/rene/dev/elastic/elasticsearch/qa/multi-cluster-search/build.gradle'.
+
+> Configure project :qa:no-bootstrap-tests
+Evaluating project ':qa:no-bootstrap-tests' using build file '/Users/rene/dev/elastic/elasticsearch/qa/no-bootstrap-tests/build.gradle'.
+
+> Configure project :qa:packaging
+Evaluating project ':qa:packaging' using build file '/Users/rene/dev/elastic/elasticsearch/qa/packaging/build.gradle'.
+
+> Configure project :qa:remote-clusters
+Evaluating project ':qa:remote-clusters' using build file '/Users/rene/dev/elastic/elasticsearch/qa/remote-clusters/build.gradle'.
+
+> Configure project :qa:repository-multi-version
+Evaluating project ':qa:repository-multi-version' using build file '/Users/rene/dev/elastic/elasticsearch/qa/repository-multi-version/build.gradle'.
+
+> Configure project :qa:restricted-loggers
+Evaluating project ':qa:restricted-loggers' using build file '/Users/rene/dev/elastic/elasticsearch/qa/restricted-loggers/build.gradle'.
+
+> Configure project :qa:rolling-upgrade
+Evaluating project ':qa:rolling-upgrade' using build file '/Users/rene/dev/elastic/elasticsearch/qa/rolling-upgrade/build.gradle'.
+
+> Configure project :qa:rolling-upgrade-legacy
+Evaluating project ':qa:rolling-upgrade-legacy' using build file '/Users/rene/dev/elastic/elasticsearch/qa/rolling-upgrade-legacy/build.gradle'.
+
+> Configure project :qa:smoke-test-http
+Evaluating project ':qa:smoke-test-http' using build file '/Users/rene/dev/elastic/elasticsearch/qa/smoke-test-http/build.gradle'.
+
+> Configure project :qa:smoke-test-ingest-disabled
+Evaluating project ':qa:smoke-test-ingest-disabled' using build file '/Users/rene/dev/elastic/elasticsearch/qa/smoke-test-ingest-disabled/build.gradle'.
+
+> Configure project :qa:smoke-test-ingest-with-all-dependencies
+Evaluating project ':qa:smoke-test-ingest-with-all-dependencies' using build file '/Users/rene/dev/elastic/elasticsearch/qa/smoke-test-ingest-with-all-dependencies/build.gradle'.
+
+> Configure project :qa:smoke-test-multinode
+Evaluating project ':qa:smoke-test-multinode' using build file '/Users/rene/dev/elastic/elasticsearch/qa/smoke-test-multinode/build.gradle'.
+
+> Configure project :qa:smoke-test-plugins
+Evaluating project ':qa:smoke-test-plugins' using build file '/Users/rene/dev/elastic/elasticsearch/qa/smoke-test-plugins/build.gradle'.
+
+> Configure project :qa:stable-api
+Evaluating project ':qa:stable-api' using build file '/Users/rene/dev/elastic/elasticsearch/qa/stable-api/build.gradle'.
+
+> Configure project :qa:system-indices
+Evaluating project ':qa:system-indices' using build file '/Users/rene/dev/elastic/elasticsearch/qa/system-indices/build.gradle'.
+
+> Configure project :qa:unconfigured-node-name
+Evaluating project ':qa:unconfigured-node-name' using build file '/Users/rene/dev/elastic/elasticsearch/qa/unconfigured-node-name/build.gradle'.
+
+> Configure project :qa:verify-version-constants
+Evaluating project ':qa:verify-version-constants' using build file '/Users/rene/dev/elastic/elasticsearch/qa/verify-version-constants/build.gradle'.
+
+> Configure project :test:external-modules
+Evaluating project ':test:external-modules' using build file '/Users/rene/dev/elastic/elasticsearch/test/external-modules/build.gradle'.
+
+> Configure project :test:fixtures
+Evaluating project ':test:fixtures' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/build.gradle'.
+
+> Configure project :test:framework
+Evaluating project ':test:framework' using build file '/Users/rene/dev/elastic/elasticsearch/test/framework/build.gradle'.
+
+> Configure project :test:immutable-collections-patch
+Evaluating project ':test:immutable-collections-patch' using build file '/Users/rene/dev/elastic/elasticsearch/test/immutable-collections-patch/build.gradle'.
+
+> Configure project :test:logger-usage
+Evaluating project ':test:logger-usage' using build file '/Users/rene/dev/elastic/elasticsearch/test/logger-usage/build.gradle'.
+
+> Configure project :test:metadata-extractor
+Evaluating project ':test:metadata-extractor' using build file '/Users/rene/dev/elastic/elasticsearch/test/metadata-extractor/build.gradle'.
+
+> Configure project :test:test-clusters
+Evaluating project ':test:test-clusters' using build file '/Users/rene/dev/elastic/elasticsearch/test/test-clusters/build.gradle'.
+
+> Configure project :test:x-content
+Evaluating project ':test:x-content' using build file '/Users/rene/dev/elastic/elasticsearch/test/x-content/build.gradle'.
+
+> Configure project :test:yaml-rest-runner
+Evaluating project ':test:yaml-rest-runner' using build file '/Users/rene/dev/elastic/elasticsearch/test/yaml-rest-runner/build.gradle'.
+
+> Configure project :x-pack:libs
+Evaluating project ':x-pack:libs' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/libs/build.gradle'.
+
+> Configure project :x-pack:license-tools
+Evaluating project ':x-pack:license-tools' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/license-tools/build.gradle'.
+
+> Configure project :x-pack:plugin
+Evaluating project ':x-pack:plugin' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/build.gradle'.
+
+> Configure project :x-pack:qa
+Evaluating project ':x-pack:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/build.gradle'.
+
+> Configure project :x-pack:rest-resources-zip
+Evaluating project ':x-pack:rest-resources-zip' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/rest-resources-zip/build.gradle'.
+
+> Configure project :x-pack:test
+Evaluating project ':x-pack:test' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/test/build.gradle'.
+
+> Configure project :distribution:archives:darwin-aarch64-tar
+Evaluating project ':distribution:archives:darwin-aarch64-tar' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/archives/darwin-aarch64-tar/build.gradle'.
+
+> Configure project :distribution:archives:darwin-tar
+Evaluating project ':distribution:archives:darwin-tar' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/archives/darwin-tar/build.gradle'.
+
+> Configure project :distribution:archives:integ-test-zip
+Evaluating project ':distribution:archives:integ-test-zip' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/archives/integ-test-zip/build.gradle'.
+
+> Configure project :distribution:archives:linux-aarch64-tar
+Evaluating project ':distribution:archives:linux-aarch64-tar' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/archives/linux-aarch64-tar/build.gradle'.
+
+> Configure project :distribution:archives:linux-tar
+Evaluating project ':distribution:archives:linux-tar' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/archives/linux-tar/build.gradle'.
+
+> Configure project :distribution:archives:windows-zip
+Evaluating project ':distribution:archives:windows-zip' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/archives/windows-zip/build.gradle'.
+
+> Configure project :distribution:bwc:bugfix
+Evaluating project ':distribution:bwc:bugfix' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/bwc/bugfix/build.gradle'.
+
+> Configure project :distribution:bwc:bugfix2
+Evaluating project ':distribution:bwc:bugfix2' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/bwc/bugfix2/build.gradle'.
+
+> Configure project :distribution:bwc:maintenance
+Evaluating project ':distribution:bwc:maintenance' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/bwc/maintenance/build.gradle'.
+
+> Configure project :distribution:bwc:minor
+Evaluating project ':distribution:bwc:minor' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/bwc/minor/build.gradle'.
+
+> Configure project :distribution:bwc:staged
+Evaluating project ':distribution:bwc:staged' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/bwc/staged/build.gradle'.
+
+> Configure project :distribution:docker:cloud-ess-docker-aarch64-export
+Evaluating project ':distribution:docker:cloud-ess-docker-aarch64-export' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/cloud-ess-docker-aarch64-export/build.gradle'.
+
+> Configure project :distribution:docker:cloud-ess-docker-export
+Evaluating project ':distribution:docker:cloud-ess-docker-export' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/cloud-ess-docker-export/build.gradle'.
+
+> Configure project :distribution:docker:docker-aarch64-export
+Evaluating project ':distribution:docker:docker-aarch64-export' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/docker-aarch64-export/build.gradle'.
+
+> Configure project :distribution:docker:docker-export
+Evaluating project ':distribution:docker:docker-export' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/docker-export/build.gradle'.
+
+> Configure project :distribution:docker:ironbank-docker-aarch64-export
+Evaluating project ':distribution:docker:ironbank-docker-aarch64-export' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/ironbank-docker-aarch64-export/build.gradle'.
+
+> Configure project :distribution:docker:ironbank-docker-export
+Evaluating project ':distribution:docker:ironbank-docker-export' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/ironbank-docker-export/build.gradle'.
+
+> Configure project :distribution:docker:wolfi-docker-aarch64-export
+Evaluating project ':distribution:docker:wolfi-docker-aarch64-export' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/wolfi-docker-aarch64-export/build.gradle'.
+
+> Configure project :distribution:docker:wolfi-docker-export
+Evaluating project ':distribution:docker:wolfi-docker-export' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/wolfi-docker-export/build.gradle'.
+
+> Configure project :distribution:packages:aarch64-deb
+Evaluating project ':distribution:packages:aarch64-deb' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/packages/aarch64-deb/build.gradle'.
+
+> Configure project :distribution:packages:aarch64-rpm
+Evaluating project ':distribution:packages:aarch64-rpm' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/packages/aarch64-rpm/build.gradle'.
+
+> Configure project :distribution:packages:deb
+Evaluating project ':distribution:packages:deb' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/packages/deb/build.gradle'.
+
+> Configure project :distribution:packages:rpm
+Evaluating project ':distribution:packages:rpm' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/packages/rpm/build.gradle'.
+
+> Configure project :distribution:tools:ansi-console
+Evaluating project ':distribution:tools:ansi-console' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/ansi-console/build.gradle'.
+
+> Configure project :distribution:tools:cli-launcher
+Evaluating project ':distribution:tools:cli-launcher' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/cli-launcher/build.gradle'.
+
+> Configure project :distribution:tools:geoip-cli
+Evaluating project ':distribution:tools:geoip-cli' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/geoip-cli/build.gradle'.
+
+> Configure project :distribution:tools:java-version-checker
+Evaluating project ':distribution:tools:java-version-checker' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/java-version-checker/build.gradle'.
+
+> Configure project :distribution:tools:keystore-cli
+Evaluating project ':distribution:tools:keystore-cli' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/keystore-cli/build.gradle'.
+
+> Configure project :distribution:tools:plugin-cli
+Evaluating project ':distribution:tools:plugin-cli' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/plugin-cli/build.gradle'.
+
+> Configure project :distribution:tools:server-cli
+Evaluating project ':distribution:tools:server-cli' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/server-cli/build.gradle'.
+
+> Configure project :distribution:tools:windows-service-cli
+Evaluating project ':distribution:tools:windows-service-cli' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/windows-service-cli/build.gradle'.
+
+> Configure project :libs:entitlement:agent
+Evaluating project ':libs:entitlement:agent' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/agent/build.gradle'.
+
+> Configure project :libs:entitlement:asm-provider
+Evaluating project ':libs:entitlement:asm-provider' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/asm-provider/build.gradle'.
+
+> Configure project :libs:entitlement:bridge
+Evaluating project ':libs:entitlement:bridge' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/bridge/build.gradle'.
+
+> Configure project :libs:entitlement:qa
+Evaluating project ':libs:entitlement:qa' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/qa/build.gradle'.
+
+> Configure project :libs:entitlement:tools
+Evaluating project ':libs:entitlement:tools' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/tools/build.gradle'.
+
+> Configure project :libs:native:native-libraries
+Evaluating project ':libs:native:native-libraries' using build file '/Users/rene/dev/elastic/elasticsearch/libs/native/libraries/build.gradle'.
+
+> Configure project :libs:x-content:impl
+Evaluating project ':libs:x-content:impl' using build file '/Users/rene/dev/elastic/elasticsearch/libs/x-content/impl/build.gradle'.
+
+> Configure project :modules:ingest-geoip:qa
+Evaluating project ':modules:ingest-geoip:qa' using build file '/Users/rene/dev/elastic/elasticsearch/modules/ingest-geoip/qa/build.gradle'.
+
+> Configure project :modules:lang-painless:spi
+Evaluating project ':modules:lang-painless:spi' using build file '/Users/rene/dev/elastic/elasticsearch/modules/lang-painless/spi/build.gradle'.
+
+> Configure project :plugins:discovery-ec2:qa
+Evaluating project ':plugins:discovery-ec2:qa' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/discovery-ec2/qa/build.gradle'.
+
+> Configure project :plugins:discovery-gce:qa
+Evaluating project ':plugins:discovery-gce:qa' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/discovery-gce/qa/build.gradle'.
+
+> Configure project :plugins:repository-hdfs:hadoop-client-api
+Evaluating project ':plugins:repository-hdfs:hadoop-client-api' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/repository-hdfs/hadoop-client-api/build.gradle'.
+
+> Configure project :qa:stable-api:logging
+Evaluating project ':qa:stable-api:logging' using build file '/Users/rene/dev/elastic/elasticsearch/qa/stable-api/logging/build.gradle'.
+
+> Configure project :qa:stable-api:plugin-analysis-api
+Evaluating project ':qa:stable-api:plugin-analysis-api' using build file '/Users/rene/dev/elastic/elasticsearch/qa/stable-api/plugin-analysis-api/build.gradle'.
+
+> Configure project :qa:stable-api:plugin-api
+Evaluating project ':qa:stable-api:plugin-api' using build file '/Users/rene/dev/elastic/elasticsearch/qa/stable-api/plugin-api/build.gradle'.
+
+> Configure project :test:external-modules:test-apm-integration
+Evaluating project ':test:external-modules:test-apm-integration' using build file '/Users/rene/dev/elastic/elasticsearch/test/external-modules/apm-integration/build.gradle'.
+
+> Configure project :test:external-modules:test-delayed-aggs
+Evaluating project ':test:external-modules:test-delayed-aggs' using build file '/Users/rene/dev/elastic/elasticsearch/test/external-modules/delayed-aggs/build.gradle'.
+
+> Configure project :test:external-modules:test-die-with-dignity
+Evaluating project ':test:external-modules:test-die-with-dignity' using build file '/Users/rene/dev/elastic/elasticsearch/test/external-modules/die-with-dignity/build.gradle'.
+
+> Configure project :test:external-modules:test-error-query
+Evaluating project ':test:external-modules:test-error-query' using build file '/Users/rene/dev/elastic/elasticsearch/test/external-modules/error-query/build.gradle'.
+
+> Configure project :test:external-modules:test-esql-heap-attack
+Evaluating project ':test:external-modules:test-esql-heap-attack' using build file '/Users/rene/dev/elastic/elasticsearch/test/external-modules/esql-heap-attack/build.gradle'.
+
+> Configure project :test:external-modules:test-jvm-crash
+Evaluating project ':test:external-modules:test-jvm-crash' using build file '/Users/rene/dev/elastic/elasticsearch/test/external-modules/jvm-crash/build.gradle'.
+
+> Configure project :test:external-modules:test-latency-simulating-directory
+Evaluating project ':test:external-modules:test-latency-simulating-directory' using build file '/Users/rene/dev/elastic/elasticsearch/test/external-modules/latency-simulating-directory/build.gradle'.
+
+> Configure project :test:fixtures:aws-sts-fixture
+Evaluating project ':test:fixtures:aws-sts-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/aws-sts-fixture/build.gradle'.
+
+> Configure project :test:fixtures:azure-fixture
+Evaluating project ':test:fixtures:azure-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/azure-fixture/build.gradle'.
+
+> Configure project :test:fixtures:ec2-imds-fixture
+Evaluating project ':test:fixtures:ec2-imds-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/ec2-imds-fixture/build.gradle'.
+
+> Configure project :test:fixtures:gcs-fixture
+Evaluating project ':test:fixtures:gcs-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/gcs-fixture/build.gradle'.
+
+> Configure project :test:fixtures:geoip-fixture
+Evaluating project ':test:fixtures:geoip-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/geoip-fixture/build.gradle'.
+
+> Configure project :test:fixtures:hdfs-fixture
+Evaluating project ':test:fixtures:hdfs-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/hdfs-fixture/build.gradle'.
+
+> Configure project :test:fixtures:krb5kdc-fixture
+Evaluating project ':test:fixtures:krb5kdc-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/krb5kdc-fixture/build.gradle'.
+
+> Configure project :test:fixtures:minio-fixture
+Evaluating project ':test:fixtures:minio-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/minio-fixture/build.gradle'.
+
+> Configure project :test:fixtures:old-elasticsearch
+Evaluating project ':test:fixtures:old-elasticsearch' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/old-elasticsearch/build.gradle'.
+
+> Configure project :test:fixtures:s3-fixture
+Evaluating project ':test:fixtures:s3-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/s3-fixture/build.gradle'.
+
+> Configure project :test:fixtures:testcontainer-utils
+Evaluating project ':test:fixtures:testcontainer-utils' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/testcontainer-utils/build.gradle'.
+
+> Configure project :test:fixtures:url-fixture
+Evaluating project ':test:fixtures:url-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/url-fixture/build.gradle'.
+
+> Configure project :x-pack:libs:es-opensaml-security-api
+Evaluating project ':x-pack:libs:es-opensaml-security-api' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/libs/es-opensaml-security-api/build.gradle'.
+
+> Configure project :x-pack:plugin:analytics
+Evaluating project ':x-pack:plugin:analytics' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/analytics/build.gradle'.
+
+> Configure project :x-pack:plugin:apm-data
+Evaluating project ':x-pack:plugin:apm-data' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/apm-data/build.gradle'.
+
+> Configure project :x-pack:plugin:async
+Evaluating project ':x-pack:plugin:async' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/async/build.gradle'.
+
+> Configure project :x-pack:plugin:async-search
+Evaluating project ':x-pack:plugin:async-search' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/async-search/build.gradle'.
+
+> Configure project :x-pack:plugin:autoscaling
+Evaluating project ':x-pack:plugin:autoscaling' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/autoscaling/build.gradle'.
+
+> Configure project :x-pack:plugin:blob-cache
+Evaluating project ':x-pack:plugin:blob-cache' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/blob-cache/build.gradle'.
+
+> Configure project :x-pack:plugin:ccr
+Evaluating project ':x-pack:plugin:ccr' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ccr/build.gradle'.
+
+> Configure project :x-pack:plugin:core
+Evaluating project ':x-pack:plugin:core' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/core/build.gradle'.
+
+> Configure project :x-pack:plugin:deprecation
+Evaluating project ':x-pack:plugin:deprecation' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/deprecation/build.gradle'.
+
+> Configure project :x-pack:plugin:downsample
+Evaluating project ':x-pack:plugin:downsample' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/downsample/build.gradle'.
+
+> Configure project :x-pack:plugin:enrich
+Evaluating project ':x-pack:plugin:enrich' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/enrich/build.gradle'.
+
+> Configure project :x-pack:plugin:ent-search
+Evaluating project ':x-pack:plugin:ent-search' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ent-search/build.gradle'.
+
+> Configure project :x-pack:plugin:eql
+Evaluating project ':x-pack:plugin:eql' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/build.gradle'.
+
+> Configure project :x-pack:plugin:esql
+Evaluating project ':x-pack:plugin:esql' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/build.gradle'.
+
+> Configure project :x-pack:plugin:esql-core
+Evaluating project ':x-pack:plugin:esql-core' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql-core/build.gradle'.
+
+> Configure project :x-pack:plugin:fleet
+Evaluating project ':x-pack:plugin:fleet' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/fleet/build.gradle'.
+
+> Configure project :x-pack:plugin:frozen-indices
+Evaluating project ':x-pack:plugin:frozen-indices' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/frozen-indices/build.gradle'.
+
+> Configure project :x-pack:plugin:geoip-enterprise-downloader
+Evaluating project ':x-pack:plugin:geoip-enterprise-downloader' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/geoip-enterprise-downloader/build.gradle'.
+
+> Configure project :x-pack:plugin:graph
+Evaluating project ':x-pack:plugin:graph' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/graph/build.gradle'.
+
+> Configure project :x-pack:plugin:identity-provider
+Evaluating project ':x-pack:plugin:identity-provider' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/identity-provider/build.gradle'.
+
+> Configure project :x-pack:plugin:ilm
+Evaluating project ':x-pack:plugin:ilm' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ilm/build.gradle'.
+
+> Configure project :x-pack:plugin:inference
+Evaluating project ':x-pack:plugin:inference' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/inference/build.gradle'.
+
+> Configure project :x-pack:plugin:kql
+Evaluating project ':x-pack:plugin:kql' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/kql/build.gradle'.
+
+> Configure project :x-pack:plugin:logsdb
+Evaluating project ':x-pack:plugin:logsdb' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/logsdb/build.gradle'.
+
+> Configure project :x-pack:plugin:logstash
+Evaluating project ':x-pack:plugin:logstash' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/logstash/build.gradle'.
+
+> Configure project :x-pack:plugin:mapper-aggregate-metric
+Evaluating project ':x-pack:plugin:mapper-aggregate-metric' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/mapper-aggregate-metric/build.gradle'.
+
+> Configure project :x-pack:plugin:mapper-constant-keyword
+Evaluating project ':x-pack:plugin:mapper-constant-keyword' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/mapper-constant-keyword/build.gradle'.
+
+> Configure project :x-pack:plugin:mapper-counted-keyword
+Evaluating project ':x-pack:plugin:mapper-counted-keyword' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/mapper-counted-keyword/build.gradle'.
+
+> Configure project :x-pack:plugin:mapper-unsigned-long
+Evaluating project ':x-pack:plugin:mapper-unsigned-long' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/mapper-unsigned-long/build.gradle'.
+
+> Configure project :x-pack:plugin:mapper-version
+Evaluating project ':x-pack:plugin:mapper-version' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/mapper-version/build.gradle'.
+
+> Configure project :x-pack:plugin:migrate
+Evaluating project ':x-pack:plugin:migrate' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/migrate/build.gradle'.
+
+> Configure project :x-pack:plugin:ml
+Evaluating project ':x-pack:plugin:ml' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/build.gradle'.
+
+> Configure project :x-pack:plugin:ml-package-loader
+Evaluating project ':x-pack:plugin:ml-package-loader' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml-package-loader/build.gradle'.
+
+> Configure project :x-pack:plugin:monitoring
+Evaluating project ':x-pack:plugin:monitoring' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/monitoring/build.gradle'.
+
+> Configure project :x-pack:plugin:old-lucene-versions
+Evaluating project ':x-pack:plugin:old-lucene-versions' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/old-lucene-versions/build.gradle'.
+
+> Configure project :x-pack:plugin:otel-data
+Evaluating project ':x-pack:plugin:otel-data' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/otel-data/build.gradle'.
+
+> Configure project :x-pack:plugin:profiling
+Evaluating project ':x-pack:plugin:profiling' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/profiling/build.gradle'.
+
+> Configure project :x-pack:plugin:ql
+Evaluating project ':x-pack:plugin:ql' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ql/build.gradle'.
+
+> Configure project :x-pack:plugin:rank-rrf
+Evaluating project ':x-pack:plugin:rank-rrf' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/rank-rrf/build.gradle'.
+
+> Configure project :x-pack:plugin:redact
+Evaluating project ':x-pack:plugin:redact' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/redact/build.gradle'.
+
+> Configure project :x-pack:plugin:repositories-metering-api
+Evaluating project ':x-pack:plugin:repositories-metering-api' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/repositories-metering-api/build.gradle'.
+
+> Configure project :x-pack:plugin:rollup
+Evaluating project ':x-pack:plugin:rollup' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/rollup/build.gradle'.
+
+> Configure project :x-pack:plugin:search-business-rules
+Evaluating project ':x-pack:plugin:search-business-rules' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/search-business-rules/build.gradle'.
+
+> Configure project :x-pack:plugin:searchable-snapshots
+Evaluating project ':x-pack:plugin:searchable-snapshots' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/build.gradle'.
+
+> Configure project :x-pack:plugin:security
+Evaluating project ':x-pack:plugin:security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/build.gradle'.
+
+> Configure project :x-pack:plugin:shutdown
+Evaluating project ':x-pack:plugin:shutdown' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/shutdown/build.gradle'.
+
+> Configure project :x-pack:plugin:slm
+Evaluating project ':x-pack:plugin:slm' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/slm/build.gradle'.
+
+> Configure project :x-pack:plugin:snapshot-based-recoveries
+Evaluating project ':x-pack:plugin:snapshot-based-recoveries' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-based-recoveries/build.gradle'.
+
+> Configure project :x-pack:plugin:snapshot-repo-test-kit
+Evaluating project ':x-pack:plugin:snapshot-repo-test-kit' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-repo-test-kit/build.gradle'.
+
+> Configure project :x-pack:plugin:spatial
+Evaluating project ':x-pack:plugin:spatial' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/spatial/build.gradle'.
+
+> Configure project :x-pack:plugin:sql
+Evaluating project ':x-pack:plugin:sql' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/build.gradle'.
+
+> Configure project :x-pack:plugin:stack
+Evaluating project ':x-pack:plugin:stack' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/stack/build.gradle'.
+
+> Configure project :x-pack:plugin:text-structure
+Evaluating project ':x-pack:plugin:text-structure' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/text-structure/build.gradle'.
+
+> Configure project :x-pack:plugin:transform
+Evaluating project ':x-pack:plugin:transform' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/transform/build.gradle'.
+
+> Configure project :x-pack:plugin:vector-tile
+Evaluating project ':x-pack:plugin:vector-tile' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/vector-tile/build.gradle'.
+
+> Configure project :x-pack:plugin:voting-only-node
+Evaluating project ':x-pack:plugin:voting-only-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/voting-only-node/build.gradle'.
+
+> Configure project :x-pack:plugin:watcher
+Evaluating project ':x-pack:plugin:watcher' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/watcher/build.gradle'.
+
+> Configure project :x-pack:plugin:wildcard
+Evaluating project ':x-pack:plugin:wildcard' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/wildcard/build.gradle'.
+
+> Configure project :x-pack:plugin:write-load-forecaster
+Evaluating project ':x-pack:plugin:write-load-forecaster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/write-load-forecaster/build.gradle'.
+
+> Configure project :x-pack:qa:core-rest-tests-with-security
+Evaluating project ':x-pack:qa:core-rest-tests-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/core-rest-tests-with-security/build.gradle'.
+
+> Configure project :x-pack:qa:evil-tests
+Evaluating project ':x-pack:qa:evil-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/evil-tests/build.gradle'.
+
+> Configure project :x-pack:qa:freeze-plugin
+Evaluating project ':x-pack:qa:freeze-plugin' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/freeze-plugin/build.gradle'.
+
+> Configure project :x-pack:qa:full-cluster-restart
+Evaluating project ':x-pack:qa:full-cluster-restart' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/full-cluster-restart/build.gradle'.
+
+> Configure project :x-pack:qa:kerberos-tests
+Evaluating project ':x-pack:qa:kerberos-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/kerberos-tests/build.gradle'.
+
+> Configure project :x-pack:qa:mixed-tier-cluster
+Evaluating project ':x-pack:qa:mixed-tier-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/mixed-tier-cluster/build.gradle'.
+
+> Configure project :x-pack:qa:multi-cluster-search-security
+Evaluating project ':x-pack:qa:multi-cluster-search-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/multi-cluster-search-security/build.gradle'.
+
+> Configure project :x-pack:qa:multi-node
+Evaluating project ':x-pack:qa:multi-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/multi-node/build.gradle'.
+
+> Configure project :x-pack:qa:oidc-op-tests
+Evaluating project ':x-pack:qa:oidc-op-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/oidc-op-tests/build.gradle'.
+
+> Configure project :x-pack:qa:openldap-tests
+Evaluating project ':x-pack:qa:openldap-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/openldap-tests/build.gradle'.
+
+> Configure project :x-pack:qa:password-protected-keystore
+Evaluating project ':x-pack:qa:password-protected-keystore' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/password-protected-keystore/build.gradle'.
+
+> Configure project :x-pack:qa:reindex-tests-with-security
+Evaluating project ':x-pack:qa:reindex-tests-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/reindex-tests-with-security/build.gradle'.
+
+> Configure project :x-pack:qa:repository-old-versions
+Evaluating project ':x-pack:qa:repository-old-versions' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/repository-old-versions/build.gradle'.
+
+> Configure project :x-pack:qa:rolling-upgrade
+Evaluating project ':x-pack:qa:rolling-upgrade' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/rolling-upgrade/build.gradle'.
+
+> Configure project :x-pack:qa:rolling-upgrade-basic
+Evaluating project ':x-pack:qa:rolling-upgrade-basic' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/rolling-upgrade-basic/build.gradle'.
+
+> Configure project :x-pack:qa:rolling-upgrade-multi-cluster
+Evaluating project ':x-pack:qa:rolling-upgrade-multi-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle'.
+
+> Configure project :x-pack:qa:runtime-fields
+Evaluating project ':x-pack:qa:runtime-fields' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/runtime-fields/build.gradle'.
+
+> Configure project :x-pack:qa:saml-idp-tests
+Evaluating project ':x-pack:qa:saml-idp-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/saml-idp-tests/build.gradle'.
+
+> Configure project :x-pack:qa:security-example-spi-extension
+Evaluating project ':x-pack:qa:security-example-spi-extension' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/security-example-spi-extension/build.gradle'.
+
+> Configure project :x-pack:qa:security-setup-password-tests
+Evaluating project ':x-pack:qa:security-setup-password-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/security-setup-password-tests/build.gradle'.
+
+> Configure project :x-pack:qa:security-tools-tests
+Evaluating project ':x-pack:qa:security-tools-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/security-tools-tests/build.gradle'.
+
+> Configure project :x-pack:qa:smoke-test-plugins
+Evaluating project ':x-pack:qa:smoke-test-plugins' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/smoke-test-plugins/build.gradle'.
+
+> Configure project :x-pack:qa:smoke-test-plugins-ssl
+Evaluating project ':x-pack:qa:smoke-test-plugins-ssl' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/smoke-test-plugins-ssl/build.gradle'.
+
+> Configure project :x-pack:qa:smoke-test-security-with-mustache
+Evaluating project ':x-pack:qa:smoke-test-security-with-mustache' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/smoke-test-security-with-mustache/build.gradle'.
+
+> Configure project :x-pack:qa:third-party
+Evaluating project ':x-pack:qa:third-party' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/third-party/build.gradle'.
+
+> Configure project :x-pack:test:idp-fixture
+Evaluating project ':x-pack:test:idp-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/test/idp-fixture/build.gradle'.
+
+> Configure project :x-pack:test:smb-fixture
+Evaluating project ':x-pack:test:smb-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/test/smb-fixture/build.gradle'.
+
+> Configure project :libs:entitlement:qa:common
+Evaluating project ':libs:entitlement:qa:common' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/qa/common/build.gradle'.
+
+> Configure project :libs:entitlement:qa:entitlement-allowed
+Evaluating project ':libs:entitlement:qa:entitlement-allowed' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/qa/entitlement-allowed/build.gradle'.
+
+> Configure project :libs:entitlement:qa:entitlement-allowed-nonmodular
+Evaluating project ':libs:entitlement:qa:entitlement-allowed-nonmodular' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/qa/entitlement-allowed-nonmodular/build.gradle'.
+
+> Configure project :libs:entitlement:qa:entitlement-denied
+Evaluating project ':libs:entitlement:qa:entitlement-denied' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/qa/entitlement-denied/build.gradle'.
+
+> Configure project :libs:entitlement:qa:entitlement-denied-nonmodular
+Evaluating project ':libs:entitlement:qa:entitlement-denied-nonmodular' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/qa/entitlement-denied-nonmodular/build.gradle'.
+
+> Configure project :libs:entitlement:tools:common
+Evaluating project ':libs:entitlement:tools:common' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/tools/common/build.gradle'.
+
+> Configure project :libs:entitlement:tools:public-callers-finder
+Evaluating project ':libs:entitlement:tools:public-callers-finder' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/tools/public-callers-finder/build.gradle'.
+
+> Configure project :libs:entitlement:tools:securitymanager-scanner
+Evaluating project ':libs:entitlement:tools:securitymanager-scanner' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/tools/securitymanager-scanner/build.gradle'.
+
+> Configure project :modules:ingest-geoip:qa:file-based-update
+Evaluating project ':modules:ingest-geoip:qa:file-based-update' using build file '/Users/rene/dev/elastic/elasticsearch/modules/ingest-geoip/qa/file-based-update/build.gradle'.
+
+> Configure project :modules:ingest-geoip:qa:full-cluster-restart
+Evaluating project ':modules:ingest-geoip:qa:full-cluster-restart' using build file '/Users/rene/dev/elastic/elasticsearch/modules/ingest-geoip/qa/full-cluster-restart/build.gradle'.
+
+> Configure project :plugins:discovery-ec2:qa:amazon-ec2
+Evaluating project ':plugins:discovery-ec2:qa:amazon-ec2' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/discovery-ec2/qa/amazon-ec2/build.gradle'.
+
+> Configure project :plugins:discovery-gce:qa:gce
+Evaluating project ':plugins:discovery-gce:qa:gce' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/discovery-gce/qa/gce/build.gradle'.
+
+> Configure project :x-pack:plugin:async-search:qa
+Evaluating project ':x-pack:plugin:async-search:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/async-search/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:autoscaling:qa
+Evaluating project ':x-pack:plugin:autoscaling:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/autoscaling/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:ccr:qa
+Evaluating project ':x-pack:plugin:ccr:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ccr/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:core:template-resources
+Evaluating project ':x-pack:plugin:core:template-resources' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/core/template-resources/build.gradle'.
+
+> Configure project :x-pack:plugin:deprecation:qa
+Evaluating project ':x-pack:plugin:deprecation:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/deprecation/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:downsample:qa
+Evaluating project ':x-pack:plugin:downsample:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/downsample/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:enrich:qa
+Evaluating project ':x-pack:plugin:enrich:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/enrich/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:ent-search:qa
+Evaluating project ':x-pack:plugin:ent-search:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ent-search/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:eql:qa
+Evaluating project ':x-pack:plugin:eql:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:esql:arrow
+Evaluating project ':x-pack:plugin:esql:arrow' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/arrow/build.gradle'.
+
+> Configure project :x-pack:plugin:esql:compute
+Evaluating project ':x-pack:plugin:esql:compute' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/compute/build.gradle'.
+
+> Configure project :x-pack:plugin:esql:qa
+Evaluating project ':x-pack:plugin:esql:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:esql-core:test-fixtures
+Evaluating project ':x-pack:plugin:esql-core:test-fixtures' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql-core/test-fixtures/build.gradle'.
+
+> Configure project :x-pack:plugin:fleet:qa
+Evaluating project ':x-pack:plugin:fleet:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/fleet/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:graph:qa
+Evaluating project ':x-pack:plugin:graph:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/graph/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:identity-provider:qa
+Evaluating project ':x-pack:plugin:identity-provider:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/identity-provider/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:ilm:qa
+Evaluating project ':x-pack:plugin:ilm:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ilm/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:inference:qa
+Evaluating project ':x-pack:plugin:inference:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/inference/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:logsdb:qa
+Evaluating project ':x-pack:plugin:logsdb:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/logsdb/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:ml:qa
+Evaluating project ':x-pack:plugin:ml:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:ql:test-fixtures
+Evaluating project ':x-pack:plugin:ql:test-fixtures' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ql/test-fixtures/build.gradle'.
+
+> Configure project :x-pack:plugin:repositories-metering-api:qa
+Evaluating project ':x-pack:plugin:repositories-metering-api:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/repositories-metering-api/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:searchable-snapshots:qa
+Evaluating project ':x-pack:plugin:searchable-snapshots:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:security:cli
+Evaluating project ':x-pack:plugin:security:cli' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/cli/build.gradle'.
+
+> Configure project :x-pack:plugin:security:lib
+Evaluating project ':x-pack:plugin:security:lib' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/lib/build.gradle'.
+
+> Configure project :x-pack:plugin:security:qa
+Evaluating project ':x-pack:plugin:security:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:shutdown:qa
+Evaluating project ':x-pack:plugin:shutdown:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/shutdown/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:slm:qa
+Evaluating project ':x-pack:plugin:slm:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/slm/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:snapshot-based-recoveries:qa
+Evaluating project ':x-pack:plugin:snapshot-based-recoveries:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-based-recoveries/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:snapshot-repo-test-kit:qa
+Evaluating project ':x-pack:plugin:snapshot-repo-test-kit:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-repo-test-kit/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:jdbc
+Evaluating project ':x-pack:plugin:sql:jdbc' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/jdbc/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:qa
+Evaluating project ':x-pack:plugin:sql:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:sql-action
+Evaluating project ':x-pack:plugin:sql:sql-action' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/sql-action/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:sql-cli
+Evaluating project ':x-pack:plugin:sql:sql-cli' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/sql-cli/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:sql-client
+Evaluating project ':x-pack:plugin:sql:sql-client' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/sql-client/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:sql-proto
+Evaluating project ':x-pack:plugin:sql:sql-proto' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/sql-proto/build.gradle'.
+
+> Configure project :x-pack:plugin:stack:qa
+Evaluating project ':x-pack:plugin:stack:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/stack/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:text-structure:qa
+Evaluating project ':x-pack:plugin:text-structure:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/text-structure/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:transform:qa
+Evaluating project ':x-pack:plugin:transform:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/transform/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:vector-tile:qa
+Evaluating project ':x-pack:plugin:vector-tile:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/vector-tile/qa/build.gradle'.
+
+> Configure project :x-pack:plugin:watcher:qa
+Evaluating project ':x-pack:plugin:watcher:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/watcher/qa/build.gradle'.
+
+> Configure project :x-pack:qa:multi-cluster-search-security:legacy-with-basic-license
+Evaluating project ':x-pack:qa:multi-cluster-search-security:legacy-with-basic-license' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle'.
+
+> Configure project :x-pack:qa:multi-cluster-search-security:legacy-with-full-license
+Evaluating project ':x-pack:qa:multi-cluster-search-security:legacy-with-full-license' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/build.gradle'.
+
+> Configure project :x-pack:qa:multi-cluster-search-security:legacy-with-restricted-trust
+Evaluating project ':x-pack:qa:multi-cluster-search-security:legacy-with-restricted-trust' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/build.gradle'.
+
+> Configure project :x-pack:qa:runtime-fields:core-with-mapped
+Evaluating project ':x-pack:qa:runtime-fields:core-with-mapped' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/runtime-fields/core-with-mapped/build.gradle'.
+
+> Configure project :x-pack:qa:runtime-fields:core-with-search
+Evaluating project ':x-pack:qa:runtime-fields:core-with-search' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/runtime-fields/core-with-search/build.gradle'.
+
+> Configure project :x-pack:qa:runtime-fields:with-security
+Evaluating project ':x-pack:qa:runtime-fields:with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/runtime-fields/with-security/build.gradle'.
+
+> Configure project :x-pack:qa:third-party:active-directory
+Evaluating project ':x-pack:qa:third-party:active-directory' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/third-party/active-directory/build.gradle'.
+
+> Configure project :x-pack:qa:third-party:jira
+Evaluating project ':x-pack:qa:third-party:jira' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/third-party/jira/build.gradle'.
+
+> Configure project :x-pack:qa:third-party:pagerduty
+Evaluating project ':x-pack:qa:third-party:pagerduty' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/third-party/pagerduty/build.gradle'.
+
+> Configure project :x-pack:qa:third-party:slack
+Evaluating project ':x-pack:qa:third-party:slack' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/third-party/slack/build.gradle'.
+
+> Configure project :x-pack:plugin:async-search:qa:rest
+Evaluating project ':x-pack:plugin:async-search:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/async-search/qa/rest/build.gradle'.
+
+> Configure project :x-pack:plugin:async-search:qa:security
+Evaluating project ':x-pack:plugin:async-search:qa:security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/async-search/qa/security/build.gradle'.
+
+> Configure project :x-pack:plugin:autoscaling:qa:rest
+Evaluating project ':x-pack:plugin:autoscaling:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/autoscaling/qa/rest/build.gradle'.
+
+> Configure project :x-pack:plugin:ccr:qa:downgrade-to-basic-license
+Evaluating project ':x-pack:plugin:ccr:qa:downgrade-to-basic-license' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle'.
+
+> Configure project :x-pack:plugin:ccr:qa:multi-cluster
+Evaluating project ':x-pack:plugin:ccr:qa:multi-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ccr/qa/multi-cluster/build.gradle'.
+
+> Configure project :x-pack:plugin:ccr:qa:non-compliant-license
+Evaluating project ':x-pack:plugin:ccr:qa:non-compliant-license' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle'.
+
+> Configure project :x-pack:plugin:ccr:qa:rest
+Evaluating project ':x-pack:plugin:ccr:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ccr/qa/rest/build.gradle'.
+
+> Configure project :x-pack:plugin:ccr:qa:restart
+Evaluating project ':x-pack:plugin:ccr:qa:restart' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ccr/qa/restart/build.gradle'.
+
+> Configure project :x-pack:plugin:ccr:qa:security
+Evaluating project ':x-pack:plugin:ccr:qa:security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ccr/qa/security/build.gradle'.
+
+> Configure project :x-pack:plugin:deprecation:qa:common
+Evaluating project ':x-pack:plugin:deprecation:qa:common' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/deprecation/qa/common/build.gradle'.
+
+> Configure project :x-pack:plugin:deprecation:qa:early-deprecation-rest
+Evaluating project ':x-pack:plugin:deprecation:qa:early-deprecation-rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/deprecation/qa/early-deprecation-rest/build.gradle'.
+
+> Configure project :x-pack:plugin:deprecation:qa:rest
+Evaluating project ':x-pack:plugin:deprecation:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/deprecation/qa/rest/build.gradle'.
+
+> Configure project :x-pack:plugin:downsample:qa:mixed-cluster
+Evaluating project ':x-pack:plugin:downsample:qa:mixed-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/downsample/qa/mixed-cluster/build.gradle'.
+
+> Configure project :x-pack:plugin:downsample:qa:rest
+Evaluating project ':x-pack:plugin:downsample:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/downsample/qa/rest/build.gradle'.
+
+> Configure project :x-pack:plugin:downsample:qa:with-security
+Evaluating project ':x-pack:plugin:downsample:qa:with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/downsample/qa/with-security/build.gradle'.
+
+> Configure project :x-pack:plugin:enrich:qa:common
+Evaluating project ':x-pack:plugin:enrich:qa:common' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/enrich/qa/common/build.gradle'.
+
+> Configure project :x-pack:plugin:enrich:qa:rest
+Evaluating project ':x-pack:plugin:enrich:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/enrich/qa/rest/build.gradle'.
+
+> Configure project :x-pack:plugin:enrich:qa:rest-with-advanced-security
+Evaluating project ':x-pack:plugin:enrich:qa:rest-with-advanced-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/enrich/qa/rest-with-advanced-security/build.gradle'.
+
+> Configure project :x-pack:plugin:enrich:qa:rest-with-security
+Evaluating project ':x-pack:plugin:enrich:qa:rest-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/enrich/qa/rest-with-security/build.gradle'.
+
+> Configure project :x-pack:plugin:ent-search:qa:full-cluster-restart
+Evaluating project ':x-pack:plugin:ent-search:qa:full-cluster-restart' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ent-search/qa/full-cluster-restart/build.gradle'.
+
+> Configure project :x-pack:plugin:ent-search:qa:rest
+Evaluating project ':x-pack:plugin:ent-search:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ent-search/qa/rest/build.gradle'.
+
+> Configure project :x-pack:plugin:eql:qa:ccs-rolling-upgrade
+Evaluating project ':x-pack:plugin:eql:qa:ccs-rolling-upgrade' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/qa/ccs-rolling-upgrade/build.gradle'.
+
+> Configure project :x-pack:plugin:eql:qa:common
+Evaluating project ':x-pack:plugin:eql:qa:common' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/qa/common/build.gradle'.
+
+> Configure project :x-pack:plugin:eql:qa:correctness
+Evaluating project ':x-pack:plugin:eql:qa:correctness' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/qa/correctness/build.gradle'.
+
+> Configure project :x-pack:plugin:eql:qa:mixed-node
+Evaluating project ':x-pack:plugin:eql:qa:mixed-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/qa/mixed-node/build.gradle'.
+
+> Configure project :x-pack:plugin:eql:qa:multi-cluster-with-security
+Evaluating project ':x-pack:plugin:eql:qa:multi-cluster-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle'.
+
+> Configure project :x-pack:plugin:eql:qa:rest
+Evaluating project ':x-pack:plugin:eql:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/qa/rest/build.gradle'.
+
+> Configure project :x-pack:plugin:eql:qa:security
+Evaluating project ':x-pack:plugin:eql:qa:security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/qa/security/build.gradle'.
+
+> Configure project :x-pack:plugin:esql:compute:ann
+Evaluating project ':x-pack:plugin:esql:compute:ann' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/compute/ann/build.gradle'.
+
+> Configure project :x-pack:plugin:esql:compute:gen
+Evaluating project ':x-pack:plugin:esql:compute:gen' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/compute/gen/build.gradle'.
+
+> Configure project :x-pack:plugin:esql:qa:action
+Evaluating project ':x-pack:plugin:esql:qa:action' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/action/build.gradle'.
+
+> Configure project :x-pack:plugin:esql:qa:security
+Evaluating project ':x-pack:plugin:esql:qa:security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/security/build.gradle'.
+
+> Configure project :x-pack:plugin:esql:qa:server
+Evaluating project ':x-pack:plugin:esql:qa:server' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/server/build.gradle'.
+
+> Configure project :x-pack:plugin:esql:qa:testFixtures
+Evaluating project ':x-pack:plugin:esql:qa:testFixtures' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/testFixtures/build.gradle'.
+
+> Configure project :x-pack:plugin:fleet:qa:rest
+Evaluating project ':x-pack:plugin:fleet:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/fleet/qa/rest/build.gradle'.
+
+> Configure project :x-pack:plugin:graph:qa:with-security
+Evaluating project ':x-pack:plugin:graph:qa:with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/graph/qa/with-security/build.gradle'.
+
+> Configure project :x-pack:plugin:identity-provider:qa:idp-rest-tests
+Evaluating project ':x-pack:plugin:identity-provider:qa:idp-rest-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle'.
+
+> Configure project :x-pack:plugin:ilm:qa:multi-cluster
+Evaluating project ':x-pack:plugin:ilm:qa:multi-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ilm/qa/multi-cluster/build.gradle'.
+
+> Configure project :x-pack:plugin:ilm:qa:multi-node
+Evaluating project ':x-pack:plugin:ilm:qa:multi-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ilm/qa/multi-node/build.gradle'.
+
+> Configure project :x-pack:plugin:ilm:qa:rest
+Evaluating project ':x-pack:plugin:ilm:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ilm/qa/rest/build.gradle'.
+
+> Configure project :x-pack:plugin:ilm:qa:with-security
+Evaluating project ':x-pack:plugin:ilm:qa:with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ilm/qa/with-security/build.gradle'.
+
+> Configure project :x-pack:plugin:inference:qa:inference-service-tests
+Evaluating project ':x-pack:plugin:inference:qa:inference-service-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/inference/qa/inference-service-tests/build.gradle'.
+
+> Configure project :x-pack:plugin:inference:qa:mixed-cluster
+Evaluating project ':x-pack:plugin:inference:qa:mixed-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/inference/qa/mixed-cluster/build.gradle'.
+
+> Configure project :x-pack:plugin:inference:qa:rolling-upgrade
+Evaluating project ':x-pack:plugin:inference:qa:rolling-upgrade' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/inference/qa/rolling-upgrade/build.gradle'.
+
+> Configure project :x-pack:plugin:inference:qa:test-service-plugin
+Evaluating project ':x-pack:plugin:inference:qa:test-service-plugin' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/inference/qa/test-service-plugin/build.gradle'.
+
+> Configure project :x-pack:plugin:logsdb:qa:with-basic
+Evaluating project ':x-pack:plugin:logsdb:qa:with-basic' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/logsdb/qa/with-basic/build.gradle'.
+
+> Configure project :x-pack:plugin:logsdb:qa:with-custom-cutoff
+Evaluating project ':x-pack:plugin:logsdb:qa:with-custom-cutoff' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/logsdb/qa/with-custom-cutoff/build.gradle'.
+
+> Configure project :x-pack:plugin:ml:qa:basic-multi-node
+Evaluating project ':x-pack:plugin:ml:qa:basic-multi-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/basic-multi-node/build.gradle'.
+
+> Configure project :x-pack:plugin:ml:qa:disabled
+Evaluating project ':x-pack:plugin:ml:qa:disabled' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/disabled/build.gradle'.
+
+> Configure project :x-pack:plugin:ml:qa:ml-inference-service-tests
+Evaluating project ':x-pack:plugin:ml:qa:ml-inference-service-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/ml-inference-service-tests/build.gradle'.
+
+> Configure project :x-pack:plugin:ml:qa:ml-with-security
+Evaluating project ':x-pack:plugin:ml:qa:ml-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/ml-with-security/build.gradle'.
+
+> Configure project :x-pack:plugin:ml:qa:multi-cluster-tests-with-security
+Evaluating project ':x-pack:plugin:ml:qa:multi-cluster-tests-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle'.
+
+> Configure project :x-pack:plugin:ml:qa:native-multi-node-tests
+Evaluating project ':x-pack:plugin:ml:qa:native-multi-node-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle'.
+
+> Configure project :x-pack:plugin:ml:qa:no-bootstrap-tests
+Evaluating project ':x-pack:plugin:ml:qa:no-bootstrap-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/no-bootstrap-tests/build.gradle'.
+
+> Configure project :x-pack:plugin:ml:qa:single-node-tests
+Evaluating project ':x-pack:plugin:ml:qa:single-node-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/single-node-tests/build.gradle'.
+
+> Configure project :x-pack:plugin:repositories-metering-api:qa:azure
+Evaluating project ':x-pack:plugin:repositories-metering-api:qa:azure' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/repositories-metering-api/qa/azure/build.gradle'.
+
+> Configure project :x-pack:plugin:repositories-metering-api:qa:gcs
+Evaluating project ':x-pack:plugin:repositories-metering-api:qa:gcs' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle'.
+
+> Configure project :x-pack:plugin:repositories-metering-api:qa:s3
+Evaluating project ':x-pack:plugin:repositories-metering-api:qa:s3' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle'.
+
+> Configure project :x-pack:plugin:searchable-snapshots:qa:azure
+Evaluating project ':x-pack:plugin:searchable-snapshots:qa:azure' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle'.
+
+> Configure project :x-pack:plugin:searchable-snapshots:qa:gcs
+Evaluating project ':x-pack:plugin:searchable-snapshots:qa:gcs' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle'.
+
+> Configure project :x-pack:plugin:searchable-snapshots:qa:hdfs
+Evaluating project ':x-pack:plugin:searchable-snapshots:qa:hdfs' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle'.
+
+> Configure project :x-pack:plugin:searchable-snapshots:qa:minio
+Evaluating project ':x-pack:plugin:searchable-snapshots:qa:minio' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle'.
+
+> Configure project :x-pack:plugin:searchable-snapshots:qa:rest
+Evaluating project ':x-pack:plugin:searchable-snapshots:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle'.
+
+> Configure project :x-pack:plugin:searchable-snapshots:qa:s3
+Evaluating project ':x-pack:plugin:searchable-snapshots:qa:s3' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle'.
+
+> Configure project :x-pack:plugin:searchable-snapshots:qa:url
+Evaluating project ':x-pack:plugin:searchable-snapshots:qa:url' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/qa/url/build.gradle'.
+
+> Configure project :x-pack:plugin:security:lib:nimbus-jose-jwt-modified
+Evaluating project ':x-pack:plugin:security:lib:nimbus-jose-jwt-modified' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/build.gradle'.
+
+> Configure project :x-pack:plugin:security:lib:nimbus-jose-jwt-modified-part1
+Evaluating project ':x-pack:plugin:security:lib:nimbus-jose-jwt-modified-part1' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/build.gradle'.
+
+> Configure project :x-pack:plugin:security:lib:nimbus-jose-jwt-modified-part2
+Evaluating project ':x-pack:plugin:security:lib:nimbus-jose-jwt-modified-part2' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part2/build.gradle'.
+
+> Configure project :x-pack:plugin:security:qa:audit
+Evaluating project ':x-pack:plugin:security:qa:audit' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/audit/build.gradle'.
+
+> Configure project :x-pack:plugin:security:qa:basic-enable-security
+Evaluating project ':x-pack:plugin:security:qa:basic-enable-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/basic-enable-security/build.gradle'.
+
+> Configure project :x-pack:plugin:security:qa:consistency-checks
+Evaluating project ':x-pack:plugin:security:qa:consistency-checks' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/consistency-checks/build.gradle'.
+
+> Configure project :x-pack:plugin:security:qa:jwt-realm
+Evaluating project ':x-pack:plugin:security:qa:jwt-realm' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/jwt-realm/build.gradle'.
+
+> Configure project :x-pack:plugin:security:qa:multi-cluster
+Evaluating project ':x-pack:plugin:security:qa:multi-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/multi-cluster/build.gradle'.
+
+> Configure project :x-pack:plugin:security:qa:operator-privileges-tests
+Evaluating project ':x-pack:plugin:security:qa:operator-privileges-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle'.
+
+> Configure project :x-pack:plugin:security:qa:profile
+Evaluating project ':x-pack:plugin:security:qa:profile' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/profile/build.gradle'.
+
+> Configure project :x-pack:plugin:security:qa:saml-rest-tests
+Evaluating project ':x-pack:plugin:security:qa:saml-rest-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/saml-rest-tests/build.gradle'.
+
+> Configure project :x-pack:plugin:security:qa:secondary-auth-actions
+Evaluating project ':x-pack:plugin:security:qa:secondary-auth-actions' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/secondary-auth-actions/build.gradle'.
+
+> Configure project :x-pack:plugin:security:qa:security-basic
+Evaluating project ':x-pack:plugin:security:qa:security-basic' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/security-basic/build.gradle'.
+
+> Configure project :x-pack:plugin:security:qa:security-disabled
+Evaluating project ':x-pack:plugin:security:qa:security-disabled' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/security-disabled/build.gradle'.
+
+> Configure project :x-pack:plugin:security:qa:security-trial
+Evaluating project ':x-pack:plugin:security:qa:security-trial' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/security-trial/build.gradle'.
+
+> Configure project :x-pack:plugin:security:qa:service-account
+Evaluating project ':x-pack:plugin:security:qa:service-account' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/service-account/build.gradle'.
+
+> Configure project :x-pack:plugin:security:qa:smoke-test-all-realms
+Evaluating project ':x-pack:plugin:security:qa:smoke-test-all-realms' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/smoke-test-all-realms/build.gradle'.
+
+> Configure project :x-pack:plugin:security:qa:tls-basic
+Evaluating project ':x-pack:plugin:security:qa:tls-basic' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/tls-basic/build.gradle'.
+
+> Configure project :x-pack:plugin:shutdown:qa:full-cluster-restart
+Evaluating project ':x-pack:plugin:shutdown:qa:full-cluster-restart' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/shutdown/qa/full-cluster-restart/build.gradle'.
+
+> Configure project :x-pack:plugin:shutdown:qa:multi-node
+Evaluating project ':x-pack:plugin:shutdown:qa:multi-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/shutdown/qa/multi-node/build.gradle'.
+
+> Configure project :x-pack:plugin:shutdown:qa:rolling-upgrade
+Evaluating project ':x-pack:plugin:shutdown:qa:rolling-upgrade' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/shutdown/qa/rolling-upgrade/build.gradle'.
+
+> Configure project :x-pack:plugin:slm:qa:multi-node
+Evaluating project ':x-pack:plugin:slm:qa:multi-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/slm/qa/multi-node/build.gradle'.
+
+> Configure project :x-pack:plugin:slm:qa:rest
+Evaluating project ':x-pack:plugin:slm:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/slm/qa/rest/build.gradle'.
+
+> Configure project :x-pack:plugin:slm:qa:with-security
+Evaluating project ':x-pack:plugin:slm:qa:with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/slm/qa/with-security/build.gradle'.
+
+> Configure project :x-pack:plugin:snapshot-based-recoveries:qa:azure
+Evaluating project ':x-pack:plugin:snapshot-based-recoveries:qa:azure' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-based-recoveries/qa/azure/build.gradle'.
+
+> Configure project :x-pack:plugin:snapshot-based-recoveries:qa:fs
+Evaluating project ':x-pack:plugin:snapshot-based-recoveries:qa:fs' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-based-recoveries/qa/fs/build.gradle'.
+
+> Configure project :x-pack:plugin:snapshot-based-recoveries:qa:gcs
+Evaluating project ':x-pack:plugin:snapshot-based-recoveries:qa:gcs' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-based-recoveries/qa/gcs/build.gradle'.
+
+> Configure project :x-pack:plugin:snapshot-based-recoveries:qa:license-enforcing
+Evaluating project ':x-pack:plugin:snapshot-based-recoveries:qa:license-enforcing' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-based-recoveries/qa/license-enforcing/build.gradle'.
+
+> Configure project :x-pack:plugin:snapshot-based-recoveries:qa:s3
+Evaluating project ':x-pack:plugin:snapshot-based-recoveries:qa:s3' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-based-recoveries/qa/s3/build.gradle'.
+
+> Configure project :x-pack:plugin:snapshot-repo-test-kit:qa:azure
+Evaluating project ':x-pack:plugin:snapshot-repo-test-kit:qa:azure' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-repo-test-kit/qa/azure/build.gradle'.
+
+> Configure project :x-pack:plugin:snapshot-repo-test-kit:qa:gcs
+Evaluating project ':x-pack:plugin:snapshot-repo-test-kit:qa:gcs' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/build.gradle'.
+
+> Configure project :x-pack:plugin:snapshot-repo-test-kit:qa:hdfs
+Evaluating project ':x-pack:plugin:snapshot-repo-test-kit:qa:hdfs' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/build.gradle'.
+
+> Configure project :x-pack:plugin:snapshot-repo-test-kit:qa:minio
+Evaluating project ':x-pack:plugin:snapshot-repo-test-kit:qa:minio' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-repo-test-kit/qa/minio/build.gradle'.
+
+> Configure project :x-pack:plugin:snapshot-repo-test-kit:qa:rest
+Evaluating project ':x-pack:plugin:snapshot-repo-test-kit:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle'.
+
+> Configure project :x-pack:plugin:snapshot-repo-test-kit:qa:s3
+Evaluating project ':x-pack:plugin:snapshot-repo-test-kit:qa:s3' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:qa:jdbc
+Evaluating project ':x-pack:plugin:sql:qa:jdbc' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/jdbc/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:qa:mixed-node
+Evaluating project ':x-pack:plugin:sql:qa:mixed-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/mixed-node/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:qa:server
+Evaluating project ':x-pack:plugin:sql:qa:server' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/server/build.gradle'.
+
+> Configure project :x-pack:plugin:stack:qa:rest
+Evaluating project ':x-pack:plugin:stack:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/stack/qa/rest/build.gradle'.
+
+> Configure project :x-pack:plugin:text-structure:qa:text-structure-with-security
+Evaluating project ':x-pack:plugin:text-structure:qa:text-structure-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/text-structure/qa/text-structure-with-security/build.gradle'.
+
+> Configure project :x-pack:plugin:transform:qa:common
+Evaluating project ':x-pack:plugin:transform:qa:common' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/transform/qa/common/build.gradle'.
+
+> Configure project :x-pack:plugin:transform:qa:multi-cluster-tests-with-security
+Evaluating project ':x-pack:plugin:transform:qa:multi-cluster-tests-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle'.
+
+> Configure project :x-pack:plugin:transform:qa:multi-node-tests
+Evaluating project ':x-pack:plugin:transform:qa:multi-node-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/transform/qa/multi-node-tests/build.gradle'.
+
+> Configure project :x-pack:plugin:transform:qa:single-node-tests
+Evaluating project ':x-pack:plugin:transform:qa:single-node-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/transform/qa/single-node-tests/build.gradle'.
+
+> Configure project :x-pack:plugin:vector-tile:qa:multi-cluster
+Evaluating project ':x-pack:plugin:vector-tile:qa:multi-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/vector-tile/qa/multi-cluster/build.gradle'.
+
+> Configure project :x-pack:plugin:watcher:qa:common
+Evaluating project ':x-pack:plugin:watcher:qa:common' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/watcher/qa/common/build.gradle'.
+
+> Configure project :x-pack:plugin:watcher:qa:rest
+Evaluating project ':x-pack:plugin:watcher:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/watcher/qa/rest/build.gradle'.
+
+> Configure project :x-pack:plugin:watcher:qa:with-security
+Evaluating project ':x-pack:plugin:watcher:qa:with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/watcher/qa/with-security/build.gradle'.
+
+> Configure project :x-pack:plugin:esql:qa:server:mixed-cluster
+Evaluating project ':x-pack:plugin:esql:qa:server:mixed-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle'.
+
+> Configure project :x-pack:plugin:esql:qa:server:multi-clusters
+Evaluating project ':x-pack:plugin:esql:qa:server:multi-clusters' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle'.
+
+> Configure project :x-pack:plugin:esql:qa:server:multi-node
+Evaluating project ':x-pack:plugin:esql:qa:server:multi-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/server/multi-node/build.gradle'.
+
+> Configure project :x-pack:plugin:esql:qa:server:single-node
+Evaluating project ':x-pack:plugin:esql:qa:server:single-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/server/single-node/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:qa:jdbc:multi-node
+Evaluating project ':x-pack:plugin:sql:qa:jdbc:multi-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/jdbc/multi-node/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:qa:jdbc:no-sql
+Evaluating project ':x-pack:plugin:sql:qa:jdbc:no-sql' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/jdbc/no-sql/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:qa:jdbc:security
+Evaluating project ':x-pack:plugin:sql:qa:jdbc:security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/jdbc/security/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:qa:jdbc:single-node
+Evaluating project ':x-pack:plugin:sql:qa:jdbc:single-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/jdbc/single-node/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:qa:server:multi-cluster-with-security
+Evaluating project ':x-pack:plugin:sql:qa:server:multi-cluster-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:qa:server:multi-node
+Evaluating project ':x-pack:plugin:sql:qa:server:multi-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/server/multi-node/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:qa:server:security
+Evaluating project ':x-pack:plugin:sql:qa:server:security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/server/security/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:qa:server:single-node
+Evaluating project ':x-pack:plugin:sql:qa:server:single-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/server/single-node/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:qa:jdbc:security:with-ssl
+Evaluating project ':x-pack:plugin:sql:qa:jdbc:security:with-ssl' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/jdbc/security/with-ssl/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:qa:jdbc:security:without-ssl
+Evaluating project ':x-pack:plugin:sql:qa:jdbc:security:without-ssl' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/jdbc/security/without-ssl/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:qa:server:security:with-ssl
+Evaluating project ':x-pack:plugin:sql:qa:server:security:with-ssl' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/server/security/with-ssl/build.gradle'.
+
+> Configure project :x-pack:plugin:sql:qa:server:security:without-ssl
+Evaluating project ':x-pack:plugin:sql:qa:server:security:without-ssl' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/server/security/without-ssl/build.gradle'.
+All projects evaluated.
+test#with21
+Task name matched 'cleanTest'
+Task name matched 'test'
+Selected primary task 'cleanTest' from project :libs:simdvec
+Selected primary task 'test' from project :libs:simdvec
+Tasks to be executed: [task ':libs:simdvec:cleanTest', task ':libs:logging:compileJava', task ':libs:core:compileJava', task ':libs:native:compileJava', task ':libs:simdvec:compileJava', task ':libs:simdvec:processResources', task ':libs:simdvec:classes', task ':libs:simdvec:compileMain21Java', task ':libs:simdvec:processMain21Resources', task ':libs:simdvec:main21Classes', task ':libs:simdvec:compileMain22Java', task ':libs:simdvec:processMain22Resources', task ':libs:simdvec:main22Classes', task ':libs:simdvec:jar', task ':libs:cli:compileJava', task ':libs:x-content:compileJava', task ':libs:entitlement:bridge:compileJava', task ':libs:entitlement:compileJava', task ':libs:geo:compileJava', task ':libs:grok:compileJava', task ':libs:lz4:compileJava', task ':libs:plugin-api:compileJava', task ':libs:plugin-analysis-api:compileJava', task ':libs:secure-sm:compileJava', task ':libs:tdigest:compileJava', task ':server:compileJava', task ':server:processResources', task ':server:classes', task ':server:generateModulesList', task ':server:generatePluginsList', task ':server:jar', task ':client:rest:compileJava', task ':client:rest:processResources', task ':client:rest:classes', task ':client:rest:jar', task ':libs:cli:processResources', task ':libs:cli:classes', task ':libs:cli:jar', task ':libs:core:processResources', task ':libs:core:classes', task ':libs:core:jar', task ':libs:entitlement:asm-provider:compileJava', task ':libs:entitlement:asm-provider:processResources', task ':libs:entitlement:asm-provider:classes', task ':libs:entitlement:asm-provider:jar', task ':libs:entitlement:generateAsm-providerProviderManifest', task ':libs:entitlement:generateAsm-providerProviderImpl', task ':libs:entitlement:processResources', task ':libs:entitlement:classes', task ':libs:entitlement:bridge:processResources', task ':libs:entitlement:bridge:classes', task ':libs:entitlement:bridge:compileMain23Java', task ':libs:entitlement:bridge:processMain23Resources', task ':libs:entitlement:bridge:main23Classes', task ':libs:entitlement:bridge:java23Jar', task ':libs:entitlement:compileMain23Java', task ':libs:entitlement:processMain23Resources', task ':libs:entitlement:main23Classes', task ':libs:entitlement:jar', task ':libs:geo:processResources', task ':libs:geo:classes', task ':libs:geo:jar', task ':libs:grok:processResources', task ':libs:grok:classes', task ':libs:grok:jar', task ':libs:logging:processResources', task ':libs:logging:classes', task ':libs:logging:jar', task ':libs:lz4:processResources', task ':libs:lz4:classes', task ':libs:lz4:jar', task ':libs:native:processResources', task ':libs:native:classes', task ':libs:native:compileMain22Java', task ':libs:native:processMain22Resources', task ':libs:native:main22Classes', task ':libs:native:jar', task ':libs:plugin-analysis-api:processResources', task ':libs:plugin-analysis-api:classes', task ':libs:plugin-analysis-api:jar', task ':libs:plugin-api:processResources', task ':libs:plugin-api:classes', task ':libs:plugin-api:jar', task ':libs:secure-sm:processResources', task ':libs:secure-sm:classes', task ':libs:secure-sm:jar', task ':libs:ssl-config:compileJava', task ':modules:transport-netty4:compileJava', task ':test:framework:compileJava', task ':libs:simdvec:compileTestJava', task ':libs:simdvec:processTestResources', task ':libs:simdvec:testClasses', task ':libs:ssl-config:processResources', task ':libs:ssl-config:classes', task ':libs:ssl-config:jar', task ':libs:tdigest:processResources', task ':libs:tdigest:classes', task ':libs:tdigest:jar', task ':libs:x-content:impl:compileJava', task ':libs:x-content:impl:processResources', task ':libs:x-content:impl:classes', task ':libs:x-content:impl:jar', task ':libs:x-content:generateImplProviderManifest', task ':libs:x-content:generateImplProviderImpl', task ':libs:x-content:processResources', task ':libs:x-content:classes', task ':libs:x-content:jar', task ':modules:transport-netty4:processResources', task ':modules:transport-netty4:classes', task ':modules:transport-netty4:jar', task ':test:framework:processResources', task ':test:framework:classes', task ':test:framework:jar', task ':test:immutable-collections-patch:compileJava', task ':test:immutable-collections-patch:processResources', task ':test:immutable-collections-patch:classes', task ':test:immutable-collections-patch:generatePatch', task ':libs:native:native-libraries:extractLibs', task ':libs:simdvec:test']
+Tasks that were excluded: []
+=======================================
+Elasticsearch Build Hamster says Hello!
+  Gradle Version        : 8.11.1
+  OS Info               : Mac OS X 15.2 (aarch64)
+  JDK Version           : 21.0.5+9-LTS-239 (Oracle)
+  JAVA_HOME             : /Users/rene/.sdkman/candidates/java/21.0.5-oracle
+  Random Testing Seed   : 7B469FBE8B6D0C65
+  In FIPS 140 mode      : false
+=======================================
+Ignoring listeners of task graph ready event, as this build (:) has already executed work.
+Resolve mutations for :libs:simdvec:cleanTest (Thread[#1322,Execution worker,5,main]) started.
+work action Unknown value (Thread[#1392,included builds Thread 2,5,main]) started.
+work action Unknown value (Thread[#1330,Execution worker Thread 9,5,main]) started.
+:libs:simdvec:cleanTest (Thread[#1322,Execution worker,5,main]) started.
+destroyer locations for task group 0 (Thread[#1332,Execution worker Thread 11,5,main]) started.
+Resolve mutations for :libs:logging:compileJava (Thread[#1332,Execution worker Thread 11,5,main]) started.
+Resolve mutations for :libs:simdvec:processResources (Thread[#1326,Execution worker Thread 5,5,main]) started.
+Resolve mutations for :libs:simdvec:processMain22Resources (Thread[#1329,Execution worker Thread 8,5,main]) started.
+Resolve mutations for :libs:simdvec:processMain21Resources (Thread[#1324,Execution worker Thread 3,5,main]) started.
+Resolve mutations for :libs:grok:compileJava (Thread[#1327,Execution worker Thread 6,5,main]) started.
+:libs:simdvec:processResources (Thread[#1326,Execution worker Thread 5,5,main]) started.
+:libs:simdvec:processMain21Resources (Thread[#1324,Execution worker Thread 3,5,main]) started.
+:libs:simdvec:processMain22Resources (Thread[#1329,Execution worker Thread 8,5,main]) started.
+Resolve mutations for :libs:geo:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started.
+Resolve mutations for :libs:secure-sm:compileJava (Thread[#1332,Execution worker Thread 11,5,main]) started.
+Resolve mutations for :libs:entitlement:bridge:compileJava (Thread[#1325,Execution worker Thread 4,5,main]) started.
+Resolve mutations for :server:generateModulesList (Thread[#1392,included builds Thread 2,5,main]) started.
+:libs:secure-sm:compileJava (Thread[#1332,Execution worker Thread 11,5,main]) started.
+Resolve mutations for :server:processResources (Thread[#1330,Execution worker Thread 9,5,main]) started.
+:libs:logging:compileJava (Thread[#1323,Execution worker Thread 2,5,main]) started.
+Resolve mutations for :libs:plugin-api:compileJava (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:server:processResources (Thread[#1330,Execution worker Thread 9,5,main]) started.
+:server:generateModulesList (Thread[#1392,included builds Thread 2,5,main]) started.
+:libs:entitlement:bridge:compileJava (Thread[#1325,Execution worker Thread 4,5,main]) started.
+:libs:geo:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:libs:plugin-api:compileJava (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:libs:grok:compileJava (Thread[#1327,Execution worker Thread 6,5,main]) started.
+
+> Task :libs:simdvec:processMain22Resources NO-SOURCE
+Skipping task ':libs:simdvec:processMain22Resources' as it has no source files and no previous output files.
+
+> Task :libs:simdvec:processMain21Resources NO-SOURCE
+Skipping task ':libs:simdvec:processMain21Resources' as it has no source files and no previous output files.
+
+> Task :server:generateModulesList UP-TO-DATE
+Caching disabled for task ':server:generateModulesList' because:
+  Build cache is disabled
+  Gradle would require more information to cache this task
+Skipping task ':server:generateModulesList' as it is up-to-date.
+
+> Task :libs:simdvec:processResources NO-SOURCE
+Skipping task ':libs:simdvec:processResources' as it has no source files and no previous output files.
+Resolve mutations for :client:rest:processResources (Thread[#1392,included builds Thread 2,5,main]) started.
+
+> Task :libs:grok:compileJava UP-TO-DATE
+Caching disabled for task ':libs:grok:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:grok:compileJava' as it is up-to-date.
+:client:rest:processResources (Thread[#1392,included builds Thread 2,5,main]) started.
+Resolve mutations for :libs:cli:processResources (Thread[#1326,Execution worker Thread 5,5,main]) started.
+Resolve mutations for :client:rest:compileJava (Thread[#1324,Execution worker Thread 3,5,main]) started.
+
+> Task :libs:plugin-api:compileJava UP-TO-DATE
+Caching disabled for task ':libs:plugin-api:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:plugin-api:compileJava' as it is up-to-date.
+No compile result for :libs:plugin-api:compileJava
+No compile result for :libs:plugin-api:compileJava
+Resolve mutations for :server:generatePluginsList (Thread[#1329,Execution worker Thread 8,5,main]) started.
+
+> Task :libs:secure-sm:compileJava UP-TO-DATE
+Caching disabled for task ':libs:secure-sm:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:secure-sm:compileJava' as it is up-to-date.
+No compile result for :libs:secure-sm:compileJava
+No compile result for :libs:secure-sm:compileJava
+No compile result for :libs:plugin-api:compileJava
+No compile result for :libs:plugin-api:compileJava
+:libs:cli:processResources (Thread[#1326,Execution worker Thread 5,5,main]) started.
+
+> Task :libs:geo:compileJava UP-TO-DATE
+Caching disabled for task ':libs:geo:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:geo:compileJava' as it is up-to-date.
+
+> Task :libs:entitlement:bridge:compileJava UP-TO-DATE
+Caching disabled for task ':libs:entitlement:bridge:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:entitlement:bridge:compileJava' as it is up-to-date.
+No compile result for :libs:geo:compileJava
+No compile result for :libs:geo:compileJava
+No compile result for :libs:grok:compileJava
+No compile result for :libs:grok:compileJava
+
+> Task :server:processResources UP-TO-DATE
+Caching disabled for task ':server:processResources' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':server:processResources' as it is up-to-date.
+No compile result for :libs:grok:compileJava
+work action resolve main (project :libs:plugin-api) (Thread[#1330,Execution worker Thread 9,5,main]) started.
+No compile result for :libs:geo:compileJava
+No compile result for :libs:geo:compileJava
+No compile result for :libs:entitlement:bridge:compileJava
+No compile result for :libs:entitlement:bridge:compileJava
+No compile result for :libs:entitlement:bridge:compileJava
+No compile result for :libs:entitlement:bridge:compileJava
+
+> Task :libs:logging:compileJava UP-TO-DATE
+Caching disabled for task ':libs:logging:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:logging:compileJava' as it is up-to-date.
+work action resolve main (project :libs:entitlement:bridge) (Thread[#1325,Execution worker Thread 4,5,main]) started.
+work action resolve main (project :libs:plugin-api) (Thread[#1331,Execution worker Thread 10,5,main]) started.
+No compile result for :libs:secure-sm:compileJava
+:server:generatePluginsList (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:client:rest:compileJava (Thread[#1324,Execution worker Thread 3,5,main]) started.
+No compile result for :libs:secure-sm:compileJava
+Resolve mutations for :libs:plugin-analysis-api:compileJava (Thread[#1331,Execution worker Thread 10,5,main]) started.
+work action resolve main (project :libs:entitlement:bridge) (Thread[#1325,Execution worker Thread 4,5,main]) started.
+No compile result for :libs:logging:compileJava
+work action resolve main (project :libs:geo) (Thread[#1328,Execution worker Thread 7,5,main]) started.
+Resolve mutations for :libs:core:processResources (Thread[#1330,Execution worker Thread 9,5,main]) started.
+No compile result for :libs:grok:compileJava
+work action resolve main (project :libs:grok) (Thread[#1330,Execution worker Thread 9,5,main]) started.
+work action resolve main (project :libs:grok) (Thread[#1327,Execution worker Thread 6,5,main]) started.
+:libs:plugin-analysis-api:compileJava (Thread[#1331,Execution worker Thread 10,5,main]) started.
+Resolve mutations for :libs:entitlement:asm-provider:processResources (Thread[#1327,Execution worker Thread 6,5,main]) started.
+work action resolve main (project :libs:secure-sm) (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:libs:entitlement:asm-provider:processResources (Thread[#1327,Execution worker Thread 6,5,main]) started.
+No compile result for :libs:logging:compileJava
+work action resolve main (project :libs:geo) (Thread[#1325,Execution worker Thread 4,5,main]) started.
+work action resolve main (project :libs:secure-sm) (Thread[#1332,Execution worker Thread 11,5,main]) started.
+Resolve mutations for :libs:entitlement:bridge:processResources (Thread[#1325,Execution worker Thread 4,5,main]) started.
+No compile result for :libs:logging:compileJava
+No compile result for :libs:logging:compileJava
+:libs:entitlement:bridge:processResources (Thread[#1325,Execution worker Thread 4,5,main]) started.
+work action resolve main (project :libs:logging) (Thread[#1323,Execution worker Thread 2,5,main]) started.
+Resolve mutations for :libs:entitlement:processResources (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:libs:entitlement:processResources (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :libs:cli:processResources NO-SOURCE
+Skipping task ':libs:cli:processResources' as it has no source files and no previous output files.
+:libs:core:processResources (Thread[#1330,Execution worker Thread 9,5,main]) started.
+Resolve mutations for :libs:entitlement:processMain23Resources (Thread[#1326,Execution worker Thread 5,5,main]) started.
+
+> Task :server:generatePluginsList UP-TO-DATE
+Caching disabled for task ':server:generatePluginsList' because:
+  Build cache is disabled
+  Gradle would require more information to cache this task
+Skipping task ':server:generatePluginsList' as it is up-to-date.
+Resolve mutations for :libs:geo:processResources (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:libs:entitlement:processMain23Resources (Thread[#1326,Execution worker Thread 5,5,main]) started.
+work action resolve main (project :libs:logging) (Thread[#1323,Execution worker Thread 2,5,main]) started.
+Resolve mutations for :libs:core:compileJava (Thread[#1323,Execution worker Thread 2,5,main]) started.
+Resolve mutations for :libs:entitlement:bridge:processMain23Resources (Thread[#1332,Execution worker Thread 11,5,main]) started.
+
+> Task :libs:entitlement:processResources NO-SOURCE
+Skipping task ':libs:entitlement:processResources' as it has no source files and no previous output files.
+Resolve mutations for :libs:grok:processResources (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :libs:entitlement:bridge:processResources NO-SOURCE
+Skipping task ':libs:entitlement:bridge:processResources' as it has no source files and no previous output files.
+
+> Task :libs:plugin-analysis-api:compileJava UP-TO-DATE
+Caching disabled for task ':libs:plugin-analysis-api:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:plugin-analysis-api:compileJava' as it is up-to-date.
+:libs:grok:processResources (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :libs:entitlement:processMain23Resources NO-SOURCE
+Skipping task ':libs:entitlement:processMain23Resources' as it has no source files and no previous output files.
+:libs:geo:processResources (Thread[#1329,Execution worker Thread 8,5,main]) started.
+
+> Task :client:rest:processResources UP-TO-DATE
+Caching disabled for task ':client:rest:processResources' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':client:rest:processResources' as it is up-to-date.
+:libs:core:compileJava (Thread[#1323,Execution worker Thread 2,5,main]) started.
+
+> Task :client:rest:compileJava UP-TO-DATE
+Caching disabled for task ':client:rest:compileJava' because:
+  Build cache is disabled
+Skipping task ':client:rest:compileJava' as it is up-to-date.
+No compile result for :client:rest:compileJava
+No compile result for :libs:plugin-analysis-api:compileJava
+Resolve mutations for :libs:entitlement:bridge:classes (Thread[#1325,Execution worker Thread 4,5,main]) started.
+:libs:entitlement:bridge:processMain23Resources (Thread[#1332,Execution worker Thread 11,5,main]) started.
+No compile result for :libs:plugin-analysis-api:compileJava
+Resolve mutations for :libs:lz4:processResources (Thread[#1326,Execution worker Thread 5,5,main]) started.
+Resolve mutations for :libs:logging:processResources (Thread[#1392,included builds Thread 2,5,main]) started.
+No compile result for :client:rest:compileJava
+:libs:logging:processResources (Thread[#1392,included builds Thread 2,5,main]) started.
+
+> Task :libs:geo:processResources NO-SOURCE
+Skipping task ':libs:geo:processResources' as it has no source files and no previous output files.
+
+> Task :libs:core:processResources NO-SOURCE
+Skipping task ':libs:core:processResources' as it has no source files and no previous output files.
+No compile result for :client:rest:compileJava
+:libs:lz4:processResources (Thread[#1326,Execution worker Thread 5,5,main]) started.
+No compile result for :libs:plugin-analysis-api:compileJava
+
+> Task :libs:entitlement:asm-provider:processResources UP-TO-DATE
+Caching disabled for task ':libs:entitlement:asm-provider:processResources' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:entitlement:asm-provider:processResources' as it is up-to-date.
+
+> Task :libs:grok:processResources UP-TO-DATE
+Caching disabled for task ':libs:grok:processResources' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:grok:processResources' as it is up-to-date.
+:libs:entitlement:bridge:classes (Thread[#1325,Execution worker Thread 4,5,main]) started.
+Resolve mutations for :libs:native:processMain22Resources (Thread[#1327,Execution worker Thread 6,5,main]) started.
+Resolve mutations for :libs:grok:classes (Thread[#1328,Execution worker Thread 7,5,main]) started.
+No compile result for :libs:plugin-analysis-api:compileJava
+
+> Task :libs:entitlement:bridge:processMain23Resources NO-SOURCE
+Skipping task ':libs:entitlement:bridge:processMain23Resources' as it has no source files and no previous output files.
+:libs:grok:classes (Thread[#1328,Execution worker Thread 7,5,main]) started.
+work action resolve main (project :libs:plugin-analysis-api) (Thread[#1331,Execution worker Thread 10,5,main]) started.
+work action resolve main (project :libs:plugin-analysis-api) (Thread[#1332,Execution worker Thread 11,5,main]) started.
+Resolve mutations for :libs:plugin-analysis-api:processResources (Thread[#1331,Execution worker Thread 10,5,main]) started.
+Resolve mutations for :libs:plugin-api:processResources (Thread[#1332,Execution worker Thread 11,5,main]) started.
+
+> Task :libs:logging:processResources NO-SOURCE
+Skipping task ':libs:logging:processResources' as it has no source files and no previous output files.
+No compile result for :client:rest:compileJava
+:libs:plugin-analysis-api:processResources (Thread[#1331,Execution worker Thread 10,5,main]) started.
+Resolve mutations for :client:rest:classes (Thread[#1324,Execution worker Thread 3,5,main]) started.
+Resolve mutations for :libs:logging:classes (Thread[#1392,included builds Thread 2,5,main]) started.
+:client:rest:classes (Thread[#1324,Execution worker Thread 3,5,main]) started.
+Resolve mutations for :libs:native:processResources (Thread[#1330,Execution worker Thread 9,5,main]) started.
+:libs:logging:classes (Thread[#1392,included builds Thread 2,5,main]) started.
+Resolve mutations for :libs:geo:classes (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:libs:native:processResources (Thread[#1330,Execution worker Thread 9,5,main]) started.
+:libs:plugin-api:processResources (Thread[#1332,Execution worker Thread 11,5,main]) started.
+
+> Task :libs:plugin-analysis-api:processResources NO-SOURCE
+Skipping task ':libs:plugin-analysis-api:processResources' as it has no source files and no previous output files.
+:libs:native:processMain22Resources (Thread[#1327,Execution worker Thread 6,5,main]) started.
+Resolve mutations for :libs:plugin-analysis-api:classes (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:libs:plugin-analysis-api:classes (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :libs:grok:classes UP-TO-DATE
+Skipping task ':libs:grok:classes' as it has no actions.
+
+> Task :libs:core:compileJava UP-TO-DATE
+Caching disabled for task ':libs:core:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:core:compileJava' as it is up-to-date.
+No compile result for :libs:core:compileJava
+:libs:geo:classes (Thread[#1329,Execution worker Thread 8,5,main]) started.
+
+> Task :libs:logging:classes UP-TO-DATE
+Skipping task ':libs:logging:classes' as it has no actions.
+
+> Task :client:rest:classes UP-TO-DATE
+Skipping task ':client:rest:classes' as it has no actions.
+
+> Task :libs:geo:classes UP-TO-DATE
+Skipping task ':libs:geo:classes' as it has no actions.
+Resolve mutations for :client:rest:jar (Thread[#1324,Execution worker Thread 3,5,main]) started.
+Resolve mutations for :libs:logging:jar (Thread[#1392,included builds Thread 2,5,main]) started.
+No compile result for :libs:core:compileJava
+
+> Task :libs:native:processMain22Resources NO-SOURCE
+Skipping task ':libs:native:processMain22Resources' as it has no source files and no previous output files.
+No compile result for :libs:core:compileJava
+:client:rest:jar (Thread[#1324,Execution worker Thread 3,5,main]) started.
+Resolve mutations for :libs:grok:jar (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :libs:native:processResources NO-SOURCE
+Skipping task ':libs:native:processResources' as it has no source files and no previous output files.
+work action resolve main (project :client:rest) (Thread[#1330,Execution worker Thread 9,5,main]) started.
+Resolve mutations for :libs:simdvec:processTestResources (Thread[#1330,Execution worker Thread 9,5,main]) started.
+
+> Task :libs:entitlement:bridge:classes UP-TO-DATE
+Skipping task ':libs:entitlement:bridge:classes' as it has no actions.
+
+> Task :libs:plugin-analysis-api:classes UP-TO-DATE
+Skipping task ':libs:plugin-analysis-api:classes' as it has no actions.
+
+> Task :libs:lz4:processResources NO-SOURCE
+Skipping task ':libs:lz4:processResources' as it has no source files and no previous output files.
+:libs:simdvec:processTestResources (Thread[#1330,Execution worker Thread 9,5,main]) started.
+Resolve mutations for :libs:entitlement:bridge:compileMain23Java (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:libs:grok:jar (Thread[#1328,Execution worker Thread 7,5,main]) started.
+Resolve mutations for :libs:ssl-config:processResources (Thread[#1326,Execution worker Thread 5,5,main]) started.
+Resolve mutations for :libs:secure-sm:processResources (Thread[#1327,Execution worker Thread 6,5,main]) started.
+:libs:logging:jar (Thread[#1392,included builds Thread 2,5,main]) started.
+
+> Task :libs:simdvec:processTestResources NO-SOURCE
+Skipping task ':libs:simdvec:processTestResources' as it has no source files and no previous output files.
+No compile result for :libs:core:compileJava
+
+> Task :libs:plugin-api:processResources NO-SOURCE
+Skipping task ':libs:plugin-api:processResources' as it has no source files and no previous output files.
+Resolve mutations for :libs:geo:jar (Thread[#1329,Execution worker Thread 8,5,main]) started.
+work action resolve main (project :libs:core) (Thread[#1332,Execution worker Thread 11,5,main]) started.
+work action resolve main (project :libs:core) (Thread[#1323,Execution worker Thread 2,5,main]) started.
+Resolve mutations for :libs:tdigest:processResources (Thread[#1330,Execution worker Thread 9,5,main]) started.
+:libs:secure-sm:processResources (Thread[#1327,Execution worker Thread 6,5,main]) started.
+:libs:ssl-config:processResources (Thread[#1326,Execution worker Thread 5,5,main]) started.
+:libs:entitlement:bridge:compileMain23Java (Thread[#1331,Execution worker Thread 10,5,main]) started.
+Resolve mutations for :libs:plugin-analysis-api:jar (Thread[#1325,Execution worker Thread 4,5,main]) started.
+Resolve mutations for :libs:lz4:compileJava (Thread[#1325,Execution worker Thread 4,5,main]) started.
+Resolve mutations for :libs:x-content:compileJava (Thread[#1330,Execution worker Thread 9,5,main]) started.
+Resolve mutations for :libs:cli:compileJava (Thread[#1323,Execution worker Thread 2,5,main]) started.
+Resolve mutations for :libs:native:compileJava (Thread[#1329,Execution worker Thread 8,5,main]) started.
+Resolve mutations for :libs:core:classes (Thread[#1332,Execution worker Thread 11,5,main]) started.
+Resolve mutations for :libs:tdigest:compileJava (Thread[#1332,Execution worker Thread 11,5,main]) started.
+:libs:lz4:compileJava (Thread[#1325,Execution worker Thread 4,5,main]) started.
+
+> Task :libs:simdvec:cleanTest
+Caching disabled for task ':libs:simdvec:cleanTest' because:
+  Build cache is disabled
+Task ':libs:simdvec:cleanTest' is not up-to-date because:
+  Task has not declared any outputs despite executing actions.
+:libs:cli:compileJava (Thread[#1323,Execution worker Thread 2,5,main]) started.
+
+> Task :libs:ssl-config:processResources NO-SOURCE
+Skipping task ':libs:ssl-config:processResources' as it has no source files and no previous output files.
+:libs:tdigest:compileJava (Thread[#1332,Execution worker Thread 11,5,main]) started.
+:libs:native:compileJava (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:libs:x-content:compileJava (Thread[#1330,Execution worker Thread 9,5,main]) started.
+
+> Task :libs:secure-sm:processResources NO-SOURCE
+Skipping task ':libs:secure-sm:processResources' as it has no source files and no previous output files.
+:libs:geo:jar (Thread[#1326,Execution worker Thread 5,5,main]) started.
+:libs:core:classes (Thread[#1322,Execution worker,5,main]) started.
+:libs:plugin-analysis-api:jar (Thread[#1327,Execution worker Thread 6,5,main]) started.
+
+> Task :libs:entitlement:bridge:compileMain23Java UP-TO-DATE
+Caching disabled for task ':libs:entitlement:bridge:compileMain23Java' because:
+  Build cache is disabled
+Skipping task ':libs:entitlement:bridge:compileMain23Java' as it is up-to-date.
+
+> Task :libs:lz4:compileJava UP-TO-DATE
+Caching disabled for task ':libs:lz4:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:lz4:compileJava' as it is up-to-date.
+No compile result for :libs:lz4:compileJava
+No compile result for :libs:lz4:compileJava
+No compile result for :libs:lz4:compileJava
+No compile result for :libs:entitlement:bridge:compileMain23Java
+
+> Task :libs:cli:compileJava UP-TO-DATE
+Caching disabled for task ':libs:cli:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:cli:compileJava' as it is up-to-date.
+No compile result for :libs:cli:compileJava
+No compile result for :libs:cli:compileJava
+No compile result for :libs:cli:compileJava
+No compile result for :libs:cli:compileJava
+work action resolve main (project :libs:cli) (Thread[#1323,Execution worker Thread 2,5,main]) started.
+No compile result for :libs:entitlement:bridge:compileMain23Java
+No compile result for :libs:entitlement:bridge:compileMain23Java
+No compile result for :libs:lz4:compileJava
+work action resolve main (project :libs:lz4) (Thread[#1325,Execution worker Thread 4,5,main]) started.
+No compile result for :libs:entitlement:bridge:compileMain23Java
+
+> Task :libs:native:compileJava UP-TO-DATE
+Caching disabled for task ':libs:native:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:native:compileJava' as it is up-to-date.
+No compile result for :libs:native:compileJava
+Resolve mutations for :libs:cli:classes (Thread[#1331,Execution worker Thread 10,5,main]) started.
+work action resolve main (project :libs:cli) (Thread[#1323,Execution worker Thread 2,5,main]) started.
+No compile result for :libs:native:compileJava
+work action resolve main (project :libs:lz4) (Thread[#1325,Execution worker Thread 4,5,main]) started.
+
+> Task :libs:core:classes UP-TO-DATE
+Skipping task ':libs:core:classes' as it has no actions.
+No compile result for :libs:native:compileJava
+:libs:cli:classes (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :libs:tdigest:compileJava UP-TO-DATE
+Caching disabled for task ':libs:tdigest:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:tdigest:compileJava' as it is up-to-date.
+Resolve mutations for :libs:entitlement:bridge:main23Classes (Thread[#1323,Execution worker Thread 2,5,main]) started.
+No compile result for :libs:tdigest:compileJava
+:libs:entitlement:bridge:main23Classes (Thread[#1323,Execution worker Thread 2,5,main]) started.
+
+> Task :libs:x-content:compileJava UP-TO-DATE
+Caching disabled for task ':libs:x-content:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:x-content:compileJava' as it is up-to-date.
+No compile result for :libs:native:compileJava
+
+> Task :libs:entitlement:bridge:main23Classes UP-TO-DATE
+Skipping task ':libs:entitlement:bridge:main23Classes' as it has no actions.
+Resolve mutations for :libs:core:jar (Thread[#1322,Execution worker,5,main]) started.
+work action resolve main (project :libs:native) (Thread[#1323,Execution worker Thread 2,5,main]) started.
+Resolve mutations for :libs:lz4:classes (Thread[#1325,Execution worker Thread 4,5,main]) started.
+Resolve mutations for :libs:entitlement:bridge:java23Jar (Thread[#1323,Execution worker Thread 2,5,main]) started.
+:libs:lz4:classes (Thread[#1325,Execution worker Thread 4,5,main]) started.
+work action resolve main (project :libs:native) (Thread[#1329,Execution worker Thread 8,5,main]) started.
+No compile result for :libs:x-content:compileJava
+:libs:entitlement:bridge:java23Jar (Thread[#1323,Execution worker Thread 2,5,main]) started.
+No compile result for :libs:tdigest:compileJava
+No compile result for :libs:tdigest:compileJava
+No compile result for :libs:tdigest:compileJava
+No compile result for :libs:x-content:compileJava
+Resolve mutations for :libs:simdvec:compileJava (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:libs:core:jar (Thread[#1322,Execution worker,5,main]) started.
+
+> Task :libs:lz4:classes UP-TO-DATE
+Skipping task ':libs:lz4:classes' as it has no actions.
+No compile result for :libs:x-content:compileJava
+work action resolve main (project :libs:tdigest) (Thread[#1332,Execution worker Thread 11,5,main]) started.
+
+> Task :libs:cli:classes UP-TO-DATE
+Skipping task ':libs:cli:classes' as it has no actions.
+Resolve mutations for :libs:lz4:jar (Thread[#1332,Execution worker Thread 11,5,main]) started.
+No compile result for :libs:x-content:compileJava
+work action resolve main (project :libs:x-content) (Thread[#1330,Execution worker Thread 9,5,main]) started.
+work action resolve main (project :libs:tdigest) (Thread[#1325,Execution worker Thread 4,5,main]) started.
+:libs:lz4:jar (Thread[#1332,Execution worker Thread 11,5,main]) started.
+Resolve mutations for :libs:native:classes (Thread[#1325,Execution worker Thread 4,5,main]) started.
+:libs:simdvec:compileJava (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:libs:native:classes (Thread[#1325,Execution worker Thread 4,5,main]) started.
+work action resolve main (project :libs:x-content) (Thread[#1330,Execution worker Thread 9,5,main]) started.
+Resolve mutations for :libs:cli:jar (Thread[#1331,Execution worker Thread 10,5,main]) started.
+Resolve mutations for :libs:entitlement:compileJava (Thread[#1330,Execution worker Thread 9,5,main]) started.
+:libs:cli:jar (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :libs:native:classes UP-TO-DATE
+Skipping task ':libs:native:classes' as it has no actions.
+Resolve mutations for :libs:native:compileMain22Java (Thread[#1325,Execution worker Thread 4,5,main]) started.
+:libs:entitlement:compileJava (Thread[#1330,Execution worker Thread 9,5,main]) started.
+
+> Task :libs:entitlement:bridge:java23Jar UP-TO-DATE
+Caching disabled for task ':libs:entitlement:bridge:java23Jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:entitlement:bridge:java23Jar' as it is up-to-date.
+work action resolve elasticsearch-entitlement-bridge-9.0.0-SNAPSHOT.jar (project :libs:entitlement:bridge) (Thread[#1323,Execution worker Thread 2,5,main]) started.
+:libs:native:compileMain22Java (Thread[#1325,Execution worker Thread 4,5,main]) started.
+Resolve mutations for :libs:plugin-api:classes (Thread[#1323,Execution worker Thread 2,5,main]) started.
+:libs:plugin-api:classes (Thread[#1323,Execution worker Thread 2,5,main]) started.
+
+> Task :libs:plugin-api:classes UP-TO-DATE
+Skipping task ':libs:plugin-api:classes' as it has no actions.
+Resolve mutations for :libs:plugin-api:jar (Thread[#1323,Execution worker Thread 2,5,main]) started.
+:libs:plugin-api:jar (Thread[#1323,Execution worker Thread 2,5,main]) started.
+
+> Task :libs:simdvec:compileJava UP-TO-DATE
+Caching disabled for task ':libs:simdvec:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:simdvec:compileJava' as it is up-to-date.
+No compile result for :libs:simdvec:compileJava
+No compile result for :libs:simdvec:compileJava
+No compile result for :libs:simdvec:compileJava
+No compile result for :libs:simdvec:compileJava
+Resolve mutations for :libs:simdvec:classes (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:libs:simdvec:classes (Thread[#1329,Execution worker Thread 8,5,main]) started.
+
+> Task :libs:entitlement:compileJava UP-TO-DATE
+Caching disabled for task ':libs:entitlement:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:entitlement:compileJava' as it is up-to-date.
+No compile result for :libs:entitlement:compileJava
+
+> Task :libs:native:compileMain22Java UP-TO-DATE
+Caching disabled for task ':libs:native:compileMain22Java' because:
+  Build cache is disabled
+Skipping task ':libs:native:compileMain22Java' as it is up-to-date.
+No compile result for :libs:entitlement:compileJava
+No compile result for :libs:native:compileMain22Java
+No compile result for :libs:native:compileMain22Java
+No compile result for :libs:entitlement:compileJava
+No compile result for :libs:native:compileMain22Java
+No compile result for :libs:native:compileMain22Java
+No compile result for :libs:entitlement:compileJava
+work action resolve main (project :libs:simdvec) (Thread[#1325,Execution worker Thread 4,5,main]) started.
+work action resolve main (project :libs:entitlement) (Thread[#1330,Execution worker Thread 9,5,main]) started.
+
+> Task :libs:simdvec:classes UP-TO-DATE
+Skipping task ':libs:simdvec:classes' as it has no actions.
+work action resolve main (project :libs:simdvec) (Thread[#1330,Execution worker Thread 9,5,main]) started.
+work action resolve main (project :libs:entitlement) (Thread[#1325,Execution worker Thread 4,5,main]) started.
+Resolve mutations for :libs:simdvec:compileMain21Java (Thread[#1329,Execution worker Thread 8,5,main]) started.
+Resolve mutations for :libs:native:main22Classes (Thread[#1330,Execution worker Thread 9,5,main]) started.
+Resolve mutations for :server:compileJava (Thread[#1325,Execution worker Thread 4,5,main]) started.
+Resolve mutations for :libs:entitlement:asm-provider:compileJava (Thread[#1330,Execution worker Thread 9,5,main]) started.
+:libs:simdvec:compileMain21Java (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:server:compileJava (Thread[#1325,Execution worker Thread 4,5,main]) started.
+:libs:entitlement:asm-provider:compileJava (Thread[#1330,Execution worker Thread 9,5,main]) started.
+
+> Task :client:rest:jar UP-TO-DATE
+Caching disabled for task ':client:rest:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':client:rest:jar' as it is up-to-date.
+:libs:native:main22Classes (Thread[#1324,Execution worker Thread 3,5,main]) started.
+
+> Task :libs:native:main22Classes UP-TO-DATE
+Skipping task ':libs:native:main22Classes' as it has no actions.
+Resolve mutations for :libs:native:jar (Thread[#1324,Execution worker Thread 3,5,main]) started.
+:libs:native:jar (Thread[#1324,Execution worker Thread 3,5,main]) started.
+
+> Task :libs:simdvec:compileMain21Java UP-TO-DATE
+Caching disabled for task ':libs:simdvec:compileMain21Java' because:
+  Build cache is disabled
+Skipping task ':libs:simdvec:compileMain21Java' as it is up-to-date.
+No compile result for :libs:simdvec:compileMain21Java
+No compile result for :libs:simdvec:compileMain21Java
+
+> Task :libs:entitlement:asm-provider:compileJava UP-TO-DATE
+Caching disabled for task ':libs:entitlement:asm-provider:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:entitlement:asm-provider:compileJava' as it is up-to-date.
+No compile result for :libs:simdvec:compileMain21Java
+No compile result for :libs:entitlement:asm-provider:compileJava
+No compile result for :libs:simdvec:compileMain21Java
+No compile result for :libs:entitlement:asm-provider:compileJava
+Resolve mutations for :libs:simdvec:main21Classes (Thread[#1329,Execution worker Thread 8,5,main]) started.
+No compile result for :libs:entitlement:asm-provider:compileJava
+No compile result for :libs:entitlement:asm-provider:compileJava
+:libs:simdvec:main21Classes (Thread[#1329,Execution worker Thread 8,5,main]) started.
+Resolve mutations for :libs:entitlement:asm-provider:classes (Thread[#1330,Execution worker Thread 9,5,main]) started.
+:libs:entitlement:asm-provider:classes (Thread[#1330,Execution worker Thread 9,5,main]) started.
+
+> Task :libs:simdvec:main21Classes UP-TO-DATE
+Skipping task ':libs:simdvec:main21Classes' as it has no actions.
+
+> Task :libs:entitlement:asm-provider:classes UP-TO-DATE
+Skipping task ':libs:entitlement:asm-provider:classes' as it has no actions.
+Resolve mutations for :libs:simdvec:compileMain22Java (Thread[#1329,Execution worker Thread 8,5,main]) started.
+Resolve mutations for :libs:entitlement:asm-provider:jar (Thread[#1330,Execution worker Thread 9,5,main]) started.
+:libs:entitlement:asm-provider:jar (Thread[#1330,Execution worker Thread 9,5,main]) started.
+:libs:simdvec:compileMain22Java (Thread[#1329,Execution worker Thread 8,5,main]) started.
+
+> Task :libs:simdvec:compileMain22Java UP-TO-DATE
+Caching disabled for task ':libs:simdvec:compileMain22Java' because:
+  Build cache is disabled
+Skipping task ':libs:simdvec:compileMain22Java' as it is up-to-date.
+No compile result for :libs:simdvec:compileMain22Java
+No compile result for :libs:simdvec:compileMain22Java
+No compile result for :libs:simdvec:compileMain22Java
+No compile result for :libs:simdvec:compileMain22Java
+Resolve mutations for :libs:simdvec:main22Classes (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:libs:simdvec:main22Classes (Thread[#1329,Execution worker Thread 8,5,main]) started.
+
+> Task :libs:simdvec:main22Classes UP-TO-DATE
+Skipping task ':libs:simdvec:main22Classes' as it has no actions.
+Resolve mutations for :libs:simdvec:jar (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:libs:simdvec:jar (Thread[#1329,Execution worker Thread 8,5,main]) started.
+
+> Task :libs:logging:jar UP-TO-DATE
+Caching disabled for task ':libs:logging:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:logging:jar' as it is up-to-date.
+Resolve mutations for :libs:secure-sm:classes (Thread[#1392,included builds Thread 2,5,main]) started.
+:libs:secure-sm:classes (Thread[#1392,included builds Thread 2,5,main]) started.
+
+> Task :libs:secure-sm:classes UP-TO-DATE
+Skipping task ':libs:secure-sm:classes' as it has no actions.
+Resolve mutations for :libs:secure-sm:jar (Thread[#1392,included builds Thread 2,5,main]) started.
+:libs:secure-sm:jar (Thread[#1392,included builds Thread 2,5,main]) started.
+
+> Task :server:compileJava UP-TO-DATE
+Caching disabled for task ':server:compileJava' because:
+  Build cache is disabled
+Skipping task ':server:compileJava' as it is up-to-date.
+No compile result for :server:compileJava
+No compile result for :server:compileJava
+No compile result for :server:compileJava
+No compile result for :server:compileJava
+Resolve mutations for :server:classes (Thread[#1325,Execution worker Thread 4,5,main]) started.
+:server:classes (Thread[#1325,Execution worker Thread 4,5,main]) started.
+
+> Task :server:classes UP-TO-DATE
+Skipping task ':server:classes' as it has no actions.
+
+> Task :libs:grok:jar UP-TO-DATE
+Caching disabled for task ':libs:grok:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:grok:jar' as it is up-to-date.
+Resolve mutations for :server:jar (Thread[#1325,Execution worker Thread 4,5,main]) started.
+Resolve mutations for :libs:ssl-config:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:server:jar (Thread[#1325,Execution worker Thread 4,5,main]) started.
+:libs:ssl-config:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :libs:ssl-config:compileJava UP-TO-DATE
+Caching disabled for task ':libs:ssl-config:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:ssl-config:compileJava' as it is up-to-date.
+No compile result for :libs:ssl-config:compileJava
+No compile result for :libs:ssl-config:compileJava
+No compile result for :libs:ssl-config:compileJava
+No compile result for :libs:ssl-config:compileJava
+work action resolve main (project :server) (Thread[#1328,Execution worker Thread 7,5,main]) started.
+work action resolve main (project :libs:ssl-config) (Thread[#1328,Execution worker Thread 7,5,main]) started.
+work action resolve main (project :server) (Thread[#1328,Execution worker Thread 7,5,main]) started.
+work action resolve main (project :libs:ssl-config) (Thread[#1328,Execution worker Thread 7,5,main]) started.
+Resolve mutations for :modules:transport-netty4:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:modules:transport-netty4:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :libs:geo:jar UP-TO-DATE
+Caching disabled for task ':libs:geo:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:geo:jar' as it is up-to-date.
+Resolve mutations for :libs:ssl-config:classes (Thread[#1326,Execution worker Thread 5,5,main]) started.
+:libs:ssl-config:classes (Thread[#1326,Execution worker Thread 5,5,main]) started.
+
+> Task :libs:ssl-config:classes UP-TO-DATE
+Skipping task ':libs:ssl-config:classes' as it has no actions.
+
+> Task :modules:transport-netty4:compileJava UP-TO-DATE
+Caching disabled for task ':modules:transport-netty4:compileJava' because:
+  Build cache is disabled
+Skipping task ':modules:transport-netty4:compileJava' as it is up-to-date.
+No compile result for :modules:transport-netty4:compileJava
+Resolve mutations for :libs:ssl-config:jar (Thread[#1326,Execution worker Thread 5,5,main]) started.
+No compile result for :modules:transport-netty4:compileJava
+No compile result for :modules:transport-netty4:compileJava
+No compile result for :modules:transport-netty4:compileJava
+:libs:ssl-config:jar (Thread[#1326,Execution worker Thread 5,5,main]) started.
+work action resolve main (project :modules:transport-netty4) (Thread[#1328,Execution worker Thread 7,5,main]) started.
+Resolve mutations for :test:framework:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:test:framework:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :test:framework:compileJava UP-TO-DATE
+Caching disabled for task ':test:framework:compileJava' because:
+  Build cache is disabled
+Skipping task ':test:framework:compileJava' as it is up-to-date.
+No compile result for :test:framework:compileJava
+No compile result for :test:framework:compileJava
+No compile result for :test:framework:compileJava
+No compile result for :test:framework:compileJava
+work action resolve main (project :test:framework) (Thread[#1328,Execution worker Thread 7,5,main]) started.
+Resolve mutations for :libs:simdvec:compileTestJava (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:libs:simdvec:compileTestJava (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :libs:plugin-analysis-api:jar UP-TO-DATE
+Caching disabled for task ':libs:plugin-analysis-api:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:plugin-analysis-api:jar' as it is up-to-date.
+:libs:tdigest:processResources (Thread[#1327,Execution worker Thread 6,5,main]) started.
+
+> Task :libs:tdigest:processResources NO-SOURCE
+Skipping task ':libs:tdigest:processResources' as it has no source files and no previous output files.
+Resolve mutations for :libs:tdigest:classes (Thread[#1327,Execution worker Thread 6,5,main]) started.
+:libs:tdigest:classes (Thread[#1327,Execution worker Thread 6,5,main]) started.
+
+> Task :libs:tdigest:classes UP-TO-DATE
+Skipping task ':libs:tdigest:classes' as it has no actions.
+Resolve mutations for :libs:tdigest:jar (Thread[#1327,Execution worker Thread 6,5,main]) started.
+:libs:tdigest:jar (Thread[#1327,Execution worker Thread 6,5,main]) started.
+
+> Task :libs:simdvec:compileTestJava UP-TO-DATE
+Caching disabled for task ':libs:simdvec:compileTestJava' because:
+  Build cache is disabled
+Skipping task ':libs:simdvec:compileTestJava' as it is up-to-date.
+No compile result for :libs:simdvec:compileTestJava
+No compile result for :libs:simdvec:compileTestJava
+No compile result for :libs:simdvec:compileTestJava
+No compile result for :libs:simdvec:compileTestJava
+Resolve mutations for :libs:simdvec:testClasses (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:libs:simdvec:testClasses (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :libs:simdvec:testClasses UP-TO-DATE
+Skipping task ':libs:simdvec:testClasses' as it has no actions.
+Resolve mutations for :libs:x-content:impl:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:libs:x-content:impl:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :libs:x-content:impl:compileJava UP-TO-DATE
+Caching disabled for task ':libs:x-content:impl:compileJava' because:
+  Build cache is disabled
+Skipping task ':libs:x-content:impl:compileJava' as it is up-to-date.
+No compile result for :libs:x-content:impl:compileJava
+No compile result for :libs:x-content:impl:compileJava
+No compile result for :libs:x-content:impl:compileJava
+No compile result for :libs:x-content:impl:compileJava
+Resolve mutations for :libs:x-content:impl:processResources (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:libs:x-content:impl:processResources (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :libs:x-content:impl:processResources UP-TO-DATE
+Caching disabled for task ':libs:x-content:impl:processResources' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:x-content:impl:processResources' as it is up-to-date.
+Resolve mutations for :libs:x-content:impl:classes (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:libs:x-content:impl:classes (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :libs:x-content:impl:classes UP-TO-DATE
+Skipping task ':libs:x-content:impl:classes' as it has no actions.
+Resolve mutations for :libs:x-content:impl:jar (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:libs:x-content:impl:jar (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :libs:lz4:jar UP-TO-DATE
+Caching disabled for task ':libs:lz4:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:lz4:jar' as it is up-to-date.
+Resolve mutations for :libs:x-content:processResources (Thread[#1332,Execution worker Thread 11,5,main]) started.
+:libs:x-content:processResources (Thread[#1332,Execution worker Thread 11,5,main]) started.
+
+> Task :libs:x-content:processResources NO-SOURCE
+Skipping task ':libs:x-content:processResources' as it has no source files and no previous output files.
+Resolve mutations for :modules:transport-netty4:processResources (Thread[#1332,Execution worker Thread 11,5,main]) started.
+:modules:transport-netty4:processResources (Thread[#1332,Execution worker Thread 11,5,main]) started.
+
+> Task :modules:transport-netty4:processResources NO-SOURCE
+Skipping task ':modules:transport-netty4:processResources' as it has no source files and no previous output files.
+Resolve mutations for :modules:transport-netty4:classes (Thread[#1332,Execution worker Thread 11,5,main]) started.
+:modules:transport-netty4:classes (Thread[#1332,Execution worker Thread 11,5,main]) started.
+
+> Task :modules:transport-netty4:classes UP-TO-DATE
+Skipping task ':modules:transport-netty4:classes' as it has no actions.
+Resolve mutations for :modules:transport-netty4:jar (Thread[#1332,Execution worker Thread 11,5,main]) started.
+:modules:transport-netty4:jar (Thread[#1332,Execution worker Thread 11,5,main]) started.
+
+> Task :libs:core:jar UP-TO-DATE
+Caching disabled for task ':libs:core:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:core:jar' as it is up-to-date.
+Resolve mutations for :test:framework:processResources (Thread[#1322,Execution worker,5,main]) started.
+:test:framework:processResources (Thread[#1322,Execution worker,5,main]) started.
+
+> Task :test:framework:processResources UP-TO-DATE
+Caching disabled for task ':test:framework:processResources' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':test:framework:processResources' as it is up-to-date.
+Resolve mutations for :test:framework:classes (Thread[#1322,Execution worker,5,main]) started.
+:test:framework:classes (Thread[#1322,Execution worker,5,main]) started.
+
+> Task :test:framework:classes UP-TO-DATE
+Skipping task ':test:framework:classes' as it has no actions.
+Resolve mutations for :test:framework:jar (Thread[#1322,Execution worker,5,main]) started.
+:test:framework:jar (Thread[#1322,Execution worker,5,main]) started.
+
+> Task :libs:cli:jar UP-TO-DATE
+Caching disabled for task ':libs:cli:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:cli:jar' as it is up-to-date.
+Resolve mutations for :test:immutable-collections-patch:compileJava (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:test:immutable-collections-patch:compileJava (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :test:immutable-collections-patch:compileJava UP-TO-DATE
+Caching disabled for task ':test:immutable-collections-patch:compileJava' because:
+  Build cache is disabled
+Skipping task ':test:immutable-collections-patch:compileJava' as it is up-to-date.
+No compile result for :test:immutable-collections-patch:compileJava
+No compile result for :test:immutable-collections-patch:compileJava
+No compile result for :test:immutable-collections-patch:compileJava
+No compile result for :test:immutable-collections-patch:compileJava
+Resolve mutations for :test:immutable-collections-patch:processResources (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:test:immutable-collections-patch:processResources (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :test:immutable-collections-patch:processResources NO-SOURCE
+Skipping task ':test:immutable-collections-patch:processResources' as it has no source files and no previous output files.
+Resolve mutations for :test:immutable-collections-patch:classes (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:test:immutable-collections-patch:classes (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :test:immutable-collections-patch:classes UP-TO-DATE
+Skipping task ':test:immutable-collections-patch:classes' as it has no actions.
+Resolve mutations for :test:immutable-collections-patch:generatePatch (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:test:immutable-collections-patch:generatePatch (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :test:immutable-collections-patch:generatePatch UP-TO-DATE
+Caching disabled for task ':test:immutable-collections-patch:generatePatch' because:
+  Build cache is disabled
+  Gradle would require more information to cache this task
+Skipping task ':test:immutable-collections-patch:generatePatch' as it is up-to-date.
+Resolve mutations for :libs:native:native-libraries:extractLibs (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:libs:native:native-libraries:extractLibs (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :libs:plugin-api:jar UP-TO-DATE
+Caching disabled for task ':libs:plugin-api:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:plugin-api:jar' as it is up-to-date.
+work action resolve jdk-patches (project :test:immutable-collections-patch) (Thread[#1323,Execution worker Thread 2,5,main]) started.
+work action resolve elasticsearch-grok-9.0.0-SNAPSHOT.jar (project :libs:grok) (Thread[#1323,Execution worker Thread 2,5,main]) started.
+work action resolve elasticsearch-plugin-api-9.0.0-SNAPSHOT.jar (project :libs:plugin-api) (Thread[#1323,Execution worker Thread 2,5,main]) started.
+work action resolve elasticsearch-geo-9.0.0-SNAPSHOT.jar (project :libs:geo) (Thread[#1323,Execution worker Thread 2,5,main]) started.
+work action resolve elasticsearch-rest-client-9.0.0-SNAPSHOT.jar (project :client:rest) (Thread[#1323,Execution worker Thread 2,5,main]) started.
+work action resolve elasticsearch-core-9.0.0-SNAPSHOT.jar (project :libs:core) (Thread[#1323,Execution worker Thread 2,5,main]) started.
+work action resolve elasticsearch-lz4-9.0.0-SNAPSHOT.jar (project :libs:lz4) (Thread[#1323,Execution worker Thread 2,5,main]) started.
+work action resolve elasticsearch-cli-9.0.0-SNAPSHOT.jar (project :libs:cli) (Thread[#1323,Execution worker Thread 2,5,main]) started.
+work action resolve elasticsearch-plugin-analysis-api-9.0.0-SNAPSHOT.jar (project :libs:plugin-analysis-api) (Thread[#1323,Execution worker Thread 2,5,main]) started.
+work action resolve elasticsearch-logging-9.0.0-SNAPSHOT.jar (project :libs:logging) (Thread[#1323,Execution worker Thread 2,5,main]) started.
+
+> Task :libs:native:native-libraries:extractLibs UP-TO-DATE
+Transforming zstd-1.5.5-darwin-aarch64.jar (org.elasticsearch:zstd:1.5.5) with UnzipTransform
+Transforming zstd-1.5.5-darwin-x86-64.jar (org.elasticsearch:zstd:1.5.5) with UnzipTransform
+Transforming zstd-1.5.5-linux-aarch64.jar (org.elasticsearch:zstd:1.5.5) with UnzipTransform
+Transforming zstd-1.5.5-linux-x86-64.jar (org.elasticsearch:zstd:1.5.5) with UnzipTransform
+Transforming zstd-1.5.5-windows-x86-64.jar (org.elasticsearch:zstd:1.5.5) with UnzipTransform
+Transforming vec-1.0.10.zip (org.elasticsearch:vec:1.0.10) with UnzipTransform
+Caching disabled for task ':libs:native:native-libraries:extractLibs' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:native:native-libraries:extractLibs' as it is up-to-date.
+work action resolve platform (project :libs:native:native-libraries) (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :libs:native:jar UP-TO-DATE
+Caching disabled for task ':libs:native:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:native:jar' as it is up-to-date.
+work action resolve elasticsearch-native-9.0.0-SNAPSHOT.jar (project :libs:native) (Thread[#1324,Execution worker Thread 3,5,main]) started.
+
+> Task :libs:entitlement:asm-provider:jar UP-TO-DATE
+Caching disabled for task ':libs:entitlement:asm-provider:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:entitlement:asm-provider:jar' as it is up-to-date.
+work action resolve elasticsearch-entitlement-asm-provider-9.0.0-SNAPSHOT.jar (project :libs:entitlement:asm-provider) (Thread[#1330,Execution worker Thread 9,5,main]) started.
+UnzipTransform (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Transform elasticsearch-entitlement-asm-provider-9.0.0-SNAPSHOT.jar (project :libs:entitlement:asm-provider) with UnzipTransform
+Transforming elasticsearch-entitlement-asm-provider-9.0.0-SNAPSHOT.jar (project :libs:entitlement:asm-provider) with UnzipTransform
+Resolve mutations for :libs:entitlement:generateAsm-providerProviderManifest (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:libs:entitlement:generateAsm-providerProviderManifest (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :libs:entitlement:generateAsm-providerProviderManifest UP-TO-DATE
+Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with UnzipTransform
+Caching disabled for task ':libs:entitlement:generateAsm-providerProviderManifest' because:
+  Build cache is disabled
+  Caching has not been enabled for the task
+Skipping task ':libs:entitlement:generateAsm-providerProviderManifest' as it is up-to-date.
+Resolve mutations for :libs:entitlement:generateAsm-providerProviderImpl (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:libs:entitlement:generateAsm-providerProviderImpl (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :libs:entitlement:generateAsm-providerProviderImpl UP-TO-DATE
+Caching disabled for task ':libs:entitlement:generateAsm-providerProviderImpl' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:entitlement:generateAsm-providerProviderImpl' as it is up-to-date.
+Resolve mutations for :libs:entitlement:classes (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:libs:entitlement:classes (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :libs:entitlement:classes UP-TO-DATE
+Skipping task ':libs:entitlement:classes' as it has no actions.
+Resolve mutations for :libs:entitlement:compileMain23Java (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:libs:entitlement:compileMain23Java (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :libs:entitlement:compileMain23Java UP-TO-DATE
+Caching disabled for task ':libs:entitlement:compileMain23Java' because:
+  Build cache is disabled
+Skipping task ':libs:entitlement:compileMain23Java' as it is up-to-date.
+No compile result for :libs:entitlement:compileMain23Java
+No compile result for :libs:entitlement:compileMain23Java
+No compile result for :libs:entitlement:compileMain23Java
+
+> Task :libs:simdvec:jar UP-TO-DATE
+Caching disabled for task ':libs:simdvec:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:simdvec:jar' as it is up-to-date.
+No compile result for :libs:entitlement:compileMain23Java
+Resolve mutations for :libs:entitlement:main23Classes (Thread[#1331,Execution worker Thread 10,5,main]) started.
+work action resolve elasticsearch-simdvec-9.0.0-SNAPSHOT.jar (project :libs:simdvec) (Thread[#1329,Execution worker Thread 8,5,main]) started.
+:libs:entitlement:main23Classes (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :libs:entitlement:main23Classes UP-TO-DATE
+Skipping task ':libs:entitlement:main23Classes' as it has no actions.
+Resolve mutations for :libs:entitlement:jar (Thread[#1331,Execution worker Thread 10,5,main]) started.
+:libs:entitlement:jar (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :libs:secure-sm:jar UP-TO-DATE
+Caching disabled for task ':libs:secure-sm:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:secure-sm:jar' as it is up-to-date.
+work action resolve elasticsearch-secure-sm-9.0.0-SNAPSHOT.jar (project :libs:secure-sm) (Thread[#1392,included builds Thread 2,5,main]) started.
+
+> Task :server:jar UP-TO-DATE
+Caching disabled for task ':server:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':server:jar' as it is up-to-date.
+work action resolve elasticsearch-9.0.0-SNAPSHOT.jar (project :server) (Thread[#1325,Execution worker Thread 4,5,main]) started.
+
+> Task :libs:ssl-config:jar UP-TO-DATE
+Caching disabled for task ':libs:ssl-config:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:ssl-config:jar' as it is up-to-date.
+work action resolve elasticsearch-ssl-config-9.0.0-SNAPSHOT.jar (project :libs:ssl-config) (Thread[#1326,Execution worker Thread 5,5,main]) started.
+
+> Task :libs:tdigest:jar UP-TO-DATE
+Caching disabled for task ':libs:tdigest:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:tdigest:jar' as it is up-to-date.
+work action resolve elasticsearch-tdigest-9.0.0-SNAPSHOT.jar (project :libs:tdigest) (Thread[#1327,Execution worker Thread 6,5,main]) started.
+
+> Task :libs:x-content:impl:jar UP-TO-DATE
+Caching disabled for task ':libs:x-content:impl:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:x-content:impl:jar' as it is up-to-date.
+work action resolve x-content-impl-9.0.0-SNAPSHOT.jar (project :libs:x-content:impl) (Thread[#1328,Execution worker Thread 7,5,main]) started.
+UnzipTransform (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Transform x-content-impl-9.0.0-SNAPSHOT.jar (project :libs:x-content:impl) with UnzipTransform
+Transforming x-content-impl-9.0.0-SNAPSHOT.jar (project :libs:x-content:impl) with UnzipTransform
+Resolve mutations for :libs:x-content:generateImplProviderManifest (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:libs:x-content:generateImplProviderManifest (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :libs:x-content:generateImplProviderManifest UP-TO-DATE
+Transforming jackson-core-2.17.2.jar (com.fasterxml.jackson.core:jackson-core:2.17.2) with UnzipTransform
+Transforming jackson-dataformat-smile-2.17.2.jar (com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.17.2) with UnzipTransform
+Transforming jackson-dataformat-yaml-2.17.2.jar (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.17.2) with UnzipTransform
+Transforming jackson-dataformat-cbor-2.17.2.jar (com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.17.2) with UnzipTransform
+Transforming snakeyaml-2.0.jar (org.yaml:snakeyaml:2.0) with UnzipTransform
+Caching disabled for task ':libs:x-content:generateImplProviderManifest' because:
+  Build cache is disabled
+  Caching has not been enabled for the task
+Skipping task ':libs:x-content:generateImplProviderManifest' as it is up-to-date.
+Resolve mutations for :libs:x-content:generateImplProviderImpl (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:libs:x-content:generateImplProviderImpl (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :libs:x-content:generateImplProviderImpl UP-TO-DATE
+Caching disabled for task ':libs:x-content:generateImplProviderImpl' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:x-content:generateImplProviderImpl' as it is up-to-date.
+Resolve mutations for :libs:x-content:classes (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:libs:x-content:classes (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :libs:x-content:classes UP-TO-DATE
+Skipping task ':libs:x-content:classes' as it has no actions.
+Resolve mutations for :libs:x-content:jar (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:libs:x-content:jar (Thread[#1328,Execution worker Thread 7,5,main]) started.
+
+> Task :modules:transport-netty4:jar UP-TO-DATE
+Caching disabled for task ':modules:transport-netty4:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':modules:transport-netty4:jar' as it is up-to-date.
+work action resolve transport-netty4-9.0.0-SNAPSHOT.jar (project :modules:transport-netty4) (Thread[#1332,Execution worker Thread 11,5,main]) started.
+
+> Task :test:framework:jar UP-TO-DATE
+Caching disabled for task ':test:framework:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':test:framework:jar' as it is up-to-date.
+work action resolve framework-9.0.0-SNAPSHOT.jar (project :test:framework) (Thread[#1322,Execution worker,5,main]) started.
+
+> Task :libs:entitlement:jar UP-TO-DATE
+Caching disabled for task ':libs:entitlement:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:entitlement:jar' as it is up-to-date.
+work action resolve elasticsearch-entitlement-9.0.0-SNAPSHOT.jar (project :libs:entitlement) (Thread[#1331,Execution worker Thread 10,5,main]) started.
+
+> Task :libs:x-content:jar UP-TO-DATE
+Caching disabled for task ':libs:x-content:jar' because:
+  Build cache is disabled
+  Not worth caching
+Skipping task ':libs:x-content:jar' as it is up-to-date.
+work action resolve elasticsearch-x-content-9.0.0-SNAPSHOT.jar (project :libs:x-content) (Thread[#1328,Execution worker Thread 7,5,main]) started.
+Resolve mutations for :libs:simdvec:test (Thread[#1328,Execution worker Thread 7,5,main]) started.
+:libs:simdvec:test (Thread[#1328,Execution worker Thread 7,5,main]) started.
+Gradle Test Executor 18 started executing tests.
+Gradle Test Executor 17 started executing tests.
+Gradle Test Executor 19 started executing tests.
+Gradle Test Executor 19 finished executing tests.
+WARNING: A terminally deprecated method in java.lang.System has been called
+WARNING: System::setSecurityManager has been called by org.gradle.api.internal.tasks.testing.worker.TestWorker (file:/Users/rene/.gradle/wrapper/dists/gradle-8.11.1-all/2qik7nd48slq1ooc2496ixf4i/gradle-8.11.1/lib/plugins/gradle-testing-base-infrastructure-8.11.1.jar)
+WARNING: Please consider reporting this to the maintainers of org.gradle.api.internal.tasks.testing.worker.TestWorker
+WARNING: System::setSecurityManager will be removed in a future release
+Gradle Test Executor 18 finished executing tests.
+WARNING: A terminally deprecated method in java.lang.System has been called
+WARNING: System::setSecurityManager has been called by org.gradle.api.internal.tasks.testing.worker.TestWorker (file:/Users/rene/.gradle/wrapper/dists/gradle-8.11.1-all/2qik7nd48slq1ooc2496ixf4i/gradle-8.11.1/lib/plugins/gradle-testing-base-infrastructure-8.11.1.jar)
+WARNING: Please consider reporting this to the maintainers of org.gradle.api.internal.tasks.testing.worker.TestWorker
+WARNING: System::setSecurityManager will be removed in a future release
+Gradle Test Executor 17 finished executing tests.
+
+> Task :libs:simdvec:test
+Caching disabled for task ':libs:simdvec:test' because:
+  Build cache is disabled
+Task ':libs:simdvec:test' is not up-to-date because:
+  Output property 'binaryResultsDirectory' file /Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/test-results/test/binary has been removed.
+  Output property 'binaryResultsDirectory' file /Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/test-results/test/binary/output.bin has been removed.
+  Output property 'binaryResultsDirectory' file /Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/test-results/test/binary/output.bin.idx has been removed.
+  and more...
+  and more...
+Starting process 'Gradle Test Executor 17'. Working directory: /Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/testrun/test Command: /Users/rene/.gradle/jdks/oracle_corporation-23-aarch64-os_x.2/jdk-23.jdk/Contents/Home/bin/java -Des.nativelibs.path=/Users/rene/dev/elastic/elasticsearch/libs/native/libraries/build/platform/darwin-aarch64 -Des.scripting.update.ctx_in_params=false -Des.search.rewrite_sort=true -Des.transport.cname_in_publish_address=true -Dgradle.dist.lib=/Users/rene/.gradle/wrapper/dists/gradle-8.11.1-all/2qik7nd48slq1ooc2496ixf4i/gradle-8.11.1/lib -Dgradle.user.home=/Users/rene/.gradle -Dgradle.worker.jar=/Users/rene/.gradle/caches/8.11.1/workerMain/gradle-worker.jar -Dio.netty.noKeySetOptimization=true -Dio.netty.noUnsafe=true -Dio.netty.recycler.maxCapacityPerThread=0 -Djava.awt.headless=true -Djava.locale.providers=CLDR -Djava.security.manager=allow -Djna.nosys=true -Dorg.gradle.internal.worker.tmpdir=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/tmp/test/work -Dtests.artifact=simdvec -Dtests.gradle=true -Dtests.hackImmutableCollections=true -Dtests.logger.level=WARN -Dtests.security.manager=true -Dtests.seed=7B469FBE8B6D0C65 -Dtests.task=:libs:simdvec:test -Dtests.testfeatures.enabled=true -Dworkspace.dir=/Users/rene/dev/elastic/elasticsearch --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.security.cert=ALL-UNNAMED --add-opens=java.base/java.nio.channels=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/javax.net.ssl=ALL-UNNAMED --add-opens=java.base/java.nio.file=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.management/java.lang.management=ALL-UNNAMED -XX:+HeapDumpOnOutOfMemoryError -esa --add-modules=jdk.incubator.vector -XX:HeapDumpPath=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/heapdump --patch-module=java.base=/Users/rene/dev/elastic/elasticsearch/test/immutable-collections-patch/build/jdk-patches/java.base --add-opens=java.base/java.util=ALL-UNNAMED @/Users/rene/.gradle/.tmp/gradle-worker-classpath15834082604113316666txt -Xms512m -Xmx512m -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/testrun/test/temp -Duser.country=DE -Duser.language=en -Duser.variant -ea worker.org.gradle.process.internal.worker.GradleWorkerMain 'Gradle Test Executor 17'
+Starting process 'Gradle Test Executor 19'. Working directory: /Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/testrun/test Command: /Users/rene/.gradle/jdks/oracle_corporation-23-aarch64-os_x.2/jdk-23.jdk/Contents/Home/bin/java -Des.nativelibs.path=/Users/rene/dev/elastic/elasticsearch/libs/native/libraries/build/platform/darwin-aarch64 -Des.scripting.update.ctx_in_params=false -Des.search.rewrite_sort=true -Des.transport.cname_in_publish_address=true -Dgradle.dist.lib=/Users/rene/.gradle/wrapper/dists/gradle-8.11.1-all/2qik7nd48slq1ooc2496ixf4i/gradle-8.11.1/lib -Dgradle.user.home=/Users/rene/.gradle -Dgradle.worker.jar=/Users/rene/.gradle/caches/8.11.1/workerMain/gradle-worker.jar -Dio.netty.noKeySetOptimization=true -Dio.netty.noUnsafe=true -Dio.netty.recycler.maxCapacityPerThread=0 -Djava.awt.headless=true -Djava.locale.providers=CLDR -Djava.security.manager=allow -Djna.nosys=true -Dorg.gradle.internal.worker.tmpdir=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/tmp/test/work -Dtests.artifact=simdvec -Dtests.gradle=true -Dtests.hackImmutableCollections=true -Dtests.logger.level=WARN -Dtests.security.manager=true -Dtests.seed=7B469FBE8B6D0C65 -Dtests.task=:libs:simdvec:test -Dtests.testfeatures.enabled=true -Dworkspace.dir=/Users/rene/dev/elastic/elasticsearch --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.security.cert=ALL-UNNAMED --add-opens=java.base/java.nio.channels=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/javax.net.ssl=ALL-UNNAMED --add-opens=java.base/java.nio.file=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.management/java.lang.management=ALL-UNNAMED -XX:+HeapDumpOnOutOfMemoryError -esa --add-modules=jdk.incubator.vector -XX:HeapDumpPath=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/heapdump --patch-module=java.base=/Users/rene/dev/elastic/elasticsearch/test/immutable-collections-patch/build/jdk-patches/java.base --add-opens=java.base/java.util=ALL-UNNAMED @/Users/rene/.gradle/.tmp/gradle-worker-classpath8760626708071365177txt -Xms512m -Xmx512m -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/testrun/test/temp -Duser.country=DE -Duser.language=en -Duser.variant -ea worker.org.gradle.process.internal.worker.GradleWorkerMain 'Gradle Test Executor 19'
+Starting process 'Gradle Test Executor 18'. Working directory: /Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/testrun/test Command: /Users/rene/.gradle/jdks/oracle_corporation-23-aarch64-os_x.2/jdk-23.jdk/Contents/Home/bin/java -Des.nativelibs.path=/Users/rene/dev/elastic/elasticsearch/libs/native/libraries/build/platform/darwin-aarch64 -Des.scripting.update.ctx_in_params=false -Des.search.rewrite_sort=true -Des.transport.cname_in_publish_address=true -Dgradle.dist.lib=/Users/rene/.gradle/wrapper/dists/gradle-8.11.1-all/2qik7nd48slq1ooc2496ixf4i/gradle-8.11.1/lib -Dgradle.user.home=/Users/rene/.gradle -Dgradle.worker.jar=/Users/rene/.gradle/caches/8.11.1/workerMain/gradle-worker.jar -Dio.netty.noKeySetOptimization=true -Dio.netty.noUnsafe=true -Dio.netty.recycler.maxCapacityPerThread=0 -Djava.awt.headless=true -Djava.locale.providers=CLDR -Djava.security.manager=allow -Djna.nosys=true -Dorg.gradle.internal.worker.tmpdir=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/tmp/test/work -Dtests.artifact=simdvec -Dtests.gradle=true -Dtests.hackImmutableCollections=true -Dtests.logger.level=WARN -Dtests.security.manager=true -Dtests.seed=7B469FBE8B6D0C65 -Dtests.task=:libs:simdvec:test -Dtests.testfeatures.enabled=true -Dworkspace.dir=/Users/rene/dev/elastic/elasticsearch --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.security.cert=ALL-UNNAMED --add-opens=java.base/java.nio.channels=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/javax.net.ssl=ALL-UNNAMED --add-opens=java.base/java.nio.file=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.management/java.lang.management=ALL-UNNAMED -XX:+HeapDumpOnOutOfMemoryError -esa --add-modules=jdk.incubator.vector -XX:HeapDumpPath=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/heapdump --patch-module=java.base=/Users/rene/dev/elastic/elasticsearch/test/immutable-collections-patch/build/jdk-patches/java.base --add-opens=java.base/java.util=ALL-UNNAMED @/Users/rene/.gradle/.tmp/gradle-worker-classpath6774643281130261042txt -Xms512m -Xmx512m -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/testrun/test/temp -Duser.country=DE -Duser.language=en -Duser.variant -ea worker.org.gradle.process.internal.worker.GradleWorkerMain 'Gradle Test Executor 18'
+Successfully started process 'Gradle Test Executor 17'
+Successfully started process 'Gradle Test Executor 18'
+Successfully started process 'Gradle Test Executor 19'
+WARNING: Using incubator modules: jdk.incubator.vector
+WARNING: Using incubator modules: jdk.incubator.vector
+WARNING: Using incubator modules: jdk.incubator.vector
+
+VectorScorerFactoryTests STANDARD_ERROR
+    WARNING: A restricted method in 
+    java.lang.foreign.Linker
+     has been called
+    WARNING: 
+    java.lang.foreign.Linker::downcallHandle
+     has been called by 
+    org.elasticsearch.nativeaccess.jdk.LinkerHelper
+     in 
+    an unnamed module
+
+    WARNING: Use --enable-native-access=
+    ALL-UNNAMED
+     to avoid a warning for callers in this module
+    WARNING: Restricted methods will be blocked in a future release unless native access is enabled
+
+
+ESVectorUtilTests STANDARD_ERROR
+    WARNING: A restricted method in 
+    java.lang.foreign.Linker
+     has been called
+    WARNING: 
+    java.lang.foreign.Linker::downcallHandle
+     has been called by 
+    org.elasticsearch.nativeaccess.jdk.LinkerHelper
+     in 
+    an unnamed module
+
+    WARNING: Use --enable-native-access=
+    ALL-UNNAMED
+     to avoid a warning for callers in this module
+    WARNING: Restricted methods will be blocked in a future release unless native access is enabled
+
+
+VectorScorerFactoryTests STANDARD_OUT
+    [2024-12-23T09:52:40,834][INFO ][o.e.n.j.JdkVectorLibrary ] [[SUITE-VectorScorerFactoryTests-seed#[7B469FBE8B6D0C65]]] vec_caps=1
+    [2024-12-23T09:52:40,837][INFO ][o.e.n.NativeAccess       ] [[SUITE-VectorScorerFactoryTests-seed#[7B469FBE8B6D0C65]]] Using native vector library; to disable start with -Dorg.elasticsearch.nativeaccess.enableVectorLibrary=false
+
+ESVectorUtilTests STANDARD_OUT
+    [2024-12-23T09:52:40,837][INFO ][o.e.n.j.JdkVectorLibrary ] [[SUITE-ESVectorUtilTests-seed#[7B469FBE8B6D0C65]]] vec_caps=1
+    [2024-12-23T09:52:40,840][INFO ][o.e.n.NativeAccess       ] [[SUITE-ESVectorUtilTests-seed#[7B469FBE8B6D0C65]]] Using native vector library; to disable start with -Dorg.elasticsearch.nativeaccess.enableVectorLibrary=false
+
+VectorScorerFactoryTests STANDARD_OUT
+    [2024-12-23T09:52:40,842][INFO ][o.e.n.NativeAccess       ] [[SUITE-VectorScorerFactoryTests-seed#[7B469FBE8B6D0C65]]] Using [jdk] native provider and native methods for [MacOS]
+
+ESVectorUtilTests STANDARD_OUT
+    [2024-12-23T09:52:40,844][INFO ][o.e.n.NativeAccess       ] [[SUITE-ESVectorUtilTests-seed#[7B469FBE8B6D0C65]]] Using [jdk] native provider and native methods for [MacOS]
+    [2024-12-23T09:52:41,180][INFO ][o.e.s.i.v.ESVectorizationProvider] [[SUITE-ESVectorUtilTests-seed#[7B469FBE8B6D0C65]]] Java vector incubator API enabled; uses preferredBitSize=128
+
+ESVectorUtilTests STANDARD_ERROR
+    Dec 23, 2024 9:52:41 AM org.apache.lucene.internal.vectorization.PanamaVectorizationProvider <init>
+    INFO: Java vector incubator API enabled; uses preferredBitSize=128
+
+VectorScorerFactoryTests STANDARD_ERROR
+    Dec 23, 2024 9:52:41 AM org.apache.lucene.internal.vectorization.PanamaVectorizationProvider <init>
+    INFO: Java vector incubator API enabled; uses preferredBitSize=128
+
+VectorScorerFactoryTests > testRandomScorerMax STANDARD_OUT
+    [2024-12-23T04:52:41,272][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerMax] JDK=23, os=Mac OS X, arch=aarch64
+
+ESVectorUtilTests > testBitAndCount STANDARD_OUT
+    [2024-12-23T20:52:41,275][INFO ][o.e.s.ESVectorUtilTests  ] [testBitAndCount] before test
+    [2024-12-23T20:52:41,282][INFO ][o.e.s.i.v.ESVectorizationProvider] [testBitAndCount] Java vector incubator API enabled; uses preferredBitSize=128
+
+VectorScorerFactoryTests > testRandomScorerMax STANDARD_OUT
+    [2024-12-23T04:52:41,282][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerMax] before test
+
+ESVectorUtilTests > testBitAndCount STANDARD_OUT
+    [2024-12-23T20:52:41,293][INFO ][o.e.s.ESVectorUtilTests  ] [testBitAndCount] after test
+
+VectorScorerFactoryTests > testRandomScorerMax STANDARD_OUT
+    [2024-12-23T04:52:41,295][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerMax] Testing testRandom-COSINE-2168.vex
+
+ESVectorUtilTests > testIpByteBin STANDARD_OUT
+    [2024-12-23T20:52:41,298][INFO ][o.e.s.ESVectorUtilTests  ] [testIpByteBin] before test
+
+VectorScorerFactoryTests > testRandomScorerMax STANDARD_OUT
+    [2024-12-23T04:52:41,494][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerMax] Testing testRandom-DOT_PRODUCT-1685.vex
+
+ESVectorUtilTests > testIpByteBin STANDARD_OUT
+    [2024-12-23T20:52:41,494][INFO ][o.e.s.ESVectorUtilTests  ] [testIpByteBin] after test
+
+ESVectorUtilTests > testIpByteBinInvariants STANDARD_OUT
+    [2024-12-23T20:52:41,498][INFO ][o.e.s.ESVectorUtilTests  ] [testIpByteBinInvariants] before test
+    [2024-12-23T20:52:41,499][INFO ][o.e.s.ESVectorUtilTests  ] [testIpByteBinInvariants] after test
+
+ESVectorUtilTests > testBasicIpByteBin STANDARD_OUT
+    [2024-12-23T20:52:41,502][INFO ][o.e.s.ESVectorUtilTests  ] [testBasicIpByteBin] before test
+    [2024-12-23T20:52:41,502][INFO ][o.e.s.ESVectorUtilTests  ] [testBasicIpByteBin] after test
+
+ESVectorUtilTests > testIpByteBit STANDARD_OUT
+    [2024-12-23T20:52:41,505][INFO ][o.e.s.ESVectorUtilTests  ] [testIpByteBit] before test
+    [2024-12-23T20:52:41,505][INFO ][o.e.s.ESVectorUtilTests  ] [testIpByteBit] after test
+
+ESVectorUtilTests > testIpFloatBit STANDARD_OUT
+    [2024-12-23T20:52:41,507][INFO ][o.e.s.ESVectorUtilTests  ] [testIpFloatBit] before test
+    [2024-12-23T20:52:41,508][INFO ][o.e.s.ESVectorUtilTests  ] [testIpFloatBit] after test
+
+VectorScorerFactoryTests > testRandomScorerMax STANDARD_OUT
+    [2024-12-23T04:52:41,517][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerMax] Testing testRandom-EUCLIDEAN-723.vex
+    [2024-12-23T04:52:41,592][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerMax] Testing testRandom-MAXIMUM_INNER_PRODUCT-2519.vex
+    [2024-12-23T04:52:41,628][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerMax] after test
+
+VectorScorerFactoryTests > testSimpleMaxChunkSizeSmall STANDARD_OUT
+    [2024-12-23T04:52:41,629][INFO ][o.e.s.VectorScorerFactoryTests] [testSimpleMaxChunkSizeSmall] JDK=23, os=Mac OS X, arch=aarch64
+    [2024-12-23T04:52:41,632][INFO ][o.e.s.VectorScorerFactoryTests] [testSimpleMaxChunkSizeSmall] before test
+    [2024-12-23T04:52:41,632][INFO ][o.e.s.VectorScorerFactoryTests] [testSimpleMaxChunkSizeSmall] maxChunkSize=14
+    [2024-12-23T04:52:41,645][INFO ][o.e.s.VectorScorerFactoryTests] [testSimpleMaxChunkSizeSmall] after test
+
+VectorScorerFactoryTests > testRace STANDARD_OUT
+    [2024-12-23T04:52:41,645][INFO ][o.e.s.VectorScorerFactoryTests] [testRace] JDK=23, os=Mac OS X, arch=aarch64
+    [2024-12-23T04:52:41,648][INFO ][o.e.s.VectorScorerFactoryTests] [testRace] before test
+    [2024-12-23T04:52:41,659][INFO ][o.e.s.VectorScorerFactoryTests] [testRace] after test
+
+VectorScorerFactoryTests > testNonNegativeDotProduct STANDARD_OUT
+    [2024-12-23T04:52:41,659][INFO ][o.e.s.VectorScorerFactoryTests] [testNonNegativeDotProduct] JDK=23, os=Mac OS X, arch=aarch64
+    [2024-12-23T04:52:41,662][INFO ][o.e.s.VectorScorerFactoryTests] [testNonNegativeDotProduct] before test
+    [2024-12-23T04:52:41,663][INFO ][o.e.s.VectorScorerFactoryTests] [testNonNegativeDotProduct] after test
+
+VectorScorerFactoryTests > testRandomSlice STANDARD_OUT
+    [2024-12-23T04:52:41,664][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] JDK=23, os=Mac OS X, arch=aarch64
+    [2024-12-23T04:52:41,666][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] before test
+    [2024-12-23T04:52:41,667][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-0-30
+    [2024-12-23T04:52:41,669][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-1-30
+    [2024-12-23T04:52:41,671][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-2-30
+    [2024-12-23T04:52:41,672][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-3-30
+    [2024-12-23T04:52:41,674][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-4-30
+    [2024-12-23T04:52:41,675][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-5-30
+    [2024-12-23T04:52:41,676][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-6-30
+    [2024-12-23T04:52:41,677][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-7-30
+    [2024-12-23T04:52:41,679][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-8-30
+    [2024-12-23T04:52:41,681][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-9-30
+    [2024-12-23T04:52:41,682][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-10-30
+    [2024-12-23T04:52:41,683][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-11-30
+    [2024-12-23T04:52:41,684][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-12-30
+    [2024-12-23T04:52:41,685][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-13-30
+    [2024-12-23T04:52:41,687][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-14-30
+    [2024-12-23T04:52:41,689][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-15-30
+    [2024-12-23T04:52:41,690][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-16-30
+    [2024-12-23T04:52:41,691][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-17-30
+    [2024-12-23T04:52:41,692][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-18-30
+    [2024-12-23T04:52:41,693][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-19-30
+    [2024-12-23T04:52:41,694][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-20-30
+    [2024-12-23T04:52:41,696][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-21-30
+    [2024-12-23T04:52:41,697][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-22-30
+    [2024-12-23T04:52:41,698][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-23-30
+    [2024-12-23T04:52:41,699][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-24-30
+    [2024-12-23T04:52:41,700][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-25-30
+    [2024-12-23T04:52:41,702][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-26-30
+    [2024-12-23T04:52:41,703][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-27-30
+    [2024-12-23T04:52:41,703][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-28-30
+    [2024-12-23T04:52:41,705][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-29-30
+    [2024-12-23T04:52:41,706][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-30-30
+    [2024-12-23T04:52:41,707][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-31-30
+    [2024-12-23T04:52:41,708][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-32-30
+    [2024-12-23T04:52:41,710][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-33-30
+    [2024-12-23T04:52:41,711][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-34-30
+    [2024-12-23T04:52:41,712][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-35-30
+    [2024-12-23T04:52:41,713][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-36-30
+    [2024-12-23T04:52:41,715][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-37-30
+    [2024-12-23T04:52:41,716][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-38-30
+    [2024-12-23T04:52:41,717][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-39-30
+    [2024-12-23T04:52:41,718][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-40-30
+    [2024-12-23T04:52:41,719][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-41-30
+    [2024-12-23T04:52:41,720][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-42-30
+    [2024-12-23T04:52:41,722][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-43-30
+    [2024-12-23T04:52:41,723][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-44-30
+    [2024-12-23T04:52:41,724][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-45-30
+    [2024-12-23T04:52:41,725][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-46-30
+    [2024-12-23T04:52:41,727][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-47-30
+    [2024-12-23T04:52:41,727][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-48-30
+    [2024-12-23T04:52:41,728][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-49-30
+    [2024-12-23T04:52:41,729][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-50-30
+    [2024-12-23T04:52:41,731][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-51-30
+    [2024-12-23T04:52:41,732][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-52-30
+    [2024-12-23T04:52:41,733][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-53-30
+    [2024-12-23T04:52:41,734][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-54-30
+    [2024-12-23T04:52:41,735][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-55-30
+    [2024-12-23T04:52:41,736][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-56-30
+    [2024-12-23T04:52:41,738][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-57-30
+    [2024-12-23T04:52:41,739][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-58-30
+    [2024-12-23T04:52:41,740][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-59-30
+    [2024-12-23T04:52:41,741][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-60-30
+    [2024-12-23T04:52:41,742][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-61-30
+    [2024-12-23T04:52:41,743][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-62-30
+    [2024-12-23T04:52:41,744][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-63-30
+    [2024-12-23T04:52:41,746][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-64-30
+    [2024-12-23T04:52:41,746][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-65-30
+    [2024-12-23T04:52:41,747][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-66-30
+    [2024-12-23T04:52:41,749][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-67-30
+    [2024-12-23T04:52:41,750][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-68-30
+    [2024-12-23T04:52:41,751][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-69-30
+    [2024-12-23T04:52:41,752][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-70-30
+    [2024-12-23T04:52:41,753][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-71-30
+    [2024-12-23T04:52:41,755][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-72-30
+    [2024-12-23T04:52:41,755][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-73-30
+    [2024-12-23T04:52:41,757][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-74-30
+    [2024-12-23T04:52:41,758][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-75-30
+    [2024-12-23T04:52:41,759][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-76-30
+    [2024-12-23T04:52:41,760][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-77-30
+    [2024-12-23T04:52:41,761][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-78-30
+    [2024-12-23T04:52:41,761][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-79-30
+    [2024-12-23T04:52:41,762][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-80-30
+    [2024-12-23T04:52:41,764][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-81-30
+    [2024-12-23T04:52:41,764][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-82-30
+    [2024-12-23T04:52:41,766][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-83-30
+    [2024-12-23T04:52:41,767][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-84-30
+    [2024-12-23T04:52:41,768][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-85-30
+    [2024-12-23T04:52:41,769][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-86-30
+    [2024-12-23T04:52:41,770][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-87-30
+    [2024-12-23T04:52:41,771][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-88-30
+    [2024-12-23T04:52:41,772][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-89-30
+    [2024-12-23T04:52:41,772][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-90-30
+    [2024-12-23T04:52:41,774][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-91-30
+    [2024-12-23T04:52:41,774][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-92-30
+    [2024-12-23T04:52:41,775][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-93-30
+    [2024-12-23T04:52:41,776][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-94-30
+    [2024-12-23T04:52:41,777][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-95-30
+    [2024-12-23T04:52:41,778][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-96-30
+    [2024-12-23T04:52:41,779][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-97-30
+    [2024-12-23T04:52:41,780][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-98-30
+    [2024-12-23T04:52:41,781][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-99-30
+    [2024-12-23T04:52:41,782][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] after test
+
+VectorScorerFactoryTests > testRandomMin STANDARD_OUT
+    [2024-12-23T04:52:41,783][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMin] JDK=23, os=Mac OS X, arch=aarch64
+    [2024-12-23T04:52:41,785][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMin] before test
+    [2024-12-23T04:52:41,786][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMin] Testing testRandom-279
+    [2024-12-23T04:52:41,799][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMin] after test
+
+VectorScorerFactoryTests > testRandomMaxChunkSizeSmall STANDARD_OUT
+    [2024-12-23T04:52:41,799][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMaxChunkSizeSmall] JDK=23, os=Mac OS X, arch=aarch64
+    [2024-12-23T04:52:41,801][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMaxChunkSizeSmall] before test
+    [2024-12-23T04:52:41,802][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMaxChunkSizeSmall] maxChunkSize=118
+    [2024-12-23T04:52:41,802][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMaxChunkSizeSmall] Testing testRandom-3935
+    [2024-12-23T04:52:41,916][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMaxChunkSizeSmall] after test
+
+VectorScorerFactoryTests > testLarge SKIPPED
+
+VectorScorerFactoryTests > testRandomMax STANDARD_OUT
+    [2024-12-23T04:52:41,917][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMax] JDK=23, os=Mac OS X, arch=aarch64
+    [2024-12-23T04:52:41,919][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMax] before test
+    [2024-12-23T04:52:41,920][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMax] Testing testRandom-3531
+    [2024-12-23T04:52:41,946][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMax] after test
+
+VectorScorerFactoryTests > testRandom STANDARD_OUT
+    [2024-12-23T04:52:41,946][INFO ][o.e.s.VectorScorerFactoryTests] [testRandom] JDK=23, os=Mac OS X, arch=aarch64
+    [2024-12-23T04:52:41,949][INFO ][o.e.s.VectorScorerFactoryTests] [testRandom] before test
+    [2024-12-23T04:52:41,949][INFO ][o.e.s.VectorScorerFactoryTests] [testRandom] Testing testRandom-1840
+    [2024-12-23T04:52:41,980][INFO ][o.e.s.VectorScorerFactoryTests] [testRandom] after test
+
+VectorScorerFactoryTests > testRandomScorer STANDARD_OUT
+    [2024-12-23T04:52:41,981][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorer] JDK=23, os=Mac OS X, arch=aarch64
+    [2024-12-23T04:52:41,985][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorer] before test
+    [2024-12-23T04:52:41,986][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorer] Testing testRandom-COSINE-199.vex
+    [2024-12-23T04:52:41,989][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorer] Testing testRandom-DOT_PRODUCT-3144.vex
+    [2024-12-23T04:52:41,992][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorer] Testing testRandom-EUCLIDEAN-3017.vex
+    [2024-12-23T04:52:42,020][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorer] Testing testRandom-MAXIMUM_INNER_PRODUCT-1915.vex
+    [2024-12-23T04:52:42,025][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorer] after test
+
+VectorScorerFactoryTests > testSimple STANDARD_OUT
+    [2024-12-23T04:52:42,025][INFO ][o.e.s.VectorScorerFactoryTests] [testSimple] JDK=23, os=Mac OS X, arch=aarch64
+    [2024-12-23T04:52:42,027][INFO ][o.e.s.VectorScorerFactoryTests] [testSimple] before test
+    [2024-12-23T04:52:42,032][INFO ][o.e.s.VectorScorerFactoryTests] [testSimple] after test
+
+VectorScorerFactoryTests > testRandomScorerChunkSizeSmall STANDARD_OUT
+    [2024-12-23T04:52:42,032][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerChunkSizeSmall] JDK=23, os=Mac OS X, arch=aarch64
+    [2024-12-23T04:52:42,034][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerChunkSizeSmall] before test
+    [2024-12-23T04:52:42,034][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerChunkSizeSmall] maxChunkSize=76
+    [2024-12-23T04:52:42,035][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerChunkSizeSmall] Testing testRandom-COSINE-2044.vex
+    [2024-12-23T04:52:42,061][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerChunkSizeSmall] Testing testRandom-DOT_PRODUCT-159.vex
+    [2024-12-23T04:52:42,063][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerChunkSizeSmall] Testing testRandom-EUCLIDEAN-455.vex
+    [2024-12-23T04:52:42,069][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerChunkSizeSmall] Testing testRandom-MAXIMUM_INNER_PRODUCT-1088.vex
+    [2024-12-23T04:52:42,071][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerChunkSizeSmall] after test
+
+VectorScorerFactoryTests > testSupport STANDARD_OUT
+    [2024-12-23T04:52:42,072][INFO ][o.e.s.VectorScorerFactoryTests] [testSupport] JDK=23, os=Mac OS X, arch=aarch64
+    [2024-12-23T04:52:42,074][INFO ][o.e.s.VectorScorerFactoryTests] [testSupport] before test
+    [2024-12-23T04:52:42,074][INFO ][o.e.s.VectorScorerFactoryTests] [testSupport] after test
+Finished generating test XML results (0.001 secs) into: /Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/test-results/test
+Generating HTML test report...
+Finished generating test html results (0.002 secs) into: /Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/reports/tests/test
+
+0 problems were found storing the configuration cache.
+
+See the complete report at file:///Users/rene/dev/elastic/elasticsearch/build/reports/configuration-cache/bf67ihcjtxveqczwzl7s4owb2/9o6bg5x67n4fyxqy871utzrqc/configuration-cache-report.html
+
+BUILD SUCCESSFUL in 4s
+86 actionable tasks: 2 executed, 84 up-to-date
+Watched directory hierarchies: [/Users/rene/dev/elastic/elasticsearch]
+
+Publishing build scan...
+https://gradle-enterprise.elastic.co/s/s75qfwti22ejk
+
+Configuration cache entry stored.
diff --git a/libs/tdigest/build.gradle b/libs/tdigest/build.gradle
index 2713df701fb44..b79a6ce0a486a 100644
--- a/libs/tdigest/build.gradle
+++ b/libs/tdigest/build.gradle
@@ -36,7 +36,7 @@ tasks.named('forbiddenApisMain').configure {
   replaceSignatureFiles 'jdk-signatures'
 }
 
-ext.projectLicenses.set(['The Apache Software License, Version 2.0': 'http://www.apache.org/licenses/LICENSE-2.0'])
+ext.projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')])
 licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt'))
 
 tasks.withType(LicenseHeadersTask.class).configureEach {
diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java
index 06724b049f821..61bec8e41b163 100644
--- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java
+++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java
@@ -111,7 +111,7 @@ public class MergingDigest extends AbstractTDigest {
     // based on accumulated k-index. This can be much faster since we
     // scale functions are more expensive than the corresponding
     // weight limits.
-    public static boolean useWeightLimit = true;
+    public static final boolean useWeightLimit = true;
 
     static MergingDigest create(TDigestArrays arrays, double compression) {
         arrays.adjustBreaker(SHALLOW_SIZE);
diff --git a/modules/aggregations/build.gradle b/modules/aggregations/build.gradle
index 3b5fb6ddecde9..d65b6e8fd2ddd 100644
--- a/modules/aggregations/build.gradle
+++ b/modules/aggregations/build.gradle
@@ -11,8 +11,8 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'Adds "built in" aggregations to Elasticsearch.'
-  classname 'org.elasticsearch.aggregations.AggregationsPlugin'
+  description = 'Adds "built in" aggregations to Elasticsearch.'
+  classname ='org.elasticsearch.aggregations.AggregationsPlugin'
   extendedPlugins = ['lang-painless']
 }
 
diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java
index 5249077bdfdbb..d225ccc9d173f 100644
--- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java
+++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.aggregations.AggregationsPlugin;
 import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder;
+import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
@@ -96,8 +97,12 @@ public void testCancellationDuringTimeSeriesAggregation() throws Exception {
         }
 
         logger.info("Executing search");
+        // we have to explicitly set error_trace=true for the later exception check for `TimeSeriesIndexSearcher`
+        Client client = client();
+        client.threadPool().getThreadContext().putHeader("error_trace", "true");
         TimeSeriesAggregationBuilder timeSeriesAggregationBuilder = new TimeSeriesAggregationBuilder("test_agg");
-        ActionFuture<SearchResponse> searchResponse = prepareSearch("test").setQuery(matchAllQuery())
+        ActionFuture<SearchResponse> searchResponse = client.prepareSearch("test")
+            .setQuery(matchAllQuery())
             .addAggregation(
                 timeSeriesAggregationBuilder.subAggregation(
                     new ScriptedMetricAggregationBuilder("sub_agg").initScript(
diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java
index eb78101c6181a..9380610d730bf 100644
--- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java
+++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java
@@ -99,17 +99,12 @@ public void setUp() throws Exception {
                 .setSettings(
                     settings.put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersions.NEW_INDEXVERSION_FORMAT).build()
                 )
-                .setMapping(mapping)
-                .get()
-        );
-
-        assertAcked(
+                .setMapping(mapping),
             indicesAdmin().prepareCreate(afterIndex)
                 .setSettings(
                     settings.put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersions.TIME_SERIES_ID_HASHING).build()
                 )
                 .setMapping(mapping)
-                .get()
         );
 
         final TimeSeriesDataset timeSeriesDataset = new TimeSeriesDataset();
diff --git a/modules/analysis-common/build.gradle b/modules/analysis-common/build.gradle
index f4f7e787d2b7b..173e1eeef60a2 100644
--- a/modules/analysis-common/build.gradle
+++ b/modules/analysis-common/build.gradle
@@ -13,8 +13,8 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'Adds "built in" analyzers to Elasticsearch.'
-  classname 'org.elasticsearch.analysis.common.CommonAnalysisPlugin'
+  description = 'Adds "built in" analyzers to Elasticsearch.'
+  classname = 'org.elasticsearch.analysis.common.CommonAnalysisPlugin'
   extendedPlugins = ['lang-painless']
 }
 
@@ -37,3 +37,16 @@ artifacts {
 tasks.named("yamlRestCompatTestTransform").configure { task ->
   task.replaceValueInMatch("tokens.0.token", "absenț", "romanian")
 }
+
+tasks.named("yamlRestTest").configure {
+  if (buildParams.getRuntimeJavaVersion().map{ it.majorVersion.toInteger() }.get() >= 24 ||
+    "-Des.entitlements.enabled=true".equals(System.getProperty("tests.jvm.argline"))) {
+    systemProperty 'tests.rest.blacklist',
+      [
+        // AWAITSFIX: this test relies on security manager, which doesn't exist in JDK 24.
+        // and entitlements don't yet replace the functionality.
+        // see https://github.com/elastic/elasticsearch/issues/119130
+        'analysis-common/40_token_filters/stemmer_override file access',
+      ].join(',')
+  }
+}
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
index a97154fd4d1ff..c980aaba71444 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
@@ -101,7 +101,12 @@
 import org.apache.lucene.analysis.tr.TurkishAnalyzer;
 import org.apache.lucene.analysis.util.ElisionFilter;
 import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.common.logging.DeprecationCategory;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.AnalyzerProvider;
 import org.elasticsearch.index.analysis.CharFilterFactory;
@@ -134,6 +139,8 @@
 
 public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin, ScriptPlugin {
 
+    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(CommonAnalysisPlugin.class);
+
     private final SetOnce<ScriptService> scriptServiceHolder = new SetOnce<>();
     private final SetOnce<SynonymsManagementAPIService> synonymsManagementServiceHolder = new SetOnce<>();
 
@@ -224,6 +231,28 @@ public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
         filters.put("dictionary_decompounder", requiresAnalysisSettings(DictionaryCompoundWordTokenFilterFactory::new));
         filters.put("dutch_stem", DutchStemTokenFilterFactory::new);
         filters.put("edge_ngram", EdgeNGramTokenFilterFactory::new);
+        filters.put("edgeNGram", (IndexSettings indexSettings, Environment environment, String name, Settings settings) -> {
+            return new EdgeNGramTokenFilterFactory(indexSettings, environment, name, settings) {
+                @Override
+                public TokenStream create(TokenStream tokenStream) {
+                    if (indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.V_8_0_0)) {
+                        throw new IllegalArgumentException(
+                            "The [edgeNGram] token filter name was deprecated in 6.4 and cannot be used in new indices. "
+                                + "Please change the filter name to [edge_ngram] instead."
+                        );
+                    } else {
+                        deprecationLogger.warn(
+                            DeprecationCategory.ANALYSIS,
+                            "edgeNGram_deprecation",
+                            "The [edgeNGram] token filter name is deprecated and will be removed in a future version. "
+                                + "Please change the filter name to [edge_ngram] instead."
+                        );
+                    }
+                    return super.create(tokenStream);
+                }
+
+            };
+        });
         filters.put("elision", requiresAnalysisSettings(ElisionTokenFilterFactory::new));
         filters.put("fingerprint", FingerprintTokenFilterFactory::new);
         filters.put("flatten_graph", FlattenGraphTokenFilterFactory::new);
@@ -243,6 +272,28 @@ public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
         filters.put("min_hash", MinHashTokenFilterFactory::new);
         filters.put("multiplexer", MultiplexerTokenFilterFactory::new);
         filters.put("ngram", NGramTokenFilterFactory::new);
+        filters.put("nGram", (IndexSettings indexSettings, Environment environment, String name, Settings settings) -> {
+            return new NGramTokenFilterFactory(indexSettings, environment, name, settings) {
+                @Override
+                public TokenStream create(TokenStream tokenStream) {
+                    if (indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.V_8_0_0)) {
+                        throw new IllegalArgumentException(
+                            "The [nGram] token filter name was deprecated in 6.4 and cannot be used in new indices. "
+                                + "Please change the filter name to [ngram] instead."
+                        );
+                    } else {
+                        deprecationLogger.warn(
+                            DeprecationCategory.ANALYSIS,
+                            "nGram_deprecation",
+                            "The [nGram] token filter name is deprecated and will be removed in a future version. "
+                                + "Please change the filter name to [ngram] instead."
+                        );
+                    }
+                    return super.create(tokenStream);
+                }
+
+            };
+        });
         filters.put("pattern_capture", requiresAnalysisSettings(PatternCaptureGroupTokenFilterFactory::new));
         filters.put("pattern_replace", requiresAnalysisSettings(PatternReplaceTokenFilterFactory::new));
         filters.put("persian_normalization", PersianNormalizationFilterFactory::new);
@@ -294,7 +345,39 @@ public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
         tokenizers.put("simple_pattern", SimplePatternTokenizerFactory::new);
         tokenizers.put("simple_pattern_split", SimplePatternSplitTokenizerFactory::new);
         tokenizers.put("thai", ThaiTokenizerFactory::new);
+        tokenizers.put("nGram", (IndexSettings indexSettings, Environment environment, String name, Settings settings) -> {
+            if (indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.V_8_0_0)) {
+                throw new IllegalArgumentException(
+                    "The [nGram] tokenizer name was deprecated in 7.6. "
+                        + "Please use the tokenizer name to [ngram] for indices created in versions 8 or higher instead."
+                );
+            } else if (indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.V_7_6_0)) {
+                deprecationLogger.warn(
+                    DeprecationCategory.ANALYSIS,
+                    "nGram_tokenizer_deprecation",
+                    "The [nGram] tokenizer name is deprecated and will be removed in a future version. "
+                        + "Please change the tokenizer name to [ngram] instead."
+                );
+            }
+            return new NGramTokenizerFactory(indexSettings, environment, name, settings);
+        });
         tokenizers.put("ngram", NGramTokenizerFactory::new);
+        tokenizers.put("edgeNGram", (IndexSettings indexSettings, Environment environment, String name, Settings settings) -> {
+            if (indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.V_8_0_0)) {
+                throw new IllegalArgumentException(
+                    "The [edgeNGram] tokenizer name was deprecated in 7.6. "
+                        + "Please use the tokenizer name to [edge_nGram] for indices created in versions 8 or higher instead."
+                );
+            } else if (indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.V_7_6_0)) {
+                deprecationLogger.warn(
+                    DeprecationCategory.ANALYSIS,
+                    "edgeNGram_tokenizer_deprecation",
+                    "The [edgeNGram] tokenizer name is deprecated and will be removed in a future version. "
+                        + "Please change the tokenizer name to [edge_ngram] instead."
+                );
+            }
+            return new EdgeNGramTokenizerFactory(indexSettings, environment, name, settings);
+        });
         tokenizers.put("edge_ngram", EdgeNGramTokenizerFactory::new);
         tokenizers.put("char_group", CharGroupTokenizerFactory::new);
         tokenizers.put("classic", ClassicTokenizerFactory::new);
@@ -505,17 +588,53 @@ public List<PreConfiguredTokenizer> getPreConfiguredTokenizers() {
         tokenizers.add(PreConfiguredTokenizer.singleton("letter", LetterTokenizer::new));
         tokenizers.add(PreConfiguredTokenizer.singleton("whitespace", WhitespaceTokenizer::new));
         tokenizers.add(PreConfiguredTokenizer.singleton("ngram", NGramTokenizer::new));
-        tokenizers.add(
-            PreConfiguredTokenizer.indexVersion(
-                "edge_ngram",
-                (version) -> new EdgeNGramTokenizer(NGramTokenizer.DEFAULT_MIN_NGRAM_SIZE, NGramTokenizer.DEFAULT_MAX_NGRAM_SIZE)
-            )
-        );
+        tokenizers.add(PreConfiguredTokenizer.indexVersion("edge_ngram", (version) -> {
+            if (version.onOrAfter(IndexVersions.V_7_3_0)) {
+                return new EdgeNGramTokenizer(NGramTokenizer.DEFAULT_MIN_NGRAM_SIZE, NGramTokenizer.DEFAULT_MAX_NGRAM_SIZE);
+            }
+            return new EdgeNGramTokenizer(EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE);
+        }));
         tokenizers.add(PreConfiguredTokenizer.singleton("pattern", () -> new PatternTokenizer(Regex.compile("\\W+", null), -1)));
         tokenizers.add(PreConfiguredTokenizer.singleton("thai", ThaiTokenizer::new));
         // TODO deprecate and remove in API
         // This is already broken with normalization, so backwards compat isn't necessary?
         tokenizers.add(PreConfiguredTokenizer.singleton("lowercase", XLowerCaseTokenizer::new));
+
+        tokenizers.add(PreConfiguredTokenizer.indexVersion("nGram", (version) -> {
+            if (version.onOrAfter(IndexVersions.V_8_0_0)) {
+                throw new IllegalArgumentException(
+                    "The [nGram] tokenizer name was deprecated in 7.6. "
+                        + "Please use the tokenizer name to [ngram] for indices created in versions 8 or higher instead."
+                );
+            } else if (version.onOrAfter(IndexVersions.V_7_6_0)) {
+                deprecationLogger.warn(
+                    DeprecationCategory.ANALYSIS,
+                    "nGram_tokenizer_deprecation",
+                    "The [nGram] tokenizer name is deprecated and will be removed in a future version. "
+                        + "Please change the tokenizer name to [ngram] instead."
+                );
+            }
+            return new NGramTokenizer();
+        }));
+        tokenizers.add(PreConfiguredTokenizer.indexVersion("edgeNGram", (version) -> {
+            if (version.onOrAfter(IndexVersions.V_8_0_0)) {
+                throw new IllegalArgumentException(
+                    "The [edgeNGram] tokenizer name was deprecated in 7.6. "
+                        + "Please use the tokenizer name to [edge_ngram] for indices created in versions 8 or higher instead."
+                );
+            } else if (version.onOrAfter(IndexVersions.V_7_6_0)) {
+                deprecationLogger.warn(
+                    DeprecationCategory.ANALYSIS,
+                    "edgeNGram_tokenizer_deprecation",
+                    "The [edgeNGram] tokenizer name is deprecated and will be removed in a future version. "
+                        + "Please change the tokenizer name to [edge_ngram] instead."
+                );
+            }
+            if (version.onOrAfter(IndexVersions.V_7_3_0)) {
+                return new EdgeNGramTokenizer(NGramTokenizer.DEFAULT_MIN_NGRAM_SIZE, NGramTokenizer.DEFAULT_MAX_NGRAM_SIZE);
+            }
+            return new EdgeNGramTokenizer(EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE);
+        }));
         tokenizers.add(PreConfiguredTokenizer.singleton("PathHierarchy", PathHierarchyTokenizer::new));
 
         return tokenizers;
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzerProvider.java
index 1fcbfb4bc2c8d..68acf46f21c24 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzerProvider.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzerProvider.java
@@ -24,11 +24,11 @@
  */
 public class FingerprintAnalyzerProvider extends AbstractIndexAnalyzerProvider<Analyzer> {
 
-    public static ParseField SEPARATOR = new ParseField("separator");
-    public static ParseField MAX_OUTPUT_SIZE = new ParseField("max_output_size");
+    public static final ParseField SEPARATOR = new ParseField("separator");
+    public static final ParseField MAX_OUTPUT_SIZE = new ParseField("max_output_size");
 
-    public static int DEFAULT_MAX_OUTPUT_SIZE = 255;
-    public static CharArraySet DEFAULT_STOP_WORDS = CharArraySet.EMPTY_SET;
+    public static final int DEFAULT_MAX_OUTPUT_SIZE = 255;
+    public static final CharArraySet DEFAULT_STOP_WORDS = CharArraySet.EMPTY_SET;
     public static final char DEFAULT_SEPARATOR = ' ';
 
     private final FingerprintAnalyzer analyzer;
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java
index a6e9fccd9d09c..362192ed40df3 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java
@@ -89,8 +89,6 @@ public class StemmerTokenFilterFactory extends AbstractTokenFilterFactory {
 
     private final String language;
 
-    private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(StemmerTokenFilterFactory.class);
-
     StemmerTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException {
         super(name);
         this.language = Strings.capitalize(settings.get("language", settings.get("name", "porter")));
@@ -192,7 +190,7 @@ public boolean incrementToken() {
             } else if ("german".equalsIgnoreCase(language)) {
                 return new SnowballFilter(tokenStream, new GermanStemmer());
             } else if ("german2".equalsIgnoreCase(language)) {
-                DEPRECATION_LOGGER.critical(
+                deprecationLogger.critical(
                     DeprecationCategory.ANALYSIS,
                     "german2_stemmer_deprecation",
                     "The 'german2' stemmer has been deprecated and folded into the 'german' Stemmer. "
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java
index cb5c01621f45f..04481515384a8 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java
@@ -13,7 +13,6 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.synonym.SynonymFilter;
 import org.apache.lucene.analysis.synonym.SynonymMap;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexService.IndexCreationContext;
@@ -130,8 +129,6 @@ public static SynonymsSource fromSettings(Settings settings) {
         }
     }
 
-    private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(SynonymTokenFilterFactory.class);
-
     private final String format;
     private final boolean expand;
     private final boolean lenient;
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisPluginTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisPluginTests.java
new file mode 100644
index 0000000000000..9972d58b2dcc1
--- /dev/null
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisPluginTests.java
@@ -0,0 +1,292 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.analysis.common;
+
+import org.apache.lucene.analysis.Tokenizer;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
+import org.elasticsearch.index.analysis.TokenizerFactory;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.test.index.IndexVersionUtils;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class CommonAnalysisPluginTests extends ESTestCase {
+
+    /**
+     * Check that the deprecated "nGram" filter throws exception for indices created since 7.0.0 and
+     * logs a warning for earlier indices when the filter is used as a custom filter
+     */
+    public void testNGramFilterInCustomAnalyzerDeprecationError() throws IOException {
+        final Settings settings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+            .put(
+                IndexMetadata.SETTING_VERSION_CREATED,
+                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_8_0_0, IndexVersion.current())
+            )
+            .put("index.analysis.analyzer.custom_analyzer.type", "custom")
+            .put("index.analysis.analyzer.custom_analyzer.tokenizer", "standard")
+            .putList("index.analysis.analyzer.custom_analyzer.filter", "my_ngram")
+            .put("index.analysis.filter.my_ngram.type", "nGram")
+            .build();
+
+        try (CommonAnalysisPlugin commonAnalysisPlugin = new CommonAnalysisPlugin()) {
+            IllegalArgumentException ex = expectThrows(
+                IllegalArgumentException.class,
+                () -> createTestAnalysis(IndexSettingsModule.newIndexSettings("index", settings), settings, commonAnalysisPlugin)
+            );
+            assertEquals(
+                "The [nGram] token filter name was deprecated in 6.4 and cannot be used in new indices. "
+                    + "Please change the filter name to [ngram] instead.",
+                ex.getMessage()
+            );
+        }
+
+        final Settings settingsPre7 = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+            .put(
+                IndexMetadata.SETTING_VERSION_CREATED,
+                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_6_0)
+            )
+            .put("index.analysis.analyzer.custom_analyzer.type", "custom")
+            .put("index.analysis.analyzer.custom_analyzer.tokenizer", "standard")
+            .putList("index.analysis.analyzer.custom_analyzer.filter", "my_ngram")
+            .put("index.analysis.filter.my_ngram.type", "nGram")
+            .build();
+        try (CommonAnalysisPlugin commonAnalysisPlugin = new CommonAnalysisPlugin()) {
+            createTestAnalysis(IndexSettingsModule.newIndexSettings("index", settingsPre7), settingsPre7, commonAnalysisPlugin);
+            assertWarnings(
+                "The [nGram] token filter name is deprecated and will be removed in a future version. "
+                    + "Please change the filter name to [ngram] instead."
+            );
+        }
+    }
+
+    /**
+     * Check that the deprecated "edgeNGram" filter throws exception for indices created since 7.0.0 and
+     * logs a warning for earlier indices when the filter is used as a custom filter
+     */
+    public void testEdgeNGramFilterInCustomAnalyzerDeprecationError() throws IOException {
+        final Settings settings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+            .put(
+                IndexMetadata.SETTING_VERSION_CREATED,
+                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_8_0_0, IndexVersion.current())
+            )
+            .put("index.analysis.analyzer.custom_analyzer.type", "custom")
+            .put("index.analysis.analyzer.custom_analyzer.tokenizer", "standard")
+            .putList("index.analysis.analyzer.custom_analyzer.filter", "my_ngram")
+            .put("index.analysis.filter.my_ngram.type", "edgeNGram")
+            .build();
+
+        try (CommonAnalysisPlugin commonAnalysisPlugin = new CommonAnalysisPlugin()) {
+            IllegalArgumentException ex = expectThrows(
+                IllegalArgumentException.class,
+                () -> createTestAnalysis(IndexSettingsModule.newIndexSettings("index", settings), settings, commonAnalysisPlugin)
+            );
+            assertEquals(
+                "The [edgeNGram] token filter name was deprecated in 6.4 and cannot be used in new indices. "
+                    + "Please change the filter name to [edge_ngram] instead.",
+                ex.getMessage()
+            );
+        }
+
+        final Settings settingsPre7 = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+            .put(
+                IndexMetadata.SETTING_VERSION_CREATED,
+                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_6_0)
+            )
+            .put("index.analysis.analyzer.custom_analyzer.type", "custom")
+            .put("index.analysis.analyzer.custom_analyzer.tokenizer", "standard")
+            .putList("index.analysis.analyzer.custom_analyzer.filter", "my_ngram")
+            .put("index.analysis.filter.my_ngram.type", "edgeNGram")
+            .build();
+
+        try (CommonAnalysisPlugin commonAnalysisPlugin = new CommonAnalysisPlugin()) {
+            createTestAnalysis(IndexSettingsModule.newIndexSettings("index", settingsPre7), settingsPre7, commonAnalysisPlugin);
+            assertWarnings(
+                "The [edgeNGram] token filter name is deprecated and will be removed in a future version. "
+                    + "Please change the filter name to [edge_ngram] instead."
+            );
+        }
+    }
+
+    /**
+     * Check that we log a deprecation warning for "nGram" and "edgeNGram" tokenizer names with 7.6 and
+     * disallow usages for indices created after 8.0
+     */
+    public void testNGramTokenizerDeprecation() throws IOException {
+        // tests for prebuilt tokenizer
+        doTestPrebuiltTokenizerDeprecation(
+            "nGram",
+            "ngram",
+            IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_5_2),
+            false
+        );
+        doTestPrebuiltTokenizerDeprecation(
+            "edgeNGram",
+            "edge_ngram",
+            IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_5_2),
+            false
+        );
+        doTestPrebuiltTokenizerDeprecation(
+            "nGram",
+            "ngram",
+            IndexVersionUtils.randomVersionBetween(
+                random(),
+                IndexVersions.V_7_6_0,
+                IndexVersion.max(IndexVersions.V_7_6_0, IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0))
+            ),
+            true
+        );
+        doTestPrebuiltTokenizerDeprecation(
+            "edgeNGram",
+            "edge_ngram",
+            IndexVersionUtils.randomVersionBetween(
+                random(),
+                IndexVersions.V_7_6_0,
+                IndexVersion.max(IndexVersions.V_7_6_0, IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0))
+            ),
+            true
+        );
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> doTestPrebuiltTokenizerDeprecation(
+                "nGram",
+                "ngram",
+                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_8_0_0, IndexVersion.current()),
+                true
+            )
+        );
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> doTestPrebuiltTokenizerDeprecation(
+                "edgeNGram",
+                "edge_ngram",
+                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_8_0_0, IndexVersion.current()),
+                true
+            )
+        );
+
+        // same batch of tests for custom tokenizer definition in the settings
+        doTestCustomTokenizerDeprecation(
+            "nGram",
+            "ngram",
+            IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_5_2),
+            false
+        );
+        doTestCustomTokenizerDeprecation(
+            "edgeNGram",
+            "edge_ngram",
+            IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_5_2),
+            false
+        );
+        doTestCustomTokenizerDeprecation(
+            "nGram",
+            "ngram",
+            IndexVersionUtils.randomVersionBetween(
+                random(),
+                IndexVersions.V_7_6_0,
+                IndexVersion.max(IndexVersions.V_7_6_0, IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0))
+            ),
+            true
+        );
+        doTestCustomTokenizerDeprecation(
+            "edgeNGram",
+            "edge_ngram",
+            IndexVersionUtils.randomVersionBetween(
+                random(),
+                IndexVersions.V_7_6_0,
+                IndexVersion.max(IndexVersions.V_7_6_0, IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0))
+            ),
+            true
+        );
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> doTestCustomTokenizerDeprecation(
+                "nGram",
+                "ngram",
+                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_8_0_0, IndexVersion.current()),
+                true
+            )
+        );
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> doTestCustomTokenizerDeprecation(
+                "edgeNGram",
+                "edge_ngram",
+                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_8_0_0, IndexVersion.current()),
+                true
+            )
+        );
+    }
+
+    public void doTestPrebuiltTokenizerDeprecation(String deprecatedName, String replacement, IndexVersion version, boolean expectWarning)
+        throws IOException {
+        final Settings settings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+            .put(IndexMetadata.SETTING_VERSION_CREATED, version)
+            .build();
+
+        try (CommonAnalysisPlugin commonAnalysisPlugin = new CommonAnalysisPlugin()) {
+            Map<String, TokenizerFactory> tokenizers = createTestAnalysis(
+                IndexSettingsModule.newIndexSettings("index", settings),
+                settings,
+                commonAnalysisPlugin
+            ).tokenizer;
+            TokenizerFactory tokenizerFactory = tokenizers.get(deprecatedName);
+
+            Tokenizer tokenizer = tokenizerFactory.create();
+            assertNotNull(tokenizer);
+            if (expectWarning) {
+                assertWarnings(
+                    "The ["
+                        + deprecatedName
+                        + "] tokenizer name is deprecated and will be removed in a future version. "
+                        + "Please change the tokenizer name to ["
+                        + replacement
+                        + "] instead."
+                );
+            }
+        }
+    }
+
+    public void doTestCustomTokenizerDeprecation(String deprecatedName, String replacement, IndexVersion version, boolean expectWarning)
+        throws IOException {
+        final Settings settings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+            .put(IndexMetadata.SETTING_VERSION_CREATED, version)
+            .put("index.analysis.analyzer.custom_analyzer.type", "custom")
+            .put("index.analysis.analyzer.custom_analyzer.tokenizer", "my_tokenizer")
+            .put("index.analysis.tokenizer.my_tokenizer.type", deprecatedName)
+            .build();
+
+        try (CommonAnalysisPlugin commonAnalysisPlugin = new CommonAnalysisPlugin()) {
+            createTestAnalysis(IndexSettingsModule.newIndexSettings("index", settings), settings, commonAnalysisPlugin);
+
+            if (expectWarning) {
+                assertWarnings(
+                    "The ["
+                        + deprecatedName
+                        + "] tokenizer name is deprecated and will be removed in a future version. "
+                        + "Please change the tokenizer name to ["
+                        + replacement
+                        + "] instead."
+                );
+            }
+        }
+    }
+}
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java
index 11d1653439e59..c998e927e25a8 100644
--- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java
@@ -34,7 +34,7 @@
 
 public class EdgeNGramTokenizerTests extends ESTokenStreamTestCase {
 
-    private static IndexAnalyzers buildAnalyzers(IndexVersion version, String tokenizer) throws IOException {
+    private IndexAnalyzers buildAnalyzers(IndexVersion version, String tokenizer) throws IOException {
         Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
         Settings indexSettings = Settings.builder()
             .put(IndexMetadata.SETTING_VERSION_CREATED, version)
@@ -54,6 +54,7 @@ public void testPreConfiguredTokenizer() throws IOException {
             assertNotNull(analyzer);
             assertAnalyzesTo(analyzer, "test", new String[] { "t", "te" });
         }
+
     }
 
     public void testCustomTokenChars() throws IOException {
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/NGramTokenizerFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/NGramTokenizerFactoryTests.java
index 8c365a1362f85..35c01b5b9296f 100644
--- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/NGramTokenizerFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/NGramTokenizerFactoryTests.java
@@ -161,7 +161,7 @@ public void testBackwardsCompatibilityEdgeNgramTokenFilter() throws Exception {
         for (int i = 0; i < iters; i++) {
             final Index index = new Index("test", "_na_");
             final String name = "ngr";
-            IndexVersion v = IndexVersionUtils.randomVersion(random());
+            IndexVersion v = IndexVersionUtils.randomVersion();
             Builder builder = newAnalysisSettingsBuilder().put("min_gram", 2).put("max_gram", 3);
             boolean reverse = random().nextBoolean();
             if (reverse) {
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PersianAnalyzerProviderTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PersianAnalyzerProviderTests.java
index 7b962538c2a10..153c3e9549285 100644
--- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PersianAnalyzerProviderTests.java
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PersianAnalyzerProviderTests.java
@@ -56,7 +56,7 @@ public void testPersianAnalyzerPostLucene10() throws IOException {
     public void testPersianAnalyzerPreLucene10() throws IOException {
         IndexVersion preLucene10Version = IndexVersionUtils.randomVersionBetween(
             random(),
-            IndexVersionUtils.getFirstVersion(),
+            IndexVersionUtils.getLowestReadCompatibleVersion(),
             IndexVersionUtils.getPreviousVersion(IndexVersions.UPGRADE_TO_LUCENE_10_0_0)
         );
         Settings settings = ESTestCase.indexSettings(1, 1)
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RomanianAnalyzerTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RomanianAnalyzerTests.java
index 1af44bc71f35d..29e27e62e3164 100644
--- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RomanianAnalyzerTests.java
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RomanianAnalyzerTests.java
@@ -57,7 +57,7 @@ public void testRomanianAnalyzerPostLucene10() throws IOException {
     public void testRomanianAnalyzerPreLucene10() throws IOException {
         IndexVersion preLucene10Version = IndexVersionUtils.randomVersionBetween(
             random(),
-            IndexVersionUtils.getFirstVersion(),
+            IndexVersionUtils.getLowestReadCompatibleVersion(),
             IndexVersionUtils.getPreviousVersion(IndexVersions.UPGRADE_TO_LUCENE_10_0_0)
         );
         Settings settings = ESTestCase.indexSettings(1, 1)
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactoryTests.java
index bb06c221873b5..4e774d92e3d62 100644
--- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactoryTests.java
@@ -39,7 +39,7 @@ public class StemmerTokenFilterFactoryTests extends ESTokenStreamTestCase {
     public void testEnglishFilterFactory() throws IOException {
         int iters = scaledRandomIntBetween(20, 100);
         for (int i = 0; i < iters; i++) {
-            IndexVersion v = IndexVersionUtils.randomVersion(random());
+            IndexVersion v = IndexVersionUtils.randomVersion();
             Settings settings = Settings.builder()
                 .put("index.analysis.filter.my_english.type", "stemmer")
                 .put("index.analysis.filter.my_english.language", "english")
@@ -66,7 +66,7 @@ public void testPorter2FilterFactory() throws IOException {
         int iters = scaledRandomIntBetween(20, 100);
         for (int i = 0; i < iters; i++) {
 
-            IndexVersion v = IndexVersionUtils.randomVersion(random());
+            IndexVersion v = IndexVersionUtils.randomVersion();
             Settings settings = Settings.builder()
                 .put("index.analysis.filter.my_porter2.type", "stemmer")
                 .put("index.analysis.filter.my_porter2.language", "porter2")
@@ -90,7 +90,7 @@ public void testPorter2FilterFactory() throws IOException {
     }
 
     public void testMultipleLanguagesThrowsException() throws IOException {
-        IndexVersion v = IndexVersionUtils.randomVersion(random());
+        IndexVersion v = IndexVersionUtils.randomVersion();
         Settings settings = Settings.builder()
             .put("index.analysis.filter.my_english.type", "stemmer")
             .putList("index.analysis.filter.my_english.language", "english", "light_english")
@@ -142,7 +142,7 @@ private static Analyzer createGermanStemmer(String variant, IndexVersion v) thro
     }
 
     public void testKpDeprecation() throws IOException {
-        IndexVersion v = IndexVersionUtils.randomVersion(random());
+        IndexVersion v = IndexVersionUtils.randomVersion();
         Settings settings = Settings.builder()
             .put("index.analysis.filter.my_kp.type", "stemmer")
             .put("index.analysis.filter.my_kp.language", "kp")
@@ -155,7 +155,7 @@ public void testKpDeprecation() throws IOException {
     }
 
     public void testLovinsDeprecation() throws IOException {
-        IndexVersion v = IndexVersionUtils.randomVersion(random());
+        IndexVersion v = IndexVersionUtils.randomVersion();
         Settings settings = Settings.builder()
             .put("index.analysis.filter.my_lovins.type", "stemmer")
             .put("index.analysis.filter.my_lovins.language", "lovins")
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java
index 4fc6ca96b5f08..af57b8270ff02 100644
--- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java
@@ -118,7 +118,7 @@ public void testSynonymWordDeleteByAnalyzer() throws IOException {
         // Test with an index version where lenient should always be false by default
         IndexVersion randomNonLenientIndexVersion = IndexVersionUtils.randomVersionBetween(
             random(),
-            IndexVersions.MINIMUM_COMPATIBLE,
+            IndexVersions.MINIMUM_READONLY_COMPATIBLE,
             IndexVersions.INDEX_SORTING_ON_NESTED
         );
         assertIsNotLenient.accept(randomNonLenientIndexVersion, false);
@@ -177,7 +177,7 @@ public void testSynonymWordDeleteByAnalyzerFromFile() throws IOException {
         // Test with an index version where lenient should always be false by default
         IndexVersion randomNonLenientIndexVersion = IndexVersionUtils.randomVersionBetween(
             random(),
-            IndexVersions.MINIMUM_COMPATIBLE,
+            IndexVersions.MINIMUM_READONLY_COMPATIBLE,
             IndexVersions.INDEX_SORTING_ON_NESTED
         );
         assertIsNotLenient.accept(randomNonLenientIndexVersion, false);
@@ -231,7 +231,7 @@ public void testExpandSynonymWordDeleteByAnalyzer() throws IOException {
         // Test with an index version where lenient should always be false by default
         IndexVersion randomNonLenientIndexVersion = IndexVersionUtils.randomVersionBetween(
             random(),
-            IndexVersions.MINIMUM_COMPATIBLE,
+            IndexVersions.MINIMUM_READONLY_COMPATIBLE,
             IndexVersions.INDEX_SORTING_ON_NESTED
         );
         assertIsNotLenient.accept(randomNonLenientIndexVersion, false);
@@ -338,7 +338,7 @@ public void testShingleFilters() {
         Settings settings = Settings.builder()
             .put(
                 IndexMetadata.SETTING_VERSION_CREATED,
-                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current())
+                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersion.current())
             )
             .put("path.home", createTempDir().toString())
             .put("index.analysis.filter.synonyms.type", "synonym")
@@ -392,7 +392,7 @@ public void testPreconfiguredTokenFilters() throws IOException {
         Settings settings = Settings.builder()
             .put(
                 IndexMetadata.SETTING_VERSION_CREATED,
-                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current())
+                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersion.current())
             )
             .put("path.home", createTempDir().toString())
             .build();
@@ -424,7 +424,7 @@ public void testDisallowedTokenFilters() throws IOException {
         Settings settings = Settings.builder()
             .put(
                 IndexMetadata.SETTING_VERSION_CREATED,
-                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current())
+                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersion.current())
             )
             .put("path.home", createTempDir().toString())
             .putList("common_words", "a", "b")
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/UniqueTokenFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/UniqueTokenFilterTests.java
index 6bec8dc1ebc62..d30e9d3c68cc9 100644
--- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/UniqueTokenFilterTests.java
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/UniqueTokenFilterTests.java
@@ -124,11 +124,7 @@ public void testOldVersionGetXUniqueTokenFilter() throws IOException {
         Settings settings = Settings.builder()
             .put(
                 IndexMetadata.SETTING_VERSION_CREATED,
-                IndexVersionUtils.randomVersionBetween(
-                    random(),
-                    IndexVersions.MINIMUM_COMPATIBLE,
-                    IndexVersionUtils.getPreviousVersion(IndexVersions.UNIQUE_TOKEN_FILTER_POS_FIX)
-                )
+                IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.UNIQUE_TOKEN_FILTER_POS_FIX)
             )
             .build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
diff --git a/modules/apm/build.gradle b/modules/apm/build.gradle
index b510e2403e933..07e6c7a042135 100644
--- a/modules/apm/build.gradle
+++ b/modules/apm/build.gradle
@@ -7,9 +7,9 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 
 esplugin {
-  name 'apm'
-  description 'Provides APM integration for Elasticsearch'
-  classname 'org.elasticsearch.telemetry.apm.APM'
+  name = 'apm'
+  description = 'Provides APM integration for Elasticsearch'
+  classname ='org.elasticsearch.telemetry.apm.APM'
 }
 
 def otelVersion = '1.31.0'
diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java
index 339a4ec24ca13..43447cfa21a62 100644
--- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java
+++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java
@@ -92,14 +92,7 @@ public List<Setting<?>> getSettings() {
             APMAgentSettings.TELEMETRY_TRACING_ENABLED_SETTING,
             APMAgentSettings.TELEMETRY_TRACING_NAMES_INCLUDE_SETTING,
             APMAgentSettings.TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING,
-            APMAgentSettings.TELEMETRY_TRACING_SANITIZE_FIELD_NAMES,
-            // The settings below are deprecated and are currently kept as fallback.
-            APMAgentSettings.TRACING_APM_SECRET_TOKEN_SETTING,
-            APMAgentSettings.TRACING_APM_API_KEY_SETTING,
-            APMAgentSettings.TRACING_APM_ENABLED_SETTING,
-            APMAgentSettings.TRACING_APM_NAMES_INCLUDE_SETTING,
-            APMAgentSettings.TRACING_APM_NAMES_EXCLUDE_SETTING,
-            APMAgentSettings.TRACING_APM_SANITIZE_FIELD_NAMES
+            APMAgentSettings.TELEMETRY_TRACING_SANITIZE_FIELD_NAMES
         );
     }
 }
diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java
index 99b2a4510bf93..8647761e2defe 100644
--- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java
+++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java
@@ -25,9 +25,7 @@
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
-import java.util.function.Function;
 
-import static org.elasticsearch.common.settings.Setting.Property.DeprecatedWarning;
 import static org.elasticsearch.common.settings.Setting.Property.NodeScope;
 import static org.elasticsearch.common.settings.Setting.Property.OperatorDynamic;
 
@@ -101,9 +99,6 @@ public void setAgentSetting(String key, String value) {
 
     private static final String TELEMETRY_SETTING_PREFIX = "telemetry.";
 
-    // The old legacy prefix
-    private static final String LEGACY_TRACING_APM_SETTING_PREFIX = "tracing.apm.";
-
     /**
      * Allow-list of APM agent config keys users are permitted to configure.
      * @see <a href="https://www.elastic.co/guide/en/apm/agent/java/current/configuration.html">APM Java Agent Configuration</a>
@@ -248,56 +243,24 @@ private static Setting<String> concreteAgentSetting(String namespace, String qua
 
     public static final Setting.AffixSetting<String> APM_AGENT_SETTINGS = Setting.prefixKeySetting(
         TELEMETRY_SETTING_PREFIX + "agent.",
-        LEGACY_TRACING_APM_SETTING_PREFIX + "agent.",
-        (namespace, qualifiedKey) -> qualifiedKey.startsWith(LEGACY_TRACING_APM_SETTING_PREFIX)
-            ? concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic, DeprecatedWarning)
-            : concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic)
+        null, // no fallback
+        (namespace, qualifiedKey) -> concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic)
     );
 
-    /**
-     * @deprecated in favor of TELEMETRY_TRACING_NAMES_INCLUDE_SETTING.
-     */
-    @Deprecated
-    public static final Setting<List<String>> TRACING_APM_NAMES_INCLUDE_SETTING = Setting.stringListSetting(
-        LEGACY_TRACING_APM_SETTING_PREFIX + "names.include",
-        OperatorDynamic,
-        NodeScope,
-        DeprecatedWarning
-    );
-
-    public static final Setting<List<String>> TELEMETRY_TRACING_NAMES_INCLUDE_SETTING = Setting.listSetting(
+    public static final Setting<List<String>> TELEMETRY_TRACING_NAMES_INCLUDE_SETTING = Setting.stringListSetting(
         TELEMETRY_SETTING_PREFIX + "tracing.names.include",
-        TRACING_APM_NAMES_INCLUDE_SETTING,
-        Function.identity(),
         OperatorDynamic,
         NodeScope
     );
 
-    /**
-     * @deprecated in favor of TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING.
-     */
-    @Deprecated
-    public static final Setting<List<String>> TRACING_APM_NAMES_EXCLUDE_SETTING = Setting.stringListSetting(
-        LEGACY_TRACING_APM_SETTING_PREFIX + "names.exclude",
-        OperatorDynamic,
-        NodeScope,
-        DeprecatedWarning
-    );
-
-    public static final Setting<List<String>> TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING = Setting.listSetting(
+    public static final Setting<List<String>> TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING = Setting.stringListSetting(
         TELEMETRY_SETTING_PREFIX + "tracing.names.exclude",
-        TRACING_APM_NAMES_EXCLUDE_SETTING,
-        Function.identity(),
         OperatorDynamic,
         NodeScope
     );
 
-    /**
-     * @deprecated in favor of TELEMETRY_TRACING_SANITIZE_FIELD_NAMES.
-     */
-    @Deprecated
-    public static final Setting<List<String>> TRACING_APM_SANITIZE_FIELD_NAMES = Setting.stringListSetting(
-        LEGACY_TRACING_APM_SETTING_PREFIX + "sanitize_field_names",
+    public static final Setting<List<String>> TELEMETRY_TRACING_SANITIZE_FIELD_NAMES = Setting.stringListSetting(
+        TELEMETRY_SETTING_PREFIX + "tracing.sanitize_field_names",
         List.of(
             "password",
             "passwd",
@@ -313,33 +276,12 @@ private static Setting<String> concreteAgentSetting(String namespace, String qua
             "set-cookie"
         ),
         OperatorDynamic,
-        NodeScope,
-        DeprecatedWarning
-    );
-
-    public static final Setting<List<String>> TELEMETRY_TRACING_SANITIZE_FIELD_NAMES = Setting.listSetting(
-        TELEMETRY_SETTING_PREFIX + "tracing.sanitize_field_names",
-        TRACING_APM_SANITIZE_FIELD_NAMES,
-        Function.identity(),
-        OperatorDynamic,
         NodeScope
     );
 
-    /**
-     * @deprecated in favor of TELEMETRY_TRACING_ENABLED_SETTING.
-     */
-    @Deprecated
-    public static final Setting<Boolean> TRACING_APM_ENABLED_SETTING = Setting.boolSetting(
-        LEGACY_TRACING_APM_SETTING_PREFIX + "enabled",
-        false,
-        OperatorDynamic,
-        NodeScope,
-        DeprecatedWarning
-    );
-
     public static final Setting<Boolean> TELEMETRY_TRACING_ENABLED_SETTING = Setting.boolSetting(
         TELEMETRY_SETTING_PREFIX + "tracing.enabled",
-        TRACING_APM_ENABLED_SETTING,
+        false,
         OperatorDynamic,
         NodeScope
     );
@@ -351,33 +293,13 @@ private static Setting<String> concreteAgentSetting(String namespace, String qua
         NodeScope
     );
 
-    /**
-     * @deprecated in favor of TELEMETRY_SECRET_TOKEN_SETTING.
-     */
-    @Deprecated
-    public static final Setting<SecureString> TRACING_APM_SECRET_TOKEN_SETTING = SecureSetting.secureString(
-        LEGACY_TRACING_APM_SETTING_PREFIX + "secret_token",
-        null,
-        DeprecatedWarning
-    );
-
     public static final Setting<SecureString> TELEMETRY_SECRET_TOKEN_SETTING = SecureSetting.secureString(
         TELEMETRY_SETTING_PREFIX + "secret_token",
-        TRACING_APM_SECRET_TOKEN_SETTING
-    );
-
-    /**
-     * @deprecated in favor of TELEMETRY_API_KEY_SETTING.
-     */
-    @Deprecated
-    public static final Setting<SecureString> TRACING_APM_API_KEY_SETTING = SecureSetting.secureString(
-        LEGACY_TRACING_APM_SETTING_PREFIX + "api_key",
-        null,
-        DeprecatedWarning
+        null
     );
 
     public static final Setting<SecureString> TELEMETRY_API_KEY_SETTING = SecureSetting.secureString(
         TELEMETRY_SETTING_PREFIX + "api_key",
-        TRACING_APM_API_KEY_SETTING
+        null
     );
 }
diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java
index cb74d62137815..f60179d533950 100644
--- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java
+++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
+import org.elasticsearch.lucene.util.automaton.MinimizationOperations;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.telemetry.apm.internal.APMAgentSettings;
 import org.elasticsearch.telemetry.tracing.TraceContext;
@@ -439,7 +440,7 @@ private static CharacterRunAutomaton buildAutomaton(List<String> includePatterns
             ? includeAutomaton
             : Operations.minus(includeAutomaton, excludeAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
 
-        return new CharacterRunAutomaton(Operations.determinize(finalAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT));
+        return new CharacterRunAutomaton(MinimizationOperations.minimize(finalAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT));
     }
 
     private static Automaton patternsToAutomaton(List<String> patterns) {
diff --git a/modules/apm/src/main/plugin-metadata/entitlement-policy.yaml b/modules/apm/src/main/plugin-metadata/entitlement-policy.yaml
new file mode 100644
index 0000000000000..9c10bafca42f9
--- /dev/null
+++ b/modules/apm/src/main/plugin-metadata/entitlement-policy.yaml
@@ -0,0 +1,4 @@
+org.elasticsearch.telemetry.apm:
+  - create_class_loader
+elastic.apm.agent:
+  - set_https_connection_properties
diff --git a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java
index a60048c82a3c9..5516672420924 100644
--- a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java
+++ b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java
@@ -11,8 +11,6 @@
 
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.common.settings.MockSecureSettings;
-import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 import org.mockito.Mockito;
@@ -21,21 +19,13 @@
 import java.util.Set;
 
 import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.APM_AGENT_SETTINGS;
-import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_API_KEY_SETTING;
 import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_METRICS_ENABLED_SETTING;
-import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_SECRET_TOKEN_SETTING;
 import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_TRACING_ENABLED_SETTING;
 import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING;
 import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_TRACING_NAMES_INCLUDE_SETTING;
 import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_TRACING_SANITIZE_FIELD_NAMES;
-import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_API_KEY_SETTING;
-import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_ENABLED_SETTING;
-import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_NAMES_EXCLUDE_SETTING;
-import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_NAMES_INCLUDE_SETTING;
-import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_SANITIZE_FIELD_NAMES;
-import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_SECRET_TOKEN_SETTING;
-import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasItem;
 import static org.mockito.Mockito.clearInvocations;
 import static org.mockito.Mockito.mock;
@@ -70,14 +60,6 @@ public void testEnableTracing() {
         }
     }
 
-    public void testEnableTracingUsingLegacySetting() {
-        Settings settings = Settings.builder().put(TRACING_APM_ENABLED_SETTING.getKey(), true).build();
-        apmAgentSettings.initAgentSystemProperties(settings);
-
-        verify(apmAgentSettings).setAgentSetting("recording", "true");
-        assertWarnings("[tracing.apm.enabled] setting was deprecated in Elasticsearch and will be removed in a future release.");
-    }
-
     public void testEnableMetrics() {
         for (boolean tracingEnabled : List.of(true, false)) {
             clearInvocations(apmAgentSettings, apmTelemetryProvider.getMeterService());
@@ -121,14 +103,6 @@ public void testDisableTracing() {
         }
     }
 
-    public void testDisableTracingUsingLegacySetting() {
-        Settings settings = Settings.builder().put(TRACING_APM_ENABLED_SETTING.getKey(), false).build();
-        apmAgentSettings.initAgentSystemProperties(settings);
-
-        verify(apmAgentSettings).setAgentSetting("recording", "false");
-        assertWarnings("[tracing.apm.enabled] setting was deprecated in Elasticsearch and will be removed in a future release.");
-    }
-
     public void testDisableMetrics() {
         for (boolean tracingEnabled : List.of(true, false)) {
             clearInvocations(apmAgentSettings, apmTelemetryProvider.getMeterService());
@@ -181,70 +155,18 @@ public void testSetAgentSettings() {
         verify(apmAgentSettings).setAgentSetting("span_compression_enabled", "true");
     }
 
-    public void testSetAgentsSettingsWithLegacyPrefix() {
-        Settings settings = Settings.builder()
-            .put(TELEMETRY_TRACING_ENABLED_SETTING.getKey(), true)
-            .put("tracing.apm.agent.span_compression_enabled", "true")
-            .build();
-        apmAgentSettings.initAgentSystemProperties(settings);
-
-        verify(apmAgentSettings).setAgentSetting("recording", "true");
-        verify(apmAgentSettings).setAgentSetting("span_compression_enabled", "true");
-        assertWarnings(
-            "[tracing.apm.agent.span_compression_enabled] setting was deprecated in Elasticsearch and will be removed in a future release."
-        );
-    }
-
     /**
      * Check that invalid or forbidden APM agent settings are rejected.
      */
     public void testRejectForbiddenOrUnknownAgentSettings() {
-        List<String> prefixes = List.of(APM_AGENT_SETTINGS.getKey(), "tracing.apm.agent.");
-        for (String prefix : prefixes) {
-            Settings settings = Settings.builder().put(prefix + "unknown", "true").build();
-            Exception exception = expectThrows(IllegalArgumentException.class, () -> APM_AGENT_SETTINGS.getAsMap(settings));
-            assertThat(exception.getMessage(), containsString("[" + prefix + "unknown]"));
-        }
-        // though, accept / ignore nested global_labels
-        for (String prefix : prefixes) {
-            Settings settings = Settings.builder().put(prefix + "global_labels.abc", "123").build();
-            APMAgentSettings.APM_AGENT_SETTINGS.getAsMap(settings);
-
-            if (prefix.startsWith("tracing.apm.agent.")) {
-                assertWarnings(
-                    "[tracing.apm.agent.global_labels.abc] setting was deprecated in Elasticsearch and will be removed in a future release."
-                );
-            }
-        }
-    }
-
-    public void testTelemetryTracingNamesIncludeFallback() {
-        Settings settings = Settings.builder().put(TRACING_APM_NAMES_INCLUDE_SETTING.getKey(), "abc,xyz").build();
-
-        List<String> included = TELEMETRY_TRACING_NAMES_INCLUDE_SETTING.get(settings);
-
-        assertThat(included, containsInAnyOrder("abc", "xyz"));
-        assertWarnings("[tracing.apm.names.include] setting was deprecated in Elasticsearch and will be removed in a future release.");
-    }
-
-    public void testTelemetryTracingNamesExcludeFallback() {
-        Settings settings = Settings.builder().put(TRACING_APM_NAMES_EXCLUDE_SETTING.getKey(), "abc,xyz").build();
-
-        List<String> included = TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING.get(settings);
-
-        assertThat(included, containsInAnyOrder("abc", "xyz"));
-        assertWarnings("[tracing.apm.names.exclude] setting was deprecated in Elasticsearch and will be removed in a future release.");
-    }
-
-    public void testTelemetryTracingSanitizeFieldNamesFallback() {
-        Settings settings = Settings.builder().put(TRACING_APM_SANITIZE_FIELD_NAMES.getKey(), "abc,xyz").build();
-
-        List<String> included = TELEMETRY_TRACING_SANITIZE_FIELD_NAMES.get(settings);
+        String prefix = APM_AGENT_SETTINGS.getKey();
+        Settings settings = Settings.builder().put(prefix + "unknown", "true").build();
+        Exception exception = expectThrows(IllegalArgumentException.class, () -> APM_AGENT_SETTINGS.getAsMap(settings));
+        assertThat(exception.getMessage(), containsString("[" + prefix + "unknown]"));
 
-        assertThat(included, containsInAnyOrder("abc", "xyz"));
-        assertWarnings(
-            "[tracing.apm.sanitize_field_names] setting was deprecated in Elasticsearch and will be removed in a future release."
-        );
+        // though, accept / ignore nested global_labels
+        var map = APMAgentSettings.APM_AGENT_SETTINGS.getAsMap(Settings.builder().put(prefix + "global_labels.abc", "123").build());
+        assertThat(map, hasEntry("global_labels.abc", "123"));
     }
 
     public void testTelemetryTracingSanitizeFieldNamesFallbackDefault() {
@@ -252,28 +174,6 @@ public void testTelemetryTracingSanitizeFieldNamesFallbackDefault() {
         assertThat(included, hasItem("password")); // and more defaults
     }
 
-    public void testTelemetrySecretTokenFallback() {
-        MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString(TRACING_APM_SECRET_TOKEN_SETTING.getKey(), "verysecret");
-        Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
-
-        try (SecureString secureString = TELEMETRY_SECRET_TOKEN_SETTING.get(settings)) {
-            assertEquals("verysecret", secureString.toString());
-        }
-        assertWarnings("[tracing.apm.secret_token] setting was deprecated in Elasticsearch and will be removed in a future release.");
-    }
-
-    public void testTelemetryApiKeyFallback() {
-        MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString(TRACING_APM_API_KEY_SETTING.getKey(), "abc");
-        Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
-
-        try (SecureString secureString = TELEMETRY_API_KEY_SETTING.get(settings)) {
-            assertEquals("abc", secureString.toString());
-        }
-        assertWarnings("[tracing.apm.api_key] setting was deprecated in Elasticsearch and will be removed in a future release.");
-    }
-
     /**
      * Check that invalid or forbidden APM agent settings are rejected if their last part resembles an allowed setting.
      */
diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle
index 8ae56101ef01e..60bc8d1dc6a92 100644
--- a/modules/data-streams/build.gradle
+++ b/modules/data-streams/build.gradle
@@ -7,8 +7,8 @@ apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.yaml-rest-compat-test'
 
 esplugin {
-  description 'Elasticsearch Expanded Pack Plugin - Data Streams'
-  classname 'org.elasticsearch.datastreams.DataStreamsPlugin'
+  description = 'Elasticsearch Expanded Pack Plugin - Data Streams'
+  classname ='org.elasticsearch.datastreams.DataStreamsPlugin'
 }
 
 restResources {
@@ -20,6 +20,7 @@ restResources {
 
 dependencies {
   testImplementation project(path: ':test:test-clusters')
+  testImplementation project(":modules:mapper-extras")
   internalClusterTestImplementation project(":modules:mapper-extras")
 }
 
@@ -70,4 +71,16 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
   task.skipTest("data_stream/200_rollover_failure_store/Lazily roll over a data stream's failure store after a shard failure", "Configuring the failure store via data stream templates is not supported anymore.")
   task.skipTest("data_stream/200_rollover_failure_store/Don't roll over a data stream's failure store when conditions aren't met", "Configuring the failure store via data stream templates is not supported anymore.")
   task.skipTest("data_stream/200_rollover_failure_store/Roll over a data stream's failure store with conditions", "Configuring the failure store via data stream templates is not supported anymore.")
+
+  task.skipTest("data_stream/200_rollover_failure_store/Rolling over a failure store on a data stream without the failure store enabled should work", "Rolling over a data stream using target_failure_store is no longer supported.")
+  task.skipTest("data_stream/200_rollover_failure_store/Rolling over an uninitialized failure store should initialize it", "Rolling over a data stream using target_failure_store is no longer supported.")
+
+  task.skipTest("data_stream/210_rollover_failure_store/A failure store marked for lazy rollover should only be rolled over when there is a failure", "Rolling over a data stream using target_failure_store is no longer supported.")
+  task.skipTest("data_stream/210_rollover_failure_store/Don't roll over a data stream's failure store when conditions aren't met", "Rolling over a data stream using target_failure_store is no longer supported.")
+  task.skipTest("data_stream/210_rollover_failure_store/Rolling over a failure store on a data stream without the failure store enabled should work", "Rolling over a data stream using target_failure_store is no longer supported.")
+  task.skipTest("data_stream/210_rollover_failure_store/Rolling over an uninitialized failure store should initialize it", "Rolling over a data stream using target_failure_store is no longer supported.")
+  task.skipTest("data_stream/210_rollover_failure_store/Roll over a data stream's failure store with conditions", "Rolling over a data stream using target_failure_store is no longer supported.")
+  task.skipTest("data_stream/210_rollover_failure_store/Lazily roll over a data stream's failure store after an ingest failure", "Rolling over a data stream using target_failure_store is no longer supported.")
+  task.skipTest("data_stream/210_rollover_failure_store/Lazily roll over a data stream's failure store after a shard failure", "Rolling over a data stream using target_failure_store is no longer supported.")
+  task.skipTest("data_stream/210_rollover_failure_store/Roll over a data stream's failure store without conditions", "Rolling over a data stream using target_failure_store is no longer supported.")
 })
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java
index ac73385a97d70..91f18ad3573fd 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java
@@ -493,7 +493,7 @@ private static ShardStats getShardStats(IndexMetadata indexMeta, int shardIndex,
         CommonStats stats = new CommonStats();
         stats.docs = new DocsStats(100, 0, randomByteSizeValue().getBytes());
         stats.store = new StoreStats();
-        stats.indexing = new IndexingStats(new IndexingStats.Stats(1, 1, 1, 1, 1, 1, 1, 1, false, 1, targetWriteLoad, 1));
+        stats.indexing = new IndexingStats(new IndexingStats.Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, false, 1, targetWriteLoad, 1));
         return new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null, false, 0);
     }
 
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java
index 0e03045a090f8..2739eb51376ea 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java
@@ -588,11 +588,15 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception {
         verifyResolvability(dataStreamName, indicesAdmin().prepareForceMerge(dataStreamName), false);
         verifyResolvability(dataStreamName, indicesAdmin().prepareValidateQuery(dataStreamName), false);
         verifyResolvability(dataStreamName, indicesAdmin().prepareRecoveries(dataStreamName), false);
-        verifyResolvability(dataStreamName, indicesAdmin().prepareGetAliases("dummy").setIndices(dataStreamName), false);
+        verifyResolvability(
+            dataStreamName,
+            indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "dummy").setIndices(dataStreamName),
+            false
+        );
         verifyResolvability(dataStreamName, indicesAdmin().prepareGetFieldMappings(dataStreamName), false);
         verifyResolvability(dataStreamName, indicesAdmin().preparePutMapping(dataStreamName).setSource("""
             {"_doc":{"properties": {"my_field":{"type":"keyword"}}}}""", XContentType.JSON), false);
-        verifyResolvability(dataStreamName, indicesAdmin().prepareGetMappings(dataStreamName), false);
+        verifyResolvability(dataStreamName, indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, dataStreamName), false);
         verifyResolvability(
             dataStreamName,
             indicesAdmin().prepareUpdateSettings(dataStreamName).setSettings(Settings.builder().put("index.number_of_replicas", 0)),
@@ -635,11 +639,11 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception {
         verifyResolvability(wildcardExpression, indicesAdmin().prepareForceMerge(wildcardExpression), false);
         verifyResolvability(wildcardExpression, indicesAdmin().prepareValidateQuery(wildcardExpression), false);
         verifyResolvability(wildcardExpression, indicesAdmin().prepareRecoveries(wildcardExpression), false);
-        verifyResolvability(wildcardExpression, indicesAdmin().prepareGetAliases(wildcardExpression), false);
+        verifyResolvability(wildcardExpression, indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, wildcardExpression), false);
         verifyResolvability(wildcardExpression, indicesAdmin().prepareGetFieldMappings(wildcardExpression), false);
         verifyResolvability(wildcardExpression, indicesAdmin().preparePutMapping(wildcardExpression).setSource("""
             {"_doc":{"properties": {"my_field":{"type":"keyword"}}}}""", XContentType.JSON), false);
-        verifyResolvability(wildcardExpression, indicesAdmin().prepareGetMappings(wildcardExpression), false);
+        verifyResolvability(wildcardExpression, indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, wildcardExpression), false);
         verifyResolvability(wildcardExpression, indicesAdmin().prepareGetSettings(wildcardExpression), false);
         verifyResolvability(
             wildcardExpression,
@@ -718,7 +722,7 @@ public void testCannotDeleteComposableTemplateUsedByDataStream() throws Exceptio
         TransportDeleteComposableIndexTemplateAction.Request deleteRequest = new TransportDeleteComposableIndexTemplateAction.Request("id");
         client().execute(TransportDeleteComposableIndexTemplateAction.TYPE, deleteRequest).get();
 
-        GetComposableIndexTemplateAction.Request getReq = new GetComposableIndexTemplateAction.Request("id");
+        GetComposableIndexTemplateAction.Request getReq = new GetComposableIndexTemplateAction.Request(TEST_REQUEST_TIMEOUT, "id");
         Exception e3 = expectThrows(Exception.class, client().execute(GetComposableIndexTemplateAction.INSTANCE, getReq));
         maybeE = ExceptionsHelper.unwrapCausesAndSuppressed(e3, err -> err.getMessage().contains("index template matching [id] not found"));
         assertTrue(maybeE.isPresent());
@@ -738,7 +742,7 @@ public void testAliasActionsOnDataStreams() throws Exception {
         IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest();
         aliasesAddRequest.addAliasAction(addAction);
         assertAcked(indicesAdmin().aliases(aliasesAddRequest).actionGet());
-        GetAliasesResponse response = indicesAdmin().getAliases(new GetAliasesRequest()).actionGet();
+        GetAliasesResponse response = indicesAdmin().getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT)).actionGet();
         assertThat(
             response.getDataStreamAliases(),
             equalTo(Map.of("metrics-foo", List.of(new DataStreamAlias("foo", List.of("metrics-foo"), null, null))))
@@ -764,7 +768,7 @@ public void testDataSteamAliasWithFilter() throws Exception {
         IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest();
         aliasesAddRequest.addAliasAction(addAction);
         assertAcked(indicesAdmin().aliases(aliasesAddRequest).actionGet());
-        GetAliasesResponse response = indicesAdmin().getAliases(new GetAliasesRequest()).actionGet();
+        GetAliasesResponse response = indicesAdmin().getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT)).actionGet();
         assertThat(
             response.getDataStreamAliases(),
             equalTo(
@@ -794,7 +798,7 @@ public void testDataSteamAliasWithFilter() throws Exception {
         aliasesAddRequest = new IndicesAliasesRequest();
         aliasesAddRequest.addAliasAction(addAction);
         assertAcked(indicesAdmin().aliases(aliasesAddRequest).actionGet());
-        response = indicesAdmin().getAliases(new GetAliasesRequest()).actionGet();
+        response = indicesAdmin().getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT)).actionGet();
         assertThat(
             response.getDataStreamAliases(),
             equalTo(
@@ -840,7 +844,7 @@ public void testSearchFilteredAndUnfilteredAlias() throws Exception {
         aliasesAddRequest.addAliasAction(addFilteredAliasAction);
         aliasesAddRequest.addAliasAction(addUnfilteredAliasAction);
         assertAcked(indicesAdmin().aliases(aliasesAddRequest).actionGet());
-        GetAliasesResponse response = indicesAdmin().getAliases(new GetAliasesRequest()).actionGet();
+        GetAliasesResponse response = indicesAdmin().getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT)).actionGet();
         assertThat(response.getDataStreamAliases(), hasKey("logs-foobar"));
         assertThat(
             response.getDataStreamAliases().get("logs-foobar"),
@@ -884,7 +888,7 @@ public void testRandomDataSteamAliasesUpdate() throws Exception {
         addAction = new AliasActions(AliasActions.Type.ADD).aliases(alias).indices(dataStreams[0]).filter(indexFilters).writeIndex(true);
         assertAcked(indicesAdmin().aliases(new IndicesAliasesRequest().addAliasAction(addAction)).actionGet());
 
-        GetAliasesResponse response = indicesAdmin().getAliases(new GetAliasesRequest()).actionGet();
+        GetAliasesResponse response = indicesAdmin().getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT)).actionGet();
         assertThat(response.getDataStreamAliases().size(), equalTo(dataStreams.length));
         List<DataStreamAlias> result = response.getDataStreamAliases()
             .values()
@@ -933,7 +937,7 @@ public void testDataSteamAliasWithMalformedFilter() throws Exception {
             indicesAdmin().aliases(new IndicesAliasesRequest().addAliasAction(addAction))
         );
         assertThat(e.getMessage(), equalTo("failed to parse filter for alias [" + alias + "]"));
-        GetAliasesResponse response = indicesAdmin().getAliases(new GetAliasesRequest()).actionGet();
+        GetAliasesResponse response = indicesAdmin().getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT)).actionGet();
         assertThat(response.getDataStreamAliases(), anEmptyMap());
     }
 
@@ -998,7 +1002,7 @@ public void testRemoveDataStreamAliasesMixedExpression() throws Exception {
         aliasesAddRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).index("metrics-foo").aliases("my-alias1"));
         aliasesAddRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).index("metrics-myindex").aliases("my-alias2"));
         assertAcked(indicesAdmin().aliases(aliasesAddRequest).actionGet());
-        GetAliasesResponse response = indicesAdmin().getAliases(new GetAliasesRequest()).actionGet();
+        GetAliasesResponse response = indicesAdmin().getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT)).actionGet();
         assertThat(
             response.getDataStreamAliases(),
             equalTo(Map.of("metrics-foo", List.of(new DataStreamAlias("my-alias1", List.of("metrics-foo"), null, null))))
@@ -1013,7 +1017,7 @@ public void testRemoveDataStreamAliasesMixedExpression() throws Exception {
             aliasesAddRequest.addAliasAction(new AliasActions(AliasActions.Type.REMOVE).index("_all").aliases("my-*"));
         }
         assertAcked(indicesAdmin().aliases(aliasesAddRequest).actionGet());
-        response = indicesAdmin().getAliases(new GetAliasesRequest()).actionGet();
+        response = indicesAdmin().getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT)).actionGet();
         assertThat(response.getDataStreamAliases(), anEmptyMap());
         assertThat(response.getAliases().get("metrics-myindex").size(), equalTo(0));
         assertThat(response.getAliases().size(), equalTo(1));
@@ -1034,7 +1038,7 @@ public void testUpdateDataStreamsWithWildcards() throws Exception {
                 new AliasActions(AliasActions.Type.ADD).index("metrics-foo").aliases("my-alias1", "my-alias2")
             );
             assertAcked(indicesAdmin().aliases(aliasesAddRequest).actionGet());
-            GetAliasesResponse response = indicesAdmin().getAliases(new GetAliasesRequest()).actionGet();
+            GetAliasesResponse response = indicesAdmin().getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT)).actionGet();
             assertThat(response.getDataStreamAliases().keySet(), containsInAnyOrder("metrics-foo"));
             assertThat(
                 response.getDataStreamAliases().get("metrics-foo"),
@@ -1060,7 +1064,7 @@ public void testUpdateDataStreamsWithWildcards() throws Exception {
                 aliasesAddRequest.addAliasAction(new AliasActions(AliasActions.Type.REMOVE).index("_all").aliases("_all"));
             }
             assertAcked(indicesAdmin().aliases(aliasesAddRequest).actionGet());
-            GetAliasesResponse response = indicesAdmin().getAliases(new GetAliasesRequest()).actionGet();
+            GetAliasesResponse response = indicesAdmin().getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT)).actionGet();
             assertThat(response.getDataStreamAliases(), anEmptyMap());
             assertThat(response.getAliases().size(), equalTo(0));
         }
@@ -1176,7 +1180,7 @@ public void testUpdateMappingViaDataStream() throws Exception {
             DataStreamTimestampFieldMapper.NAME,
             Map.of("enabled", true)
         );
-        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("logs-foobar").get();
+        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "logs-foobar").get();
         assertThat(getMappingsResponse.getMappings().size(), equalTo(2));
         assertThat(getMappingsResponse.getMappings().get(backingIndex1).getSourceAsMap(), equalTo(expectedMapping));
         assertThat(getMappingsResponse.getMappings().get(backingIndex2).getSourceAsMap(), equalTo(expectedMapping));
@@ -1191,7 +1195,7 @@ public void testUpdateMappingViaDataStream() throws Exception {
             .setSource("{\"properties\":{\"my_field\":{\"type\":\"keyword\"}}}", XContentType.JSON)
             .get();
         // The mappings of all backing indices should be updated:
-        getMappingsResponse = indicesAdmin().prepareGetMappings("logs-foobar").get();
+        getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "logs-foobar").get();
         assertThat(getMappingsResponse.getMappings().size(), equalTo(2));
         assertThat(getMappingsResponse.getMappings().get(backingIndex1).getSourceAsMap(), equalTo(expectedMapping));
         assertThat(getMappingsResponse.getMappings().get(backingIndex2).getSourceAsMap(), equalTo(expectedMapping));
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java
index 286ad68896797..f6c703b96888c 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java
@@ -31,11 +31,13 @@
 import org.elasticsearch.action.datastreams.CreateDataStreamAction;
 import org.elasticsearch.action.datastreams.DeleteDataStreamAction;
 import org.elasticsearch.action.datastreams.GetDataStreamAction;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamAlias;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.index.Index;
@@ -60,7 +62,6 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -77,6 +78,7 @@
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
@@ -136,27 +138,17 @@ public void setup() throws Exception {
         assertTrue(response.isAcknowledged());
 
         // Initialize the failure store.
-        RolloverRequest rolloverRequest = new RolloverRequest("with-fs", null);
-        rolloverRequest.setIndicesOptions(
-            IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build()
-        );
+        RolloverRequest rolloverRequest = new RolloverRequest("with-fs::failures", null);
         response = client.execute(RolloverAction.INSTANCE, rolloverRequest).get();
         assertTrue(response.isAcknowledged());
 
         // Resolve backing index names after data streams have been created:
         // (these names have a date component, and running around midnight could lead to test failures otherwise)
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
-        GetDataStreamAction.Response getDataStreamResponse = client.execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet();
-        dsBackingIndexName = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().get(0).getName();
-        otherDsBackingIndexName = getDataStreamResponse.getDataStreams().get(1).getDataStream().getIndices().get(0).getName();
-        fsBackingIndexName = getDataStreamResponse.getDataStreams().get(2).getDataStream().getIndices().get(0).getName();
-        fsFailureIndexName = getDataStreamResponse.getDataStreams()
-            .get(2)
-            .getDataStream()
-            .getFailureIndices()
-            .getIndices()
-            .get(0)
-            .getName();
+        List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = getDataStreamInfo("*");
+        dsBackingIndexName = dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName();
+        otherDsBackingIndexName = dataStreamInfos.get(1).getDataStream().getIndices().get(0).getName();
+        fsBackingIndexName = dataStreamInfos.get(2).getDataStream().getIndices().get(0).getName();
+        fsFailureIndexName = dataStreamInfos.get(2).getDataStream().getFailureIndices().getIndices().get(0).getName();
 
         // Will be used in some tests, to test renaming while restoring a snapshot:
         ds2BackingIndexName = dsBackingIndexName.replace("-ds-", "-ds2-");
@@ -198,9 +190,7 @@ public void testSnapshotAndRestore() throws Exception {
 
         assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());
 
-        assertAcked(
-            client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" }))
-        );
+        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "ds")));
 
         RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
             .cluster()
@@ -218,15 +208,15 @@ public void testSnapshotAndRestore() throws Exception {
             assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap());
         });
 
-        GetDataStreamAction.Response ds = client.execute(
-            GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" })
-        ).get();
-        assertEquals(1, ds.getDataStreams().size());
-        assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
-        assertEquals(dsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
+        List<GetDataStreamAction.Response.DataStreamInfo> ds = getDataStreamInfo("ds");
+        assertEquals(1, ds.size());
+        assertEquals(1, ds.get(0).getDataStream().getIndices().size());
+        assertEquals(dsBackingIndexName, ds.get(0).getDataStream().getIndices().get(0).getName());
 
-        GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet();
+        GetAliasesResponse getAliasesResponse = client.admin()
+            .indices()
+            .getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "my-alias"))
+            .actionGet();
         assertThat(getAliasesResponse.getDataStreamAliases().keySet(), containsInAnyOrder("ds", "other-ds"));
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").size(), equalTo(1));
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").get(0).getName(), equalTo("my-alias"));
@@ -278,19 +268,18 @@ public void testSnapshotAndRestoreAllDataStreamsInPlace() throws Exception {
             assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap());
         });
 
-        GetDataStreamAction.Request getDataSteamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
-        GetDataStreamAction.Response ds = client.execute(GetDataStreamAction.INSTANCE, getDataSteamRequest).get();
+        List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = getDataStreamInfo("*");
         assertThat(
-            ds.getDataStreams().stream().map(e -> e.getDataStream().getName()).collect(Collectors.toList()),
+            dataStreamInfos.stream().map(e -> e.getDataStream().getName()).collect(Collectors.toList()),
             contains(equalTo("ds"), equalTo("other-ds"), equalTo("with-fs"))
         );
-        List<Index> backingIndices = ds.getDataStreams().get(0).getDataStream().getIndices();
+        List<Index> backingIndices = dataStreamInfos.get(0).getDataStream().getIndices();
         assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(dsBackingIndexName));
-        backingIndices = ds.getDataStreams().get(1).getDataStream().getIndices();
+        backingIndices = dataStreamInfos.get(1).getDataStream().getIndices();
         assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(otherDsBackingIndexName));
-        backingIndices = ds.getDataStreams().get(2).getDataStream().getIndices();
+        backingIndices = dataStreamInfos.get(2).getDataStream().getIndices();
         assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(fsBackingIndexName));
-        List<Index> failureIndices = ds.getDataStreams().get(2).getDataStream().getFailureIndices().getIndices();
+        List<Index> failureIndices = dataStreamInfos.get(2).getDataStream().getFailureIndices().getIndices();
         assertThat(failureIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(fsFailureIndexName));
     }
 
@@ -337,14 +326,10 @@ public void testSnapshotAndRestoreInPlace() {
             assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap());
         });
 
-        GetDataStreamAction.Request getDataSteamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" });
-        GetDataStreamAction.Response ds = client.execute(GetDataStreamAction.INSTANCE, getDataSteamRequest).actionGet();
-        assertThat(
-            ds.getDataStreams().stream().map(e -> e.getDataStream().getName()).collect(Collectors.toList()),
-            contains(equalTo("ds"))
-        );
-        List<Index> backingIndices = ds.getDataStreams().get(0).getDataStream().getIndices();
-        assertThat(ds.getDataStreams().get(0).getDataStream().getIndices(), hasSize(1));
+        List<GetDataStreamAction.Response.DataStreamInfo> dsInfo = getDataStreamInfo("ds");
+        assertThat(dsInfo.stream().map(e -> e.getDataStream().getName()).collect(Collectors.toList()), contains(equalTo("ds")));
+        List<Index> backingIndices = dsInfo.get(0).getDataStream().getIndices();
+        assertThat(dsInfo.get(0).getDataStream().getIndices(), hasSize(1));
         assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(equalTo(dsBackingIndexName)));
 
         // The backing index created as part of rollover should still exist (but just not part of the data stream)
@@ -357,39 +342,40 @@ public void testSnapshotAndRestoreInPlace() {
     }
 
     public void testFailureStoreSnapshotAndRestore() throws Exception {
+        String dataStreamName = "with-fs";
         CreateSnapshotResponse createSnapshotResponse = client.admin()
             .cluster()
             .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
             .setWaitForCompletion(true)
-            .setIndices("with-fs")
+            .setIndices(IndexNameExpressionResolver.combineSelector(dataStreamName, IndexComponentSelector.ALL_APPLICABLE))
             .setIncludeGlobalState(false)
             .get();
 
         RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
         assertEquals(RestStatus.OK, status);
 
+        assertThat(getSnapshot(REPO, SNAPSHOT).dataStreams(), containsInAnyOrder(dataStreamName));
         assertThat(getSnapshot(REPO, SNAPSHOT).indices(), containsInAnyOrder(fsBackingIndexName, fsFailureIndexName));
 
-        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "with-fs")));
+        assertAcked(
+            client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, dataStreamName))
+        );
 
         {
             RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
                 .cluster()
                 .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
                 .setWaitForCompletion(true)
-                .setIndices("with-fs")
+                .setIndices(dataStreamName)
                 .get();
 
             assertEquals(2, restoreSnapshotResponse.getRestoreInfo().successfulShards());
 
-            GetDataStreamAction.Response ds = client.execute(
-                GetDataStreamAction.INSTANCE,
-                new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "with-fs" })
-            ).get();
-            assertEquals(1, ds.getDataStreams().size());
-            assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
-            assertEquals(fsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
-            assertEquals(fsFailureIndexName, ds.getDataStreams().get(0).getDataStream().getFailureIndices().getIndices().get(0).getName());
+            List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = getDataStreamInfo(dataStreamName);
+            assertEquals(1, dataStreamInfos.size());
+            assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size());
+            assertEquals(fsBackingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName());
+            assertEquals(fsFailureIndexName, dataStreamInfos.get(0).getDataStream().getFailureIndices().getIndices().get(0).getName());
         }
         {
             // With rename pattern
@@ -397,21 +383,18 @@ public void testFailureStoreSnapshotAndRestore() throws Exception {
                 .cluster()
                 .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
                 .setWaitForCompletion(true)
-                .setIndices("with-fs")
+                .setIndices(dataStreamName)
                 .setRenamePattern("-fs")
                 .setRenameReplacement("-fs2")
                 .get();
 
             assertEquals(2, restoreSnapshotResponse.getRestoreInfo().successfulShards());
 
-            GetDataStreamAction.Response ds = client.execute(
-                GetDataStreamAction.INSTANCE,
-                new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "with-fs2" })
-            ).get();
-            assertEquals(1, ds.getDataStreams().size());
-            assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
-            assertEquals(fs2BackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
-            assertEquals(fs2FailureIndexName, ds.getDataStreams().get(0).getDataStream().getFailureIndices().getIndices().get(0).getName());
+            List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = getDataStreamInfo("with-fs2");
+            assertEquals(1, dataStreamInfos.size());
+            assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size());
+            assertEquals(fs2BackingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName());
+            assertEquals(fs2FailureIndexName, dataStreamInfos.get(0).getDataStream().getFailureIndices().getIndices().get(0).getName());
         }
     }
 
@@ -477,15 +460,15 @@ public void testSnapshotAndRestoreAllIncludeSpecificDataStream() throws Exceptio
             assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap());
         });
 
-        GetDataStreamAction.Response ds = client.execute(
-            GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamToSnapshot })
-        ).get();
-        assertEquals(1, ds.getDataStreams().size());
-        assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
-        assertEquals(backingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
+        List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = getDataStreamInfo(dataStreamToSnapshot);
+        assertEquals(1, dataStreamInfos.size());
+        assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size());
+        assertEquals(backingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName());
 
-        GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet();
+        GetAliasesResponse getAliasesResponse = client.admin()
+            .indices()
+            .getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "my-alias"))
+            .actionGet();
         assertThat(getAliasesResponse.getDataStreamAliases().keySet(), contains(dataStreamToSnapshot));
         assertThat(
             getAliasesResponse.getDataStreamAliases().get(dataStreamToSnapshot),
@@ -536,17 +519,17 @@ public void testSnapshotAndRestoreReplaceAll() throws Exception {
             assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap());
         });
 
-        GetDataStreamAction.Response ds = client.execute(
-            GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" })
-        ).get();
-        assertEquals(3, ds.getDataStreams().size());
+        List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = getDataStreamInfo("*");
+        assertEquals(3, dataStreamInfos.size());
         assertThat(
-            ds.getDataStreams().stream().map(i -> i.getDataStream().getName()).collect(Collectors.toList()),
+            dataStreamInfos.stream().map(i -> i.getDataStream().getName()).collect(Collectors.toList()),
             containsInAnyOrder("ds", "other-ds", "with-fs")
         );
 
-        GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet();
+        GetAliasesResponse getAliasesResponse = client.admin()
+            .indices()
+            .getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "my-alias"))
+            .actionGet();
         assertThat(getAliasesResponse.getDataStreamAliases().keySet(), containsInAnyOrder("ds", "other-ds"));
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").size(), equalTo(1));
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").get(0).getName(), equalTo("my-alias"));
@@ -596,21 +579,21 @@ public void testSnapshotAndRestoreAll() throws Exception {
             assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap());
         });
 
-        GetDataStreamAction.Response ds = client.execute(
-            GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" })
-        ).get();
-        assertEquals(3, ds.getDataStreams().size());
-        assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
-        assertEquals(dsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
-        assertEquals(1, ds.getDataStreams().get(1).getDataStream().getIndices().size());
-        assertEquals(otherDsBackingIndexName, ds.getDataStreams().get(1).getDataStream().getIndices().get(0).getName());
-        assertEquals(1, ds.getDataStreams().get(2).getDataStream().getIndices().size());
-        assertEquals(fsBackingIndexName, ds.getDataStreams().get(2).getDataStream().getIndices().get(0).getName());
-        assertEquals(1, ds.getDataStreams().get(2).getDataStream().getFailureIndices().getIndices().size());
-        assertEquals(fsFailureIndexName, ds.getDataStreams().get(2).getDataStream().getFailureIndices().getIndices().get(0).getName());
-
-        GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet();
+        List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = getDataStreamInfo("*");
+        assertEquals(3, dataStreamInfos.size());
+        assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size());
+        assertEquals(dsBackingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName());
+        assertEquals(1, dataStreamInfos.get(1).getDataStream().getIndices().size());
+        assertEquals(otherDsBackingIndexName, dataStreamInfos.get(1).getDataStream().getIndices().get(0).getName());
+        assertEquals(1, dataStreamInfos.get(2).getDataStream().getIndices().size());
+        assertEquals(fsBackingIndexName, dataStreamInfos.get(2).getDataStream().getIndices().get(0).getName());
+        assertEquals(1, dataStreamInfos.get(2).getDataStream().getFailureIndices().getIndices().size());
+        assertEquals(fsFailureIndexName, dataStreamInfos.get(2).getDataStream().getFailureIndices().getIndices().get(0).getName());
+
+        GetAliasesResponse getAliasesResponse = client.admin()
+            .indices()
+            .getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "my-alias"))
+            .actionGet();
         assertThat(getAliasesResponse.getDataStreamAliases().keySet(), containsInAnyOrder("ds", "other-ds"));
 
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").size(), equalTo(1));
@@ -667,21 +650,21 @@ public void testSnapshotAndRestoreIncludeAliasesFalse() throws Exception {
             assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap());
         });
 
-        GetDataStreamAction.Response ds = client.execute(
-            GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" })
-        ).get();
-        assertEquals(3, ds.getDataStreams().size());
-        assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
-        assertEquals(dsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
-        assertEquals(1, ds.getDataStreams().get(1).getDataStream().getIndices().size());
-        assertEquals(otherDsBackingIndexName, ds.getDataStreams().get(1).getDataStream().getIndices().get(0).getName());
-        assertEquals(1, ds.getDataStreams().get(2).getDataStream().getIndices().size());
-        assertEquals(fsBackingIndexName, ds.getDataStreams().get(2).getDataStream().getIndices().get(0).getName());
-        assertEquals(1, ds.getDataStreams().get(2).getDataStream().getIndices().size());
-        assertEquals(fsFailureIndexName, ds.getDataStreams().get(2).getDataStream().getFailureIndices().getIndices().get(0).getName());
-
-        GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("*")).actionGet();
+        List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = getDataStreamInfo("*");
+        assertEquals(3, dataStreamInfos.size());
+        assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size());
+        assertEquals(dsBackingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName());
+        assertEquals(1, dataStreamInfos.get(1).getDataStream().getIndices().size());
+        assertEquals(otherDsBackingIndexName, dataStreamInfos.get(1).getDataStream().getIndices().get(0).getName());
+        assertEquals(1, dataStreamInfos.get(2).getDataStream().getIndices().size());
+        assertEquals(fsBackingIndexName, dataStreamInfos.get(2).getDataStream().getIndices().get(0).getName());
+        assertEquals(1, dataStreamInfos.get(2).getDataStream().getIndices().size());
+        assertEquals(fsFailureIndexName, dataStreamInfos.get(2).getDataStream().getFailureIndices().getIndices().get(0).getName());
+
+        GetAliasesResponse getAliasesResponse = client.admin()
+            .indices()
+            .getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "*"))
+            .actionGet();
         assertThat(getAliasesResponse.getDataStreamAliases(), anEmptyMap());
         assertAcked(
             client().execute(
@@ -721,20 +704,20 @@ public void testRename() throws Exception {
             .setRenameReplacement("ds2")
             .get();
 
-        GetDataStreamAction.Response ds = client.execute(
-            GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds2" })
-        ).get();
-        assertEquals(1, ds.getDataStreams().size());
-        assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
-        assertEquals(ds2BackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
+        List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = getDataStreamInfo("ds2");
+        assertEquals(1, dataStreamInfos.size());
+        assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size());
+        assertEquals(ds2BackingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName());
         assertResponse(
             client.prepareSearch("ds2"),
             response -> assertEquals(DOCUMENT_SOURCE, response.getHits().getHits()[0].getSourceAsMap())
         );
         assertEquals(DOCUMENT_SOURCE, client.prepareGet(ds2BackingIndexName, id).get().getSourceAsMap());
 
-        GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet();
+        GetAliasesResponse getAliasesResponse = client.admin()
+            .indices()
+            .getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "my-alias"))
+            .actionGet();
         assertThat(getAliasesResponse.getDataStreamAliases().keySet(), containsInAnyOrder("ds", "ds2", "other-ds"));
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds2").size(), equalTo(1));
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds2").get(0).getName(), equalTo("my-alias"));
@@ -779,15 +762,15 @@ public void testRenameWriteDataStream() throws Exception {
             .setRenameReplacement("other-ds2")
             .get();
 
-        GetDataStreamAction.Response ds = client.execute(
-            GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "other-ds2" })
-        ).get();
-        assertEquals(1, ds.getDataStreams().size());
-        assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
-        assertEquals(otherDs2BackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
+        List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = getDataStreamInfo("other-ds2");
+        assertEquals(1, dataStreamInfos.size());
+        assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size());
+        assertEquals(otherDs2BackingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName());
 
-        GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet();
+        GetAliasesResponse getAliasesResponse = client.admin()
+            .indices()
+            .getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "my-alias"))
+            .actionGet();
         assertThat(getAliasesResponse.getDataStreamAliases().keySet(), containsInAnyOrder("ds", "other-ds", "other-ds2"));
         assertThat(getAliasesResponse.getDataStreamAliases().get("other-ds2").size(), equalTo(1));
         assertThat(getAliasesResponse.getDataStreamAliases().get("other-ds2").get(0).getName(), equalTo("my-alias"));
@@ -849,9 +832,8 @@ public void testBackingIndexIsNotRenamedWhenRestoringDataStream() {
 
         assertThat(restoreSnapshotResponse.status(), is(RestStatus.OK));
 
-        GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" });
-        GetDataStreamAction.Response response = client.execute(GetDataStreamAction.INSTANCE, getDSRequest).actionGet();
-        assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(dsBackingIndexName));
+        List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = getDataStreamInfo("ds");
+        assertThat(dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName(), is(dsBackingIndexName));
     }
 
     public void testDataStreamAndBackingIndicesAreRenamedUsingRegex() {
@@ -888,17 +870,15 @@ public void testDataStreamAndBackingIndicesAreRenamedUsingRegex() {
         assertThat(restoreSnapshotResponse.status(), is(RestStatus.OK));
 
         // assert "ds" was restored as "test-ds" and the backing index has a valid name
-        GetDataStreamAction.Request getRenamedDS = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "test-ds" });
-        GetDataStreamAction.Response response = client.execute(GetDataStreamAction.INSTANCE, getRenamedDS).actionGet();
+        List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = getDataStreamInfo("test-ds");
         assertThat(
-            response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(),
+            dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName(),
             is(DataStream.getDefaultBackingIndexName("test-ds", 1L))
         );
 
         // data stream "ds" should still exist in the system
-        GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" });
-        response = client.execute(GetDataStreamAction.INSTANCE, getDSRequest).actionGet();
-        assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(dsBackingIndexName));
+        dataStreamInfos = getDataStreamInfo("ds");
+        assertThat(dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName(), is(dsBackingIndexName));
     }
 
     public void testWildcards() throws Exception {
@@ -924,16 +904,13 @@ public void testWildcards() throws Exception {
 
         assertEquals(RestStatus.OK, restoreSnapshotResponse.status());
 
-        GetDataStreamAction.Response ds = client.execute(
-            GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds2" })
-        ).get();
-        assertEquals(1, ds.getDataStreams().size());
-        assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
-        assertEquals(ds2BackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
+        List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = getDataStreamInfo("ds2");
+        assertEquals(1, dataStreamInfos.size());
+        assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size());
+        assertEquals(ds2BackingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName());
         assertThat(
             "we renamed the restored data stream to one that doesn't match any existing composable template",
-            ds.getDataStreams().get(0).getIndexTemplate(),
+            dataStreamInfos.get(0).getIndexTemplate(),
             is(nullValue())
         );
     }
@@ -955,7 +932,7 @@ public void testDataStreamNotStoredWhenIndexRequested() {
         );
     }
 
-    public void testDataStreamNotRestoredWhenIndexRequested() throws Exception {
+    public void testDataStreamNotRestoredWhenIndexRequested() {
         CreateSnapshotResponse createSnapshotResponse = client.admin()
             .cluster()
             .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO, "snap2")
@@ -984,7 +961,7 @@ public void testDataStreamNotRestoredWhenIndexRequested() throws Exception {
         expectThrows(ResourceNotFoundException.class, client.execute(GetDataStreamAction.INSTANCE, getRequest));
     }
 
-    public void testDataStreamNotIncludedInLimitedSnapshot() throws ExecutionException, InterruptedException {
+    public void testDataStreamNotIncludedInLimitedSnapshot() {
         final String snapshotName = "test-snap";
         CreateSnapshotResponse createSnapshotResponse = client.admin()
             .cluster()
@@ -1042,12 +1019,7 @@ public void testDeleteDataStreamDuringSnapshot() throws Exception {
         assertDocCount(dataStream, 100L);
         // Resolve backing index name after the data stream has been created because it has a date component,
         // and running around midnight could lead to test failures otherwise
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
-            TEST_REQUEST_TIMEOUT,
-            new String[] { dataStream }
-        );
-        GetDataStreamAction.Response getDataStreamResponse = client.execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet();
-        String backingIndexName = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().get(0).getName();
+        String backingIndexName = getDataStreamInfo(dataStream).get(0).getDataStream().getIndices().get(0).getName();
 
         logger.info("--> snapshot");
         ActionFuture<CreateSnapshotResponse> future = client1.admin()
@@ -1235,7 +1207,7 @@ public void testSnapshotDSDuringRolloverAndDeleteOldIndex() throws Exception {
         assertEquals(restoreSnapshotResponse.failedShards(), 0);
     }
 
-    public void testExcludeDSFromSnapshotWhenExcludingItsIndices() {
+    public void testExcludeDSFromSnapshotWhenExcludingAnyOfItsIndices() {
         final String snapshot = "test-snapshot";
         final String indexWithoutDataStream = "test-idx-no-ds";
         createIndexWithContent(indexWithoutDataStream);
@@ -1251,10 +1223,47 @@ public void testExcludeDSFromSnapshotWhenExcludingItsIndices() {
             .getRestoreInfo();
         assertThat(restoreInfo.failedShards(), is(0));
         assertThat(restoreInfo.successfulShards(), is(1));
+
+        // Exclude only failure store indices
+        {
+            String dataStreamName = "with-fs";
+            CreateSnapshotResponse createSnapshotResponse = client.admin()
+                .cluster()
+                .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
+                .setWaitForCompletion(true)
+                .setIndices(dataStreamName + "*", "-.fs*")
+                .setIncludeGlobalState(false)
+                .get();
+
+            RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
+            assertEquals(RestStatus.OK, status);
+
+            SnapshotInfo retrievedSnapshot = getSnapshot(REPO, SNAPSHOT);
+            assertThat(retrievedSnapshot.dataStreams(), contains(dataStreamName));
+            assertThat(retrievedSnapshot.indices(), containsInAnyOrder(fsBackingIndexName));
+
+            assertAcked(
+                safeGet(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "*")))
+            );
+
+            RestoreInfo restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
+                .setWaitForCompletion(true)
+                .setIndices(dataStreamName)
+                .get()
+                .getRestoreInfo();
+
+            assertThat(restoreSnapshotResponse, notNullValue());
+            assertThat(restoreSnapshotResponse.successfulShards(), equalTo(restoreSnapshotResponse.totalShards()));
+            assertThat(restoreSnapshotResponse.failedShards(), is(0));
+
+            GetDataStreamAction.Response.DataStreamInfo dataStream = getDataStreamInfo(dataStreamName).getFirst();
+            assertThat(dataStream.getDataStream().getBackingIndices().getIndices(), not(empty()));
+            assertThat(dataStream.getDataStream().getFailureIndices().getIndices(), empty());
+        }
     }
 
     /**
-     * This test is a copy of the {@link #testExcludeDSFromSnapshotWhenExcludingItsIndices()} the only difference
+     * This test is a copy of the {@link #testExcludeDSFromSnapshotWhenExcludingAnyOfItsIndices()} ()} the only difference
      * is that one include the global state and one doesn't. In general this shouldn't matter that's why it used to be
      * a random parameter of the test, but because of #107515 it fails when we include the global state. Keep them
      * separate until this is fixed.
@@ -1284,10 +1293,7 @@ public void testRestoreSnapshotFully() throws Exception {
         createIndexWithContent(indexName);
         createFullSnapshot(REPO, snapshotName);
 
-        assertAcked(
-            client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" }))
-                .get()
-        );
+        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "*")).get());
         assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.lenientExpandOpenHidden()).get());
 
         RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
@@ -1297,8 +1303,7 @@ public void testRestoreSnapshotFully() throws Exception {
             .get();
         assertEquals(RestStatus.OK, restoreSnapshotResponse.status());
 
-        GetDataStreamAction.Request getRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
-        assertThat(client.execute(GetDataStreamAction.INSTANCE, getRequest).get().getDataStreams(), hasSize(3));
+        assertThat(getDataStreamInfo("*"), hasSize(3));
         assertNotNull(client.admin().indices().prepareGetIndex().setIndices(indexName).get());
     }
 
@@ -1326,7 +1331,7 @@ public void testRestoreDataStreamAliasWithConflictingDataStream() throws Excepti
         }
     }
 
-    public void testRestoreDataStreamAliasWithConflictingIndicesAlias() throws Exception {
+    public void testRestoreDataStreamAliasWithConflictingIndicesAlias() {
         var snapshotName = "test-snapshot";
         createFullSnapshot(REPO, snapshotName);
         client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "*")).actionGet();
@@ -1484,4 +1489,8 @@ public void testWarningHeaderOnRestoreTemplateFromSnapshot() throws Exception {
 
     }
 
+    protected List<GetDataStreamAction.Response.DataStreamInfo> getDataStreamInfo(String... dataStreamNames) {
+        GetDataStreamAction.Request getRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, dataStreamNames);
+        return safeGet(client.execute(GetDataStreamAction.INSTANCE, getRequest)).getDataStreams();
+    }
 }
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java
index e9eaf7b5faddb..bee3989d20ff0 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java
@@ -20,11 +20,12 @@
 import org.elasticsearch.action.bulk.FailureStoreMetrics;
 import org.elasticsearch.action.datastreams.CreateDataStreamAction;
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.core.Strings;
@@ -194,9 +195,9 @@ public void testRejectionFromFailureStore() throws IOException {
         createDataStream();
 
         // Initialize failure store.
-        var rolloverRequest = new RolloverRequest(dataStream, null);
-        rolloverRequest.setIndicesOptions(
-            IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build()
+        var rolloverRequest = new RolloverRequest(
+            IndexNameExpressionResolver.combineSelector(dataStream, IndexComponentSelector.FAILURES),
+            null
         );
         var rolloverResponse = client().execute(RolloverAction.INSTANCE, rolloverRequest).actionGet();
         var failureStoreIndex = rolloverResponse.getNewIndex();
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java
index de6b7a682324e..54e21d5155ed1 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java
@@ -12,6 +12,9 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.RestStatus;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -57,7 +60,7 @@ public void setup() throws IOException {
 
         assertOK(client().performRequest(new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME)));
         // Initialize the failure store.
-        assertOK(client().performRequest(new Request("POST", DATA_STREAM_NAME + "/_rollover?target_failure_store")));
+        assertOK(client().performRequest(new Request("POST", DATA_STREAM_NAME + "::failures/_rollover")));
         ensureGreen(DATA_STREAM_NAME);
 
         final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME));
@@ -122,13 +125,25 @@ public void testExplicitlyResetDataStreamOptions() throws IOException {
         assertOK(client().performRequest(otherRequest));
     }
 
-    public void testEnableDisableFailureStore() throws IOException {
+    public void testBehaviorWithEachFailureStoreOptionAndClusterSetting() throws IOException {
         {
+            // Default data stream options
             assertAcknowledged(client().performRequest(new Request("DELETE", "/_data_stream/" + DATA_STREAM_NAME + "/_options")));
-            assertFailureStore(false, 1);
+            setDataStreamFailureStoreClusterSetting(DATA_STREAM_NAME);
             assertDataStreamOptions(null);
+            assertFailureStoreValuesInGetDataStreamResponse(true, 1);
+            assertRedirectsDocWithBadMappingToFailureStore();
+            setDataStreamFailureStoreClusterSetting("does-not-match-failure-data-stream");
+            assertDataStreamOptions(null);
+            assertFailureStoreValuesInGetDataStreamResponse(false, 1);
+            assertFailsDocWithBadMapping();
+            setDataStreamFailureStoreClusterSetting(null); // should get same behaviour as when we set it to something non-matching
+            assertDataStreamOptions(null);
+            assertFailureStoreValuesInGetDataStreamResponse(false, 1);
+            assertFailsDocWithBadMapping();
         }
         {
+            // Data stream options with failure store enabled
             Request enableRequest = new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME + "/_options");
             enableRequest.setJsonEntity("""
                 {
@@ -137,11 +152,21 @@ public void testEnableDisableFailureStore() throws IOException {
                   }
                 }""");
             assertAcknowledged(client().performRequest(enableRequest));
-            assertFailureStore(true, 1);
+            setDataStreamFailureStoreClusterSetting(DATA_STREAM_NAME);
+            assertDataStreamOptions(true);
+            assertFailureStoreValuesInGetDataStreamResponse(true, 1);
+            assertRedirectsDocWithBadMappingToFailureStore();
+            setDataStreamFailureStoreClusterSetting("does-not-match-failure-data-stream"); // should have no effect as enabled in options
             assertDataStreamOptions(true);
+            assertFailureStoreValuesInGetDataStreamResponse(true, 1);
+            assertRedirectsDocWithBadMappingToFailureStore();
+            setDataStreamFailureStoreClusterSetting(null); // same as previous
+            assertDataStreamOptions(true);
+            assertFailureStoreValuesInGetDataStreamResponse(true, 1);
+            assertRedirectsDocWithBadMappingToFailureStore();
         }
-
         {
+            // Data stream options with failure store disabled
             Request disableRequest = new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME + "/_options");
             disableRequest.setJsonEntity("""
                 {
@@ -150,13 +175,23 @@ public void testEnableDisableFailureStore() throws IOException {
                   }
                 }""");
             assertAcknowledged(client().performRequest(disableRequest));
-            assertFailureStore(false, 1);
+            setDataStreamFailureStoreClusterSetting(DATA_STREAM_NAME); // should have no effect as disabled in options
             assertDataStreamOptions(false);
+            assertFailureStoreValuesInGetDataStreamResponse(false, 1);
+            assertFailsDocWithBadMapping();
+            setDataStreamFailureStoreClusterSetting("does-not-match-failure-data-stream");
+            assertDataStreamOptions(false);
+            assertFailureStoreValuesInGetDataStreamResponse(false, 1);
+            assertFailsDocWithBadMapping();
+            setDataStreamFailureStoreClusterSetting(null);
+            assertDataStreamOptions(false);
+            assertFailureStoreValuesInGetDataStreamResponse(false, 1);
+            assertFailsDocWithBadMapping();
         }
     }
 
     @SuppressWarnings("unchecked")
-    private void assertFailureStore(boolean failureStoreEnabled, int failureStoreSize) throws IOException {
+    private void assertFailureStoreValuesInGetDataStreamResponse(boolean failureStoreEnabled, int failureStoreSize) throws IOException {
         final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME));
         List<Object> dataStreams = (List<Object>) entityAsMap(dataStreamResponse).get("data_streams");
         assertThat(dataStreams.size(), is(1));
@@ -198,4 +233,32 @@ private List<String> getIndices(Map<String, Object> response) {
         List<Map<String, String>> indices = (List<Map<String, String>>) response.get("indices");
         return indices.stream().map(index -> index.get("index_name")).toList();
     }
+
+    private static void setDataStreamFailureStoreClusterSetting(String value) throws IOException {
+        updateClusterSettings(
+            Settings.builder().put(DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), value).build()
+        );
+    }
+
+    private Response putDocumentWithBadMapping() throws IOException {
+        Request request = new Request("POST", DATA_STREAM_NAME + "/_doc");
+        request.setJsonEntity("""
+            {
+              "@timestamp": "not a timestamp",
+              "foo": "bar"
+            }
+            """);
+        return client().performRequest(request);
+    }
+
+    private void assertRedirectsDocWithBadMappingToFailureStore() throws IOException {
+        Response response = putDocumentWithBadMapping();
+        String failureStoreResponse = (String) entityAsMap(response).get("failure_store");
+        assertThat(failureStoreResponse, is("used"));
+    }
+
+    private void assertFailsDocWithBadMapping() {
+        ResponseException e = assertThrows(ResponseException.class, this::putDocumentWithBadMapping);
+        assertThat(e.getResponse().getStatusLine().getStatusCode(), is(RestStatus.BAD_REQUEST.getStatus()));
+    }
 }
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java
deleted file mode 100644
index 85b914be30b2c..0000000000000
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.datastreams;
-
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.ResponseException;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-
-/**
- * This should be a yaml test, but in order to write one we would need to expose the new parameter in the rest-api-spec.
- * We do not want to do that until the feature flag is removed. For this reason, we temporarily, test the affected APIs here.
- * Please convert this to a yaml test when the feature flag is removed.
- */
-public class FailureStoreQueryParamIT extends DisabledSecurityDataStreamTestCase {
-
-    private static final String DATA_STREAM_NAME = "failure-data-stream";
-    private String backingIndex;
-    private String failureStoreIndex;
-
-    @SuppressWarnings("unchecked")
-    @Before
-    public void setup() throws IOException {
-        Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/ds-template");
-        putComposableIndexTemplateRequest.setJsonEntity("""
-            {
-              "index_patterns": ["failure-data-stream"],
-              "template": {
-                "settings": {
-                  "number_of_replicas": 0
-                },
-                "data_stream_options": {
-                  "failure_store": {
-                      "enabled": true
-                  }
-                }
-              },
-              "data_stream": {
-              }
-            }
-            """);
-        assertOK(client().performRequest(putComposableIndexTemplateRequest));
-
-        assertOK(client().performRequest(new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME)));
-        // Initialize the failure store.
-        assertOK(client().performRequest(new Request("POST", DATA_STREAM_NAME + "/_rollover?target_failure_store")));
-        ensureGreen(DATA_STREAM_NAME);
-
-        final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME));
-        List<Object> dataStreams = (List<Object>) entityAsMap(dataStreamResponse).get("data_streams");
-        assertThat(dataStreams.size(), is(1));
-        Map<String, Object> dataStream = (Map<String, Object>) dataStreams.get(0);
-        assertThat(dataStream.get("name"), equalTo(DATA_STREAM_NAME));
-        List<String> backingIndices = getIndices(dataStream);
-        assertThat(backingIndices.size(), is(1));
-        List<String> failureStore = getFailureStore(dataStream);
-        assertThat(failureStore.size(), is(1));
-        backingIndex = backingIndices.get(0);
-        failureStoreIndex = failureStore.get(0);
-    }
-
-    public void testGetIndexApi() throws IOException {
-        {
-            final Response indicesResponse = client().performRequest(new Request("GET", "/" + DATA_STREAM_NAME));
-            Map<String, Object> indices = entityAsMap(indicesResponse);
-            assertThat(indices.size(), is(2));
-            assertThat(indices.containsKey(backingIndex), is(true));
-            assertThat(indices.containsKey(failureStoreIndex), is(true));
-        }
-        {
-            final Response indicesResponse = client().performRequest(new Request("GET", "/" + DATA_STREAM_NAME + "?failure_store=exclude"));
-            Map<String, Object> indices = entityAsMap(indicesResponse);
-            assertThat(indices.size(), is(1));
-            assertThat(indices.containsKey(backingIndex), is(true));
-        }
-        {
-            final Response indicesResponse = client().performRequest(new Request("GET", "/" + DATA_STREAM_NAME + "?failure_store=only"));
-            Map<String, Object> indices = entityAsMap(indicesResponse);
-            assertThat(indices.size(), is(1));
-            assertThat(indices.containsKey(failureStoreIndex), is(true));
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testGetIndexStatsApi() throws IOException {
-        {
-            final Response statsResponse = client().performRequest(new Request("GET", "/" + DATA_STREAM_NAME + "/_stats"));
-            Map<String, Object> indices = (Map<String, Object>) entityAsMap(statsResponse).get("indices");
-            assertThat(indices.size(), is(1));
-            assertThat(indices.containsKey(backingIndex), is(true));
-        }
-        {
-            final Response statsResponse = client().performRequest(
-                new Request("GET", "/" + DATA_STREAM_NAME + "/_stats?failure_store=include")
-            );
-            Map<String, Object> indices = (Map<String, Object>) entityAsMap(statsResponse).get("indices");
-            assertThat(indices.size(), is(2));
-            assertThat(indices.containsKey(backingIndex), is(true));
-            assertThat(indices.containsKey(failureStoreIndex), is(true));
-        }
-        {
-            final Response statsResponse = client().performRequest(
-                new Request("GET", "/" + DATA_STREAM_NAME + "/_stats?failure_store=only")
-            );
-            Map<String, Object> indices = (Map<String, Object>) entityAsMap(statsResponse).get("indices");
-            assertThat(indices.size(), is(1));
-            assertThat(indices.containsKey(failureStoreIndex), is(true));
-        }
-    }
-
-    public void testGetIndexSettingsApi() throws IOException {
-        {
-            final Response indicesResponse = client().performRequest(new Request("GET", "/" + DATA_STREAM_NAME + "/_settings"));
-            Map<String, Object> indices = entityAsMap(indicesResponse);
-            assertThat(indices.size(), is(1));
-            assertThat(indices.containsKey(backingIndex), is(true));
-        }
-        {
-            final Response indicesResponse = client().performRequest(
-                new Request("GET", "/" + DATA_STREAM_NAME + "/_settings?failure_store=include")
-            );
-            Map<String, Object> indices = entityAsMap(indicesResponse);
-            assertThat(indices.size(), is(2));
-            assertThat(indices.containsKey(backingIndex), is(true));
-            assertThat(indices.containsKey(failureStoreIndex), is(true));
-        }
-        {
-            final Response indicesResponse = client().performRequest(
-                new Request("GET", "/" + DATA_STREAM_NAME + "/_settings?failure_store=only")
-            );
-            Map<String, Object> indices = entityAsMap(indicesResponse);
-            assertThat(indices.size(), is(1));
-            assertThat(indices.containsKey(failureStoreIndex), is(true));
-        }
-    }
-
-    public void testGetIndexMappingApi() throws IOException {
-        {
-            final Response indicesResponse = client().performRequest(new Request("GET", "/" + DATA_STREAM_NAME + "/_mapping"));
-            Map<String, Object> indices = entityAsMap(indicesResponse);
-            assertThat(indices.size(), is(1));
-            assertThat(indices.containsKey(backingIndex), is(true));
-        }
-        {
-            final Response indicesResponse = client().performRequest(
-                new Request("GET", "/" + DATA_STREAM_NAME + "/_mapping?failure_store=include")
-            );
-            Map<String, Object> indices = entityAsMap(indicesResponse);
-            assertThat(indices.size(), is(2));
-            assertThat(indices.containsKey(backingIndex), is(true));
-            assertThat(indices.containsKey(failureStoreIndex), is(true));
-        }
-        {
-            final Response indicesResponse = client().performRequest(
-                new Request("GET", "/" + DATA_STREAM_NAME + "/_mapping?failure_store=only")
-            );
-            Map<String, Object> indices = entityAsMap(indicesResponse);
-            assertThat(indices.size(), is(1));
-            assertThat(indices.containsKey(failureStoreIndex), is(true));
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testPutIndexMappingApi() throws IOException {
-        {
-            final Request mappingRequest = new Request("PUT", "/" + DATA_STREAM_NAME + "/_mapping");
-            mappingRequest.setJsonEntity("""
-                {
-                  "properties": {
-                    "email": {
-                      "type": "keyword"
-                    }
-                  }
-                }
-                """);
-            assertAcknowledged(client().performRequest(mappingRequest));
-        }
-        {
-            final Request mappingRequest = new Request("PUT", "/" + DATA_STREAM_NAME + "/_mapping?failure_store=include");
-            mappingRequest.setJsonEntity("""
-                {
-                  "properties": {
-                    "email": {
-                      "type": "keyword"
-                    }
-                  }
-                }
-                """);
-            ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(mappingRequest));
-            Map<String, Object> response = entityAsMap(responseException.getResponse());
-            assertThat(((Map<String, Object>) response.get("error")).get("reason"), is("failure index not supported"));
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    private List<String> getFailureStore(Map<String, Object> response) {
-        var failureStore = (Map<String, Object>) response.get("failure_store");
-        return getIndices(failureStore);
-
-    }
-
-    @SuppressWarnings("unchecked")
-    private List<String> getIndices(Map<String, Object> response) {
-        List<Map<String, String>> indices = (List<Map<String, String>>) response.get("indices");
-        return indices.stream().map(index -> index.get("index_name")).toList();
-    }
-}
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java
index f090186480b76..8026ec641d040 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java
@@ -9,10 +9,6 @@
 
 package org.elasticsearch.datastreams;
 
-import org.elasticsearch.action.admin.indices.rollover.LazyRolloverAction;
-import org.elasticsearch.action.datastreams.autosharding.DataStreamAutoShardingService;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
-import org.elasticsearch.datastreams.lifecycle.health.DataStreamLifecycleHealthInfoPublisher;
 import org.elasticsearch.features.FeatureSpecification;
 import org.elasticsearch.features.NodeFeature;
 
@@ -27,12 +23,7 @@ public class DataStreamFeatures implements FeatureSpecification {
 
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(
-            DataStreamLifecycleHealthInfoPublisher.DSL_HEALTH_INFO_FEATURE,  // Added in 8.12
-            LazyRolloverAction.DATA_STREAM_LAZY_ROLLOVER,                    // Added in 8.13
-            DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE,
-            DataStreamGlobalRetention.GLOBAL_RETENTION                       // Added in 8.14
-        );
+        return Set.of();
     }
 
     @Override
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java
index cb7445705537a..7d5f4bbee32be 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java
@@ -197,8 +197,7 @@ public Collection<?> createComponents(PluginServices services) {
                 settings,
                 services.client(),
                 services.clusterService(),
-                errorStoreInitialisationService.get(),
-                services.featureService()
+                errorStoreInitialisationService.get()
             )
         );
         dataLifecycleInitialisationService.set(
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java
index 1d3b1b676282a..cc5e00d8283ad 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.action.datastreams.DataStreamsStatsAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -102,10 +103,11 @@ protected ClusterBlockException checkRequestBlock(
 
     @Override
     protected String[] resolveConcreteIndexNames(ClusterState clusterState, DataStreamsStatsAction.Request request) {
-        return DataStreamsActionUtil.resolveConcreteIndexNames(
+        return DataStreamsActionUtil.resolveConcreteIndexNamesWithSelector(
             indexNameExpressionResolver,
             clusterState,
             request.indices(),
+            IndexComponentSelector.ALL_APPLICABLE,
             request.indicesOptions()
         ).toArray(String[]::new);
     }
@@ -163,13 +165,17 @@ protected DataStreamsStatsAction.DataStreamShardStats readShardResult(StreamInpu
             request.indicesOptions(),
             request.indices()
         );
-        for (String abstractionName : abstractionNames) {
-            IndexAbstraction indexAbstraction = indicesLookup.get(abstractionName);
+        for (String abstraction : abstractionNames) {
+            IndexAbstraction indexAbstraction = indicesLookup.get(abstraction);
             assert indexAbstraction != null;
             if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) {
                 DataStream dataStream = (DataStream) indexAbstraction;
                 AggregatedStats stats = aggregatedDataStreamsStats.computeIfAbsent(dataStream.getName(), s -> new AggregatedStats());
-                dataStream.getIndices().stream().map(Index::getName).forEach(index -> {
+                dataStream.getBackingIndices().getIndices().stream().map(Index::getName).forEach(index -> {
+                    stats.backingIndices.add(index);
+                    allBackingIndices.add(index);
+                });
+                dataStream.getFailureIndices().getIndices().stream().map(Index::getName).forEach(index -> {
                     stats.backingIndices.add(index);
                     allBackingIndices.add(index);
                 });
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java
index ffa2447f5f5aa..2d310fef0be7e 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -64,6 +65,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadAction
     private final SystemIndices systemIndices;
     private final ClusterSettings clusterSettings;
     private final DataStreamGlobalRetentionSettings globalRetentionSettings;
+    private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings;
     private final Client client;
 
     @Inject
@@ -75,6 +77,7 @@ public TransportGetDataStreamsAction(
         IndexNameExpressionResolver indexNameExpressionResolver,
         SystemIndices systemIndices,
         DataStreamGlobalRetentionSettings globalRetentionSettings,
+        DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
         Client client
     ) {
         super(
@@ -91,6 +94,7 @@ public TransportGetDataStreamsAction(
         this.systemIndices = systemIndices;
         this.globalRetentionSettings = globalRetentionSettings;
         clusterSettings = clusterService.getClusterSettings();
+        this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings;
         this.client = new OriginSettingClient(client, "stack");
     }
 
@@ -122,6 +126,7 @@ public void onResponse(DataStreamsStatsAction.Response response) {
                             systemIndices,
                             clusterSettings,
                             globalRetentionSettings,
+                            dataStreamFailureStoreSettings,
                             maxTimestamps
                         )
                     );
@@ -134,7 +139,16 @@ public void onFailure(Exception e) {
             });
         } else {
             listener.onResponse(
-                innerOperation(state, request, indexNameExpressionResolver, systemIndices, clusterSettings, globalRetentionSettings, null)
+                innerOperation(
+                    state,
+                    request,
+                    indexNameExpressionResolver,
+                    systemIndices,
+                    clusterSettings,
+                    globalRetentionSettings,
+                    dataStreamFailureStoreSettings,
+                    null
+                )
             );
         }
     }
@@ -146,11 +160,16 @@ static GetDataStreamAction.Response innerOperation(
         SystemIndices systemIndices,
         ClusterSettings clusterSettings,
         DataStreamGlobalRetentionSettings globalRetentionSettings,
+        DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
         @Nullable Map<String, Long> maxTimestamps
     ) {
         List<DataStream> dataStreams = getDataStreams(state, indexNameExpressionResolver, request);
         List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = new ArrayList<>(dataStreams.size());
         for (DataStream dataStream : dataStreams) {
+            // For this action, we are returning whether the failure store is effectively enabled, either in metadata or by cluster setting.
+            // Users can use the get data stream options API to find out whether it is explicitly enabled in metadata.
+            boolean failureStoreEffectivelyEnabled = DataStream.isFailureStoreFeatureFlagEnabled()
+                && dataStream.isFailureStoreEffectivelyEnabled(dataStreamFailureStoreSettings);
             final String indexTemplate;
             boolean indexTemplatePreferIlmValue = true;
             String ilmPolicyName = null;
@@ -254,6 +273,7 @@ public int compareTo(IndexInfo o) {
             dataStreamInfos.add(
                 new GetDataStreamAction.Response.DataStreamInfo(
                     dataStream,
+                    failureStoreEffectivelyEnabled,
                     streamHealth.getStatus(),
                     indexTemplate,
                     ilmPolicyName,
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java
index 7d2828e30d5ab..7de3f180753f8 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java
@@ -33,7 +33,7 @@
 import org.elasticsearch.action.downsample.DownsampleAction;
 import org.elasticsearch.action.downsample.DownsampleConfig;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
-import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
@@ -49,6 +49,9 @@
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SelectorResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -944,11 +947,6 @@ private Set<Index> maybeExecuteForceMerge(ClusterState state, List<Index> indice
             if ((configuredFloorSegmentMerge == null || configuredFloorSegmentMerge.equals(targetMergePolicyFloorSegment) == false)
                 || (configuredMergeFactor == null || configuredMergeFactor.equals(targetMergePolicyFactor) == false)) {
                 UpdateSettingsRequest updateMergePolicySettingsRequest = new UpdateSettingsRequest();
-                updateMergePolicySettingsRequest.indicesOptions(
-                    IndicesOptions.builder(updateMergePolicySettingsRequest.indicesOptions())
-                        .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
-                        .build()
-                );
                 updateMergePolicySettingsRequest.indices(indexName);
                 updateMergePolicySettingsRequest.settings(
                     Settings.builder()
@@ -998,8 +996,11 @@ private Set<Index> maybeExecuteForceMerge(ClusterState state, List<Index> indice
 
     private void rolloverDataStream(String writeIndexName, RolloverRequest rolloverRequest, ActionListener<Void> listener) {
         // "saving" the rollover target name here so we don't capture the entire request
-        String rolloverTarget = rolloverRequest.getRolloverTarget();
-        logger.trace("Data stream lifecycle issues rollover request for data stream [{}]", rolloverTarget);
+        ResolvedExpression resolvedRolloverTarget = SelectorResolver.parseExpression(
+            rolloverRequest.getRolloverTarget(),
+            rolloverRequest.indicesOptions()
+        );
+        logger.trace("Data stream lifecycle issues rollover request for data stream [{}]", rolloverRequest.getRolloverTarget());
         client.admin().indices().rolloverIndex(rolloverRequest, new ActionListener<>() {
             @Override
             public void onResponse(RolloverResponse rolloverResponse) {
@@ -1014,7 +1015,7 @@ public void onResponse(RolloverResponse rolloverResponse) {
                     logger.info(
                         "Data stream lifecycle successfully rolled over datastream [{}] due to the following met rollover "
                             + "conditions {}. The new index is [{}]",
-                        rolloverTarget,
+                        rolloverRequest.getRolloverTarget(),
                         metConditions,
                         rolloverResponse.getNewIndex()
                     );
@@ -1024,7 +1025,7 @@ public void onResponse(RolloverResponse rolloverResponse) {
 
             @Override
             public void onFailure(Exception e) {
-                DataStream dataStream = clusterService.state().metadata().dataStreams().get(rolloverTarget);
+                DataStream dataStream = clusterService.state().metadata().dataStreams().get(resolvedRolloverTarget.resource());
                 if (dataStream == null || dataStream.getWriteIndex().getName().equals(writeIndexName) == false) {
                     // the data stream has another write index so no point in recording an error for the previous write index we were
                     // attempting to roll over
@@ -1407,9 +1408,7 @@ static RolloverRequest getDefaultRolloverRequest(
     ) {
         RolloverRequest rolloverRequest = new RolloverRequest(dataStream, null).masterNodeTimeout(TimeValue.MAX_VALUE);
         if (rolloverFailureStore) {
-            rolloverRequest.setIndicesOptions(
-                IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build()
-            );
+            rolloverRequest.setRolloverTarget(IndexNameExpressionResolver.combineSelector(dataStream, IndexComponentSelector.FAILURES));
         }
         rolloverRequest.setConditions(rolloverConfiguration.resolveRolloverConditions(dataRetention));
         return rolloverRequest;
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java
index 1595348649528..7992362d791b1 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java
@@ -50,7 +50,7 @@ public static final class Request extends AcknowledgedRequest<Request> implement
                     .allowAliasToMultipleIndices(false)
                     .allowClosedIndices(true)
                     .ignoreThrottled(false)
-                    .allowFailureIndices(false)
+                    .allowSelectors(false)
                     .build()
             )
             .build();
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisher.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisher.java
index 552bf75adae23..71575ee88aa7d 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisher.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisher.java
@@ -19,8 +19,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleErrorStore;
-import org.elasticsearch.features.FeatureService;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.health.node.DataStreamLifecycleHealthInfo;
 import org.elasticsearch.health.node.DslErrorInfo;
 import org.elasticsearch.health.node.UpdateHealthInfoCacheAction;
@@ -45,12 +43,10 @@ public class DataStreamLifecycleHealthInfoPublisher {
         Setting.Property.Dynamic,
         Setting.Property.NodeScope
     );
-    public static final NodeFeature DSL_HEALTH_INFO_FEATURE = new NodeFeature("health.dsl.info");
 
     private final Client client;
     private final ClusterService clusterService;
     private final DataStreamLifecycleErrorStore errorStore;
-    private final FeatureService featureService;
     private volatile int signallingErrorRetryInterval;
     private volatile int maxNumberOfErrorsToPublish;
 
@@ -58,13 +54,11 @@ public DataStreamLifecycleHealthInfoPublisher(
         Settings settings,
         Client client,
         ClusterService clusterService,
-        DataStreamLifecycleErrorStore errorStore,
-        FeatureService featureService
+        DataStreamLifecycleErrorStore errorStore
     ) {
         this.client = client;
         this.clusterService = clusterService;
         this.errorStore = errorStore;
-        this.featureService = featureService;
         this.signallingErrorRetryInterval = DATA_STREAM_SIGNALLING_ERROR_RETRY_INTERVAL_SETTING.get(settings);
         this.maxNumberOfErrorsToPublish = DATA_STREAM_LIFECYCLE_MAX_ERRORS_TO_PUBLISH_SETTING.get(settings);
     }
@@ -89,9 +83,6 @@ private void updateNumberOfErrorsToPublish(int newValue) {
      * {@link org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService#DATA_STREAM_SIGNALLING_ERROR_RETRY_INTERVAL_SETTING}
      */
     public void publishDslErrorEntries(ActionListener<AcknowledgedResponse> actionListener) {
-        if (featureService.clusterHasFeature(clusterService.state(), DSL_HEALTH_INFO_FEATURE) == false) {
-            return;
-        }
         // fetching the entries that persist in the error store for more than the signalling retry interval
         // note that we're reporting this view into the error store on every publishing iteration
         List<DslErrorInfo> errorEntriesToSignal = errorStore.getErrorsInfo(
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java
index 98a29dd636ddf..860bcb5bf2fbe 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java
@@ -39,7 +39,9 @@ public static final class Request extends AcknowledgedRequest<Request> implement
             .wildcardOptions(
                 IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false)
             )
-            .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true))
+            .gatekeeperOptions(
+                IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true).allowSelectors(false)
+            )
             .build();
 
         public Request(StreamInput in) throws IOException {
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java
index c1354da1129ca..45bda1abd5c02 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java
@@ -50,7 +50,9 @@ public static class Request extends MasterNodeReadRequest<Request> implements In
             .wildcardOptions(
                 IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false)
             )
-            .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true))
+            .gatekeeperOptions(
+                IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true).allowSelectors(false)
+            )
             .build();
         private boolean includeDefaults = false;
 
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java
index d055a6972312a..d66b45665d4e2 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java
@@ -71,7 +71,9 @@ public static Request parseRequest(XContentParser parser, Factory factory) {
             .wildcardOptions(
                 IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false)
             )
-            .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true))
+            .gatekeeperOptions(
+                IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true).allowSelectors(false)
+            )
             .build();
         private final DataStreamOptions options;
 
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java
index b61e38297397d..be157608b1c3f 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.action.datastreams.GetDataStreamAction;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.set.Sets;
@@ -42,8 +41,7 @@ public class RestGetDataStreamsAction extends BaseRestHandler {
                 IndicesOptions.WildcardOptions.ALLOW_NO_INDICES,
                 IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED,
                 "verbose"
-            ),
-            DataStream.isFailureStoreFeatureFlagEnabled() ? Set.of(IndicesOptions.FAILURE_STORE_QUERY_PARAM) : Set.of()
+            )
         )
     );
 
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java
index d5c5193948213..e32636fe40d84 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteComposableIndexTemplateAction;
 import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
+import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.datastreams.CreateDataStreamAction;
 import org.elasticsearch.action.datastreams.DataStreamsStatsAction;
 import org.elasticsearch.action.datastreams.DeleteDataStreamAction;
@@ -22,8 +23,12 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.DataStreamFailureStore;
+import org.elasticsearch.cluster.metadata.DataStreamOptions;
+import org.elasticsearch.cluster.metadata.ResettableValue;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.xcontent.json.JsonXContent;
@@ -40,12 +45,14 @@
 
 import static java.lang.Math.max;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
 
 public class DataStreamsStatsTests extends ESSingleNodeTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return List.of(DataStreamsPlugin.class);
+        return List.of(DataStreamsPlugin.class, MapperExtrasPlugin.class);
     }
 
     private final Set<String> createdDataStreams = new HashSet<>();
@@ -107,8 +114,30 @@ public void testStatsExistingDataStream() throws Exception {
         assertEquals(stats.getTotalStoreSize().getBytes(), stats.getDataStreams()[0].getStoreSize().getBytes());
     }
 
+    public void testStatsExistingDataStreamWithFailureStores() throws Exception {
+        String dataStreamName = createDataStream(false, true);
+        createFailedDocument(dataStreamName);
+
+        DataStreamsStatsAction.Response stats = getDataStreamsStats();
+
+        assertEquals(2, stats.getSuccessfulShards());
+        assertEquals(0, stats.getFailedShards());
+        assertEquals(1, stats.getDataStreamCount());
+        assertEquals(2, stats.getBackingIndices());
+        assertNotEquals(0L, stats.getTotalStoreSize().getBytes());
+        assertEquals(1, stats.getDataStreams().length);
+        assertEquals(dataStreamName, stats.getDataStreams()[0].getDataStream());
+        assertEquals(2, stats.getDataStreams()[0].getBackingIndices());
+        // The timestamp is going to not be something we can validate because
+        // it captures the time of failure which is uncontrolled in the test
+        // Just make sure it exists by ensuring it isn't zero
+        assertThat(stats.getDataStreams()[0].getMaximumTimestamp(), is(greaterThan(0L)));
+        assertNotEquals(0L, stats.getDataStreams()[0].getStoreSize().getBytes());
+        assertEquals(stats.getTotalStoreSize().getBytes(), stats.getDataStreams()[0].getStoreSize().getBytes());
+    }
+
     public void testStatsExistingHiddenDataStream() throws Exception {
-        String dataStreamName = createDataStream(true);
+        String dataStreamName = createDataStream(true, false);
         long timestamp = createDocument(dataStreamName);
 
         DataStreamsStatsAction.Response stats = getDataStreamsStats(true);
@@ -221,14 +250,19 @@ public void testStatsMultipleDataStreams() throws Exception {
     }
 
     private String createDataStream() throws Exception {
-        return createDataStream(false);
+        return createDataStream(false, false);
     }
 
-    private String createDataStream(boolean hidden) throws Exception {
+    private String createDataStream(boolean hidden, boolean failureStore) throws Exception {
         String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.getDefault());
+        ResettableValue<DataStreamOptions.Template> failureStoreOptions = failureStore == false
+            ? ResettableValue.undefined()
+            : ResettableValue.create(
+                new DataStreamOptions.Template(ResettableValue.create(new DataStreamFailureStore.Template(ResettableValue.create(true))))
+            );
         Template idxTemplate = new Template(null, new CompressedXContent("""
             {"properties":{"@timestamp":{"type":"date"},"data":{"type":"keyword"}}}
-            """), null);
+            """), null, null, failureStoreOptions);
         ComposableIndexTemplate template = ComposableIndexTemplate.builder()
             .indexPatterns(List.of(dataStreamName + "*"))
             .template(idxTemplate)
@@ -269,6 +303,27 @@ private long createDocument(String dataStreamName) throws Exception {
         return timestamp;
     }
 
+    private long createFailedDocument(String dataStreamName) throws Exception {
+        // Get some randomized but reasonable timestamps on the data since not all of it is guaranteed to arrive in order.
+        long timeSeed = System.currentTimeMillis();
+        long timestamp = randomLongBetween(timeSeed - TimeUnit.HOURS.toMillis(5), timeSeed);
+        client().bulk(
+            new BulkRequest(dataStreamName).add(
+                new IndexRequest().opType(DocWriteRequest.OpType.CREATE)
+                    .source(
+                        JsonXContent.contentBuilder()
+                            .startObject()
+                            .field("@timestamp", timestamp)
+                            .object("data", b -> b.field("garbage", randomAlphaOfLength(25)))
+                            .endObject()
+                    )
+            )
+        ).get();
+        indicesAdmin().refresh(new RefreshRequest(".fs-" + dataStreamName + "*").indicesOptions(IndicesOptions.lenientExpandOpenHidden()))
+            .get();
+        return timestamp;
+    }
+
     private DataStreamsStatsAction.Response getDataStreamsStats() throws Exception {
         return getDataStreamsStats(false);
     }
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java
index 710ea8c15b66e..9414943cbb439 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java
@@ -102,6 +102,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
 
             Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo(
                 logs,
+                true,
                 ClusterHealthStatus.GREEN,
                 "index-template",
                 null,
@@ -205,6 +206,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
 
             Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo(
                 logs,
+                true,
                 ClusterHealthStatus.GREEN,
                 "index-template",
                 null,
@@ -282,6 +284,7 @@ public void testManagedByDisplayValuesDontAccidentalyChange() {
 
     private Response.DataStreamInfo mutateInstance(Response.DataStreamInfo instance) {
         var dataStream = instance.getDataStream();
+        var failureStoreEffectivelyEnabled = instance.isFailureStoreEffectivelyEnabled();
         var status = instance.getDataStreamStatus();
         var indexTemplate = instance.getIndexTemplate();
         var ilmPolicyName = instance.getIlmPolicy();
@@ -289,7 +292,7 @@ private Response.DataStreamInfo mutateInstance(Response.DataStreamInfo instance)
         var indexSettings = instance.getIndexSettingsValues();
         var templatePreferIlm = instance.templatePreferIlmValue();
         var maximumTimestamp = instance.getMaximumTimestamp();
-        switch (randomIntBetween(0, 7)) {
+        switch (randomIntBetween(0, 8)) {
             case 0 -> dataStream = randomValueOtherThan(dataStream, DataStreamTestHelper::randomInstance);
             case 1 -> status = randomValueOtherThan(status, () -> randomFrom(ClusterHealthStatus.values()));
             case 2 -> indexTemplate = randomBoolean() && indexTemplate != null ? null : randomAlphaOfLengthBetween(2, 10);
@@ -314,9 +317,11 @@ private Response.DataStreamInfo mutateInstance(Response.DataStreamInfo instance)
             case 7 -> maximumTimestamp = (maximumTimestamp == null)
                 ? randomNonNegativeLong()
                 : (usually() ? randomValueOtherThan(maximumTimestamp, ESTestCase::randomNonNegativeLong) : null);
+            case 8 -> failureStoreEffectivelyEnabled = failureStoreEffectivelyEnabled ? false : true;
         }
         return new Response.DataStreamInfo(
             dataStream,
+            failureStoreEffectivelyEnabled,
             status,
             indexTemplate,
             ilmPolicyName,
@@ -355,6 +360,7 @@ private Response.DataStreamInfo generateRandomDataStreamInfo() {
         List<Tuple<Instant, Instant>> timeSeries = randomBoolean() ? generateRandomTimeSeries() : null;
         return new Response.DataStreamInfo(
             DataStreamTestHelper.randomInstance(),
+            randomBoolean(),
             ClusterHealthStatus.GREEN,
             randomAlphaOfLengthBetween(2, 10),
             randomAlphaOfLengthBetween(2, 10),
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java
index 2efe881266c1b..ba4627f8955a1 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
@@ -39,6 +40,8 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
 public class TransportGetDataStreamsActionTests extends ESTestCase {
@@ -48,6 +51,9 @@ public class TransportGetDataStreamsActionTests extends ESTestCase {
     private final DataStreamGlobalRetentionSettings dataStreamGlobalRetentionSettings = DataStreamGlobalRetentionSettings.create(
         ClusterSettings.createBuiltInClusterSettings()
     );
+    private final DataStreamFailureStoreSettings emptyDataStreamFailureStoreSettings = DataStreamFailureStoreSettings.create(
+        ClusterSettings.createBuiltInClusterSettings()
+    );
 
     public void testGetDataStream() {
         final String dataStreamName = "my-data-stream";
@@ -166,6 +172,7 @@ public void testGetTimeSeriesDataStream() {
             systemIndices,
             ClusterSettings.createBuiltInClusterSettings(),
             dataStreamGlobalRetentionSettings,
+            emptyDataStreamFailureStoreSettings,
             null
         );
         assertThat(
@@ -197,6 +204,7 @@ public void testGetTimeSeriesDataStream() {
             systemIndices,
             ClusterSettings.createBuiltInClusterSettings(),
             dataStreamGlobalRetentionSettings,
+            emptyDataStreamFailureStoreSettings,
             null
         );
         assertThat(
@@ -248,6 +256,7 @@ public void testGetTimeSeriesDataStreamWithOutOfOrderIndices() {
             systemIndices,
             ClusterSettings.createBuiltInClusterSettings(),
             dataStreamGlobalRetentionSettings,
+            emptyDataStreamFailureStoreSettings,
             null
         );
         assertThat(
@@ -292,6 +301,7 @@ public void testGetTimeSeriesMixedDataStream() {
             systemIndices,
             ClusterSettings.createBuiltInClusterSettings(),
             dataStreamGlobalRetentionSettings,
+            emptyDataStreamFailureStoreSettings,
             null
         );
 
@@ -338,6 +348,7 @@ public void testPassingGlobalRetention() {
             systemIndices,
             ClusterSettings.createBuiltInClusterSettings(),
             dataStreamGlobalRetentionSettings,
+            emptyDataStreamFailureStoreSettings,
             null
         );
         assertThat(response.getGlobalRetention(), nullValue());
@@ -363,8 +374,102 @@ public void testPassingGlobalRetention() {
             systemIndices,
             ClusterSettings.createBuiltInClusterSettings(),
             withGlobalRetentionSettings,
+            emptyDataStreamFailureStoreSettings,
             null
         );
         assertThat(response.getGlobalRetention(), equalTo(globalRetention));
     }
+
+    public void testDataStreamIsFailureStoreEffectivelyEnabled_disabled() {
+        var metadata = new Metadata.Builder();
+        DataStreamTestHelper.getClusterStateWithDataStreams(
+            metadata,
+            List.of(Tuple.tuple("data-stream-1", 2)),
+            List.of(),
+            System.currentTimeMillis(),
+            Settings.EMPTY,
+            0,
+            false,
+            false
+        );
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(metadata).build();
+
+        var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {});
+        var response = TransportGetDataStreamsAction.innerOperation(
+            state,
+            req,
+            resolver,
+            systemIndices,
+            ClusterSettings.createBuiltInClusterSettings(),
+            dataStreamGlobalRetentionSettings,
+            emptyDataStreamFailureStoreSettings,
+            null
+        );
+        assertThat(response.getDataStreams(), hasSize(1));
+        assertThat(response.getDataStreams().getFirst().isFailureStoreEffectivelyEnabled(), is(false));
+    }
+
+    public void testDataStreamIsFailureStoreEffectivelyEnabled_enabledExplicitly() {
+        var metadata = new Metadata.Builder();
+        DataStreamTestHelper.getClusterStateWithDataStreams(
+            metadata,
+            List.of(Tuple.tuple("data-stream-1", 2)),
+            List.of(),
+            System.currentTimeMillis(),
+            Settings.EMPTY,
+            0,
+            false,
+            true
+        );
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(metadata).build();
+
+        var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {});
+        var response = TransportGetDataStreamsAction.innerOperation(
+            state,
+            req,
+            resolver,
+            systemIndices,
+            ClusterSettings.createBuiltInClusterSettings(),
+            dataStreamGlobalRetentionSettings,
+            emptyDataStreamFailureStoreSettings,
+            null
+        );
+        assertThat(response.getDataStreams(), hasSize(1));
+        assertThat(response.getDataStreams().getFirst().isFailureStoreEffectivelyEnabled(), is(true));
+    }
+
+    public void testDataStreamIsFailureStoreEffectivelyEnabled_enabledByClusterSetting() {
+        var metadata = new Metadata.Builder();
+        DataStreamTestHelper.getClusterStateWithDataStreams(
+            metadata,
+            List.of(Tuple.tuple("data-stream-1", 2)),
+            List.of(),
+            System.currentTimeMillis(),
+            Settings.EMPTY,
+            0,
+            false,
+            false
+        );
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(metadata).build();
+
+        var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {});
+        var response = TransportGetDataStreamsAction.innerOperation(
+            state,
+            req,
+            resolver,
+            systemIndices,
+            ClusterSettings.createBuiltInClusterSettings(),
+            dataStreamGlobalRetentionSettings,
+            DataStreamFailureStoreSettings.create(
+                ClusterSettings.createBuiltInClusterSettings(
+                    Settings.builder()
+                        .put(DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), "data-stream-*")
+                        .build()
+                )
+            ),
+            null
+        );
+        assertThat(response.getDataStreams(), hasSize(1));
+        assertThat(response.getDataStreams().getFirst().isFailureStoreEffectivelyEnabled(), is(true));
+    }
 }
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
index 698ab427ab040..0bb990e544892 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.action.downsample.DownsampleAction;
 import org.elasticsearch.action.downsample.DownsampleConfig;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
-import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterName;
@@ -46,6 +46,7 @@
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -66,9 +67,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.datastreams.DataStreamFeatures;
 import org.elasticsearch.datastreams.lifecycle.health.DataStreamLifecycleHealthInfoPublisher;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
@@ -182,13 +181,7 @@ public void setupServices() {
             () -> now,
             errorStore,
             allocationService,
-            new DataStreamLifecycleHealthInfoPublisher(
-                Settings.EMPTY,
-                client,
-                clusterService,
-                errorStore,
-                new FeatureService(List.of(new DataStreamFeatures()))
-            ),
+            new DataStreamLifecycleHealthInfoPublisher(Settings.EMPTY, client, clusterService, errorStore),
             globalRetentionSettings
         );
         clientDelegate = null;
@@ -225,11 +218,12 @@ public void testOperationsExecutedOnce() {
         assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class));
         RolloverRequest rolloverBackingIndexRequest = (RolloverRequest) clientSeenRequests.get(0);
         assertThat(rolloverBackingIndexRequest.getRolloverTarget(), is(dataStreamName));
-        assertThat(rolloverBackingIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.DATA));
         assertThat(clientSeenRequests.get(1), instanceOf(RolloverRequest.class));
         RolloverRequest rolloverFailureIndexRequest = (RolloverRequest) clientSeenRequests.get(1);
-        assertThat(rolloverFailureIndexRequest.getRolloverTarget(), is(dataStreamName));
-        assertThat(rolloverFailureIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.FAILURES));
+        assertThat(
+            rolloverFailureIndexRequest.getRolloverTarget(),
+            is(IndexNameExpressionResolver.combineSelector(dataStreamName, IndexComponentSelector.FAILURES))
+        );
         List<DeleteIndexRequest> deleteRequests = clientSeenRequests.subList(2, 5)
             .stream()
             .map(transportRequest -> (DeleteIndexRequest) transportRequest)
@@ -1463,13 +1457,7 @@ public void testTrackingTimeStats() {
             () -> now.getAndAdd(delta),
             errorStore,
             mock(AllocationService.class),
-            new DataStreamLifecycleHealthInfoPublisher(
-                Settings.EMPTY,
-                getTransportRequestsRecordingClient(),
-                clusterService,
-                errorStore,
-                new FeatureService(List.of(new DataStreamFeatures()))
-            ),
+            new DataStreamLifecycleHealthInfoPublisher(Settings.EMPTY, getTransportRequestsRecordingClient(), clusterService, errorStore),
             globalRetentionSettings
         );
         assertThat(service.getLastRunDuration(), is(nullValue()));
@@ -1546,11 +1534,12 @@ public void testFailureStoreIsManagedEvenWhenDisabled() {
         assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class));
         RolloverRequest rolloverBackingIndexRequest = (RolloverRequest) clientSeenRequests.get(0);
         assertThat(rolloverBackingIndexRequest.getRolloverTarget(), is(dataStreamName));
-        assertThat(rolloverBackingIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.DATA));
         assertThat(clientSeenRequests.get(1), instanceOf(RolloverRequest.class));
         RolloverRequest rolloverFailureIndexRequest = (RolloverRequest) clientSeenRequests.get(1);
-        assertThat(rolloverFailureIndexRequest.getRolloverTarget(), is(dataStreamName));
-        assertThat(rolloverFailureIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.FAILURES));
+        assertThat(
+            rolloverFailureIndexRequest.getRolloverTarget(),
+            is(IndexNameExpressionResolver.combineSelector(dataStreamName, IndexComponentSelector.FAILURES))
+        );
         assertThat(
             ((DeleteIndexRequest) clientSeenRequests.get(2)).indices()[0],
             is(dataStream.getFailureIndices().getIndices().get(0).getName())
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisherTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisherTests.java
index cff6127e0729e..f8a2ac3c61029 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisherTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisherTests.java
@@ -24,10 +24,8 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.datastreams.DataStreamFeatures;
 import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleErrorStore;
 import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.health.node.DataStreamLifecycleHealthInfo;
 import org.elasticsearch.health.node.DslErrorInfo;
 import org.elasticsearch.health.node.UpdateHealthInfoCacheAction;
@@ -40,7 +38,6 @@
 
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
 
@@ -83,13 +80,7 @@ public void setupServices() {
 
         final Client client = getTransportRequestsRecordingClient();
         errorStore = new DataStreamLifecycleErrorStore(() -> now);
-        dslHealthInfoPublisher = new DataStreamLifecycleHealthInfoPublisher(
-            Settings.EMPTY,
-            client,
-            clusterService,
-            errorStore,
-            new FeatureService(List.of(new DataStreamFeatures()))
-        );
+        dslHealthInfoPublisher = new DataStreamLifecycleHealthInfoPublisher(Settings.EMPTY, client, clusterService, errorStore);
     }
 
     @After
@@ -105,16 +96,6 @@ public void testPublishDslErrorEntries() {
         }
         errorStore.recordError("testIndex", new IllegalStateException("bad state"));
         ClusterState stateWithHealthNode = ClusterStateCreationUtils.state(node1, node1, node1, allNodes);
-        stateWithHealthNode = ClusterState.builder(stateWithHealthNode)
-            .nodeFeatures(
-                Map.of(
-                    node1.getId(),
-                    Set.of(DataStreamLifecycleHealthInfoPublisher.DSL_HEALTH_INFO_FEATURE.id()),
-                    node2.getId(),
-                    Set.of(DataStreamLifecycleHealthInfoPublisher.DSL_HEALTH_INFO_FEATURE.id())
-                )
-            )
-            .build();
         ClusterServiceUtils.setState(clusterService, stateWithHealthNode);
         dslHealthInfoPublisher.publishDslErrorEntries(new ActionListener<>() {
             @Override
@@ -143,16 +124,6 @@ public void testPublishDslErrorEntriesNoHealthNode() {
         errorStore.recordError("testIndex", new IllegalStateException("bad state"));
 
         ClusterState stateNoHealthNode = ClusterStateCreationUtils.state(node1, node1, null, allNodes);
-        stateNoHealthNode = ClusterState.builder(stateNoHealthNode)
-            .nodeFeatures(
-                Map.of(
-                    node1.getId(),
-                    Set.of(DataStreamLifecycleHealthInfoPublisher.DSL_HEALTH_INFO_FEATURE.id()),
-                    node2.getId(),
-                    Set.of(DataStreamLifecycleHealthInfoPublisher.DSL_HEALTH_INFO_FEATURE.id())
-                )
-            )
-            .build();
         ClusterServiceUtils.setState(clusterService, stateNoHealthNode);
         dslHealthInfoPublisher.publishDslErrorEntries(new ActionListener<>() {
             @Override
@@ -170,16 +141,6 @@ public void onFailure(Exception e) {
     public void testPublishDslErrorEntriesEmptyErrorStore() {
         // publishes the empty error store (this is the "back to healthy" state where all errors have been fixed)
         ClusterState state = ClusterStateCreationUtils.state(node1, node1, node1, allNodes);
-        state = ClusterState.builder(state)
-            .nodeFeatures(
-                Map.of(
-                    node1.getId(),
-                    Set.of(DataStreamLifecycleHealthInfoPublisher.DSL_HEALTH_INFO_FEATURE.id()),
-                    node2.getId(),
-                    Set.of(DataStreamLifecycleHealthInfoPublisher.DSL_HEALTH_INFO_FEATURE.id())
-                )
-            )
-            .build();
         ClusterServiceUtils.setState(clusterService, state);
         dslHealthInfoPublisher.publishDslErrorEntries(new ActionListener<>() {
             @Override
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml
index 9ea3bfefabdf8..884adb5458102 100644
--- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml
@@ -300,9 +300,6 @@ index without timestamp with pipeline:
 
 ---
 dynamic templates:
-  - requires:
-      cluster_features: ["mapper.pass_through_priority"]
-      reason: support for priority in passthrough objects
   - do:
       allowed_warnings:
         - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
@@ -450,9 +447,6 @@ dynamic templates:
 
 ---
 dynamic templates - conflicting aliases:
-  - requires:
-      cluster_features: ["mapper.pass_through_priority"]
-      reason: support for priority in passthrough objects
   - do:
       allowed_warnings:
         - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
@@ -549,9 +543,6 @@ dynamic templates - conflicting aliases:
 
 ---
 dynamic templates - conflicting aliases with top-level field:
-  - requires:
-      cluster_features: ["mapper.pass_through_priority"]
-      reason: support for priority in passthrough objects
   - do:
       allowed_warnings:
         - "index template [my-dynamic-template] has index patterns [otel] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
@@ -632,9 +623,6 @@ dynamic templates - conflicting aliases with top-level field:
 
 ---
 dynamic templates with nesting:
-  - requires:
-      cluster_features: ["mapper.pass_through_priority"]
-      reason: support for priority in passthrough objects
   - do:
       allowed_warnings:
         - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
@@ -810,10 +798,6 @@ dynamic templates with nesting:
 
 ---
 dynamic templates with incremental indexing:
-  - requires:
-      cluster_features: ["mapper.pass_through_priority"]
-      reason: support for priority in passthrough objects
-
   - do:
       allowed_warnings:
         - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
@@ -1038,9 +1022,6 @@ dynamic templates with incremental indexing:
 
 ---
 subobject in passthrough object auto flatten:
-  - requires:
-      cluster_features: ["mapper.pass_through_priority"]
-      reason: support for priority in passthrough objects
   - do:
       allowed_warnings:
         - "index template [my-passthrough-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-passthrough-template] will take precedence during new index creation"
@@ -1108,9 +1089,6 @@ enable subobjects in passthrough object:
 
 ---
 passthrough objects with duplicate priority:
-  - requires:
-      cluster_features: ["mapper.pass_through_priority"]
-      reason: support for priority in passthrough objects
   - do:
       catch: /has a conflicting param/
       indices.put_index_template:
@@ -1135,9 +1113,6 @@ passthrough objects with duplicate priority:
 
 ---
 dimensions with ignore_malformed and ignore_above:
-  - requires:
-      cluster_features: ["mapper.keyword_dimension_ignore_above"]
-      reason: support for ignore_above on keyword dimensions
   - do:
       allowed_warnings:
         - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
@@ -1229,9 +1204,6 @@ dimensions with ignore_malformed and ignore_above:
 
 ---
 non string dimension fields:
-  - requires:
-      cluster_features: ["mapper.pass_through_priority", "routing.boolean_routing_path", "mapper.boolean_dimension"]
-      reason: support for priority in passthrough objects
   - do:
       allowed_warnings:
         - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
@@ -1339,10 +1311,6 @@ non string dimension fields:
 
 ---
 multi value dimensions:
-  - requires:
-      cluster_features: ["routing.multi_value_routing_path"]
-      reason: support for multi-value dimensions
-
   - do:
       allowed_warnings:
         - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml
index 13f79e95d99f4..f439cf59bf2d3 100644
--- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml
@@ -148,8 +148,7 @@
   # rollover data stream to create new failure store index
   - do:
       indices.rollover:
-        alias: "data-stream-for-modification"
-        target_failure_store: true
+        alias: "data-stream-for-modification::failures"
   - is_true: acknowledged
 
   # save index names for later use
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_require_data_stream.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_require_data_stream.yml
similarity index 100%
rename from modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_require_data_stream.yml
rename to modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_require_data_stream.yml
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/20_unsupported_apis.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/20_unsupported_apis.yml
index 709eb41ccb61d..a1c952d4c0fad 100644
--- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/20_unsupported_apis.yml
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/20_unsupported_apis.yml
@@ -45,8 +45,6 @@
   - do:
       indices.close:
         index: logs-*
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
   - length: { indices: 0 }
 
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/210_rollover_failure_store.yml
similarity index 95%
rename from modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml
rename to modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/210_rollover_failure_store.yml
index cc3a11ffde5e8..51a1e96b1e937 100644
--- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/210_rollover_failure_store.yml
@@ -9,7 +9,7 @@ setup:
           capabilities: [ 'failure_store_in_template' ]
         - method: POST
           path: /{index}/_rollover
-          capabilities: [ 'lazy-rollover-failure-store' ]
+          capabilities: [ 'lazy-rollover-failure-store', 'index-expression-selectors' ]
 
   - do:
       allowed_warnings:
@@ -58,8 +58,7 @@ teardown:
 
   - do:
       indices.rollover:
-        alias: "data-stream-for-rollover"
-        target_failure_store: true
+        alias: "data-stream-for-rollover::failures"
 
   - match: { acknowledged: true }
   - match: { old_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" }
@@ -92,8 +91,7 @@ teardown:
 
   - do:
       indices.rollover:
-        alias: "data-stream-for-rollover"
-        target_failure_store: true
+        alias: "data-stream-for-rollover::failures"
         body:
           conditions:
             max_docs: 1
@@ -130,8 +128,7 @@ teardown:
 
   - do:
       indices.rollover:
-        alias: "data-stream-for-rollover"
-        target_failure_store: true
+        alias: "data-stream-for-rollover::failures"
         body:
           conditions:
             max_primary_shard_docs: 2
@@ -165,8 +162,7 @@ teardown:
   # Mark the failure store for lazy rollover
   - do:
       indices.rollover:
-        alias: "data-stream-for-rollover"
-        target_failure_store: true
+        alias: "data-stream-for-rollover::failures"
         lazy: true
 
   - match: { acknowledged: true }
@@ -263,8 +259,7 @@ teardown:
   # Mark the failure store for lazy rollover
   - do:
       indices.rollover:
-        alias: data-stream-for-lazy-rollover
-        target_failure_store: true
+        alias: data-stream-for-lazy-rollover::failures
         lazy: true
 
   - match: { acknowledged: true }
@@ -332,8 +327,7 @@ teardown:
   # Mark the failure store for lazy rollover
   - do:
       indices.rollover:
-        alias: "data-stream-for-rollover"
-        target_failure_store: true
+        alias: "data-stream-for-rollover::failures"
         lazy: true
 
   - match: { acknowledged: true }
@@ -377,16 +371,14 @@ teardown:
   - do:
       catch: /Rolling over\/initializing an empty failure store is only supported without conditions\./
       indices.rollover:
-        alias: "data-stream-for-rollover"
-        target_failure_store: true
+        alias: "data-stream-for-rollover::failures"
         body:
           conditions:
             max_docs: 1
 
   - do:
       indices.rollover:
-        alias: "data-stream-for-rollover"
-        target_failure_store: true
+        alias: "data-stream-for-rollover::failures"
 
   - match: { acknowledged: true }
   - match: { old_index: "_none_" }
@@ -424,8 +416,7 @@ teardown:
   # Initializing should work
   - do:
       indices.rollover:
-        alias: "other-data-stream-for-rollover"
-        target_failure_store: true
+        alias: "other-data-stream-for-rollover::failures"
 
   - match: { acknowledged: true }
   - match: { old_index: "_none_" }
@@ -448,8 +439,7 @@ teardown:
   # And "regular" rollover should work
   - do:
       indices.rollover:
-        alias: "other-data-stream-for-rollover"
-        target_failure_store: true
+        alias: "other-data-stream-for-rollover::failures"
 
   - match: { acknowledged: true }
   - match: { old_index: "/\\.fs-other-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" }
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/220_failure_store_cluster_setting.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/220_failure_store_cluster_setting.yml
new file mode 100644
index 0000000000000..90bd6fe406b57
--- /dev/null
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/220_failure_store_cluster_setting.yml
@@ -0,0 +1,222 @@
+setup:
+  - requires:
+      reason: "Data stream options was added in 8.18+"
+      test_runner_features: [ capabilities, allowed_warnings, contains ]
+      capabilities:
+        - method: POST
+          path: /{index}/_doc
+          capabilities: [ 'failure_store_status' ]
+        - method: POST
+          path: /_index_template/{template}
+          capabilities: [ 'failure_store_in_template' ]
+        - method: PUT
+          path: /_cluster/settings
+          capabilities: [ 'data_stream_failure_store_cluster_setting' ]
+
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            data_streams.failure_store.enabled: '*-matching'
+
+  - do:
+      ingest.put_pipeline:
+        id: "failing_pipeline"
+        body:  >
+          {
+            "processors": [
+              {
+                "fail": {
+                  "message" : "pipeline go boom"
+                }
+              }
+            ]
+          }
+
+  - do:
+      indices.put_index_template:
+        name: index_template_default_fs
+        body:
+          index_patterns: default-fs-*
+          data_stream: {}
+          template:
+            settings:
+              number_of_shards:   1
+              number_of_replicas: 1
+
+  - do:
+      cluster.put_component_template:
+        name: component_template_disabled_fs
+        body:
+          template:
+            data_stream_options:
+              failure_store:
+                enabled: false
+
+  - do:
+      indices.put_index_template:
+        name: index_template_disabled_fs
+        body:
+          index_patterns: disabled-fs-*
+          data_stream: {}
+          composed_of:
+            - component_template_disabled_fs
+          template:
+            settings:
+              number_of_shards:   1
+              number_of_replicas: 1
+
+---
+teardown:
+  - do:
+      indices.delete_data_stream:
+        name: default-fs-matching
+        ignore: 404
+
+  - do:
+      indices.delete_data_stream:
+        name: disabled-fs-matching
+        ignore: 404
+
+  - do:
+      indices.delete_index_template:
+        name: index_template_disabled_fs
+        ignore: 404
+
+  - do:
+      cluster.delete_component_template:
+        name: component_template_disabled_fs
+        ignore: 404
+
+  - do:
+      indices.delete_index_template:
+        name: index_template_default_fs
+        ignore: 404
+
+  - do:
+      ingest.delete_pipeline:
+        id: "failing_pipeline"
+        ignore: 404
+
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            data_streams.failure_store.enabled: null
+
+---
+"Redirect ingest failure when auto-creating data stream to failure store when enabled by setting":
+  - do:
+      index:
+        index:  default-fs-matching
+        refresh: true
+        pipeline: 'failing_pipeline'
+        body:
+          '@timestamp': '2020-12-12'
+          foo: bar
+  - match: { failure_store: used }
+  - match: { _index: '/\.fs-default-fs-matching-(\d{4}\.\d{2}\.\d{2}-)?\d{6}/' }
+
+---
+"Redirect ingest failure into pre-existing data stream to failure store when enabled by setting":
+  - do:
+      indices.create_data_stream:
+        name: default-fs-matching
+
+  - do:
+      index:
+        index:  default-fs-matching
+        refresh: true
+        pipeline: 'failing_pipeline'
+        body:
+          '@timestamp': '2020-12-12'
+          foo: bar
+  - match: { failure_store: used }
+  - match: { _index: '/\.fs-default-fs-matching-(\d{4}\.\d{2}\.\d{2}-)?\d{6}/' }
+
+---
+"Do not redirect ingest failure when auto-creating data stream to failure store when enabled by setting but disabled in template":
+  - do:
+      index:
+        index:  disabled-fs-matching
+        refresh: true
+        pipeline: 'failing_pipeline'
+        body:
+          '@timestamp': '2020-12-12'
+          foo: bar
+      catch: '/pipeline go boom/'
+
+---
+"Do not redirect ingest failure into pre-existing data stream to failure store when enabled by setting but disabled in template":
+  - do:
+      indices.create_data_stream:
+        name: disabled-fs-matching
+
+  - do:
+      index:
+        index:  disabled-fs-matching
+        refresh: true
+        pipeline: 'failing_pipeline'
+        body:
+          '@timestamp': '2020-12-12'
+          foo: bar
+      catch: '/pipeline go boom/'
+
+---
+"Redirect mapping failure when auto-creating data stream to failure store when enabled by setting":
+  - do:
+      index:
+        index:  default-fs-matching
+        refresh: true
+        body:
+          '@timestamp': 'not a timestamp'
+          foo: bar
+  - match: { failure_store: used }
+  - match: { _index: '/\.fs-default-fs-matching-(\d{4}\.\d{2}\.\d{2}-)?\d{6}/' }
+
+---
+"Redirect mapping failure into pre-existing data stream to failure store when enabled by setting":
+  - do:
+      indices.create_data_stream:
+        name: default-fs-matching
+
+  - do:
+      index:
+        index:  default-fs-matching
+        refresh: true
+        body:
+          '@timestamp': 'not a timestamp'
+          foo: bar
+  - match: { failure_store: used }
+  - match: { _index: '/\.fs-default-fs-matching-(\d{4}\.\d{2}\.\d{2}-)?\d{6}/' }
+
+---
+"Do not redirect mapping failure when auto-creating data stream to failure store when enabled by setting but disabled in template":
+  - do:
+      index:
+        index:  disabled-fs-matching
+        refresh: true
+        body:
+          '@timestamp': 'not a timestamp'
+          foo: bar
+      catch: '/failed to parse field/'
+
+---
+"Do not redirect mapping failure into pre-existing data stream to failure store when enabled by setting but disabled in template":
+  - do:
+      indices.create_data_stream:
+        name: disabled-fs-matching
+
+  - do:
+      index:
+        index:  disabled-fs-matching
+        refresh: true
+        body:
+          '@timestamp': 'not a timestamp'
+          foo: bar
+      catch: '/failed to parse field/'
+
+# See also DataStreamOptionsIT for tests of the interaction between the failure store cluster setting and using
+# the /_data_stream/{name}/_options API to explicitly enable and disable the failure store. (At time of writing, these
+# can only be done in a Java REST test case, not a YAML one, because the failure store is behind a feature gate and so
+# the REST API spec has not been added.)
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/40_supported_apis.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/40_supported_apis.yml
index 72fce418d8899..8cbb707034546 100644
--- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/40_supported_apis.yml
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/40_supported_apis.yml
@@ -230,8 +230,6 @@ teardown:
   - do:
       indices.close:
         index: ".ds-simple-data-stream1-*000001,.ds-simple-data-stream1-*000002"
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
 
   - do:
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/80_resolve_index_data_streams.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/80_resolve_index_data_streams.yml
index 810b0e43a0da6..eb07438661330 100644
--- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/80_resolve_index_data_streams.yml
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/80_resolve_index_data_streams.yml
@@ -54,8 +54,6 @@ setup:
   - do:
       indices.close:
         index: test_index2
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       indices.create:
diff --git a/modules/dot-prefix-validation/build.gradle b/modules/dot-prefix-validation/build.gradle
index 6e232570b4a22..bbbbbb5609f1e 100644
--- a/modules/dot-prefix-validation/build.gradle
+++ b/modules/dot-prefix-validation/build.gradle
@@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'Validation for dot-prefixed indices for non-operator users'
-  classname 'org.elasticsearch.validation.DotPrefixValidationPlugin'
+  description = 'Validation for dot-prefixed indices for non-operator users'
+  classname = 'org.elasticsearch.validation.DotPrefixValidationPlugin'
 }
 
 restResources {
diff --git a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java
index 7f65bbb6937d9..80fb8a3b2e364 100644
--- a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java
+++ b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java
@@ -67,7 +67,7 @@ public abstract class DotPrefixValidator<RequestType> implements MappedActionFil
         ".ml-state",
         ".ml-anomalies-unrelated"
     );
-    public static Setting<List<String>> IGNORED_INDEX_PATTERNS_SETTING = Setting.stringListSetting(
+    public static final Setting<List<String>> IGNORED_INDEX_PATTERNS_SETTING = Setting.stringListSetting(
         "cluster.indices.validate_ignored_dot_patterns",
         List.of(
             "\\.ml-state-\\d+",
diff --git a/modules/health-shards-availability/build.gradle b/modules/health-shards-availability/build.gradle
index 4bf914fb5fdd6..62686a44a9336 100644
--- a/modules/health-shards-availability/build.gradle
+++ b/modules/health-shards-availability/build.gradle
@@ -11,8 +11,8 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'Health report API extension providing the shards_availability output'
-  classname 'org.elasticsearch.health.plugin.ShardsAvailabilityPlugin'
+  description = 'Health report API extension providing the shards_availability output'
+  classname = 'org.elasticsearch.health.plugin.ShardsAvailabilityPlugin'
 }
 
 restResources {
diff --git a/modules/ingest-attachment/build.gradle b/modules/ingest-attachment/build.gradle
index 8fe2b82fe21fb..a172112948fd3 100644
--- a/modules/ingest-attachment/build.gradle
+++ b/modules/ingest-attachment/build.gradle
@@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.yaml-rest-compat-test'
 
 esplugin {
-  description 'Ingest processor that uses Apache Tika to extract contents'
-  classname 'org.elasticsearch.ingest.attachment.IngestAttachmentPlugin'
+  description = 'Ingest processor that uses Apache Tika to extract contents'
+  classname = 'org.elasticsearch.ingest.attachment.IngestAttachmentPlugin'
 }
 
 // this overwrites the 'versions' map from Elasticsearch itself, but indeed we want that -- we're interested in managing our dependencies
diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle
index 7cfdba4d33744..c49e0ea4b7180 100644
--- a/modules/ingest-common/build.gradle
+++ b/modules/ingest-common/build.gradle
@@ -13,8 +13,8 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources'
-  classname 'org.elasticsearch.ingest.common.IngestCommonPlugin'
+  description = 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources'
+  classname ='org.elasticsearch.ingest.common.IngestCommonPlugin'
   extendedPlugins = ['lang-painless']
 }
 
diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle
index 4312221b33937..988ca317b4730 100644
--- a/modules/ingest-geoip/build.gradle
+++ b/modules/ingest-geoip/build.gradle
@@ -14,8 +14,8 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'Ingest processor that uses lookup geo data based on IP addresses using the MaxMind geo database'
-  classname 'org.elasticsearch.ingest.geoip.IngestGeoIpPlugin'
+  description = 'Ingest processor that uses lookup geo data based on IP addresses using the MaxMind geo database'
+  classname ='org.elasticsearch.ingest.geoip.IngestGeoIpPlugin'
 
   bundleSpec.from("${project.buildDir}/ingest-geoip") {
     into '/'
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java
index a50fe7dee9008..58ff64c97b2e0 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java
@@ -15,9 +15,10 @@
 import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -90,8 +91,8 @@ public static IngestGeoIpMetadata fromXContent(XContentParser parser) throws IOE
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).xContentObjectFields(DATABASES_FIELD.getPreferredName(), databases);
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.concat(ChunkedToXContentHelper.xContentObjectFields(DATABASES_FIELD.getPreferredName(), databases));
     }
 
     @Override
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java
index aa48c73cf1d73..08efe87e6fde5 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java
@@ -160,11 +160,6 @@ public void writeTo(StreamOutput out) throws IOException {
             if (provider instanceof Maxmind maxmind) {
                 out.writeString(maxmind.accountId);
             } else {
-                /*
-                 * The existence of a non-Maxmind providers is gated on the feature get_database_configuration_action.multi_node, and
-                 * get_database_configuration_action.multi_node is only available on or after
-                 * TransportVersions.INGEST_GEO_DATABASE_PROVIDERS.
-                 */
                 assert false : "non-maxmind DatabaseConfiguration.Provider [" + provider.getWriteableName() + "]";
             }
         }
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java
index 1970883e91b3e..68b3ce279a89d 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java
@@ -25,6 +25,7 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 import static org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata.DATABASE;
@@ -91,6 +92,11 @@ protected Response(StreamInput in) throws IOException {
             this.databases = in.readCollectionAsList(DatabaseConfigurationMetadata::new);
         }
 
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeCollection(databases);
+        }
+
         @Override
         protected List<NodeResponse> readNodesFrom(StreamInput in) throws IOException {
             return in.readCollectionAsList(NodeResponse::new);
@@ -122,6 +128,63 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.endObject();
             return builder;
         }
+
+        /*
+         * This implementation of equals exists solely for testing the serialization of this object.
+         */
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Response response = (Response) o;
+            return Objects.equals(databases, response.databases)
+                && Objects.equals(getClusterName(), response.getClusterName())
+                && Objects.equals(equalsHashCodeFailures(), response.equalsHashCodeFailures())
+                && Objects.equals(getNodes(), response.getNodes())
+                && Objects.equals(equalsHashCodeNodesMap(), response.equalsHashCodeNodesMap());
+        }
+
+        /*
+         * This implementation of hashCode exists solely for testing the serialization of this object.
+         */
+        @Override
+        public int hashCode() {
+            return Objects.hash(databases, getClusterName(), equalsHashCodeFailures(), getNodes(), equalsHashCodeNodesMap());
+        }
+
+        /*
+         * FailedNodeException does not implement equals or hashCode, making it difficult to test the serialization of this class. This
+         * helper method wraps the failures() list with a class that does implement equals and hashCode.
+         */
+        private List<EqualsHashCodeFailedNodeException> equalsHashCodeFailures() {
+            return failures().stream().map(EqualsHashCodeFailedNodeException::new).toList();
+        }
+
+        private record EqualsHashCodeFailedNodeException(FailedNodeException failedNodeException) {
+            @Override
+            public boolean equals(Object o) {
+                if (o == this) return true;
+                if (o == null || getClass() != o.getClass()) return false;
+                EqualsHashCodeFailedNodeException other = (EqualsHashCodeFailedNodeException) o;
+                return Objects.equals(failedNodeException.nodeId(), other.failedNodeException.nodeId())
+                    && Objects.equals(failedNodeException.getMessage(), other.failedNodeException.getMessage());
+            }
+
+            @Override
+            public int hashCode() {
+                return Objects.hash(failedNodeException.nodeId(), failedNodeException.getMessage());
+            }
+        }
+
+        /*
+         * The getNodesMap method changes the value of the nodesMap, causing failures when testing the concurrent serialization and
+         * deserialization of this class. Since this is a response object, we do not actually care about concurrency since it will not
+         * happen in practice. So this helper method synchronizes access to getNodesMap, which can be used from equals and hashCode for
+         * tests.
+         */
+        private synchronized Map<String, NodeResponse> equalsHashCodeNodesMap() {
+            return getNodesMap();
+        }
     }
 
     public static class NodeRequest extends TransportRequest {
@@ -186,6 +249,7 @@ public List<DatabaseConfigurationMetadata> getDatabases() {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
             out.writeCollection(databases);
         }
 
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java
index c83c40e56b749..a1faaf1bb0196 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.regex.Regex;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.ingest.geoip.DatabaseNodeService;
 import org.elasticsearch.ingest.geoip.GeoIpTaskState;
 import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata;
@@ -41,8 +40,6 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.ingest.IngestGeoIpFeatures.GET_DATABASE_CONFIGURATION_ACTION_MULTI_NODE;
-
 public class TransportGetDatabaseConfigurationAction extends TransportNodesAction<
     GetDatabaseConfigurationAction.Request,
     GetDatabaseConfigurationAction.Response,
@@ -50,7 +47,6 @@ public class TransportGetDatabaseConfigurationAction extends TransportNodesActio
     GetDatabaseConfigurationAction.NodeResponse,
     List<DatabaseConfigurationMetadata>> {
 
-    private final FeatureService featureService;
     private final DatabaseNodeService databaseNodeService;
 
     @Inject
@@ -59,7 +55,6 @@ public TransportGetDatabaseConfigurationAction(
         ClusterService clusterService,
         ThreadPool threadPool,
         ActionFilters actionFilters,
-        FeatureService featureService,
         DatabaseNodeService databaseNodeService
     ) {
         super(
@@ -70,39 +65,9 @@ public TransportGetDatabaseConfigurationAction(
             GetDatabaseConfigurationAction.NodeRequest::new,
             threadPool.executor(ThreadPool.Names.MANAGEMENT)
         );
-        this.featureService = featureService;
         this.databaseNodeService = databaseNodeService;
     }
 
-    @Override
-    protected void doExecute(
-        Task task,
-        GetDatabaseConfigurationAction.Request request,
-        ActionListener<GetDatabaseConfigurationAction.Response> listener
-    ) {
-        if (featureService.clusterHasFeature(clusterService.state(), GET_DATABASE_CONFIGURATION_ACTION_MULTI_NODE) == false) {
-            /*
-             * TransportGetDatabaseConfigurationAction used to be a TransportMasterNodeAction, and not all nodes in the cluster have been
-             * updated. So we don't want to send node requests to the other nodes because they will blow up. Instead, we just return
-             * the information that we used to return from the master node (it doesn't make any difference that this might not be the master
-             * node, because we're only reading the cluster state). Because older nodes only know about the Maxmind provider type, we filter
-             * out all others here to avoid causing problems on those nodes.
-             */
-            newResponseAsync(
-                task,
-                request,
-                createActionContext(task, request).stream()
-                    .filter(database -> database.database().provider() instanceof DatabaseConfiguration.Maxmind)
-                    .toList(),
-                List.of(),
-                List.of(),
-                listener
-            );
-        } else {
-            super.doExecute(task, request, listener);
-        }
-    }
-
     protected List<DatabaseConfigurationMetadata> createActionContext(Task task, GetDatabaseConfigurationAction.Request request) {
         final Set<String> ids;
         if (request.getDatabaseIds().length == 0) {
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java
index dfb8fa78089d2..e68bb9d82e91b 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata;
 import org.elasticsearch.ingest.geoip.direct.PutDatabaseConfigurationAction.Request;
 import org.elasticsearch.injection.guice.Inject;
@@ -42,8 +41,6 @@
 import java.util.Map;
 import java.util.Optional;
 
-import static org.elasticsearch.ingest.IngestGeoIpFeatures.PUT_DATABASE_CONFIGURATION_ACTION_IPINFO;
-
 public class TransportPutDatabaseConfigurationAction extends TransportMasterNodeAction<Request, AcknowledgedResponse> {
 
     private static final Logger logger = LogManager.getLogger(TransportPutDatabaseConfigurationAction.class);
@@ -61,7 +58,6 @@ public void taskSucceeded(UpdateDatabaseConfigurationTask task, Void unused) {
         }
     };
 
-    private final FeatureService featureService;
     private final MasterServiceTaskQueue<UpdateDatabaseConfigurationTask> updateDatabaseConfigurationTaskQueue;
 
     @Inject
@@ -70,8 +66,7 @@ public TransportPutDatabaseConfigurationAction(
         ClusterService clusterService,
         ThreadPool threadPool,
         ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver,
-        FeatureService featureService
+        IndexNameExpressionResolver indexNameExpressionResolver
     ) {
         super(
             PutDatabaseConfigurationAction.NAME,
@@ -84,7 +79,6 @@ public TransportPutDatabaseConfigurationAction(
             AcknowledgedResponse::readFrom,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
-        this.featureService = featureService;
         this.updateDatabaseConfigurationTaskQueue = clusterService.createTaskQueue(
             "update-geoip-database-configuration-state-update",
             Priority.NORMAL,
@@ -96,18 +90,6 @@ public TransportPutDatabaseConfigurationAction(
     protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<AcknowledgedResponse> listener) {
         final String id = request.getDatabase().id();
 
-        // if this is an ipinfo configuration, then make sure the whole cluster supports that feature
-        if (request.getDatabase().provider() instanceof DatabaseConfiguration.Ipinfo
-            && featureService.clusterHasFeature(clusterService.state(), PUT_DATABASE_CONFIGURATION_ACTION_IPINFO) == false) {
-            listener.onFailure(
-                new IllegalArgumentException(
-                    "Unable to use ipinfo database configurations in mixed-clusters with nodes that do not support feature "
-                        + PUT_DATABASE_CONFIGURATION_ACTION_IPINFO.id()
-                )
-            );
-            return;
-        }
-
         updateDatabaseConfigurationTaskQueue.submitTask(
             Strings.format("update-geoip-database-configuration-[%s]", id),
             new UpdateDatabaseConfigurationTask(listener, request.getDatabase()),
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionNodeResponseTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionNodeResponseTests.java
new file mode 100644
index 0000000000000..12fb08a5a1abf
--- /dev/null
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionNodeResponseTests.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.ingest.geoip.direct;
+
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.io.IOException;
+import java.util.List;
+
+import static java.util.Collections.emptySet;
+
+public class GetDatabaseConfigurationActionNodeResponseTests extends AbstractWireSerializingTestCase<
+    GetDatabaseConfigurationAction.NodeResponse> {
+    @Override
+    protected Writeable.Reader<GetDatabaseConfigurationAction.NodeResponse> instanceReader() {
+        return GetDatabaseConfigurationAction.NodeResponse::new;
+    }
+
+    @Override
+    protected GetDatabaseConfigurationAction.NodeResponse createTestInstance() {
+        return getRandomDatabaseConfigurationActionNodeResponse();
+    }
+
+    static GetDatabaseConfigurationAction.NodeResponse getRandomDatabaseConfigurationActionNodeResponse() {
+        return new GetDatabaseConfigurationAction.NodeResponse(randomDiscoveryNode(), getRandomDatabaseConfigurationMetadata());
+    }
+
+    private static DiscoveryNode randomDiscoveryNode() {
+        return DiscoveryNodeUtils.builder(randomAlphaOfLength(6)).roles(emptySet()).build();
+    }
+
+    static List<DatabaseConfigurationMetadata> getRandomDatabaseConfigurationMetadata() {
+        return randomList(
+            0,
+            20,
+            () -> new DatabaseConfigurationMetadata(
+                new DatabaseConfiguration(
+                    randomAlphaOfLength(20),
+                    randomAlphaOfLength(20),
+                    randomFrom(
+                        List.of(
+                            new DatabaseConfiguration.Local(randomAlphaOfLength(10)),
+                            new DatabaseConfiguration.Web(),
+                            new DatabaseConfiguration.Ipinfo(),
+                            new DatabaseConfiguration.Maxmind(randomAlphaOfLength(10))
+                        )
+                    )
+                ),
+                randomNonNegativeLong(),
+                randomNonNegativeLong()
+            )
+        );
+    }
+
+    @Override
+    protected GetDatabaseConfigurationAction.NodeResponse mutateInstance(GetDatabaseConfigurationAction.NodeResponse instance)
+        throws IOException {
+        return null;
+    }
+
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(
+            List.of(
+                new NamedWriteableRegistry.Entry(
+                    DatabaseConfiguration.Provider.class,
+                    DatabaseConfiguration.Maxmind.NAME,
+                    DatabaseConfiguration.Maxmind::new
+                ),
+                new NamedWriteableRegistry.Entry(
+                    DatabaseConfiguration.Provider.class,
+                    DatabaseConfiguration.Ipinfo.NAME,
+                    DatabaseConfiguration.Ipinfo::new
+                ),
+                new NamedWriteableRegistry.Entry(
+                    DatabaseConfiguration.Provider.class,
+                    DatabaseConfiguration.Local.NAME,
+                    DatabaseConfiguration.Local::new
+                ),
+                new NamedWriteableRegistry.Entry(
+                    DatabaseConfiguration.Provider.class,
+                    DatabaseConfiguration.Web.NAME,
+                    DatabaseConfiguration.Web::new
+                )
+            )
+        );
+    }
+}
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionResponseTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionResponseTests.java
new file mode 100644
index 0000000000000..1b48a409d7876
--- /dev/null
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionResponseTests.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.ingest.geoip.direct;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.FailedNodeException;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.io.IOException;
+import java.util.List;
+
+public class GetDatabaseConfigurationActionResponseTests extends AbstractWireSerializingTestCase<GetDatabaseConfigurationAction.Response> {
+    @Override
+    protected Writeable.Reader<GetDatabaseConfigurationAction.Response> instanceReader() {
+        return GetDatabaseConfigurationAction.Response::new;
+    }
+
+    @Override
+    protected GetDatabaseConfigurationAction.Response createTestInstance() {
+        return new GetDatabaseConfigurationAction.Response(
+            GetDatabaseConfigurationActionNodeResponseTests.getRandomDatabaseConfigurationMetadata(),
+            getTestClusterName(),
+            getTestNodeResponses(),
+            getTestFailedNodeExceptions()
+        );
+    }
+
+    @Override
+    protected GetDatabaseConfigurationAction.Response mutateInstance(GetDatabaseConfigurationAction.Response instance) throws IOException {
+        return null;
+    }
+
+    private ClusterName getTestClusterName() {
+        return new ClusterName(randomAlphaOfLength(30));
+    }
+
+    private List<GetDatabaseConfigurationAction.NodeResponse> getTestNodeResponses() {
+        return randomList(0, 20, GetDatabaseConfigurationActionNodeResponseTests::getRandomDatabaseConfigurationActionNodeResponse);
+    }
+
+    private List<FailedNodeException> getTestFailedNodeExceptions() {
+        return randomList(
+            0,
+            5,
+            () -> new FailedNodeException(
+                randomAlphaOfLength(10),
+                randomAlphaOfLength(20),
+                new ElasticsearchException(randomAlphaOfLength(10))
+            )
+        );
+    }
+
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(
+            List.of(
+                new NamedWriteableRegistry.Entry(
+                    DatabaseConfiguration.Provider.class,
+                    DatabaseConfiguration.Maxmind.NAME,
+                    DatabaseConfiguration.Maxmind::new
+                ),
+                new NamedWriteableRegistry.Entry(
+                    DatabaseConfiguration.Provider.class,
+                    DatabaseConfiguration.Ipinfo.NAME,
+                    DatabaseConfiguration.Ipinfo::new
+                ),
+                new NamedWriteableRegistry.Entry(
+                    DatabaseConfiguration.Provider.class,
+                    DatabaseConfiguration.Local.NAME,
+                    DatabaseConfiguration.Local::new
+                ),
+                new NamedWriteableRegistry.Entry(
+                    DatabaseConfiguration.Provider.class,
+                    DatabaseConfiguration.Web.NAME,
+                    DatabaseConfiguration.Web::new
+                )
+            )
+        );
+    }
+}
diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml
index a1104505bc240..007c82db4c923 100644
--- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml
+++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml
@@ -1,9 +1,3 @@
----
-setup:
-  - requires:
-      cluster_features: ["geoip.downloader.database.configuration", "get_database_configuration_action.multi_node"]
-      reason: "geoip downloader database configuration APIs added in 8.15, and updated in 8.16 to return more results"
-
 ---
 teardown:
   - do:
diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml
index fd73c715a5ac5..0947984769529 100644
--- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml
+++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml
@@ -1,9 +1,3 @@
-setup:
-  - requires:
-      cluster_features:
-        - "put_database_configuration_action.ipinfo"
-      reason: "ipinfo support added in 8.16"
-
 ---
 "Test ip_location processor with defaults":
   - do:
diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml
index e2e9a1fdb5e28..47f09392df60e 100644
--- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml
+++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml
@@ -1,10 +1,3 @@
----
-setup:
-  - requires:
-      cluster_features:
-        - "put_database_configuration_action.ipinfo"
-      reason: "ip location downloader database configuration APIs added in 8.16 to support more types"
-
 ---
 teardown:
   - do:
diff --git a/modules/ingest-user-agent/build.gradle b/modules/ingest-user-agent/build.gradle
index a504b5a64807c..76f88544ea10f 100644
--- a/modules/ingest-user-agent/build.gradle
+++ b/modules/ingest-user-agent/build.gradle
@@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.yaml-rest-compat-test'
 
 esplugin {
-  description 'Ingest processor that extracts information from a user agent'
-  classname 'org.elasticsearch.ingest.useragent.IngestUserAgentPlugin'
+  description = 'Ingest processor that extracts information from a user agent'
+  classname ='org.elasticsearch.ingest.useragent.IngestUserAgentPlugin'
 }
 
 restResources {
diff --git a/modules/kibana/build.gradle b/modules/kibana/build.gradle
index 6ead4cc68506c..deaf2664236ba 100644
--- a/modules/kibana/build.gradle
+++ b/modules/kibana/build.gradle
@@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-java-rest-test'
 
 esplugin {
-  description 'Plugin exposing APIs for Kibana system indices'
-  classname 'org.elasticsearch.kibana.KibanaPlugin'
+  description = 'Plugin exposing APIs for Kibana system indices'
+  classname ='org.elasticsearch.kibana.KibanaPlugin'
 }
 
 dependencies {
diff --git a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java
index 553e4696af316..a9ab0c02612f6 100644
--- a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java
+++ b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java
@@ -16,6 +16,8 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor;
@@ -23,7 +25,6 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPoolStats;
 
@@ -49,10 +50,6 @@
  * threads that wait on a phaser. This lets us verify that operations on system indices
  * are being directed to other thread pools.</p>
  */
-@TestLogging(
-    reason = "investigate",
-    value = "org.elasticsearch.kibana.KibanaThreadPoolIT:DEBUG,org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor:TRACE"
-)
 public class KibanaThreadPoolIT extends ESIntegTestCase {
     private static final Logger logger = LogManager.getLogger(KibanaThreadPoolIT.class);
 
@@ -68,6 +65,8 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
             .put("thread_pool.write.queue_size", 1)
             .put("thread_pool.get.size", 1)
             .put("thread_pool.get.queue_size", 1)
+            // a rejected GET may retry on an INITIALIZING shard (the target of a relocation) and unexpectedly succeed, so block rebalancing
+            .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)
             .build();
     }
 
@@ -112,7 +111,12 @@ public void testKibanaThreadPoolByPassesBlockedThreadPools() throws Exception {
     }
 
     public void testBlockedThreadPoolsRejectUserRequests() throws Exception {
-        assertAcked(client().admin().indices().prepareCreate(USER_INDEX));
+        assertAcked(
+            client().admin()
+                .indices()
+                .prepareCreate(USER_INDEX)
+                .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)) // avoid retrying rejected actions
+        );
 
         runWithBlockedThreadPools(this::assertThreadPoolsBlocked);
 
diff --git a/modules/lang-expression/build.gradle b/modules/lang-expression/build.gradle
index 1342450755152..a7f30c4fbb5cd 100644
--- a/modules/lang-expression/build.gradle
+++ b/modules/lang-expression/build.gradle
@@ -11,8 +11,8 @@ apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'Lucene expressions integration for Elasticsearch'
-  classname 'org.elasticsearch.script.expression.ExpressionPlugin'
+  description = 'Lucene expressions integration for Elasticsearch'
+  classname ='org.elasticsearch.script.expression.ExpressionPlugin'
 }
 
 dependencies {
diff --git a/modules/lang-expression/src/main/plugin-metadata/entitlement-policy.yaml b/modules/lang-expression/src/main/plugin-metadata/entitlement-policy.yaml
new file mode 100644
index 0000000000000..b05e6e3a7bf7c
--- /dev/null
+++ b/modules/lang-expression/src/main/plugin-metadata/entitlement-policy.yaml
@@ -0,0 +1,2 @@
+org.elasticsearch.script.expression:
+  - create_class_loader
diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle
index 45037857db093..e8e539d2d6b55 100644
--- a/modules/lang-mustache/build.gradle
+++ b/modules/lang-mustache/build.gradle
@@ -12,8 +12,8 @@ apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'Mustache scripting integration for Elasticsearch'
-  classname 'org.elasticsearch.script.mustache.MustachePlugin'
+  description = 'Mustache scripting integration for Elasticsearch'
+  classname ='org.elasticsearch.script.mustache.MustachePlugin'
 }
 
 dependencies {
diff --git a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MustacheSettingsIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MustacheSettingsIT.java
new file mode 100644
index 0000000000000..7ab3f1bd3b809
--- /dev/null
+++ b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MustacheSettingsIT.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.script.mustache;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.equalTo;
+
+public class MustacheSettingsIT extends ESSingleNodeTestCase {
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return List.of(MustachePlugin.class);
+    }
+
+    @Override
+    protected Settings nodeSettings() {
+        return Settings.builder().put(MustacheScriptEngine.MUSTACHE_RESULT_SIZE_LIMIT.getKey(), "10b").build();
+    }
+
+    public void testResultSizeLimit() throws Exception {
+        createIndex("test");
+        prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("text", "value1").endObject()).get();
+        indicesAdmin().prepareRefresh().get();
+
+        String query = """
+            { "query": {"match_all": {}}, "size" : "{{my_size}}"  }""";
+        SearchRequest searchRequest = new SearchRequest();
+        searchRequest.indices("test");
+        var e = expectThrows(
+            ElasticsearchParseException.class,
+            () -> new SearchTemplateRequestBuilder(client()).setRequest(searchRequest)
+                .setScript(query)
+                .setScriptType(ScriptType.INLINE)
+                .setScriptParams(Collections.singletonMap("my_size", 1))
+                .get()
+        );
+        assertThat(e.getMessage(), equalTo("Mustache script result size limit exceeded"));
+    }
+}
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
index b24d60cb8d887..cfdd57db4dd54 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.features.NodeFeature;
@@ -73,4 +74,9 @@ public List<RestHandler> getRestHandlers(
             new RestRenderSearchTemplateAction()
         );
     }
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return List.of(MustacheScriptEngine.MUSTACHE_RESULT_SIZE_LIMIT);
+    }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java
index 9bb80d5688b5f..f8a56dcfc990e 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java
@@ -27,7 +27,7 @@
 import java.io.InputStream;
 
 public class SearchTemplateResponse extends ActionResponse implements ToXContentObject {
-    public static ParseField TEMPLATE_OUTPUT_FIELD = new ParseField("template_output");
+    public static final ParseField TEMPLATE_OUTPUT_FIELD = new ParseField("template_output");
 
     /** Contains the source of the rendered template **/
     private BytesReference source;
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java
index 990a92fb0dbf8..19069c876e7e7 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.script.ScriptType;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.StreamsUtils;
@@ -102,7 +101,7 @@ public void testMultiSearchTemplateToJson() throws Exception {
             String[] indices = { "test" };
             SearchRequest searchRequest = new SearchRequest(indices);
             // scroll is not supported in the current msearch or msearchtemplate api, so unset it:
-            searchRequest.scroll((Scroll) null);
+            searchRequest.scroll(null);
             // batched reduce size is currently not set-able on a per-request basis as it is a query string parameter only
             searchRequest.setBatchedReduceSize(SearchRequest.DEFAULT_BATCHED_REDUCE_SIZE);
             SearchTemplateRequest searchTemplateRequest = new SearchTemplateRequest(searchRequest);
diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle
index b01c9201dcff1..b030d3c2d5e16 100644
--- a/modules/lang-painless/build.gradle
+++ b/modules/lang-painless/build.gradle
@@ -15,8 +15,8 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-    description 'An easy, safe and fast scripting language for Elasticsearch'
-    classname 'org.elasticsearch.painless.PainlessPlugin'
+    description = 'An easy, safe and fast scripting language for Elasticsearch'
+    classname ='org.elasticsearch.painless.PainlessPlugin'
 }
 
 testClusters.configureEach {
diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java
index 2e7f0de027de7..37bff97a07ae2 100644
--- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java
@@ -9,8 +9,10 @@
 
 package org.elasticsearch.painless.spi;
 
+import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.painless.spi.annotation.WhitelistAnnotationParser;
 
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.LineNumberReader;
 import java.lang.reflect.Constructor;
@@ -140,7 +142,7 @@ public static Whitelist loadFromResourceFiles(Class<?> resource, String... filep
      * }
      * }
      */
-    public static Whitelist loadFromResourceFiles(Class<?> resource, Map<String, WhitelistAnnotationParser> parsers, String... filepaths) {
+    public static Whitelist loadFromResourceFiles(Class<?> owner, Map<String, WhitelistAnnotationParser> parsers, String... filepaths) {
         List<WhitelistClass> whitelistClasses = new ArrayList<>();
         List<WhitelistMethod> whitelistStatics = new ArrayList<>();
         List<WhitelistClassBinding> whitelistClassBindings = new ArrayList<>();
@@ -153,7 +155,7 @@ public static Whitelist loadFromResourceFiles(Class<?> resource, Map<String, Whi
 
             try (
                 LineNumberReader reader = new LineNumberReader(
-                    new InputStreamReader(resource.getResourceAsStream(filepath), StandardCharsets.UTF_8)
+                    new InputStreamReader(getResourceAsStream(owner, filepath), StandardCharsets.UTF_8)
                 )
             ) {
 
@@ -483,16 +485,40 @@ public static Whitelist loadFromResourceFiles(Class<?> resource, Map<String, Whi
                 if (javaClassName != null) {
                     throw new IllegalArgumentException("invalid definition: expected closing bracket");
                 }
+            } catch (ResourceNotFoundException e) {
+                throw e; // rethrow
             } catch (Exception exception) {
                 throw new RuntimeException("error in [" + filepath + "] at line [" + number + "]", exception);
             }
         }
 
-        ClassLoader loader = AccessController.doPrivileged((PrivilegedAction<ClassLoader>) resource::getClassLoader);
+        ClassLoader loader = AccessController.doPrivileged((PrivilegedAction<ClassLoader>) owner::getClassLoader);
 
         return new Whitelist(loader, whitelistClasses, whitelistStatics, whitelistClassBindings, Collections.emptyList());
     }
 
+    private static InputStream getResourceAsStream(Class<?> owner, String name) {
+        InputStream stream = owner.getResourceAsStream(name);
+        if (stream == null) {
+            String msg = "Whitelist file ["
+                + owner.getPackageName().replace(".", "/")
+                + "/"
+                + name
+                + "] not found from owning class ["
+                + owner.getName()
+                + "].";
+            if (owner.getModule().isNamed()) {
+                msg += " Check that the file exists and the package ["
+                    + owner.getPackageName()
+                    + "] is opened "
+                    + "to module "
+                    + WhitelistLoader.class.getModule().getName();
+            }
+            throw new ResourceNotFoundException(msg);
+        }
+        return stream;
+    }
+
     private static List<Object> parseWhitelistAnnotations(Map<String, WhitelistAnnotationParser> parsers, String line) {
 
         List<Object> annotations;
diff --git a/modules/lang-painless/spi/src/test/java/org/elasticsearch/painless/WhitelistLoaderTests.java b/modules/lang-painless/spi/src/test/java/org/elasticsearch/painless/WhitelistLoaderTests.java
index e62d0b438b098..b46bc118e0913 100644
--- a/modules/lang-painless/spi/src/test/java/org/elasticsearch/painless/WhitelistLoaderTests.java
+++ b/modules/lang-painless/spi/src/test/java/org/elasticsearch/painless/WhitelistLoaderTests.java
@@ -9,6 +9,7 @@
 
 package org.elasticsearch.painless;
 
+import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.painless.spi.Whitelist;
 import org.elasticsearch.painless.spi.WhitelistClass;
 import org.elasticsearch.painless.spi.WhitelistLoader;
@@ -17,10 +18,18 @@
 import org.elasticsearch.painless.spi.annotation.NoImportAnnotation;
 import org.elasticsearch.painless.spi.annotation.WhitelistAnnotationParser;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.compiler.InMemoryJavaCompiler;
+import org.elasticsearch.test.jar.JarUtils;
 
+import java.lang.ModuleLayer.Controller;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+
 public class WhitelistLoaderTests extends ESTestCase {
 
     public void testUnknownAnnotations() {
@@ -96,4 +105,52 @@ public void testAnnotations() {
 
         assertEquals(3, count);
     }
+
+    public void testMissingWhitelistResource() {
+        var e = expectThrows(ResourceNotFoundException.class, () -> WhitelistLoader.loadFromResourceFiles(Whitelist.class, "missing.txt"));
+        assertThat(
+            e.getMessage(),
+            equalTo(
+                "Whitelist file [org/elasticsearch/painless/spi/missing.txt] not found"
+                    + " from owning class [org.elasticsearch.painless.spi.Whitelist]."
+            )
+        );
+    }
+
+    public void testMissingWhitelistResourceInModule() throws Exception {
+        Map<String, CharSequence> sources = new HashMap<>();
+        sources.put("module-info", "module m {}");
+        sources.put("p.TestOwner", "package p; public class TestOwner { }");
+        var classToBytes = InMemoryJavaCompiler.compile(sources);
+
+        Path dir = createTempDir(getTestName());
+        Path jar = dir.resolve("m.jar");
+        Map<String, byte[]> jarEntries = new HashMap<>();
+        jarEntries.put("module-info.class", classToBytes.get("module-info"));
+        jarEntries.put("p/TestOwner.class", classToBytes.get("p.TestOwner"));
+        jarEntries.put("p/resource.txt", "# test resource".getBytes(StandardCharsets.UTF_8));
+        JarUtils.createJarWithEntries(jar, jarEntries);
+
+        try (var loader = JarUtils.loadJar(jar)) {
+            Controller controller = JarUtils.loadModule(jar, loader.classloader(), "m");
+            Module module = controller.layer().findModule("m").orElseThrow();
+
+            Class<?> ownerClass = module.getClassLoader().loadClass("p.TestOwner");
+
+            // first check we get a nice error message when accessing the resource
+            var e = expectThrows(ResourceNotFoundException.class, () -> WhitelistLoader.loadFromResourceFiles(ownerClass, "resource.txt"));
+            assertThat(
+                e.getMessage(),
+                equalTo(
+                    "Whitelist file [p/resource.txt] not found from owning class [p.TestOwner]."
+                        + " Check that the file exists and the package [p] is opened to module null"
+                )
+            );
+
+            // now check we can actually read it once the package is opened to us
+            controller.addOpens(module, "p", WhitelistLoader.class.getModule());
+            var whitelist = WhitelistLoader.loadFromResourceFiles(ownerClass, "resource.txt");
+            assertThat(whitelist, notNullValue());
+        }
+    }
 }
diff --git a/modules/lang-painless/src/main/plugin-metadata/entitlement-policy.yaml b/modules/lang-painless/src/main/plugin-metadata/entitlement-policy.yaml
new file mode 100644
index 0000000000000..d7e4ad872fc32
--- /dev/null
+++ b/modules/lang-painless/src/main/plugin-metadata/entitlement-policy.yaml
@@ -0,0 +1,2 @@
+org.elasticsearch.painless:
+  - create_class_loader
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt
index a5118db4876cb..e76db7cfb1d26 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt
@@ -50,7 +50,5 @@ static_import {
     double cosineSimilarity(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$CosineSimilarity
     double dotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$DotProduct
     double hamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$Hamming
-    double maxSimDotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.RankVectorsScoreScriptUtils$MaxSimDotProduct
-    double maxSimInvHamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.RankVectorsScoreScriptUtils$MaxSimInvHamming
 }
 
diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/140_dense_vector_basic.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/140_dense_vector_basic.yml
index 25088f51e2b59..1434450b65a6a 100644
--- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/140_dense_vector_basic.yml
+++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/140_dense_vector_basic.yml
@@ -221,9 +221,6 @@ setup:
   - close_to: {hits.hits.2._score: {value: 186.34454, error: 0.01}}
 ---
 "Test hamming distance fails on float":
-  - requires:
-      cluster_features: ["script.hamming"]
-      reason: "support for hamming distance added in 8.15"
   - do:
       headers:
         Content-Type: application/json
diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml
index cdd65ca0eb296..05a10ffdbccdb 100644
--- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml
+++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml
@@ -1,7 +1,5 @@
 setup:
   - requires:
-      cluster_features: ["mapper.vectors.bit_vectors"]
-      reason: "support for bit vectors added in 8.15"
       test_runner_features: headers
 
   - do:
diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/151_dense_vector_byte_hamming.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/151_dense_vector_byte_hamming.yml
index 373f048e7be78..a6c111be681f9 100644
--- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/151_dense_vector_byte_hamming.yml
+++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/151_dense_vector_byte_hamming.yml
@@ -1,7 +1,5 @@
 setup:
   - requires:
-      cluster_features: ["script.hamming"]
-      reason: "support for hamming distance added in 8.15"
       test_runner_features: headers
 
   - do:
diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/190_term_statistics_script_score.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/190_term_statistics_script_score.yml
index f82b844f01588..3a869640993f4 100644
--- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/190_term_statistics_script_score.yml
+++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/190_term_statistics_script_score.yml
@@ -1,8 +1,4 @@
 setup:
-  - requires:
-      cluster_features: ["script.term_stats"]
-      reason: "support for term stats has been added in 8.16"
-
   - do:
       indices.create:
         index: test-index
diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/191_term_statistics_function_score.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/191_term_statistics_function_score.yml
index de4d6530f4a92..3a9c71e3c2bab 100644
--- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/191_term_statistics_function_score.yml
+++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/191_term_statistics_function_score.yml
@@ -1,8 +1,4 @@
 setup:
-  - requires:
-      cluster_features: ["script.term_stats"]
-      reason: "support for term stats has been added in 8.16"
-
   - do:
       indices.create:
         index: test-index
diff --git a/modules/legacy-geo/build.gradle b/modules/legacy-geo/build.gradle
index 55171221396a3..4ed8d84ab96bc 100644
--- a/modules/legacy-geo/build.gradle
+++ b/modules/legacy-geo/build.gradle
@@ -10,8 +10,8 @@
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'Placeholder plugin for geospatial features in ES'
-  classname 'org.elasticsearch.legacygeo.LegacyGeoPlugin'
+  description = 'Placeholder plugin for geospatial features in ES'
+  classname = 'org.elasticsearch.legacygeo.LegacyGeoPlugin'
 }
 
 dependencies {
diff --git a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/GeoBoundingBoxQueryLegacyGeoShapeIT.java b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/GeoBoundingBoxQueryLegacyGeoShapeIT.java
index 37c31c8af47b0..d2dd5b7442dd2 100644
--- a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/GeoBoundingBoxQueryLegacyGeoShapeIT.java
+++ b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/GeoBoundingBoxQueryLegacyGeoShapeIT.java
@@ -45,6 +45,6 @@ public XContentBuilder getMapping() throws IOException {
 
     @Override
     public IndexVersion randomSupportedVersion() {
-        return IndexVersionUtils.randomCompatibleVersion(random());
+        return IndexVersionUtils.randomCompatibleWriteVersion(random());
     }
 }
diff --git a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java
index 918c343b79b7b..73b7c07c45fe5 100644
--- a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java
+++ b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java
@@ -41,7 +41,7 @@ protected void getGeoShapeMapping(XContentBuilder b) throws IOException {
 
     @Override
     protected IndexVersion randomSupportedVersion() {
-        return IndexVersionUtils.randomCompatibleVersion(random());
+        return IndexVersionUtils.randomCompatibleWriteVersion(random());
     }
 
     @Override
diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java
index 1616d2727bf8a..6127b4beb71ff 100644
--- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java
+++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java
@@ -46,6 +46,7 @@
 import org.elasticsearch.legacygeo.builders.ShapeBuilder;
 import org.elasticsearch.legacygeo.parsers.ShapeParser;
 import org.elasticsearch.legacygeo.query.LegacyGeoShapeQueryProcessor;
+import org.elasticsearch.lucene.spatial.CoordinateEncoder;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.locationtech.spatial4j.shape.Point;
@@ -358,7 +359,7 @@ public LegacyGeoShapeFieldMapper build(MapperBuilderContext context) {
     }
 
     @Deprecated
-    public static Mapper.TypeParser PARSER = (name, node, parserContext) -> {
+    public static final Mapper.TypeParser PARSER = (name, node, parserContext) -> {
         boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(parserContext.getSettings());
         boolean coerceByDefault = COERCE_SETTING.get(parserContext.getSettings());
         FieldMapper.Builder builder = new LegacyGeoShapeFieldMapper.Builder(
@@ -530,6 +531,17 @@ public PrefixTreeStrategy resolvePrefixTreeStrategy(String strategyName) {
         protected Function<List<ShapeBuilder<?, ?, ?>>, List<Object>> getFormatter(String format) {
             return GeometryFormatterFactory.getFormatter(format, ShapeBuilder::buildGeometry);
         }
+
+        @Override
+        protected boolean isBoundsExtractionSupported() {
+            // Extracting bounds for geo shapes is not implemented yet.
+            return false;
+        }
+
+        @Override
+        protected CoordinateEncoder coordinateEncoder() {
+            return CoordinateEncoder.GEO;
+        }
     }
 
     private final IndexVersion indexCreatedVersion;
diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoJsonShapeParserTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoJsonShapeParserTests.java
index 9b83cd9ffdb2b..bd5b289abc588 100644
--- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoJsonShapeParserTests.java
+++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoJsonShapeParserTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.common.geo.GeometryNormalizer;
 import org.elasticsearch.common.geo.GeometryParser;
 import org.elasticsearch.common.geo.Orientation;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.GeometryCollection;
 import org.elasticsearch.geometry.Line;
@@ -344,8 +343,6 @@ public void testParsePolygon() throws IOException, ParseException {
         assertGeometryEquals(p, polygonGeoJson, false);
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_ANALYTICS)
-    @AwaitsFix(bugUrl = "this test is using pre 8.0.0 index versions so needs to be removed or updated")
     public void testParse3DPolygon() throws IOException, ParseException {
         XContentBuilder polygonGeoJson = XContentFactory.jsonBuilder()
             .startObject()
diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoWKTShapeParserTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoWKTShapeParserTests.java
index 5d0df9215ef25..f944a368b2a6c 100644
--- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoWKTShapeParserTests.java
+++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoWKTShapeParserTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeometryNormalizer;
 import org.elasticsearch.common.geo.Orientation;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.Line;
 import org.elasticsearch.geometry.MultiLine;
@@ -303,8 +302,6 @@ public void testParseMixedDimensionPolyWithHole() throws IOException, ParseExcep
         assertThat(e, hasToString(containsString("coordinate dimensions do not match")));
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_ANALYTICS)
-    @AwaitsFix(bugUrl = "this test is using pre 8.0.0 index versions so needs to be removed or updated")
     public void testParseMixedDimensionPolyWithHoleStoredZ() throws IOException {
         List<Coordinate> shellCoordinates = new ArrayList<>();
         shellCoordinates.add(new Coordinate(100, 0));
@@ -338,8 +335,6 @@ public void testParseMixedDimensionPolyWithHoleStoredZ() throws IOException {
         assertThat(e, hasToString(containsString("unable to add coordinate to CoordinateBuilder: coordinate dimensions do not match")));
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_ANALYTICS)
-    @AwaitsFix(bugUrl = "this test is using pre 8.0.0 index versions so needs to be removed or updated")
     public void testParsePolyWithStoredZ() throws IOException {
         List<Coordinate> shellCoordinates = new ArrayList<>();
         shellCoordinates.add(new Coordinate(100, 0, 0));
@@ -363,8 +358,6 @@ public void testParsePolyWithStoredZ() throws IOException {
         assertEquals(shapeBuilder.numDimensions(), 3);
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_ANALYTICS)
-    @AwaitsFix(bugUrl = "this test is using pre 8.0.0 index versions so needs to be removed or updated")
     public void testParseOpenPolygon() throws IOException {
         String openPolygon = "POLYGON ((100 5, 100 10, 90 10, 90 5))";
 
diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java
index 7352b4d88a42b..c97b0a28d22de 100644
--- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java
+++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java
@@ -13,7 +13,6 @@
 import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
 import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
 import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.geo.GeoUtils;
@@ -21,7 +20,6 @@
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.geo.SpatialStrategy;
 import org.elasticsearch.core.CheckedConsumer;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.geometry.Point;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
@@ -56,8 +54,6 @@
 import static org.mockito.Mockito.when;
 
 @SuppressWarnings("deprecation")
-@UpdateForV9(owner = UpdateForV9.Owner.SEARCH_ANALYTICS)
-@AwaitsFix(bugUrl = "this is testing legacy functionality so can likely be removed in 9.0")
 public class LegacyGeoShapeFieldMapperTests extends MapperTestCase {
 
     @Override
diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldTypeTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldTypeTests.java
index bf616c8190324..f5e09f19c1a71 100644
--- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldTypeTests.java
+++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldTypeTests.java
@@ -8,9 +8,7 @@
  */
 package org.elasticsearch.legacygeo.mapper;
 
-import org.apache.lucene.tests.util.LuceneTestCase;
 import org.elasticsearch.common.geo.SpatialStrategy;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
@@ -23,8 +21,6 @@
 import java.util.List;
 import java.util.Map;
 
-@UpdateForV9(owner = UpdateForV9.Owner.SEARCH_ANALYTICS)
-@LuceneTestCase.AwaitsFix(bugUrl = "this is testing legacy functionality so can likely be removed in 9.0")
 public class LegacyGeoShapeFieldTypeTests extends FieldTypeTestCase {
 
     /**
diff --git a/modules/mapper-extras/build.gradle b/modules/mapper-extras/build.gradle
index 992f39a22b28c..4ce00ab873034 100644
--- a/modules/mapper-extras/build.gradle
+++ b/modules/mapper-extras/build.gradle
@@ -12,8 +12,8 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'Adds advanced field mappers'
-  classname 'org.elasticsearch.index.mapper.extras.MapperExtrasPlugin'
+  description = 'Adds advanced field mappers'
+  classname ='org.elasticsearch.index.mapper.extras.MapperExtrasPlugin'
 }
 
 restResources {
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java
index fd0098851c5f8..db2762a028e6a 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java
+++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java
@@ -127,7 +127,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
         }
     }
 
-    public static TypeParser PARSER = new TypeParser((n, c) -> new Builder(n));
+    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n));
 
     private final boolean index;
     private final boolean hasDocValues;
diff --git a/modules/parent-join/build.gradle b/modules/parent-join/build.gradle
index 0d34b5f6e3b40..efe52225121cd 100644
--- a/modules/parent-join/build.gradle
+++ b/modules/parent-join/build.gradle
@@ -11,8 +11,8 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'This module adds the support parent-child queries and aggregations'
-  classname 'org.elasticsearch.join.ParentJoinPlugin'
+  description = 'This module adds the support parent-child queries and aggregations'
+  classname = 'org.elasticsearch.join.ParentJoinPlugin'
 }
 
 restResources {
diff --git a/modules/percolator/build.gradle b/modules/percolator/build.gradle
index 2d2f6767f5e62..3206834521797 100644
--- a/modules/percolator/build.gradle
+++ b/modules/percolator/build.gradle
@@ -11,8 +11,8 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'Percolator module adds capability to index queries and query these queries by specifying documents'
-  classname 'org.elasticsearch.percolator.PercolatorPlugin'
+  description = 'Percolator module adds capability to index queries and query these queries by specifying documents'
+  classname = 'org.elasticsearch.percolator.PercolatorPlugin'
 }
 
 dependencies {
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
index 85af5b120f6fd..c150f01153d35 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
@@ -43,7 +43,6 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.IndexVersion;
@@ -80,7 +79,6 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
-import java.util.function.BiConsumer;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
@@ -88,20 +86,12 @@
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBuilder> {
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(ParseField.class);
-    static final String DOCUMENT_TYPE_DEPRECATION_MESSAGE = "[types removal] Types are deprecated in [percolate] queries. "
-        + "The [document_type] should no longer be specified.";
-    static final String TYPE_DEPRECATION_MESSAGE = "[types removal] Types are deprecated in [percolate] queries. "
-        + "The [type] of the indexed document should no longer be specified.";
-
     public static final String NAME = "percolate";
 
     static final ParseField DOCUMENT_FIELD = new ParseField("document");
     static final ParseField DOCUMENTS_FIELD = new ParseField("documents");
     private static final ParseField NAME_FIELD = new ParseField("name");
     private static final ParseField QUERY_FIELD = new ParseField("field");
-    private static final ParseField DOCUMENT_TYPE_FIELD = new ParseField("document_type");
-    private static final ParseField INDEXED_DOCUMENT_FIELD_TYPE = new ParseField("type");
     private static final ParseField INDEXED_DOCUMENT_FIELD_INDEX = new ParseField("index");
     private static final ParseField INDEXED_DOCUMENT_FIELD_ID = new ParseField("id");
     private static final ParseField INDEXED_DOCUMENT_FIELD_ROUTING = new ParseField("routing");
@@ -368,10 +358,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         );
     }
 
-    private static BiConsumer<PercolateQueryBuilder, String> deprecateAndIgnoreType(String key, String message) {
-        return (target, type) -> deprecationLogger.compatibleCritical(key, message);
-    }
-
     private static BytesReference parseDocument(XContentParser parser) throws IOException {
         try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
             builder.copyCurrentStructure(parser);
diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle
index c9016798c18b9..fe7453750647b 100644
--- a/modules/rank-eval/build.gradle
+++ b/modules/rank-eval/build.gradle
@@ -11,8 +11,8 @@ apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'The Rank Eval module adds APIs to evaluate ranking quality.'
-  classname 'org.elasticsearch.index.rankeval.RankEvalPlugin'
+  description = 'The Rank Eval module adds APIs to evaluate ranking quality.'
+  classname = 'org.elasticsearch.index.rankeval.RankEvalPlugin'
 }
 
 restResources {
diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle
index 7281c161e2c4a..47a3f51115b1d 100644
--- a/modules/reindex/build.gradle
+++ b/modules/reindex/build.gradle
@@ -21,8 +21,8 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'The Reindex module adds APIs to reindex from one index to another or update documents in place.'
-  classname 'org.elasticsearch.reindex.ReindexPlugin'
+  description = 'The Reindex module adds APIs to reindex from one index to another or update documents in place.'
+  classname = 'org.elasticsearch.reindex.ReindexPlugin'
 }
 
 testClusters.configureEach {
@@ -135,7 +135,8 @@ if (OS.current() == OS.WINDOWS) {
     TaskProvider<AntFixture> fixture = tasks.register("oldEs${version}Fixture", AntFixture) {
       dependsOn project.configurations.oldesFixture, jdks.legacy, oldEsDependency
       executable = "${buildParams.runtimeJavaHome.get()}/bin/java"
-      env 'CLASSPATH', "${-> project.configurations.oldesFixture.asPath}"
+      def oldesFixtureConfiguration = project.configurations.oldesFixture
+      env 'CLASSPATH', "${-> oldesFixtureConfiguration.asPath}"
       // old versions of Elasticsearch need JAVA_HOME
       env 'JAVA_HOME', jdks.legacy.javaHomePath
       // If we are running on certain arm systems we need to explicitly set the stack size to overcome JDK page size bug
diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexNodeShutdownIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexNodeShutdownIT.java
index a4b030e3c793f..4f56b6d88b634 100644
--- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexNodeShutdownIT.java
+++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexNodeShutdownIT.java
@@ -59,10 +59,10 @@ public void testReindexWithShutdown() throws Exception {
         final String dataNodeName = internalCluster().startDataOnlyNode();
 
         /* Maximum time to wait for reindexing tasks to complete before shutdown */
-        final Settings COORD_SETTINGS = Settings.builder()
+        final Settings coordSettings = Settings.builder()
             .put(MAXIMUM_REINDEXING_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(60))
             .build();
-        final String coordNodeName = internalCluster().startCoordinatingOnlyNode(Settings.EMPTY);
+        final String coordNodeName = internalCluster().startCoordinatingOnlyNode(coordSettings);
 
         ensureStableCluster(3);
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java
index fe591387e9b35..de90ff97e6a95 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java
@@ -43,7 +43,6 @@
 import org.elasticsearch.script.Metadata;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -212,7 +211,7 @@ static <Request extends AbstractBulkByScrollRequest<Request>> SearchRequest prep
         if (mainRequest.getMaxDocs() != MAX_DOCS_ALL_MATCHES
             && mainRequest.getMaxDocs() <= preparedSearchRequest.source().size()
             && mainRequest.isAbortOnVersionConflict()) {
-            preparedSearchRequest.scroll((Scroll) null);
+            preparedSearchRequest.scroll(null);
         }
 
         return preparedSearchRequest;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteRequestBuilders.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteRequestBuilders.java
index ddf73e313e830..bb2073849edc3 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteRequestBuilders.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteRequestBuilders.java
@@ -53,7 +53,7 @@ static Request initialSearch(SearchRequest searchRequest, BytesReference query,
         Request request = new Request("POST", path.toString());
 
         if (searchRequest.scroll() != null) {
-            TimeValue keepAlive = searchRequest.scroll().keepAlive();
+            TimeValue keepAlive = searchRequest.scroll();
             // V_5_0_0
             if (remoteVersion.before(Version.fromId(5000099))) {
                 /* Versions of Elasticsearch before 5.0 couldn't parse nanos or micros
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSource.java
index 5c3db5aaa6cda..cc08357aa4081 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSource.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSource.java
@@ -100,7 +100,7 @@ private void onStartResponse(RejectAwareActionListener<Response> searchListener,
 
     @Override
     protected void doStartNextScroll(String scrollId, TimeValue extraKeepAlive, RejectAwareActionListener<Response> searchListener) {
-        TimeValue keepAlive = timeValueNanos(searchRequest.scroll().keepAlive().nanos() + extraKeepAlive.nanos());
+        TimeValue keepAlive = timeValueNanos(searchRequest.scroll().nanos() + extraKeepAlive.nanos());
         execute(RemoteRequestBuilders.scroll(scrollId, keepAlive, remoteVersion), RESPONSE_PARSER, searchListener);
     }
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java
index 83e4695829373..28f2eafc20a6e 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java
@@ -199,7 +199,6 @@ public void testStartRetriesOnRejectionButFailsOnTooManyRejections() throws Exce
         DummyAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
         action.start();
         assertBusy(() -> assertEquals(testRequest.getMaxRetries() + 1, client.searchAttempts.get()));
-        assertBusy(() -> assertTrue(listener.isDone()));
         ExecutionException e = expectThrows(ExecutionException.class, () -> listener.get());
         assertThat(ExceptionsHelper.stackTrace(e), containsString(EsRejectedExecutionException.class.getSimpleName()));
         assertNull("There shouldn't be a search attempt pending that we didn't reject", client.lastSearch.get());
@@ -598,7 +597,7 @@ protected RequestWrapper<?> buildRequest(Hit doc) {
             capturedCommand.get().run();
 
             // So the next request is going to have to wait an extra 100 seconds or so (base was 10 seconds, so 110ish)
-            assertThat(client.lastScroll.get().request.scroll().keepAlive().seconds(), either(equalTo(110L)).or(equalTo(109L)));
+            assertThat(client.lastScroll.get().request.scroll().seconds(), either(equalTo(110L)).or(equalTo(109L)));
 
             // Now we can simulate a response and check the delay that we used for the task
             if (randomBoolean()) {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java
index 1c104cbd08197..5f4e2b3a55156 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java
@@ -155,10 +155,7 @@ public void testScrollKeepAlive() {
         );
 
         hitSource.startNextScroll(timeValueSeconds(100));
-        client.validateRequest(
-            TransportSearchScrollAction.TYPE,
-            (SearchScrollRequest r) -> assertEquals(r.scroll().keepAlive().seconds(), 110)
-        );
+        client.validateRequest(TransportSearchScrollAction.TYPE, (SearchScrollRequest r) -> assertEquals(r.scroll().seconds(), 110));
     }
 
     private SearchResponse createSearchResponse() {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexRestClientSslTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexRestClientSslTests.java
index 766c3ff695f84..bcc6177f8363c 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexRestClientSslTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexRestClientSslTests.java
@@ -113,14 +113,20 @@ private static SSLContext buildServerSslContext() throws Exception {
     }
 
     public void testClientFailsWithUntrustedCertificate() throws IOException {
-        assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm());
         final List<Thread> threads = new ArrayList<>();
         final Settings.Builder builder = Settings.builder().put("path.home", createTempDir());
         final Settings settings = builder.build();
         final Environment environment = TestEnvironment.newEnvironment(settings);
         final ReindexSslConfig ssl = new ReindexSslConfig(settings, environment, mock(ResourceWatcherService.class));
         try (RestClient client = Reindexer.buildRestClient(getRemoteInfo(), ssl, 1L, threads)) {
-            expectThrows(SSLHandshakeException.class, () -> client.performRequest(new Request("GET", "/")));
+            if (inFipsJvm()) {
+                // Bouncy Castle throws a different exception
+                IOException exception = expectThrows(IOException.class, () -> client.performRequest(new Request("GET", "/")));
+                assertThat(exception.getCause(), Matchers.instanceOf(javax.net.ssl.SSLException.class));
+            } else {
+                expectThrows(SSLHandshakeException.class, () -> client.performRequest(new Request("GET", "/")));
+
+            }
         }
     }
 
diff --git a/modules/repository-azure/build.gradle b/modules/repository-azure/build.gradle
index 8c1ca3891bc1e..ded138efb4af1 100644
--- a/modules/repository-azure/build.gradle
+++ b/modules/repository-azure/build.gradle
@@ -15,8 +15,8 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'The Azure Repository plugin adds support for Azure storage repositories.'
-  classname 'org.elasticsearch.repositories.azure.AzureRepositoryPlugin'
+  description = 'The Azure Repository plugin adds support for Azure storage repositories.'
+  classname ='org.elasticsearch.repositories.azure.AzureRepositoryPlugin'
 }
 
 versions << [
@@ -56,7 +56,7 @@ dependencies {
   api "io.projectreactor.netty:reactor-netty-core:${versions.azureReactorNetty}"
   api "io.projectreactor.netty:reactor-netty-http:${versions.azureReactorNetty}"
   api "io.projectreactor:reactor-core:3.4.38"
-  api "org.reactivestreams:reactive-streams:1.0.4"
+  api "org.reactivestreams:reactive-streams:${versions.reactive_streams}"
 
   // Others
   api "com.fasterxml.woodstox:woodstox-core:6.7.0"
diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
index 7373ed9485784..4c7d42e6080cd 100644
--- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
+++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
@@ -37,7 +37,7 @@ public class AzureStorageService {
      * The maximum size of a BlockBlob block.
      * See https://docs.microsoft.com/en-us/rest/api/storageservices/understanding-block-blobs--append-blobs--and-page-blobs
      */
-    public static ByteSizeValue MAX_BLOCK_SIZE = new ByteSizeValue(100, ByteSizeUnit.MB);
+    public static final ByteSizeValue MAX_BLOCK_SIZE = new ByteSizeValue(100, ByteSizeUnit.MB);
 
     /**
      * The maximum number of blocks.
diff --git a/modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml b/modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml
index 968e93cf9fc55..175abe183106b 100644
--- a/modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml
+++ b/modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml
@@ -251,11 +251,6 @@ setup:
 
 ---
 "Usage stats":
-  - requires:
-      cluster_features:
-        - repositories.supports_usage_stats
-      reason: requires this feature
-
   - do:
       cluster.stats: {}
 
diff --git a/modules/repository-gcs/build.gradle b/modules/repository-gcs/build.gradle
index 811645d154c7a..d23a0f4a7e44d 100644
--- a/modules/repository-gcs/build.gradle
+++ b/modules/repository-gcs/build.gradle
@@ -17,8 +17,8 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'The GCS repository plugin adds Google Cloud Storage support for repositories.'
-  classname 'org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin'
+  description = 'The GCS repository plugin adds Google Cloud Storage support for repositories.'
+  classname ='org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin'
 }
 
 dependencies {
diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java
index edcf03580da09..3ed492881afa9 100644
--- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java
+++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java
@@ -145,13 +145,11 @@ public void compareAndExchangeRegister(
         BytesReference updated,
         ActionListener<OptionalBytesReference> listener
     ) {
-        if (skipCas(listener)) return;
         ActionListener.completeWith(listener, () -> blobStore.compareAndExchangeRegister(buildKey(key), path, key, expected, updated));
     }
 
     @Override
     public void getRegister(OperationPurpose purpose, String key, ActionListener<OptionalBytesReference> listener) {
-        if (skipCas(listener)) return;
         ActionListener.completeWith(listener, () -> blobStore.getRegister(buildKey(key), path, key));
     }
 }
diff --git a/modules/repository-gcs/src/main/plugin-metadata/entitlement-policy.yaml b/modules/repository-gcs/src/main/plugin-metadata/entitlement-policy.yaml
new file mode 100644
index 0000000000000..a1ff54f02d969
--- /dev/null
+++ b/modules/repository-gcs/src/main/plugin-metadata/entitlement-policy.yaml
@@ -0,0 +1,2 @@
+ALL-UNNAMED:
+  - set_https_connection_properties # required by google-http-client
diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java
index a53ec71f66376..5700fa6de63fa 100644
--- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java
+++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java
@@ -59,9 +59,6 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static fixture.gcs.GoogleCloudStorageHttpHandler.getContentRangeEnd;
-import static fixture.gcs.GoogleCloudStorageHttpHandler.getContentRangeLimit;
-import static fixture.gcs.GoogleCloudStorageHttpHandler.getContentRangeStart;
 import static fixture.gcs.GoogleCloudStorageHttpHandler.parseMultipartRequestBody;
 import static fixture.gcs.TestUtils.createServiceAccount;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -369,14 +366,14 @@ public void testWriteLargeBlob() throws IOException {
 
                     assertThat(Math.toIntExact(requestBody.length()), anyOf(equalTo(defaultChunkSize), equalTo(lastChunkSize)));
 
-                    final int rangeStart = getContentRangeStart(range);
-                    final int rangeEnd = getContentRangeEnd(range);
+                    final HttpHeaderParser.ContentRange contentRange = HttpHeaderParser.parseContentRangeHeader(range);
+                    final int rangeStart = Math.toIntExact(contentRange.start());
+                    final int rangeEnd = Math.toIntExact(contentRange.end());
                     assertThat(rangeEnd + 1 - rangeStart, equalTo(Math.toIntExact(requestBody.length())));
                     assertThat(new BytesArray(data, rangeStart, rangeEnd - rangeStart + 1), is(requestBody));
                     bytesReceived.updateAndGet(existing -> Math.max(existing, rangeEnd));
 
-                    final Integer limit = getContentRangeLimit(range);
-                    if (limit != null) {
+                    if (contentRange.size() != null) {
                         exchange.sendResponseHeaders(RestStatus.OK.getStatus(), -1);
                         return;
                     } else {
diff --git a/modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml b/modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml
index e8c34a4b6a20b..d2370919297a3 100644
--- a/modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml
+++ b/modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml
@@ -234,11 +234,6 @@ setup:
 
 ---
 "Usage stats":
-  - requires:
-      cluster_features:
-        - repositories.supports_usage_stats
-      reason: requires this feature
-
   - do:
       cluster.stats: {}
 
diff --git a/modules/repository-s3/build.gradle b/modules/repository-s3/build.gradle
index f0dc1ca714958..1db83b9e9bc42 100644
--- a/modules/repository-s3/build.gradle
+++ b/modules/repository-s3/build.gradle
@@ -14,8 +14,8 @@ apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-java-rest-test'
 
 esplugin {
-  description 'The S3 repository plugin adds S3 repositories'
-  classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin'
+  description = 'The S3 repository plugin adds S3 repositories'
+  classname ='org.elasticsearch.repositories.s3.S3RepositoryPlugin'
 }
 
 dependencies {
@@ -44,12 +44,14 @@ dependencies {
   internalClusterTestRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}"
 
   yamlRestTestImplementation project(':modules:repository-s3')
+  yamlRestTestImplementation project(':test:fixtures:aws-fixture-utils')
   yamlRestTestImplementation project(':test:fixtures:s3-fixture')
   yamlRestTestImplementation project(':test:fixtures:testcontainer-utils')
   yamlRestTestImplementation project(':test:framework')
   yamlRestTestRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}"
 
   javaRestTestImplementation project(':modules:repository-s3')
+  javaRestTestImplementation project(':test:fixtures:aws-fixture-utils')
   javaRestTestImplementation project(':test:fixtures:aws-sts-fixture')
   javaRestTestImplementation project(':test:fixtures:ec2-imds-fixture')
   javaRestTestImplementation project(':test:fixtures:minio-fixture')
diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3BasicCredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3BasicCredentialsRestIT.java
index 45844703683bb..10901424512a3 100644
--- a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3BasicCredentialsRestIT.java
+++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3BasicCredentialsRestIT.java
@@ -20,6 +20,8 @@
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 
+import static fixture.aws.AwsCredentialsUtils.fixedAccessKey;
+
 @ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
 @ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482
 public class RepositoryS3BasicCredentialsRestIT extends AbstractRepositoryS3RestTestCase {
@@ -31,7 +33,7 @@ public class RepositoryS3BasicCredentialsRestIT extends AbstractRepositoryS3Rest
     private static final String SECRET_KEY = PREFIX + "secret-key";
     private static final String CLIENT = "basic_credentials_client";
 
-    private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, S3HttpFixture.fixedAccessKey(ACCESS_KEY));
+    private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, fixedAccessKey(ACCESS_KEY));
 
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsCredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsCredentialsRestIT.java
index 4f0bf83000642..c525c2ea42f58 100644
--- a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsCredentialsRestIT.java
+++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsCredentialsRestIT.java
@@ -9,10 +9,10 @@
 
 package org.elasticsearch.repositories.s3;
 
+import fixture.aws.DynamicAwsCredentials;
 import fixture.aws.imds.Ec2ImdsHttpFixture;
 import fixture.aws.imds.Ec2ImdsServiceBuilder;
 import fixture.aws.imds.Ec2ImdsVersion;
-import fixture.s3.DynamicS3Credentials;
 import fixture.s3.S3HttpFixture;
 
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
@@ -35,14 +35,14 @@ public class RepositoryS3EcsCredentialsRestIT extends AbstractRepositoryS3RestTe
     private static final String BASE_PATH = PREFIX + "base_path";
     private static final String CLIENT = "ecs_credentials_client";
 
-    private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
+    private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials();
 
     private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
-        new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).newCredentialsConsumer(dynamicS3Credentials::addValidCredentials)
+        new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).newCredentialsConsumer(dynamicCredentials::addValidCredentials)
             .alternativeCredentialsEndpoints(Set.of("/ecs_credentials_endpoint"))
     );
 
-    private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicS3Credentials::isAuthorized);
+    private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicCredentials::isAuthorized);
 
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV1CredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV1CredentialsRestIT.java
index bc41b9fd62ca9..bc689ea52ca32 100644
--- a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV1CredentialsRestIT.java
+++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV1CredentialsRestIT.java
@@ -9,10 +9,10 @@
 
 package org.elasticsearch.repositories.s3;
 
+import fixture.aws.DynamicAwsCredentials;
 import fixture.aws.imds.Ec2ImdsHttpFixture;
 import fixture.aws.imds.Ec2ImdsServiceBuilder;
 import fixture.aws.imds.Ec2ImdsVersion;
-import fixture.s3.DynamicS3Credentials;
 import fixture.s3.S3HttpFixture;
 
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
@@ -33,13 +33,13 @@ public class RepositoryS3ImdsV1CredentialsRestIT extends AbstractRepositoryS3Res
     private static final String BASE_PATH = PREFIX + "base_path";
     private static final String CLIENT = "imdsv1_credentials_client";
 
-    private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
+    private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials();
 
     private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
-        new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).newCredentialsConsumer(dynamicS3Credentials::addValidCredentials)
+        new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).newCredentialsConsumer(dynamicCredentials::addValidCredentials)
     );
 
-    private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicS3Credentials::isAuthorized);
+    private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicCredentials::isAuthorized);
 
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV2CredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV2CredentialsRestIT.java
index 34500ff5227f1..dedf205d3def2 100644
--- a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV2CredentialsRestIT.java
+++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV2CredentialsRestIT.java
@@ -9,10 +9,10 @@
 
 package org.elasticsearch.repositories.s3;
 
+import fixture.aws.DynamicAwsCredentials;
 import fixture.aws.imds.Ec2ImdsHttpFixture;
 import fixture.aws.imds.Ec2ImdsServiceBuilder;
 import fixture.aws.imds.Ec2ImdsVersion;
-import fixture.s3.DynamicS3Credentials;
 import fixture.s3.S3HttpFixture;
 
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
@@ -33,13 +33,13 @@ public class RepositoryS3ImdsV2CredentialsRestIT extends AbstractRepositoryS3Res
     private static final String BASE_PATH = PREFIX + "base_path";
     private static final String CLIENT = "imdsv2_credentials_client";
 
-    private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
+    private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials();
 
     private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
-        new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V2).newCredentialsConsumer(dynamicS3Credentials::addValidCredentials)
+        new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V2).newCredentialsConsumer(dynamicCredentials::addValidCredentials)
     );
 
-    private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicS3Credentials::isAuthorized);
+    private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicCredentials::isAuthorized);
 
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java
index 93915e8491d5b..3d7c8dd150610 100644
--- a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java
+++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java
@@ -19,13 +19,15 @@
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 
+import java.util.Locale;
+
 @ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
 @ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482
 public class RepositoryS3MinioBasicCredentialsRestIT extends AbstractRepositoryS3RestTestCase {
 
-    private static final String PREFIX = getIdentifierPrefix("RepositoryS3MinioBasicCredentialsRestIT");
+    private static final String PREFIX = getIdentifierPrefix("RepositoryS3MinioBasicCredentialsRestIT").toLowerCase(Locale.ROOT);
     private static final String BUCKET = PREFIX + "bucket";
-    private static final String BASE_PATH = PREFIX + "base_path";
+    private static final String BASE_PATH = PREFIX + "base-path";
     private static final String ACCESS_KEY = PREFIX + "access-key";
     private static final String SECRET_KEY = PREFIX + "secret-key";
     private static final String CLIENT = "minio_client";
diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestReloadCredentialsIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestReloadCredentialsIT.java
index 1f09fa6b081b9..065d1c6c9ea27 100644
--- a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestReloadCredentialsIT.java
+++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestReloadCredentialsIT.java
@@ -26,6 +26,7 @@
 
 import java.io.IOException;
 
+import static fixture.aws.AwsCredentialsUtils.mutableAccessKey;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.equalTo;
@@ -38,12 +39,7 @@ public class RepositoryS3RestReloadCredentialsIT extends ESRestTestCase {
 
     private static volatile String repositoryAccessKey;
 
-    public static final S3HttpFixture s3Fixture = new S3HttpFixture(
-        true,
-        BUCKET,
-        BASE_PATH,
-        S3HttpFixture.mutableAccessKey(() -> repositoryAccessKey)
-    );
+    public static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, mutableAccessKey(() -> repositoryAccessKey));
 
     private static final MutableSettingsProvider keystoreSettings = new MutableSettingsProvider();
 
diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3SessionCredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3SessionCredentialsRestIT.java
index 84a327ee131ae..a8009d594926f 100644
--- a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3SessionCredentialsRestIT.java
+++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3SessionCredentialsRestIT.java
@@ -20,6 +20,8 @@
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 
+import static fixture.aws.AwsCredentialsUtils.fixedAccessKeyAndToken;
+
 @ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
 @ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482
 public class RepositoryS3SessionCredentialsRestIT extends AbstractRepositoryS3RestTestCase {
@@ -36,7 +38,7 @@ public class RepositoryS3SessionCredentialsRestIT extends AbstractRepositoryS3Re
         true,
         BUCKET,
         BASE_PATH,
-        S3HttpFixture.fixedAccessKeyAndToken(ACCESS_KEY, SESSION_TOKEN)
+        fixedAccessKeyAndToken(ACCESS_KEY, SESSION_TOKEN)
     );
 
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3StsCredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3StsCredentialsRestIT.java
index de80e4179ef5e..61f0acdb16154 100644
--- a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3StsCredentialsRestIT.java
+++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3StsCredentialsRestIT.java
@@ -9,8 +9,8 @@
 
 package org.elasticsearch.repositories.s3;
 
+import fixture.aws.DynamicAwsCredentials;
 import fixture.aws.sts.AwsStsHttpFixture;
-import fixture.s3.DynamicS3Credentials;
 import fixture.s3.S3HttpFixture;
 
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
@@ -32,9 +32,9 @@ public class RepositoryS3StsCredentialsRestIT extends AbstractRepositoryS3RestTe
     private static final String BASE_PATH = PREFIX + "base_path";
     private static final String CLIENT = "sts_credentials_client";
 
-    private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
+    private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials();
 
-    private static final S3HttpFixture s3HttpFixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicS3Credentials::isAuthorized);
+    private static final S3HttpFixture s3HttpFixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicCredentials::isAuthorized);
 
     private static final String WEB_IDENTITY_TOKEN_FILE_CONTENTS = """
         Atza|IQEBLjAsAhRFiXuWpUXuRvQ9PZL3GMFcYevydwIUFAHZwXZXXXXXXXXJnrulxKDHwy87oGKPznh0D6bEQZTSCzyoCtL_8S07pLpr0zMbn6w1lfVZKNTBdDans\
@@ -42,7 +42,7 @@ public class RepositoryS3StsCredentialsRestIT extends AbstractRepositoryS3RestTe
         zTQxod27L9CqnOLio7N3gZAGpsp6n1-AJBOCJckcyXe2c6uD0srOJeZlKUm2eTDVMf8IehDVI0r1QOnTV6KzzAI3OY87Vd_cVMQ""";
 
     private static final AwsStsHttpFixture stsHttpFixture = new AwsStsHttpFixture(
-        dynamicS3Credentials::addValidCredentials,
+        dynamicCredentials::addValidCredentials,
         WEB_IDENTITY_TOKEN_FILE_CONTENTS
     );
 
diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoriesMetrics.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoriesMetrics.java
index 03106c26c9a29..6f28eda81a9b9 100644
--- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoriesMetrics.java
+++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoriesMetrics.java
@@ -21,7 +21,7 @@ public record S3RepositoriesMetrics(
     LongHistogram retryDeletesHistogram
 ) {
 
-    public static S3RepositoriesMetrics NOOP = new S3RepositoriesMetrics(RepositoriesMetrics.NOOP);
+    public static final S3RepositoriesMetrics NOOP = new S3RepositoriesMetrics(RepositoriesMetrics.NOOP);
 
     public static final String METRIC_RETRY_EVENT_TOTAL = "es.repositories.s3.input_stream.retry.event.total";
     public static final String METRIC_RETRY_SUCCESS_TOTAL = "es.repositories.s3.input_stream.retry.success.total";
diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java
index 1a66f5782fc03..a8a6986ccbb7a 100644
--- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java
+++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java
@@ -434,7 +434,7 @@ public void onFileCreated(Path file) {
                     public void onFileChanged(Path file) {
                         if (file.equals(webIdentityTokenFileSymlink)) {
                             LOGGER.debug("WS web identity token file [{}] changed, updating credentials", file);
-                            credentialsProvider.refresh();
+                            SocketAccess.doPrivilegedVoid(credentialsProvider::refresh);
                         }
                     }
                 });
diff --git a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java
index 3d34934e54945..89919f0f3ddf1 100644
--- a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java
+++ b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java
@@ -23,6 +23,8 @@
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 
+import static fixture.aws.AwsCredentialsUtils.fixedAccessKey;
+
 @ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
 @ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482
 public class RepositoryS3ClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT {
@@ -34,7 +36,7 @@ public class RepositoryS3ClientYamlTestSuiteIT extends AbstractRepositoryS3Clien
         true,
         "bucket",
         "base_path_integration_tests",
-        S3HttpFixture.fixedAccessKey(ACCESS_KEY)
+        fixedAccessKey(ACCESS_KEY)
     );
 
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle
index fd1c5089cdf1c..857a9b00f8985 100644
--- a/modules/repository-url/build.gradle
+++ b/modules/repository-url/build.gradle
@@ -14,8 +14,8 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'Module for URL repository'
-  classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin'
+  description = 'Module for URL repository'
+  classname ='org.elasticsearch.plugin.repository.url.URLRepositoryPlugin'
 }
 
 restResources {
diff --git a/modules/rest-root/build.gradle b/modules/rest-root/build.gradle
index adb8aeb02863f..a387cc4c5dd5e 100644
--- a/modules/rest-root/build.gradle
+++ b/modules/rest-root/build.gradle
@@ -11,8 +11,8 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'Adds HEAD and GET / endpoint to Elasticsearch'
-  classname 'org.elasticsearch.rest.root.MainRestPlugin'
+  description = 'Adds HEAD and GET / endpoint to Elasticsearch'
+  classname ='org.elasticsearch.rest.root.MainRestPlugin'
 }
 
 restResources {
diff --git a/modules/runtime-fields-common/build.gradle b/modules/runtime-fields-common/build.gradle
index e8e06f0a9c4c7..0cbb6a4d43c97 100644
--- a/modules/runtime-fields-common/build.gradle
+++ b/modules/runtime-fields-common/build.gradle
@@ -12,8 +12,8 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.yaml-rest-compat-test'
 
 esplugin {
-  description 'Module for runtime fields features and extensions that have large dependencies'
-  classname 'org.elasticsearch.runtimefields.RuntimeFieldsCommonPlugin'
+  description = 'Module for runtime fields features and extensions that have large dependencies'
+  classname = 'org.elasticsearch.runtimefields.RuntimeFieldsCommonPlugin'
   extendedPlugins = ['lang-painless']
 }
 
diff --git a/modules/systemd/build.gradle b/modules/systemd/build.gradle
index 8eb48e1d5f638..9155554797587 100644
--- a/modules/systemd/build.gradle
+++ b/modules/systemd/build.gradle
@@ -8,8 +8,8 @@
  */
 
 esplugin {
-  description 'Integrates Elasticsearch with systemd'
-  classname 'org.elasticsearch.systemd.SystemdPlugin'
+  description = 'Integrates Elasticsearch with systemd'
+  classname ='org.elasticsearch.systemd.SystemdPlugin'
 }
 
 dependencies {
diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle
index 13dfdf2b3c7bc..4b64b9c56917d 100644
--- a/modules/transport-netty4/build.gradle
+++ b/modules/transport-netty4/build.gradle
@@ -23,8 +23,8 @@ apply plugin: 'elasticsearch.publish'
    * fix the hack in the build framework that copies transport-netty4 into the integ test cluster
  */
 esplugin {
-  description 'Netty 4 based transport implementation'
-  classname 'org.elasticsearch.transport.netty4.Netty4Plugin'
+  description = 'Netty 4 based transport implementation'
+  classname = 'org.elasticsearch.transport.netty4.Netty4Plugin'
 }
 
 // exclude transitively pulled in version via the esplugin plugin to always build from fresh sources and make jar-hell checks pass
diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java
index 4bb27af4bd0f5..ab2fb41d5a22b 100644
--- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java
+++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java
@@ -594,7 +594,7 @@ record Ctx(String testName, String nodeName, Bootstrap clientBootstrap, Channel
         @Override
         public void close() throws Exception {
             safeGet(clientChannel.close());
-            safeGet(clientBootstrap.config().group().shutdownGracefully());
+            safeGet(clientBootstrap.config().group().shutdownGracefully(0, 0, TimeUnit.SECONDS));
             clientRespQueue.forEach(o -> { if (o instanceof FullHttpResponse resp) resp.release(); });
             for (var opaqueId : ControlServerRequestPlugin.handlers.keySet()) {
                 if (opaqueId.startsWith(testName)) {
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java
index 1a391a05add58..4809f1a1a275b 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java
@@ -139,7 +139,8 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) {
                     } else {
                         var contentStream = new Netty4HttpRequestBodyStream(
                             ctx.channel(),
-                            serverTransport.getThreadPool().getThreadContext()
+                            serverTransport.getThreadPool().getThreadContext(),
+                            activityTracker
                         );
                         currentRequestStream = contentStream;
                         netty4HttpRequest = new Netty4HttpRequest(readSequence++, request, contentStream);
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java
index ac3e3aecf97b9..0902e707b706e 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java
@@ -16,6 +16,7 @@
 import io.netty.handler.codec.http.HttpContent;
 import io.netty.handler.codec.http.LastHttpContent;
 
+import org.elasticsearch.common.network.ThreadWatchdog;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.http.HttpBody;
@@ -36,6 +37,7 @@ public class Netty4HttpRequestBodyStream implements HttpBody.Stream {
     private final ChannelFutureListener closeListener = future -> doClose();
     private final List<ChunkHandler> tracingHandlers = new ArrayList<>(4);
     private final ThreadContext threadContext;
+    private final ThreadWatchdog.ActivityTracker activityTracker;
     private ByteBuf buf;
     private boolean requested = false;
     private boolean closing = false;
@@ -46,10 +48,11 @@ public class Netty4HttpRequestBodyStream implements HttpBody.Stream {
     private volatile int bufSize = 0;
     private volatile boolean hasLast = false;
 
-    public Netty4HttpRequestBodyStream(Channel channel, ThreadContext threadContext) {
+    public Netty4HttpRequestBodyStream(Channel channel, ThreadContext threadContext, ThreadWatchdog.ActivityTracker activityTracker) {
         this.channel = channel;
         this.threadContext = threadContext;
         this.requestContext = threadContext.newStoredContext();
+        this.activityTracker = activityTracker;
         Netty4Utils.addListener(channel.closeFuture(), closeListener);
         channel.config().setAutoRead(false);
     }
@@ -76,15 +79,18 @@ public void next() {
         assert handler != null : "handler must be set before requesting next chunk";
         requestContext = threadContext.newStoredContext();
         channel.eventLoop().submit(() -> {
+            activityTracker.startActivity();
             requested = true;
-            if (buf == null) {
-                channel.read();
-            } else {
-                try {
+            try {
+                if (buf == null) {
+                    channel.read();
+                } else {
                     send();
-                } catch (Exception e) {
-                    channel.pipeline().fireExceptionCaught(e);
                 }
+            } catch (Throwable e) {
+                channel.pipeline().fireExceptionCaught(e);
+            } finally {
+                activityTracker.stopActivity();
             }
         });
     }
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Plugin.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Plugin.java
index 449fd72669dad..3feaa2874ebdc 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Plugin.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Plugin.java
@@ -91,7 +91,7 @@ public class Netty4Plugin extends Plugin implements NetworkPlugin {
      */
     private static final ByteSizeValue MTU = ByteSizeValue.ofBytes(Long.parseLong(System.getProperty("es.net.mtu", "1500")));
     private static final String SETTING_KEY_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = "http.netty.max_composite_buffer_components";
-    public static Setting<Integer> SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = new Setting<>(
+    public static final Setting<Integer> SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = new Setting<>(
         SETTING_KEY_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS,
         (s) -> {
             ByteSizeValue maxContentLength = SETTING_HTTP_MAX_CONTENT_LENGTH.get(s);
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java
index d456bbecfbd20..7492737d4f877 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java
@@ -11,6 +11,8 @@
 
 import io.netty.buffer.Unpooled;
 import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.channel.DefaultEventLoop;
 import io.netty.channel.SimpleChannelInboundHandler;
 import io.netty.channel.embedded.EmbeddedChannel;
 import io.netty.handler.codec.http.DefaultHttpContent;
@@ -19,6 +21,7 @@
 import io.netty.handler.flow.FlowControlHandler;
 
 import org.elasticsearch.common.bytes.ReleasableBytesReference;
+import org.elasticsearch.common.network.ThreadWatchdog;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.http.HttpBody;
@@ -27,6 +30,8 @@
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
@@ -35,17 +40,18 @@
 
 public class Netty4HttpRequestBodyStreamTests extends ESTestCase {
 
+    static HttpBody.ChunkHandler discardHandler = (chunk, isLast) -> chunk.close();
     private final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
     private EmbeddedChannel channel;
     private Netty4HttpRequestBodyStream stream;
-    static HttpBody.ChunkHandler discardHandler = (chunk, isLast) -> chunk.close();
+    private ThreadWatchdog.ActivityTracker activityTracker;
 
     @Override
     public void setUp() throws Exception {
         super.setUp();
         channel = new EmbeddedChannel();
-        threadContext.putHeader("header1", "value1");
-        stream = new Netty4HttpRequestBodyStream(channel, threadContext);
+        activityTracker = new ThreadWatchdog.ActivityTracker();
+        stream = new Netty4HttpRequestBodyStream(channel, threadContext, activityTracker);
         stream.setHandler(discardHandler); // set default handler, each test might override one
         channel.pipeline().addLast(new SimpleChannelInboundHandler<HttpContent>(false) {
             @Override
@@ -128,57 +134,112 @@ public void testReadFromChannel() {
     }
 
     public void testReadFromHasCorrectThreadContext() throws InterruptedException {
-        var gotLast = new AtomicBoolean(false);
         AtomicReference<Map<String, String>> headers = new AtomicReference<>();
-        stream.setHandler(new HttpBody.ChunkHandler() {
-            @Override
-            public void onNext(ReleasableBytesReference chunk, boolean isLast) {
-                headers.set(threadContext.getHeaders());
-                gotLast.set(isLast);
-                chunk.close();
-            }
-
-            @Override
-            public void close() {
-                headers.set(threadContext.getHeaders());
-            }
-        });
-        channel.pipeline().addFirst(new FlowControlHandler()); // block all incoming messages, need explicit channel.read()
+        var eventLoop = new DefaultEventLoop();
+        var gotLast = new AtomicBoolean(false);
         var chunkSize = 1024;
+        threadContext.putHeader("header1", "value1");
+        try {
+            // activity tracker requires stream execution in the same thread, setting up stream inside event-loop
+            eventLoop.submit(() -> {
+                channel = new EmbeddedChannel();
+                stream = new Netty4HttpRequestBodyStream(channel, threadContext, new ThreadWatchdog.ActivityTracker());
+                channel.pipeline().addLast(new SimpleChannelInboundHandler<HttpContent>(false) {
+                    @Override
+                    protected void channelRead0(ChannelHandlerContext ctx, HttpContent msg) {
+                        stream.handleNettyContent(msg);
+                    }
+                });
+                stream.setHandler(new HttpBody.ChunkHandler() {
+                    @Override
+                    public void onNext(ReleasableBytesReference chunk, boolean isLast) {
+                        headers.set(threadContext.getHeaders());
+                        gotLast.set(isLast);
+                        chunk.close();
+                    }
+
+                    @Override
+                    public void close() {
+                        headers.set(threadContext.getHeaders());
+                    }
+                });
+                channel.pipeline().addFirst(new FlowControlHandler()); // block all incoming messages, need explicit channel.read()
+            }).await();
 
-        channel.writeInbound(randomContent(chunkSize));
-        channel.writeInbound(randomLastContent(chunkSize));
+            channel.writeInbound(randomContent(chunkSize));
+            channel.writeInbound(randomLastContent(chunkSize));
 
-        threadContext.putHeader("header2", "value2");
-        stream.next();
+            threadContext.putHeader("header2", "value2");
+            stream.next();
 
-        Thread thread = new Thread(() -> channel.runPendingTasks());
-        thread.start();
-        thread.join();
+            eventLoop.submit(() -> channel.runPendingTasks()).await();
+            assertThat(headers.get(), hasEntry("header1", "value1"));
+            assertThat(headers.get(), hasEntry("header2", "value2"));
 
-        assertThat(headers.get(), hasEntry("header1", "value1"));
-        assertThat(headers.get(), hasEntry("header2", "value2"));
+            threadContext.putHeader("header3", "value3");
+            stream.next();
 
-        threadContext.putHeader("header3", "value3");
-        stream.next();
+            eventLoop.submit(() -> channel.runPendingTasks()).await();
+            assertThat(headers.get(), hasEntry("header1", "value1"));
+            assertThat(headers.get(), hasEntry("header2", "value2"));
+            assertThat(headers.get(), hasEntry("header3", "value3"));
 
-        thread = new Thread(() -> channel.runPendingTasks());
-        thread.start();
-        thread.join();
+            assertTrue("should receive last content", gotLast.get());
 
-        assertThat(headers.get(), hasEntry("header1", "value1"));
-        assertThat(headers.get(), hasEntry("header2", "value2"));
-        assertThat(headers.get(), hasEntry("header3", "value3"));
+            headers.set(new HashMap<>());
 
-        assertTrue("should receive last content", gotLast.get());
+            stream.close();
+
+            assertThat(headers.get(), hasEntry("header1", "value1"));
+            assertThat(headers.get(), hasEntry("header2", "value2"));
+            assertThat(headers.get(), hasEntry("header3", "value3"));
+        } finally {
+            eventLoop.shutdownGracefully(0, 0, TimeUnit.SECONDS);
+        }
+    }
 
-        headers.set(new HashMap<>());
+    public void testStreamNextActivityTracker() {
+        var t0 = activityTracker.get();
+        var N = between(1, 10);
+        for (int i = 0; i < N; i++) {
+            channel.writeInbound(randomContent(1024));
+            stream.next();
+            channel.runPendingTasks();
+        }
+        var t1 = activityTracker.get();
+        assertEquals("stream#next() must trigger activity tracker: N*step=" + N + "*2=" + N * 2L + " times", t1, t0 + N * 2L);
+    }
 
-        stream.close();
+    // ensure that we catch all exceptions and throw them into channel pipeline
+    public void testCatchExceptions() {
+        var gotExceptions = new CountDownLatch(3); // number of tests below
+
+        channel.pipeline().addLast(new ChannelInboundHandlerAdapter() {
+            @Override
+            public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
+                gotExceptions.countDown();
+            }
+        });
+
+        // catch exception for not buffered chunk, will be thrown on channel.fireChannelRead()
+        stream.setHandler((a, b) -> { throw new RuntimeException(); });
+        stream.next();
+        channel.runPendingTasks();
+        channel.writeInbound(randomContent(1));
+
+        // catch exception for buffered chunk, will be thrown from eventLoop.submit()
+        channel.writeInbound(randomContent(1));
+        stream.next();
+        channel.runPendingTasks();
+
+        // should catch OOM exceptions too, see DieWithDignity
+        // swallowing exceptions can result in dangling streams, hanging channels, and delayed shutdowns
+        stream.setHandler((a, b) -> { throw new OutOfMemoryError(); });
+        channel.writeInbound(randomContent(1));
+        stream.next();
+        channel.runPendingTasks();
 
-        assertThat(headers.get(), hasEntry("header1", "value1"));
-        assertThat(headers.get(), hasEntry("header2", "value2"));
-        assertThat(headers.get(), hasEntry("header3", "value3"));
+        safeAwait(gotExceptions);
     }
 
     HttpContent randomContent(int size, boolean isLast) {
diff --git a/muted-tests.yml b/muted-tests.yml
index 2689f02cc92cd..9766d3ed35f18 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -40,9 +40,6 @@ tests:
 - class: org.elasticsearch.packaging.test.WindowsServiceTests
   method: test33JavaChanged
   issue: https://github.com/elastic/elasticsearch/issues/113177
-- class: org.elasticsearch.smoketest.MlWithSecurityIT
-  method: test {yaml=ml/sparse_vector_search/Test sparse_vector search with query vector and pruning config}
-  issue: https://github.com/elastic/elasticsearch/issues/108997
 - class: org.elasticsearch.packaging.test.WindowsServiceTests
   method: test80JavaOptsInEnvVar
   issue: https://github.com/elastic/elasticsearch/issues/113219
@@ -55,9 +52,6 @@ tests:
 - class: org.elasticsearch.xpack.transform.integration.TransformIT
   method: testStopWaitForCheckpoint
   issue: https://github.com/elastic/elasticsearch/issues/106113
-- class: org.elasticsearch.kibana.KibanaThreadPoolIT
-  method: testBlockedThreadPoolsRejectUserRequests
-  issue: https://github.com/elastic/elasticsearch/issues/113939
 - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT
   method: testPutE5Small_withPlatformAgnosticVariant
   issue: https://github.com/elastic/elasticsearch/issues/113983
@@ -67,9 +61,6 @@ tests:
 - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT
   method: testPutE5Small_withPlatformSpecificVariant
   issue: https://github.com/elastic/elasticsearch/issues/113950
-- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
-  method: test {yaml=reference/rest-api/usage/line_38}
-  issue: https://github.com/elastic/elasticsearch/issues/113694
 - class: org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityWithApmTracingRestIT
   method: testTracingCrossCluster
   issue: https://github.com/elastic/elasticsearch/issues/112731
@@ -90,12 +81,6 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/115816
 - class: org.elasticsearch.xpack.application.connector.ConnectorIndexServiceTests
   issue: https://github.com/elastic/elasticsearch/issues/116087
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=cat.shards/10_basic/Help}
-  issue: https://github.com/elastic/elasticsearch/issues/116110
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT
-  method: testLookbackWithIndicesOptions
-  issue: https://github.com/elastic/elasticsearch/issues/116127
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_start_stop/Test start already started transform}
   issue: https://github.com/elastic/elasticsearch/issues/98802
@@ -111,9 +96,6 @@ tests:
 - class: org.elasticsearch.xpack.shutdown.NodeShutdownIT
   method: testAllocationPreventedForRemoval
   issue: https://github.com/elastic/elasticsearch/issues/116363
-- class: org.elasticsearch.xpack.downsample.ILMDownsampleDisruptionIT
-  method: testILMDownsampleRollingRestart
-  issue: https://github.com/elastic/elasticsearch/issues/114233
 - class: org.elasticsearch.reservedstate.service.RepositoriesFileSettingsIT
   method: testSettingsApplied
   issue: https://github.com/elastic/elasticsearch/issues/116694
@@ -122,8 +104,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=snapshot/20_operator_privileges_disabled/Operator only settings can be set and restored by non-operator user when operator privileges is disabled}
   issue: https://github.com/elastic/elasticsearch/issues/116775
-- class: org.elasticsearch.xpack.searchablesnapshots.hdfs.SecureHdfsSearchableSnapshotsIT
-  issue: https://github.com/elastic/elasticsearch/issues/116851
 - class: org.elasticsearch.search.basic.SearchWithRandomIOExceptionsIT
   method: testRandomDirectoryIOExceptions
   issue: https://github.com/elastic/elasticsearch/issues/114824
@@ -154,122 +134,42 @@ tests:
 - class: org.elasticsearch.xpack.inference.InferenceRestIT
   method: test {p0=inference/30_semantic_text_inference/Calculates embeddings using the default ELSER 2 endpoint}
   issue: https://github.com/elastic/elasticsearch/issues/117349
+- class: org.elasticsearch.xpack.inference.InferenceRestIT
+  method: test {p0=inference/30_semantic_text_inference_bwc/Calculates embeddings using the default ELSER 2 endpoint}
+  issue: https://github.com/elastic/elasticsearch/issues/117349
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_reset/Test reset running transform}
   issue: https://github.com/elastic/elasticsearch/issues/117473
-- class: org.elasticsearch.repositories.s3.RepositoryS3EcsClientYamlTestSuiteIT
-  issue: https://github.com/elastic/elasticsearch/issues/117525
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set}
-  issue: https://github.com/elastic/elasticsearch/issues/116777
-- class: "org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT"
-  method: "test {scoring.*}"
-  issue: https://github.com/elastic/elasticsearch/issues/117641
-- class: "org.elasticsearch.xpack.esql.qa.single_node.EsqlSpecIT"
-  method: "test {scoring.*}"
-  issue: https://github.com/elastic/elasticsearch/issues/117641
-- class: "org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT"
-  method: "test {scoring.*}"
-  issue: https://github.com/elastic/elasticsearch/issues/117641
-- class: "org.elasticsearch.xpack.esql.qa.mixed.MultiClusterEsqlSpecIT"
-  method: "test {scoring.*}"
-  issue: https://github.com/elastic/elasticsearch/issues/118460
-- class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT"
-  method: "test {scoring.*}"
-  issue: https://github.com/elastic/elasticsearch/issues/118460
-- class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
-  method: test {scoring.QstrWithFieldAndScoringSortedEval}
-  issue: https://github.com/elastic/elasticsearch/issues/117751
 - class: org.elasticsearch.search.ccs.CrossClusterIT
   method: testCancel
   issue: https://github.com/elastic/elasticsearch/issues/108061
 - class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
   method: test {p0=search.highlight/50_synthetic_source/text multi unified from vectors}
   issue: https://github.com/elastic/elasticsearch/issues/117815
-- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT
-  issue: https://github.com/elastic/elasticsearch/issues/111319
-- class: org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests
-  method: test20NoAutoGenerationWhenAutoConfigurationDisabled
-  issue: https://github.com/elastic/elasticsearch/issues/117891
-- class: org.elasticsearch.packaging.test.BootstrapCheckTests
-  method: test20RunWithBootstrapChecks
-  issue: https://github.com/elastic/elasticsearch/issues/117890
 - class: org.elasticsearch.xpack.esql.plugin.ClusterRequestTests
   method: testFallbackIndicesOptions
   issue: https://github.com/elastic/elasticsearch/issues/117937
 - class: org.elasticsearch.xpack.ml.integration.RegressionIT
   method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet
   issue: https://github.com/elastic/elasticsearch/issues/117805
-- class: org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests
-  method: test30NoAutogenerationWhenDaemonized
-  issue: https://github.com/elastic/elasticsearch/issues/117956
-- class: org.elasticsearch.packaging.test.CertGenCliTests
-  method: test40RunWithCert
-  issue: https://github.com/elastic/elasticsearch/issues/117955
-- class: org.elasticsearch.upgrades.QueryBuilderBWCIT
-  method: testQueryBuilderBWC {cluster=UPGRADED}
-  issue: https://github.com/elastic/elasticsearch/issues/116990
-- class: org.elasticsearch.xpack.restart.QueryBuilderBWCIT
-  method: testQueryBuilderBWC {p0=UPGRADED}
-  issue: https://github.com/elastic/elasticsearch/issues/116989
-- class: org.elasticsearch.index.reindex.ReindexNodeShutdownIT
-  method: testReindexWithShutdown
-  issue: https://github.com/elastic/elasticsearch/issues/118040
-- class: org.elasticsearch.packaging.test.ConfigurationTests
-  method: test20HostnameSubstitution
-  issue: https://github.com/elastic/elasticsearch/issues/118028
-- class: org.elasticsearch.packaging.test.ArchiveTests
-  method: test40AutoconfigurationNotTriggeredWhenNodeIsMeantToJoinExistingCluster
-  issue: https://github.com/elastic/elasticsearch/issues/118029
-- class: org.elasticsearch.packaging.test.ConfigurationTests
-  method: test30SymlinkedDataPath
-  issue: https://github.com/elastic/elasticsearch/issues/118111
-- class: org.elasticsearch.packaging.test.KeystoreManagementTests
-  method: test30KeystorePasswordFromFile
-  issue: https://github.com/elastic/elasticsearch/issues/118123
-- class: org.elasticsearch.packaging.test.KeystoreManagementTests
-  method: test31WrongKeystorePasswordFromFile
-  issue: https://github.com/elastic/elasticsearch/issues/118123
-- class: org.elasticsearch.packaging.test.ArchiveTests
-  method: test41AutoconfigurationNotTriggeredWhenNodeCannotContainData
-  issue: https://github.com/elastic/elasticsearch/issues/118110
 - class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS2UnavailableRemotesIT
   method: testEsqlRcs2UnavailableRemoteScenarios
   issue: https://github.com/elastic/elasticsearch/issues/117419
-- class: org.elasticsearch.packaging.test.DebPreservationTests
-  method: test40RestartOnUpgrade
-  issue: https://github.com/elastic/elasticsearch/issues/118170
 - class: org.elasticsearch.xpack.inference.DefaultEndPointsIT
   method: testInferDeploysDefaultRerank
   issue: https://github.com/elastic/elasticsearch/issues/118184
 - class: org.elasticsearch.xpack.esql.action.EsqlActionTaskIT
   method: testCancelRequestWhenFailingFetchingPages
   issue: https://github.com/elastic/elasticsearch/issues/118193
-- class: org.elasticsearch.packaging.test.MemoryLockingTests
-  method: test20MemoryLockingEnabled
-  issue: https://github.com/elastic/elasticsearch/issues/118195
-- class: org.elasticsearch.packaging.test.ArchiveTests
-  method: test42AutoconfigurationNotTriggeredWhenNodeCannotBecomeMaster
-  issue: https://github.com/elastic/elasticsearch/issues/118196
-- class: org.elasticsearch.packaging.test.ArchiveTests
-  method: test43AutoconfigurationNotTriggeredWhenTlsAlreadyConfigured
-  issue: https://github.com/elastic/elasticsearch/issues/118202
 - class: org.elasticsearch.packaging.test.ArchiveTests
   method: test44AutoConfigurationNotTriggeredOnNotWriteableConfDir
   issue: https://github.com/elastic/elasticsearch/issues/118208
 - class: org.elasticsearch.packaging.test.ArchiveTests
   method: test51AutoConfigurationWithPasswordProtectedKeystore
   issue: https://github.com/elastic/elasticsearch/issues/118212
-- class: org.elasticsearch.ingest.common.IngestCommonClientYamlTestSuiteIT
-  issue: https://github.com/elastic/elasticsearch/issues/118215
 - class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
   method: test {p0=data_stream/120_data_streams_stats/Multiple data stream}
   issue: https://github.com/elastic/elasticsearch/issues/118217
-- class: org.elasticsearch.validation.DotPrefixClientYamlTestSuiteIT
-  issue: https://github.com/elastic/elasticsearch/issues/118224
-- class: org.elasticsearch.packaging.test.ArchiveTests
-  method: test60StartAndStop
-  issue: https://github.com/elastic/elasticsearch/issues/118216
 - class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests
   method: testBottomFieldSort
   issue: https://github.com/elastic/elasticsearch/issues/118214
@@ -279,29 +179,90 @@ tests:
 - class: org.elasticsearch.xpack.searchablesnapshots.RetrySearchIntegTests
   method: testSearcherId
   issue: https://github.com/elastic/elasticsearch/issues/118374
-- class: org.elasticsearch.docker.test.DockerYmlTestSuiteIT
-  method: test {p0=/10_info/Info}
-  issue: https://github.com/elastic/elasticsearch/issues/118394
-- class: org.elasticsearch.docker.test.DockerYmlTestSuiteIT
-  method: test {p0=/11_nodes/Additional disk information}
-  issue: https://github.com/elastic/elasticsearch/issues/118395
-- class: org.elasticsearch.docker.test.DockerYmlTestSuiteIT
-  method: test {p0=/11_nodes/Test cat nodes output with full_id set}
-  issue: https://github.com/elastic/elasticsearch/issues/118396
-- class: org.elasticsearch.docker.test.DockerYmlTestSuiteIT
-  method: test {p0=/11_nodes/Test cat nodes output}
-  issue: https://github.com/elastic/elasticsearch/issues/118397
-- class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT
-  method: testEveryActionIsEitherOperatorOnlyOrNonOperator
-  issue: https://github.com/elastic/elasticsearch/issues/118220
 - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT
   issue: https://github.com/elastic/elasticsearch/issues/118238
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test011SecurityEnabledStatus
-  issue: https://github.com/elastic/elasticsearch/issues/118517
 - class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
   method: testInvalidJSON
   issue: https://github.com/elastic/elasticsearch/issues/116521
+- class: org.elasticsearch.xpack.ccr.rest.ShardChangesRestIT
+  method: testShardChangesNoOperation
+  issue: https://github.com/elastic/elasticsearch/issues/118800
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/indices/shard-stores/line_150}
+  issue: https://github.com/elastic/elasticsearch/issues/118896
+- class: org.elasticsearch.cluster.service.MasterServiceTests
+  method: testThreadContext
+  issue: https://github.com/elastic/elasticsearch/issues/118914
+- class: org.elasticsearch.xpack.security.authc.AuthenticationServiceTests
+  method: testInvalidToken
+  issue: https://github.com/elastic/elasticsearch/issues/119019
+- class: org.elasticsearch.xpack.security.authc.ldap.ActiveDirectoryRunAsIT
+  issue: https://github.com/elastic/elasticsearch/issues/115727
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/search/search-your-data/retrievers-examples/line_98}
+  issue: https://github.com/elastic/elasticsearch/issues/119155
+- class: org.elasticsearch.xpack.esql.action.EsqlNodeFailureIT
+  method: testFailureLoadingFields
+  issue: https://github.com/elastic/elasticsearch/issues/118000
+- class: org.elasticsearch.index.mapper.AbstractShapeGeometryFieldMapperTests
+  method: testCartesianBoundsBlockLoader
+  issue: https://github.com/elastic/elasticsearch/issues/119201
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=transform/transforms_start_stop/Test start/stop/start transform}
+  issue: https://github.com/elastic/elasticsearch/issues/119508
+- class: org.elasticsearch.smoketest.MlWithSecurityIT
+  method: test {yaml=ml/sparse_vector_search/Test sparse_vector search with query vector and pruning config}
+  issue: https://github.com/elastic/elasticsearch/issues/119548
+- class: org.elasticsearch.index.engine.LuceneSyntheticSourceChangesSnapshotTests
+  method: testSkipNonRootOfNestedDocuments
+  issue: https://github.com/elastic/elasticsearch/issues/119553
+- class: org.elasticsearch.xpack.ml.integration.ForecastIT
+  method: testOverflowToDisk
+  issue: https://github.com/elastic/elasticsearch/issues/117740
+- class: org.elasticsearch.xpack.security.authc.ldap.MultiGroupMappingIT
+  issue: https://github.com/elastic/elasticsearch/issues/119599
+- class: org.elasticsearch.search.profile.dfs.DfsProfilerIT
+  method: testProfileDfs
+  issue: https://github.com/elastic/elasticsearch/issues/119711
+- class: org.elasticsearch.xpack.inference.InferenceCrudIT
+  method: testGetServicesWithCompletionTaskType
+  issue: https://github.com/elastic/elasticsearch/issues/119959
+- class: org.elasticsearch.multi_cluster.MultiClusterYamlTestSuiteIT
+  issue: https://github.com/elastic/elasticsearch/issues/119983
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=transform/transforms_unattended/Test unattended put and start}
+  issue: https://github.com/elastic/elasticsearch/issues/120019
+- class: org.elasticsearch.index.mapper.IntervalThrottlerTests
+  method: testThrottling
+  issue: https://github.com/elastic/elasticsearch/issues/120023
+- class: org.elasticsearch.entitlement.qa.EntitlementsDeniedIT
+  method: testCheckThrows {pathPrefix=denied actionName=sslSessionImpl_getSessionContext}
+  issue: https://github.com/elastic/elasticsearch/issues/120053
+- class: org.elasticsearch.entitlement.qa.EntitlementsDeniedIT
+  method: testCheckThrows {pathPrefix=denied_nonmodular actionName=sslSessionImpl_getSessionContext}
+  issue: https://github.com/elastic/elasticsearch/issues/120054
+- class: org.elasticsearch.xpack.ilm.actions.SearchableSnapshotActionIT
+  method: testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped
+  issue: https://github.com/elastic/elasticsearch/issues/118406
+- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT
+  issue: https://github.com/elastic/elasticsearch/issues/120088
+- class: org.elasticsearch.xpack.searchablesnapshots.minio.MinioSearchableSnapshotsIT
+  issue: https://github.com/elastic/elasticsearch/issues/120101
+- class: org.elasticsearch.repositories.s3.S3RepositoryThirdPartyTests
+  issue: https://github.com/elastic/elasticsearch/issues/120115
+- class: org.elasticsearch.repositories.s3.RepositoryS3MinioBasicCredentialsRestIT
+  issue: https://github.com/elastic/elasticsearch/issues/120117
+- class: org.elasticsearch.repositories.blobstore.testkit.analyze.MinioRepositoryAnalysisRestIT
+  issue: https://github.com/elastic/elasticsearch/issues/118548
+- class: org.elasticsearch.xpack.security.QueryableReservedRolesIT
+  method: testConfiguredReservedRolesAfterClosingAndOpeningIndex
+  issue: https://github.com/elastic/elasticsearch/issues/120127
+- class: org.elasticsearch.oldrepos.OldRepositoryAccessIT
+  method: testOldRepoAccess
+  issue: https://github.com/elastic/elasticsearch/issues/120148
+- class: org.elasticsearch.oldrepos.OldRepositoryAccessIT
+  method: testOldSourceOnlyRepoAccess
+  issue: https://github.com/elastic/elasticsearch/issues/120080
 
 # Examples:
 #
diff --git a/plugins/analysis-icu/build.gradle b/plugins/analysis-icu/build.gradle
index 05cd2cb44124c..0d576d316f855 100644
--- a/plugins/analysis-icu/build.gradle
+++ b/plugins/analysis-icu/build.gradle
@@ -11,8 +11,8 @@ apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'The ICU Analysis plugin integrates the Lucene ICU module into Elasticsearch, adding ICU-related analysis components.'
-  classname 'org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin'
+  description = 'The ICU Analysis plugin integrates the Lucene ICU module into Elasticsearch, adding ICU-related analysis components.'
+  classname ='org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin'
 }
 
 tasks.named("forbiddenApisMain").configure {
diff --git a/plugins/analysis-kuromoji/build.gradle b/plugins/analysis-kuromoji/build.gradle
index bb59fed0db2ed..82fa59e5773c3 100644
--- a/plugins/analysis-kuromoji/build.gradle
+++ b/plugins/analysis-kuromoji/build.gradle
@@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
-  description 'The Japanese (kuromoji) Analysis plugin integrates Lucene kuromoji analysis module into elasticsearch.'
-  classname 'org.elasticsearch.plugin.analysis.kuromoji.AnalysisKuromojiPlugin'
+  description = 'The Japanese (kuromoji) Analysis plugin integrates Lucene kuromoji analysis module into elasticsearch.'
+  classname ='org.elasticsearch.plugin.analysis.kuromoji.AnalysisKuromojiPlugin'
 }
 
 dependencies {
diff --git a/plugins/analysis-nori/build.gradle b/plugins/analysis-nori/build.gradle
index 0e602886d29ce..6254a56f0657f 100644
--- a/plugins/analysis-nori/build.gradle
+++ b/plugins/analysis-nori/build.gradle
@@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
-  description 'The Korean (nori) Analysis plugin integrates Lucene nori analysis module into elasticsearch.'
-  classname 'org.elasticsearch.plugin.analysis.nori.AnalysisNoriPlugin'
+  description = 'The Korean (nori) Analysis plugin integrates Lucene nori analysis module into elasticsearch.'
+  classname ='org.elasticsearch.plugin.analysis.nori.AnalysisNoriPlugin'
 }
 
 dependencies {
diff --git a/plugins/analysis-phonetic/build.gradle b/plugins/analysis-phonetic/build.gradle
index e34a179a384ff..018e2c0e52b8d 100644
--- a/plugins/analysis-phonetic/build.gradle
+++ b/plugins/analysis-phonetic/build.gradle
@@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
-  description 'The Phonetic Analysis plugin integrates phonetic token filter analysis with elasticsearch.'
-  classname 'org.elasticsearch.plugin.analysis.phonetic.AnalysisPhoneticPlugin'
+  description = 'The Phonetic Analysis plugin integrates phonetic token filter analysis with elasticsearch.'
+  classname ='org.elasticsearch.plugin.analysis.phonetic.AnalysisPhoneticPlugin'
 }
 
 dependencies {
diff --git a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/plugin/analysis/phonetic/AnalysisPhoneticFactoryTests.java b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/plugin/analysis/phonetic/AnalysisPhoneticFactoryTests.java
index 483c8ccef1202..e51d1f24a88ad 100644
--- a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/plugin/analysis/phonetic/AnalysisPhoneticFactoryTests.java
+++ b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/plugin/analysis/phonetic/AnalysisPhoneticFactoryTests.java
@@ -44,7 +44,7 @@ public void testDisallowedWithSynonyms() throws IOException {
         Settings settings = Settings.builder()
             .put(
                 IndexMetadata.SETTING_VERSION_CREATED,
-                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current())
+                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersion.current())
             )
             .put("path.home", createTempDir().toString())
             .build();
diff --git a/plugins/analysis-smartcn/build.gradle b/plugins/analysis-smartcn/build.gradle
index cbc45f32bf596..b4ac03935aab5 100644
--- a/plugins/analysis-smartcn/build.gradle
+++ b/plugins/analysis-smartcn/build.gradle
@@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
-  description 'Smart Chinese Analysis plugin integrates Lucene Smart Chinese analysis module into elasticsearch.'
-  classname 'org.elasticsearch.plugin.analysis.smartcn.AnalysisSmartChinesePlugin'
+  description = 'Smart Chinese Analysis plugin integrates Lucene Smart Chinese analysis module into elasticsearch.'
+  classname ='org.elasticsearch.plugin.analysis.smartcn.AnalysisSmartChinesePlugin'
 }
 
 dependencies {
diff --git a/plugins/analysis-stempel/build.gradle b/plugins/analysis-stempel/build.gradle
index 2d2c187970d81..0fb15ec7d36d1 100644
--- a/plugins/analysis-stempel/build.gradle
+++ b/plugins/analysis-stempel/build.gradle
@@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
-  description 'The Stempel (Polish) Analysis plugin integrates Lucene stempel (polish) analysis module into elasticsearch.'
-  classname 'org.elasticsearch.plugin.analysis.stempel.AnalysisStempelPlugin'
+  description = 'The Stempel (Polish) Analysis plugin integrates Lucene stempel (polish) analysis module into elasticsearch.'
+  classname ='org.elasticsearch.plugin.analysis.stempel.AnalysisStempelPlugin'
 }
 
 dependencies {
diff --git a/plugins/analysis-ukrainian/build.gradle b/plugins/analysis-ukrainian/build.gradle
index 2794e5bcfe338..2be48240a8875 100644
--- a/plugins/analysis-ukrainian/build.gradle
+++ b/plugins/analysis-ukrainian/build.gradle
@@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
-  description 'The Ukrainian Analysis plugin integrates the Lucene UkrainianMorfologikAnalyzer into elasticsearch.'
-  classname 'org.elasticsearch.plugin.analysis.ukrainian.AnalysisUkrainianPlugin'
+  description = 'The Ukrainian Analysis plugin integrates the Lucene UkrainianMorfologikAnalyzer into elasticsearch.'
+  classname ='org.elasticsearch.plugin.analysis.ukrainian.AnalysisUkrainianPlugin'
 }
 
 dependencies {
diff --git a/plugins/build.gradle b/plugins/build.gradle
index 60c65885639ed..32fd646ef0be8 100644
--- a/plugins/build.gradle
+++ b/plugins/build.gradle
@@ -22,10 +22,10 @@ configure(subprojects.findAll { it.parent.path == project.path }) {
 
   esplugin {
     // for local ES plugins, the name of the plugin is the same as the directory
-    name project.name
+    name = project.name
 
-    licenseFile rootProject.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
-    noticeFile rootProject.file('NOTICE.txt')
+    licenseFile = rootProject.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
+    noticeFile = rootProject.file('NOTICE.txt')
   }
 
 
diff --git a/plugins/discovery-azure-classic/build.gradle b/plugins/discovery-azure-classic/build.gradle
index 9549236775bfe..4455e798e3a22 100644
--- a/plugins/discovery-azure-classic/build.gradle
+++ b/plugins/discovery-azure-classic/build.gradle
@@ -13,8 +13,8 @@ apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'The Azure Classic Discovery plugin allows to use Azure Classic API for the unicast discovery mechanism'
-  classname 'org.elasticsearch.plugin.discovery.azure.classic.AzureDiscoveryPlugin'
+  description = 'The Azure Classic Discovery plugin allows to use Azure Classic API for the unicast discovery mechanism'
+  classname ='org.elasticsearch.plugin.discovery.azure.classic.AzureDiscoveryPlugin'
 }
 
 def localVersions = versions + [
diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle
index 95715217fa59a..a8255c1b54517 100644
--- a/plugins/discovery-ec2/build.gradle
+++ b/plugins/discovery-ec2/build.gradle
@@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism.'
-  classname 'org.elasticsearch.discovery.ec2.Ec2DiscoveryPlugin'
+  description = 'The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism.'
+  classname ='org.elasticsearch.discovery.ec2.Ec2DiscoveryPlugin'
 }
 
 dependencies {
diff --git a/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java
index 73213090b6f93..af3b3d67951dd 100644
--- a/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java
+++ b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java
@@ -9,30 +9,24 @@
 
 package org.elasticsearch.discovery.ec2;
 
-import com.amazonaws.util.EC2MetadataUtils;
-
 import org.elasticsearch.client.Request;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
-import org.elasticsearch.test.rest.ESRestTestCase;
 import org.junit.ClassRule;
 
 import java.io.IOException;
 
-public class DiscoveryEc2AvailabilityZoneAttributeNoImdsIT extends ESRestTestCase {
+public class DiscoveryEc2AvailabilityZoneAttributeNoImdsIT extends DiscoveryEc2AvailabilityZoneAttributeTestCase {
     @ClassRule
-    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
-        .plugin("discovery-ec2")
-        .setting(AwsEc2Service.AUTO_ATTRIBUTE_SETTING.getKey(), "true")
-        .build();
+    // use an address which definitely isn't running an IMDS, just in case we're running these tests in EC2
+    public static ElasticsearchCluster cluster = DiscoveryEc2AvailabilityZoneAttributeTestCase.buildCluster(() -> "http://127.0.0.1:1");
 
     @Override
     protected String getTestRestCluster() {
         return cluster.getHttpAddresses();
     }
 
+    @Override // the base class asserts that the attribute is set, but we don't want that here
     public void testAvailabilityZoneAttribute() throws IOException {
-        assumeTrue("test only in non-AWS environment", EC2MetadataUtils.getInstanceId() == null);
-
         final var nodesInfoResponse = assertOKAndCreateObjectPath(client().performRequest(new Request("GET", "/_nodes/_all/_none")));
         for (final var nodeId : nodesInfoResponse.evaluateMapKeys("nodes")) {
             assertNull(nodesInfoResponse.evaluateExact("nodes", nodeId, "attributes", "aws_availability_zone"));
diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle
index 530070f9e0073..c6beaf3f332ca 100644
--- a/plugins/discovery-gce/build.gradle
+++ b/plugins/discovery-gce/build.gradle
@@ -2,8 +2,8 @@ apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'The Google Compute Engine (GCE) Discovery plugin allows to use GCE API for the unicast discovery mechanism.'
-  classname 'org.elasticsearch.plugin.discovery.gce.GceDiscoveryPlugin'
+  description = 'The Google Compute Engine (GCE) Discovery plugin allows to use GCE API for the unicast discovery mechanism.'
+  classname ='org.elasticsearch.plugin.discovery.gce.GceDiscoveryPlugin'
 }
 
 versions << [
diff --git a/plugins/discovery-gce/qa/gce/build.gradle b/plugins/discovery-gce/qa/gce/build.gradle
index 72cb429b49072..c9d87cd6beb1e 100644
--- a/plugins/discovery-gce/qa/gce/build.gradle
+++ b/plugins/discovery-gce/qa/gce/build.gradle
@@ -28,8 +28,9 @@ restResources {
 
 /** A task to start the GCEFixture which emulates a GCE service **/
 def gceFixtureProvider = tasks.register("gceFixture", AntFixture) {
-  dependsOn project.sourceSets.yamlRestTest.runtimeClasspath
-  env 'CLASSPATH', "${-> project.sourceSets.yamlRestTest.runtimeClasspath.asPath}"
+  def runtimeClasspath = project.sourceSets.yamlRestTest.runtimeClasspath
+  dependsOn runtimeClasspath
+  env 'CLASSPATH', "${-> runtimeClasspath.asPath}"
   executable = "${buildParams.runtimeJavaHome.get()}/bin/java"
   args 'org.elasticsearch.cloud.gce.GCEFixture', baseDir, "${buildDir}/testclusters/yamlRestTest-1/config/unicast_hosts.txt"
 }
diff --git a/plugins/discovery-gce/src/main/plugin-metadata/entitlement-policy.yaml b/plugins/discovery-gce/src/main/plugin-metadata/entitlement-policy.yaml
new file mode 100644
index 0000000000000..a1ff54f02d969
--- /dev/null
+++ b/plugins/discovery-gce/src/main/plugin-metadata/entitlement-policy.yaml
@@ -0,0 +1,2 @@
+ALL-UNNAMED:
+  - set_https_connection_properties # required by google-http-client
diff --git a/plugins/examples/custom-processor/build.gradle b/plugins/examples/custom-processor/build.gradle
index 1aa15e2ed5142..6c0281d899a4e 100644
--- a/plugins/examples/custom-processor/build.gradle
+++ b/plugins/examples/custom-processor/build.gradle
@@ -10,11 +10,11 @@ apply plugin: 'elasticsearch.esplugin'
 apply plugin: 'elasticsearch.yaml-rest-test'
 
 esplugin {
-  name 'custom-processor'
-  description 'An example plugin showing how to register a custom ingest processor'
-  classname 'org.elasticsearch.example.customprocessor.ExampleProcessorPlugin'
-  licenseFile rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
-  noticeFile rootProject.file('NOTICE.txt')
+  name = 'custom-processor'
+  description = 'An example plugin showing how to register a custom ingest processor'
+  classname ='org.elasticsearch.example.customprocessor.ExampleProcessorPlugin'
+  licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
+  noticeFile = rootProject.file('NOTICE.txt')
 }
 
 dependencies {
diff --git a/plugins/examples/custom-settings/build.gradle b/plugins/examples/custom-settings/build.gradle
index 4fc1f25e0d636..2774bf6e75c78 100644
--- a/plugins/examples/custom-settings/build.gradle
+++ b/plugins/examples/custom-settings/build.gradle
@@ -10,11 +10,11 @@ apply plugin: 'elasticsearch.esplugin'
 apply plugin: 'elasticsearch.yaml-rest-test'
 
 esplugin {
-  name 'custom-settings'
-  description 'An example plugin showing how to register custom settings'
-  classname 'org.elasticsearch.example.customsettings.ExampleCustomSettingsPlugin'
-  licenseFile rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
-  noticeFile rootProject.file('NOTICE.txt')
+  name = 'custom-settings'
+  description = 'An example plugin showing how to register custom settings'
+  classname ='org.elasticsearch.example.customsettings.ExampleCustomSettingsPlugin'
+  licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
+  noticeFile = rootProject.file('NOTICE.txt')
 }
 
 testClusters.configureEach {
diff --git a/plugins/examples/custom-significance-heuristic/build.gradle b/plugins/examples/custom-significance-heuristic/build.gradle
index 6491e9384d55e..f2f0cefa6d6f5 100644
--- a/plugins/examples/custom-significance-heuristic/build.gradle
+++ b/plugins/examples/custom-significance-heuristic/build.gradle
@@ -10,11 +10,11 @@ apply plugin: 'elasticsearch.esplugin'
 apply plugin: 'elasticsearch.yaml-rest-test'
 
 esplugin {
-  name 'custom-significance-heuristic'
-  description 'An example plugin showing how to write and register a custom significance heuristic'
-  classname 'org.elasticsearch.example.customsigheuristic.CustomSignificanceHeuristicPlugin'
-  licenseFile rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
-  noticeFile rootProject.file('NOTICE.txt')
+  name = 'custom-significance-heuristic'
+  description = 'An example plugin showing how to write and register a custom significance heuristic'
+  classname ='org.elasticsearch.example.customsigheuristic.CustomSignificanceHeuristicPlugin'
+  licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
+  noticeFile = rootProject.file('NOTICE.txt')
 }
 
 dependencies {
diff --git a/plugins/examples/custom-suggester/build.gradle b/plugins/examples/custom-suggester/build.gradle
index 6c9210f74327f..a1cf345f5e819 100644
--- a/plugins/examples/custom-suggester/build.gradle
+++ b/plugins/examples/custom-suggester/build.gradle
@@ -10,11 +10,11 @@ apply plugin: 'elasticsearch.esplugin'
 apply plugin: 'elasticsearch.yaml-rest-test'
 
 esplugin {
-  name 'custom-suggester'
-  description 'An example plugin showing how to write and register a custom suggester'
-  classname 'org.elasticsearch.example.customsuggester.CustomSuggesterPlugin'
-  licenseFile rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
-  noticeFile rootProject.file('NOTICE.txt')
+  name = 'custom-suggester'
+  description = 'An example plugin showing how to write and register a custom suggester'
+  classname ='org.elasticsearch.example.customsuggester.CustomSuggesterPlugin'
+  licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
+  noticeFile = rootProject.file('NOTICE.txt')
 }
 
 testClusters.configureEach {
diff --git a/plugins/examples/gradle/wrapper/gradle-wrapper.properties b/plugins/examples/gradle/wrapper/gradle-wrapper.properties
index 22286c90de3d1..e712035eabc7b 100644
--- a/plugins/examples/gradle/wrapper/gradle-wrapper.properties
+++ b/plugins/examples/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=89d4e70e4e84e2d2dfbb63e4daa53e21b25017cc70c37e4eea31ee51fb15098a
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-all.zip
+distributionSha256Sum=7ebdac923867a3cec0098302416d1e3c6c0c729fc4e2e05c10637a8af33a76c5
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
diff --git a/plugins/examples/painless-whitelist/build.gradle b/plugins/examples/painless-whitelist/build.gradle
index f6bbe0fc2b439..f501bd466ebe5 100644
--- a/plugins/examples/painless-whitelist/build.gradle
+++ b/plugins/examples/painless-whitelist/build.gradle
@@ -10,12 +10,12 @@ apply plugin: 'elasticsearch.esplugin'
 apply plugin: 'elasticsearch.yaml-rest-test'
 
 esplugin {
-  name 'painless-whitelist'
-  description 'An example whitelisting additional classes and methods in painless'
-  classname 'org.elasticsearch.example.painlesswhitelist.MyWhitelistPlugin'
+  name = 'painless-whitelist'
+  description = 'An example whitelisting additional classes and methods in painless'
+  classname ='org.elasticsearch.example.painlesswhitelist.MyWhitelistPlugin'
   extendedPlugins = ['lang-painless']
-  licenseFile rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
-  noticeFile rootProject.file('NOTICE.txt')
+  licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
+  noticeFile = rootProject.file('NOTICE.txt')
 }
 
 dependencies {
diff --git a/plugins/examples/rescore/build.gradle b/plugins/examples/rescore/build.gradle
index 93139b9b54d98..023033349dd8c 100644
--- a/plugins/examples/rescore/build.gradle
+++ b/plugins/examples/rescore/build.gradle
@@ -10,11 +10,11 @@ apply plugin: 'elasticsearch.esplugin'
 apply plugin: 'elasticsearch.yaml-rest-test'
 
 esplugin {
-  name 'example-rescore'
-  description 'An example plugin implementing rescore and verifying that plugins *can* implement rescore'
-  classname 'org.elasticsearch.example.rescore.ExampleRescorePlugin'
-  licenseFile rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
-  noticeFile rootProject.file('NOTICE.txt')
+  name = 'example-rescore'
+  description = 'An example plugin implementing rescore and verifying that plugins *can* implement rescore'
+  classname ='org.elasticsearch.example.rescore.ExampleRescorePlugin'
+  licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
+  noticeFile = rootProject.file('NOTICE.txt')
 }
 
 dependencies {
diff --git a/plugins/examples/rest-handler/build.gradle b/plugins/examples/rest-handler/build.gradle
index b53ef6be39d53..43590b166a545 100644
--- a/plugins/examples/rest-handler/build.gradle
+++ b/plugins/examples/rest-handler/build.gradle
@@ -10,11 +10,11 @@ apply plugin: 'elasticsearch.esplugin'
 apply plugin: 'elasticsearch.yaml-rest-test'
 
 esplugin {
-  name 'rest-handler'
-  description 'An example plugin showing how to register a REST handler'
-  classname 'org.elasticsearch.example.resthandler.ExampleRestHandlerPlugin'
-  licenseFile rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
-  noticeFile rootProject.file('NOTICE.txt')
+  name = 'rest-handler'
+  description = 'An example plugin showing how to register a REST handler'
+  classname ='org.elasticsearch.example.resthandler.ExampleRestHandlerPlugin'
+  licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
+  noticeFile = rootProject.file('NOTICE.txt')
 }
 
 dependencies {
diff --git a/plugins/examples/script-expert-scoring/build.gradle b/plugins/examples/script-expert-scoring/build.gradle
index 8947938e227a2..0fb1baaea2f03 100644
--- a/plugins/examples/script-expert-scoring/build.gradle
+++ b/plugins/examples/script-expert-scoring/build.gradle
@@ -10,11 +10,11 @@ apply plugin: 'elasticsearch.esplugin'
 apply plugin: 'elasticsearch.yaml-rest-test'
 
 esplugin {
-  name 'script-expert-scoring'
-  description 'An example script engine to use low level Lucene internals for expert scoring'
-  classname 'org.elasticsearch.example.expertscript.ExpertScriptPlugin'
-  licenseFile rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
-  noticeFile rootProject.file('NOTICE.txt')
+  name = 'script-expert-scoring'
+  description = 'An example script engine to use low level Lucene internals for expert scoring'
+  classname ='org.elasticsearch.example.expertscript.ExpertScriptPlugin'
+  licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
+  noticeFile = rootProject.file('NOTICE.txt')
 }
 
 dependencies {
diff --git a/plugins/examples/security-authorization-engine/build.gradle b/plugins/examples/security-authorization-engine/build.gradle
index 3b49a7eb4db30..faf32774a20ac 100644
--- a/plugins/examples/security-authorization-engine/build.gradle
+++ b/plugins/examples/security-authorization-engine/build.gradle
@@ -2,12 +2,12 @@ apply plugin: 'elasticsearch.esplugin'
 apply plugin: 'elasticsearch.java-rest-test'
 
 esplugin {
-  name 'security-authorization-engine'
-  description 'An example spi extension plugin for security that implements an Authorization Engine'
-  classname 'org.elasticsearch.example.AuthorizationEnginePlugin'
+  name = 'security-authorization-engine'
+  description = 'An example spi extension plugin for security that implements an Authorization Engine'
+  classname ='org.elasticsearch.example.AuthorizationEnginePlugin'
   extendedPlugins = ['x-pack-security']
-  licenseFile rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
-  noticeFile rootProject.file('NOTICE.txt')
+  licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
+  noticeFile = rootProject.file('NOTICE.txt')
 }
 
 dependencies {
diff --git a/plugins/examples/stable-analysis/build.gradle b/plugins/examples/stable-analysis/build.gradle
index 358e2585623e3..57126853da8e8 100644
--- a/plugins/examples/stable-analysis/build.gradle
+++ b/plugins/examples/stable-analysis/build.gradle
@@ -2,8 +2,8 @@ apply plugin: 'elasticsearch.stable-esplugin'
 apply plugin: 'elasticsearch.yaml-rest-test'
 
 esplugin {
-  name 'stable-analysis-plugin'
-  description 'An example analysis plugin using stable plugin api'
+  name = 'stable-analysis-plugin'
+  description = 'An example analysis plugin using stable plugin api'
 }
 //TODO write module-info
 
diff --git a/plugins/mapper-annotated-text/build.gradle b/plugins/mapper-annotated-text/build.gradle
index 435ad83974efa..63eb75865794c 100644
--- a/plugins/mapper-annotated-text/build.gradle
+++ b/plugins/mapper-annotated-text/build.gradle
@@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.yaml-rest-compat-test'
 
 esplugin {
-  description 'The Mapper Annotated_text plugin adds support for text fields with markup used to inject annotation tokens into the index.'
-  classname 'org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextPlugin'
+  description = 'The Mapper Annotated_text plugin adds support for text fields with markup used to inject annotation tokens into the index.'
+  classname ='org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextPlugin'
 }
 
 restResources {
diff --git a/plugins/mapper-annotated-text/src/main/java/module-info.java b/plugins/mapper-annotated-text/src/main/java/module-info.java
index 13f2bd66418be..58aca0d2857fe 100644
--- a/plugins/mapper-annotated-text/src/main/java/module-info.java
+++ b/plugins/mapper-annotated-text/src/main/java/module-info.java
@@ -15,6 +15,4 @@
     requires org.apache.lucene.highlighter;
 
     // exports nothing
-
-    provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.index.mapper.annotatedtext.Features;
 }
diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
index c12849d545b33..4b2006430b89e 100644
--- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
+++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
@@ -22,7 +22,6 @@
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
@@ -64,8 +63,6 @@
  **/
 public class AnnotatedTextFieldMapper extends FieldMapper {
 
-    public static final NodeFeature SYNTHETIC_SOURCE_SUPPORT = new NodeFeature("mapper.annotated_text.synthetic_source");
-
     public static final String CONTENT_TYPE = "annotated_text";
 
     private static Builder builder(FieldMapper in) {
@@ -167,7 +164,7 @@ public AnnotatedTextFieldMapper build(MapperBuilderContext context) {
         }
     }
 
-    public static TypeParser PARSER = new TypeParser(
+    public static final TypeParser PARSER = new TypeParser(
         (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), SourceFieldMapper.isSynthetic(c.getIndexSettings()))
     );
 
diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighter.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighter.java
index 8b4a9d6544b75..f636335701da5 100644
--- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighter.java
+++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighter.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.lucene.search.uhighlight.CustomUnifiedHighlighter;
+import org.elasticsearch.lucene.search.uhighlight.QueryMaxAnalyzedOffset;
 import org.elasticsearch.search.fetch.FetchSubPhase.HitContext;
 import org.elasticsearch.search.fetch.subphase.highlight.DefaultHighlighter;
 import org.elasticsearch.search.fetch.subphase.highlight.SearchHighlightContext;
@@ -52,7 +53,7 @@ protected List<Object> loadFieldValues(
     }
 
     @Override
-    protected Analyzer wrapAnalyzer(Analyzer analyzer, Integer maxAnalyzedOffset) {
+    protected Analyzer wrapAnalyzer(Analyzer analyzer, QueryMaxAnalyzedOffset maxAnalyzedOffset) {
         return new AnnotatedHighlighterAnalyzer(super.wrapAnalyzer(analyzer, maxAnalyzedOffset));
     }
 
diff --git a/plugins/mapper-annotated-text/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/plugins/mapper-annotated-text/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
deleted file mode 100644
index 1fc11da18fc3c..0000000000000
--- a/plugins/mapper-annotated-text/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
+++ /dev/null
@@ -1,10 +0,0 @@
-#
- # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- # or more contributor license agreements. Licensed under the "Elastic License
- # 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- # Public License v 1"; you may not use this file except in compliance with, at
- # your election, the "Elastic License 2.0", the "GNU Affero General Public
- # License v3.0 only", or the "Server Side Public License, v 1".
-#
-
-org.elasticsearch.index.mapper.annotatedtext.Features
diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java
index d4c4ccfaa442d..53bb87bdf0acb 100644
--- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java
+++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText;
 import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotationAnalyzerWrapper;
 import org.elasticsearch.lucene.search.uhighlight.CustomUnifiedHighlighter;
+import org.elasticsearch.lucene.search.uhighlight.QueryMaxAnalyzedOffset;
 import org.elasticsearch.lucene.search.uhighlight.Snippet;
 import org.elasticsearch.search.fetch.subphase.highlight.LimitTokenOffsetAnalyzer;
 import org.elasticsearch.test.ESTestCase;
@@ -85,7 +86,7 @@ private void assertHighlightOneDoc(
         int noMatchSize,
         String[] expectedPassages,
         int maxAnalyzedOffset,
-        Integer queryMaxAnalyzedOffset
+        Integer queryMaxAnalyzedOffsetIn
     ) throws Exception {
 
         try (Directory dir = newDirectory()) {
@@ -116,8 +117,9 @@ private void assertHighlightOneDoc(
                 for (int i = 0; i < markedUpInputs.length; i++) {
                     annotations[i] = AnnotatedText.parse(markedUpInputs[i]);
                 }
+                QueryMaxAnalyzedOffset queryMaxAnalyzedOffset = QueryMaxAnalyzedOffset.create(queryMaxAnalyzedOffsetIn, maxAnalyzedOffset);
                 if (queryMaxAnalyzedOffset != null) {
-                    wrapperAnalyzer = new LimitTokenOffsetAnalyzer(wrapperAnalyzer, queryMaxAnalyzedOffset);
+                    wrapperAnalyzer = new LimitTokenOffsetAnalyzer(wrapperAnalyzer, queryMaxAnalyzedOffset.getNotNull());
                 }
                 AnnotatedHighlighterAnalyzer hiliteAnalyzer = new AnnotatedHighlighterAnalyzer(wrapperAnalyzer);
                 hiliteAnalyzer.setAnnotations(annotations);
@@ -311,6 +313,19 @@ public void testExceedMaxAnalyzedOffset() throws Exception {
             e.getMessage()
         );
 
+        // Same as before, but force using index maxOffset (20) as queryMaxOffset by passing -1.
+        assertHighlightOneDoc(
+            "text",
+            new String[] { "[Long Text exceeds](Long+Text+exceeds) MAX analyzed offset)" },
+            query,
+            Locale.ROOT,
+            breakIterator,
+            0,
+            new String[] { "Long Text [exceeds](_hit_term=exceeds) MAX analyzed offset)" },
+            20,
+            -1
+        );
+
         assertHighlightOneDoc(
             "text",
             new String[] { "[Long Text Exceeds](Long+Text+Exceeds) MAX analyzed offset [Long Text Exceeds](Long+Text+Exceeds)" },
diff --git a/plugins/mapper-murmur3/build.gradle b/plugins/mapper-murmur3/build.gradle
index 0271296df934d..54423b2b990dd 100644
--- a/plugins/mapper-murmur3/build.gradle
+++ b/plugins/mapper-murmur3/build.gradle
@@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.yaml-rest-compat-test'
 
 esplugin {
-  description 'The Mapper Murmur3 plugin allows to compute hashes of a field\'s values at index-time and to store them in the index.'
-  classname 'org.elasticsearch.plugin.mapper.MapperMurmur3Plugin'
+  description = 'The Mapper Murmur3 plugin allows to compute hashes of a field\'s values at index-time and to store them in the index.'
+  classname ='org.elasticsearch.plugin.mapper.MapperMurmur3Plugin'
   extendedPlugins = ['lang-painless']
 }
 
diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java
index 2440668ff57be..738fb7ab7c25e 100644
--- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java
+++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java
@@ -63,7 +63,7 @@ public Murmur3FieldMapper build(MapperBuilderContext context) {
         }
     }
 
-    public static TypeParser PARSER = new TypeParser((n, c) -> new Builder(n));
+    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n));
 
     // this only exists so a check can be done to match the field type to using murmur3 hashing...
     public static class Murmur3FieldType extends MappedFieldType {
diff --git a/plugins/mapper-size/build.gradle b/plugins/mapper-size/build.gradle
index 93fa6fc320c1f..0f9bf59767d90 100644
--- a/plugins/mapper-size/build.gradle
+++ b/plugins/mapper-size/build.gradle
@@ -11,8 +11,8 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'The Mapper Size plugin allows document to record their uncompressed size at index time.'
-  classname 'org.elasticsearch.plugin.mapper.MapperSizePlugin'
+  description = 'The Mapper Size plugin allows document to record their uncompressed size at index time.'
+  classname ='org.elasticsearch.plugin.mapper.MapperSizePlugin'
 }
 
 restResources {
diff --git a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java b/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java
index c2251910c3122..435849821691e 100644
--- a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java
+++ b/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java
@@ -91,7 +91,7 @@ private void assertSizeMappingEnabled(String index, boolean enabled) throws IOEx
             "Expected size field mapping to be " + (enabled ? "enabled" : "disabled") + " for %s",
             index
         );
-        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(index).get();
+        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, index).get();
         Map<String, Object> mappingSource = getMappingsResponse.getMappings().get(index).getSourceAsMap();
         assertThat(errMsg, mappingSource, hasKey("_size"));
         String sizeAsString = mappingSource.get("_size").toString();
diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index 4da7c24de80f1..7df46699b79e2 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -14,8 +14,8 @@ apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
 esplugin {
-  description 'The HDFS repository plugin adds support for Hadoop Distributed File-System (HDFS) repositories.'
-  classname 'org.elasticsearch.repositories.hdfs.HdfsPlugin'
+  description = 'The HDFS repository plugin adds support for Hadoop Distributed File-System (HDFS) repositories.'
+  classname ='org.elasticsearch.repositories.hdfs.HdfsPlugin'
 }
 
 versions << [
@@ -28,7 +28,7 @@ configurations {
 }
 
 dependencies {
-  api project(path: 'hadoop-client-api', configuration: 'shadow')
+  api project(path: 'hadoop-client-api', configuration: 'default')
   if (isEclipse) {
     /*
      * Eclipse can't pick up the shadow dependency so we point it at *something*
diff --git a/plugins/repository-hdfs/hadoop-client-api/build.gradle b/plugins/repository-hdfs/hadoop-client-api/build.gradle
index 24e4213780fe2..46b0d949cdee2 100644
--- a/plugins/repository-hdfs/hadoop-client-api/build.gradle
+++ b/plugins/repository-hdfs/hadoop-client-api/build.gradle
@@ -1,16 +1,54 @@
-apply plugin: 'elasticsearch.build'
-apply plugin: 'com.gradleup.shadow'
+import org.gradle.api.file.ArchiveOperations
+
+apply plugin: 'elasticsearch.java'
+
+sourceSets {
+  patcher
+}
+
+configurations {
+  thejar {
+    canBeResolved = true
+  }
+}
 
 dependencies {
-  implementation "org.apache.hadoop:hadoop-client-api:${project.parent.versions.hadoop}"
+  thejar("org.apache.hadoop:hadoop-client-api:${project.parent.versions.hadoop}") {
+    transitive = false
+  }
+
+  patcherImplementation 'org.ow2.asm:asm:9.7.1'
+  patcherImplementation 'org.ow2.asm:asm-tree:9.7.1'
+}
+
+def outputDir = layout.buildDirectory.dir("patched-classes")
+
+def patchTask = tasks.register("patchClasses", JavaExec) {
+  inputs.files(configurations.thejar).withPathSensitivity(PathSensitivity.RELATIVE)
+  inputs.files(sourceSets.patcher.output).withPathSensitivity(PathSensitivity.RELATIVE)
+  outputs.dir(outputDir)
+  classpath = sourceSets.patcher.runtimeClasspath
+  mainClass = 'org.elasticsearch.hdfs.patch.HdfsClassPatcher'
+  def thejar = configurations.thejar
+  doFirst {
+    args(thejar.singleFile, outputDir.get().asFile)
+  }
 }
 
-tasks.named('shadowJar').configure {
-  exclude 'org/apache/hadoop/util/ShutdownHookManager$*.class'
+interface InjectedArchiveOps {
+  @Inject ArchiveOperations getArchiveOperations()
 }
 
-['jarHell', 'thirdPartyAudit', 'forbiddenApisMain', 'splitPackagesAudit'].each {
-  tasks.named(it).configure {
-    enabled = false
+tasks.named('jar').configure {
+  dependsOn(configurations.thejar)
+  def injected = project.objects.newInstance(InjectedArchiveOps)
+  def thejar = configurations.thejar
+  from(patchTask)
+  from({ injected.getArchiveOperations().zipTree(thejar.singleFile) }) {
+    eachFile {
+      if (outputDir.get().file(it.relativePath.pathString).asFile.exists()) {
+        it.exclude()
+      }
+    }
   }
 }
diff --git a/plugins/repository-hdfs/hadoop-client-api/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java b/plugins/repository-hdfs/hadoop-client-api/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
deleted file mode 100644
index c3d15dc06e7c1..0000000000000
--- a/plugins/repository-hdfs/hadoop-client-api/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.apache.hadoop.util;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * A replacement for the ShutdownHookManager from hadoop.
- *
- * This class does not actually add a shutdown hook. Hadoop's shutdown hook
- * manager does not fail gracefully when it lacks security manager permissions
- * to add shutdown hooks. This implements the same api as the hadoop class, but
- * with no-ops.
- */
-public class ShutdownHookManager {
-    private static final ShutdownHookManager MGR = new ShutdownHookManager();
-
-    public static ShutdownHookManager get() {
-        return MGR;
-    }
-
-    private ShutdownHookManager() {}
-
-    public void addShutdownHook(Runnable shutdownHook, int priority) {}
-
-    public void addShutdownHook(Runnable shutdownHook, int priority, long timeout, TimeUnit unit) {}
-
-    public boolean removeShutdownHook(Runnable shutdownHook) {
-        return false;
-    }
-
-    public boolean hasShutdownHook(Runnable shutdownHook) {
-        return false;
-    }
-
-    public boolean isShutdownInProgress() {
-        return false;
-    }
-
-    public void clearShutdownHooks() {}
-}
diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java
new file mode 100644
index 0000000000000..732c55929454e
--- /dev/null
+++ b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.hdfs.patch;
+
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.ClassWriter;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+
+public class HdfsClassPatcher {
+    static final Map<String, Function<ClassWriter, ClassVisitor>> patchers = Map.of(
+        "org/apache/hadoop/util/ShutdownHookManager.class",
+        ShutdownHookManagerPatcher::new,
+        "org/apache/hadoop/util/Shell.class",
+        ShellPatcher::new,
+        "org/apache/hadoop/security/UserGroupInformation.class",
+        SubjectGetSubjectPatcher::new,
+        "org/apache/hadoop/security/authentication/client/KerberosAuthenticator.class",
+        SubjectGetSubjectPatcher::new
+    );
+
+    public static void main(String[] args) throws Exception {
+        String jarPath = args[0];
+        Path outputDir = Paths.get(args[1]);
+
+        try (JarFile jarFile = new JarFile(new File(jarPath))) {
+            for (var patcher : patchers.entrySet()) {
+                JarEntry jarEntry = jarFile.getJarEntry(patcher.getKey());
+                if (jarEntry == null) {
+                    throw new IllegalArgumentException("path [" + patcher.getKey() + "] not found in [" + jarPath + "]");
+                }
+                byte[] classToPatch = jarFile.getInputStream(jarEntry).readAllBytes();
+
+                ClassReader classReader = new ClassReader(classToPatch);
+                ClassWriter classWriter = new ClassWriter(classReader, 0);
+                classReader.accept(patcher.getValue().apply(classWriter), 0);
+
+                Path outputFile = outputDir.resolve(patcher.getKey());
+                Files.createDirectories(outputFile.getParent());
+                Files.write(outputFile, classWriter.toByteArray());
+            }
+        }
+    }
+}
diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/MethodReplacement.java b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/MethodReplacement.java
new file mode 100644
index 0000000000000..e07a32cc294a5
--- /dev/null
+++ b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/MethodReplacement.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.hdfs.patch;
+
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+
+public class MethodReplacement extends MethodVisitor {
+    private final MethodVisitor delegate;
+    private final Runnable bodyWriter;
+
+    MethodReplacement(MethodVisitor delegate, Runnable bodyWriter) {
+        super(Opcodes.ASM9);
+        this.delegate = delegate;
+        this.bodyWriter = bodyWriter;
+    }
+
+    @Override
+    public void visitCode() {
+        // delegate.visitCode();
+        bodyWriter.run();
+        // delegate.visitEnd();
+    }
+
+    @Override
+    public void visitMaxs(int maxStack, int maxLocals) {
+        delegate.visitMaxs(maxStack, maxLocals);
+    }
+}
diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShellPatcher.java b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShellPatcher.java
new file mode 100644
index 0000000000000..397b63e434ba2
--- /dev/null
+++ b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShellPatcher.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.hdfs.patch;
+
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+
+class ShellPatcher extends ClassVisitor {
+
+    ShellPatcher(ClassWriter classWriter) {
+        super(Opcodes.ASM9, classWriter);
+    }
+
+    @Override
+    public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
+        MethodVisitor mv = super.visitMethod(access, name, descriptor, signature, exceptions);
+        if (name.equals("isSetsidSupported")) {
+            return new MethodReplacement(mv, () -> {
+                mv.visitInsn(Opcodes.ICONST_0);
+                mv.visitInsn(Opcodes.IRETURN);
+            });
+        }
+        return mv;
+    }
+}
diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShutdownHookManagerPatcher.java b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShutdownHookManagerPatcher.java
new file mode 100644
index 0000000000000..1235b5af9002f
--- /dev/null
+++ b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShutdownHookManagerPatcher.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.hdfs.patch;
+
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.Type;
+
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+class ShutdownHookManagerPatcher extends ClassVisitor {
+    private static final String CLASSNAME = "org/apache/hadoop/util/ShutdownHookManager";
+    private static final Set<String> VOID_METHODS = Set.of("addShutdownHook", "clearShutdownHooks");
+    private static final Set<String> BOOLEAN_METHODS = Set.of("removeShutdownHook", "hasShutdownHook", "isShutdownInProgress");
+
+    ShutdownHookManagerPatcher(ClassWriter classWriter) {
+        super(Opcodes.ASM9, classWriter);
+    }
+
+    @Override
+    public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
+        MethodVisitor mv = super.visitMethod(access, name, descriptor, signature, exceptions);
+        if (VOID_METHODS.contains(name)) {
+            // make void methods noops
+            return new MethodReplacement(mv, () -> { mv.visitInsn(Opcodes.RETURN); });
+        } else if (BOOLEAN_METHODS.contains(name)) {
+            // make boolean methods always return false
+            return new MethodReplacement(mv, () -> {
+                mv.visitInsn(Opcodes.ICONST_0);
+                mv.visitInsn(Opcodes.IRETURN);
+            });
+        } else if (name.equals("<clinit>")) {
+            return new MethodReplacement(mv, () -> {
+                // just initialize the statics, don't actually get runtime to add shutdown hook
+
+                var classType = Type.getObjectType(CLASSNAME);
+                mv.visitTypeInsn(Opcodes.NEW, CLASSNAME);
+                mv.visitInsn(Opcodes.DUP);
+                mv.visitMethodInsn(Opcodes.INVOKESPECIAL, CLASSNAME, "<init>", "()V", false);
+                mv.visitFieldInsn(Opcodes.PUTSTATIC, CLASSNAME, "MGR", classType.getDescriptor());
+
+                var timeUnitType = Type.getType(TimeUnit.class);
+                mv.visitFieldInsn(Opcodes.GETSTATIC, timeUnitType.getInternalName(), "SECONDS", timeUnitType.getDescriptor());
+                mv.visitFieldInsn(Opcodes.PUTSTATIC, CLASSNAME, "TIME_UNIT_DEFAULT", timeUnitType.getDescriptor());
+
+                var executorServiceType = Type.getType(ExecutorService.class);
+                mv.visitInsn(Opcodes.ACONST_NULL);
+                mv.visitFieldInsn(Opcodes.PUTSTATIC, CLASSNAME, "EXECUTOR", executorServiceType.getDescriptor());
+
+                mv.visitInsn(Opcodes.RETURN);
+            });
+        }
+        return mv;
+    }
+}
diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/SubjectGetSubjectPatcher.java b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/SubjectGetSubjectPatcher.java
new file mode 100644
index 0000000000000..3fb8a23be794d
--- /dev/null
+++ b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/SubjectGetSubjectPatcher.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.hdfs.patch;
+
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Type;
+
+import static org.objectweb.asm.Opcodes.ASM9;
+import static org.objectweb.asm.Opcodes.INVOKESTATIC;
+import static org.objectweb.asm.Opcodes.POP;
+
+class SubjectGetSubjectPatcher extends ClassVisitor {
+    SubjectGetSubjectPatcher(ClassWriter classWriter) {
+        super(ASM9, classWriter);
+    }
+
+    @Override
+    public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
+        return new ReplaceCallMethodVisitor(super.visitMethod(access, name, descriptor, signature, exceptions), name, access, descriptor);
+    }
+
+    /**
+     * Replaces calls to Subject.getSubject(context); with calls to Subject.current();
+     */
+    private static class ReplaceCallMethodVisitor extends MethodVisitor {
+        private static final String SUBJECT_CLASS_INTERNAL_NAME = "javax/security/auth/Subject";
+        private static final String METHOD_NAME = "getSubject";
+
+        ReplaceCallMethodVisitor(MethodVisitor methodVisitor, String name, int access, String descriptor) {
+            super(ASM9, methodVisitor);
+        }
+
+        @Override
+        public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) {
+            if (opcode == INVOKESTATIC && SUBJECT_CLASS_INTERNAL_NAME.equals(owner) && METHOD_NAME.equals(name)) {
+                // Get rid of the extra arg on the stack
+                mv.visitInsn(POP);
+                // Call Subject.current()
+                mv.visitMethodInsn(
+                    INVOKESTATIC,
+                    SUBJECT_CLASS_INTERNAL_NAME,
+                    "current",
+                    Type.getMethodDescriptor(Type.getObjectType(SUBJECT_CLASS_INTERNAL_NAME)),
+                    false
+                );
+            } else {
+                super.visitMethodInsn(opcode, owner, name, descriptor, isInterface);
+            }
+        }
+    }
+}
diff --git a/plugins/repository-hdfs/src/yamlRestTest/resources/rest-api-spec/test/hdfs_repository/40_restore.yml b/plugins/repository-hdfs/src/yamlRestTest/resources/rest-api-spec/test/hdfs_repository/40_restore.yml
index 716857781b758..a13ad970576fb 100644
--- a/plugins/repository-hdfs/src/yamlRestTest/resources/rest-api-spec/test/hdfs_repository/40_restore.yml
+++ b/plugins/repository-hdfs/src/yamlRestTest/resources/rest-api-spec/test/hdfs_repository/40_restore.yml
@@ -49,8 +49,6 @@
   - do:
       indices.close:
         index : test_index
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   # Restore index
   - do:
diff --git a/plugins/repository-hdfs/src/yamlRestTest/resources/rest-api-spec/test/secure_hdfs_repository/40_restore.yml b/plugins/repository-hdfs/src/yamlRestTest/resources/rest-api-spec/test/secure_hdfs_repository/40_restore.yml
index c7a298d72d6c9..1a5d346562181 100644
--- a/plugins/repository-hdfs/src/yamlRestTest/resources/rest-api-spec/test/secure_hdfs_repository/40_restore.yml
+++ b/plugins/repository-hdfs/src/yamlRestTest/resources/rest-api-spec/test/secure_hdfs_repository/40_restore.yml
@@ -51,8 +51,6 @@
   - do:
       indices.close:
         index : test_index
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   # Restore index
   - do:
diff --git a/plugins/store-smb/build.gradle b/plugins/store-smb/build.gradle
index 1186f5359ad5c..727f9ed588673 100644
--- a/plugins/store-smb/build.gradle
+++ b/plugins/store-smb/build.gradle
@@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'The Store SMB plugin adds support for SMB stores.'
-  classname 'org.elasticsearch.plugin.store.smb.SMBStorePlugin'
+  description = 'The Store SMB plugin adds support for SMB stores.'
+  classname ='org.elasticsearch.plugin.store.smb.SMBStorePlugin'
 }
 restResources {
   restApi {
diff --git a/qa/ccs-rolling-upgrade-remote-cluster/build.gradle b/qa/ccs-rolling-upgrade-remote-cluster/build.gradle
index e63b1629db39c..5bbade8cf6fce 100644
--- a/qa/ccs-rolling-upgrade-remote-cluster/build.gradle
+++ b/qa/ccs-rolling-upgrade-remote-cluster/build.gradle
@@ -50,8 +50,6 @@ buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
       nonInputProperties.systemProperty('tests.rest.cluster', localCluster.map(c -> c.allHttpSocketURI.join(",")))
       nonInputProperties.systemProperty('tests.rest.remote_cluster', remoteCluster.map(c -> c.allHttpSocketURI.join(",")))
     }
-
-    onlyIf("FIPS mode disabled") { buildParams.inFipsJvm == false }
   }
 
   tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) {
diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java
index aac2c661dea9f..8b74657becb24 100644
--- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java
+++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java
@@ -12,6 +12,8 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
@@ -21,6 +23,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
 import org.elasticsearch.index.query.DisMaxQueryBuilder;
@@ -51,6 +54,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.cluster.ClusterState.VERSION_INTRODUCING_TRANSPORT_VERSIONS;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 
 /**
@@ -245,7 +249,19 @@ public void testQueryBuilderBWC() throws Exception {
                     InputStream in = new ByteArrayInputStream(qbSource, 0, qbSource.length);
                     StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in), registry)
                 ) {
-                    input.setTransportVersion(TransportVersion.readVersion(input));
+                    @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS)    // won't need to read <8.8 data anymore
+                    boolean originalClusterHasTransportVersion = parseLegacyVersion(getOldClusterVersion()).map(
+                        v -> v.onOrAfter(VERSION_INTRODUCING_TRANSPORT_VERSIONS)
+                    ).orElse(true);
+                    TransportVersion transportVersion;
+                    if (originalClusterHasTransportVersion == false) {
+                        transportVersion = TransportVersion.fromId(
+                            parseLegacyVersion(getOldClusterVersion()).map(Version::id).orElse(TransportVersions.MINIMUM_COMPATIBLE.id())
+                        );
+                    } else {
+                        transportVersion = TransportVersion.readVersion(input);
+                    }
+                    input.setTransportVersion(transportVersion);
                     QueryBuilder queryBuilder = input.readNamedWriteable(QueryBuilder.class);
                     assert in.read() == -1;
                     assertEquals(expectedQueryBuilder, queryBuilder);
diff --git a/qa/logging-spi/build.gradle b/qa/logging-spi/build.gradle
index 04f09b9638bfe..d6b707eb24dd5 100644
--- a/qa/logging-spi/build.gradle
+++ b/qa/logging-spi/build.gradle
@@ -3,9 +3,9 @@ apply plugin: 'elasticsearch.base-internal-es-plugin'
 apply plugin: 'elasticsearch.internal-java-rest-test'
 
 esplugin {
-  name 'logging-spi-test'
-  description 'An test plugin to test the SPI behaviour of ES logging'
-  classname 'org.elasticsearch.test.logging.plugin.TestLoggingPlugin'
+  name = 'logging-spi-test'
+  description = 'An test plugin to test the SPI behaviour of ES logging'
+  classname ='org.elasticsearch.test.logging.plugin.TestLoggingPlugin'
 }
 
 dependencies {
diff --git a/qa/lucene-index-compatibility/build.gradle b/qa/lucene-index-compatibility/build.gradle
index 37e5eea85a08b..3b2e69ec9859f 100644
--- a/qa/lucene-index-compatibility/build.gradle
+++ b/qa/lucene-index-compatibility/build.gradle
@@ -14,7 +14,9 @@ buildParams.bwcVersions.withLatestReadOnlyIndexCompatible { bwcVersion ->
   tasks.named("javaRestTest").configure {
     systemProperty("tests.minimum.index.compatible", bwcVersion)
     usesBwcDistribution(bwcVersion)
-    enabled = true
+
+    // Tests rely on unreleased code in 8.18 branch
+    enabled = buildParams.isSnapshotBuild()
   }
 }
 
@@ -22,4 +24,3 @@ tasks.withType(Test).configureEach {
   // CI doesn't like it when there's multiple clusters running at once
   maxParallelForks = 1
 }
-
diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractIndexCompatibilityTestCase.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractIndexCompatibilityTestCase.java
new file mode 100644
index 0000000000000..13c647983fad5
--- /dev/null
+++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractIndexCompatibilityTestCase.java
@@ -0,0 +1,241 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.lucene;
+
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.InputStreamEntity;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.test.XContentTestUtils;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.util.Version;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xcontent.XContentType;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestRule;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.IntStream;
+
+import static org.elasticsearch.test.cluster.util.Version.CURRENT;
+import static org.elasticsearch.test.cluster.util.Version.fromString;
+import static org.elasticsearch.test.rest.ObjectPath.createFromResponse;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+
+public abstract class AbstractIndexCompatibilityTestCase extends ESRestTestCase {
+
+    protected static final Version VERSION_MINUS_2 = fromString(System.getProperty("tests.minimum.index.compatible"));
+    protected static final Version VERSION_MINUS_1 = fromString(System.getProperty("tests.minimum.wire.compatible"));
+    protected static final Version VERSION_CURRENT = CURRENT;
+
+    protected static final int NODES = 3;
+
+    private static TemporaryFolder REPOSITORY_PATH = new TemporaryFolder();
+
+    protected static LocalClusterConfigProvider clusterConfig = c -> {};
+    private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .version(VERSION_MINUS_2)
+        .nodes(NODES)
+        .setting("path.repo", () -> REPOSITORY_PATH.getRoot().getPath())
+        .setting("xpack.security.enabled", "false")
+        .setting("xpack.ml.enabled", "false")
+        .setting("path.repo", () -> REPOSITORY_PATH.getRoot().getPath())
+        .apply(() -> clusterConfig)
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(REPOSITORY_PATH).around(cluster);
+
+    private static boolean upgradeFailed = false;
+
+    @Before
+    public final void maybeUpgradeBeforeTest() throws Exception {
+        // We want to use this test suite for the V9 upgrade, but we are not fully committed to necessarily having N-2 support
+        // in V10, so we add a check here to ensure we'll revisit this decision once V10 exists.
+        assertThat("Explicit check that N-2 version is Elasticsearch 7", VERSION_MINUS_2.getMajor(), equalTo(7));
+
+        if (upgradeFailed == false) {
+            try {
+                maybeUpgrade();
+            } catch (Exception e) {
+                upgradeFailed = true;
+                throw e;
+            }
+        }
+
+        // Skip remaining tests if upgrade failed
+        assumeFalse("Cluster upgrade failed", upgradeFailed);
+    }
+
+    protected abstract void maybeUpgrade() throws Exception;
+
+    @After
+    public final void deleteSnapshotBlobCache() throws IOException {
+        // TODO ES-10475: The .snapshot-blob-cache created in legacy version can block upgrades, we should probably delete it automatically
+        try {
+            var request = new Request("DELETE", "/.snapshot-blob-cache");
+            request.setOptions(
+                expectWarnings(
+                    "this request accesses system indices: [.snapshot-blob-cache], but in a future major version, "
+                        + "direct access to system indices will be prevented by default"
+                )
+            );
+            adminClient().performRequest(request);
+        } catch (IOException e) {
+            if (isNotFoundResponseException(e) == false) {
+                throw e;
+            }
+        }
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected boolean preserveClusterUponCompletion() {
+        return true;
+    }
+
+    protected ElasticsearchCluster cluster() {
+        return cluster;
+    }
+
+    protected String suffix(String name) {
+        return name + '-' + getTestName().split(" ")[0].toLowerCase(Locale.ROOT);
+    }
+
+    protected Settings repositorySettings() {
+        return Settings.builder()
+            .put("location", REPOSITORY_PATH.getRoot().toPath().resolve(suffix("location")).toFile().getPath())
+            .build();
+    }
+
+    protected static Map<String, Version> nodesVersions() throws Exception {
+        var nodesInfos = getNodesInfo(adminClient());
+        assertThat(nodesInfos.size(), equalTo(NODES));
+        var versions = new HashMap<String, Version>();
+        for (var nodeInfos : nodesInfos.values()) {
+            versions.put((String) nodeInfos.get("name"), Version.fromString((String) nodeInfos.get("version")));
+        }
+        return versions;
+    }
+
+    protected static boolean isFullyUpgradedTo(Version version) throws Exception {
+        return nodesVersions().values().stream().allMatch(v -> v.equals(version));
+    }
+
+    protected static Version indexVersion(String indexName) throws Exception {
+        var response = assertOK(client().performRequest(new Request("GET", "/" + indexName + "/_settings")));
+        int id = Integer.parseInt(createFromResponse(response).evaluate(indexName + ".settings.index.version.created"));
+        return new Version((byte) ((id / 1000000) % 100), (byte) ((id / 10000) % 100), (byte) ((id / 100) % 100));
+    }
+
+    protected static void indexDocs(String indexName, int numDocs) throws Exception {
+        var request = new Request("POST", "/_bulk");
+        var docs = new StringBuilder();
+        IntStream.range(0, numDocs).forEach(n -> docs.append(Strings.format("""
+            {"index":{"_index":"%s"}}
+            {"field_0":"%s","field_1":%d,"field_2":"%s"}
+            """, indexName, Integer.toString(n), n, randomFrom(Locale.getAvailableLocales()).getDisplayName())));
+        request.setJsonEntity(docs.toString());
+        var response = assertOK(client().performRequest(request));
+        assertThat(entityAsMap(response).get("errors"), allOf(notNullValue(), is(false)));
+    }
+
+    protected static void mountIndex(String repository, String snapshot, String indexName, boolean partial, String renamedIndexName)
+        throws Exception {
+        var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_mount");
+        request.addParameter("wait_for_completion", "true");
+        var storage = partial ? "shared_cache" : "full_copy";
+        request.addParameter("storage", storage);
+        request.setJsonEntity(Strings.format("""
+            {
+              "index": "%s",
+              "renamed_index": "%s"
+            }""", indexName, renamedIndexName));
+        var responseBody = createFromResponse(client().performRequest(request));
+        assertThat(responseBody.evaluate("snapshot.shards.total"), equalTo((int) responseBody.evaluate("snapshot.shards.successful")));
+        assertThat(responseBody.evaluate("snapshot.shards.failed"), equalTo(0));
+    }
+
+    protected static void restoreIndex(String repository, String snapshot, String indexName, String renamedIndexName) throws Exception {
+        var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_restore");
+        request.addParameter("wait_for_completion", "true");
+        request.setJsonEntity(org.elasticsearch.common.Strings.format("""
+            {
+              "indices": "%s",
+              "include_global_state": false,
+              "rename_pattern": "(.+)",
+              "rename_replacement": "%s",
+              "include_aliases": false
+            }""", indexName, renamedIndexName));
+        var responseBody = createFromResponse(client().performRequest(request));
+        assertThat(responseBody.evaluate("snapshot.shards.total"), equalTo((int) responseBody.evaluate("snapshot.shards.failed")));
+        assertThat(responseBody.evaluate("snapshot.shards.successful"), equalTo(0));
+    }
+
+    protected static void updateRandomIndexSettings(String indexName) throws IOException {
+        final var settings = Settings.builder();
+        int updates = randomIntBetween(1, 3);
+        for (int i = 0; i < updates; i++) {
+            switch (i) {
+                case 0 -> settings.putList(IndexSettings.DEFAULT_FIELD_SETTING.getKey(), "field_" + randomInt(2));
+                case 1 -> settings.put(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), randomIntBetween(1, 100));
+                case 2 -> settings.put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), randomLongBetween(100L, 1000L));
+                case 3 -> settings.put(IndexSettings.MAX_SLICES_PER_SCROLL.getKey(), randomIntBetween(1, 1024));
+                default -> throw new IllegalStateException();
+            }
+        }
+        updateIndexSettings(indexName, settings);
+    }
+
+    protected static void updateRandomMappings(String indexName) throws IOException {
+        final var runtime = new HashMap<>();
+        runtime.put("field_" + randomInt(2), Map.of("type", "keyword"));
+        final var properties = new HashMap<>();
+        properties.put(randomIdentifier(), Map.of("type", "long"));
+        var body = XContentTestUtils.convertToXContent(Map.of("runtime", runtime, "properties", properties), XContentType.JSON);
+        var request = new Request("PUT", indexName + "/_mappings");
+        request.setEntity(
+            new InputStreamEntity(
+                body.streamInput(),
+                body.length(),
+
+                ContentType.create(XContentType.JSON.mediaTypeWithoutParameters())
+            )
+        );
+        assertOK(client().performRequest(request));
+    }
+
+    protected static boolean isIndexClosed(String indexName) throws Exception {
+        var responseBody = createFromResponse(client().performRequest(new Request("GET", "_cluster/state/metadata/" + indexName)));
+        var state = responseBody.evaluate("metadata.indices." + indexName + ".state");
+        return IndexMetadata.State.fromString((String) state) == IndexMetadata.State.CLOSE;
+    }
+}
diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java
deleted file mode 100644
index c42e879f84892..0000000000000
--- a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.lucene;
-
-import com.carrotsearch.randomizedtesting.TestMethodAndParams;
-import com.carrotsearch.randomizedtesting.annotations.Name;
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
-
-import org.elasticsearch.client.Request;
-import org.elasticsearch.test.cluster.ElasticsearchCluster;
-import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
-import org.elasticsearch.test.cluster.local.distribution.DistributionType;
-import org.elasticsearch.test.cluster.util.Version;
-import org.elasticsearch.test.rest.ESRestTestCase;
-import org.junit.Before;
-import org.junit.ClassRule;
-import org.junit.rules.RuleChain;
-import org.junit.rules.TemporaryFolder;
-import org.junit.rules.TestRule;
-
-import java.util.Comparator;
-import java.util.Locale;
-import java.util.stream.Stream;
-
-import static org.elasticsearch.test.cluster.util.Version.CURRENT;
-import static org.elasticsearch.test.cluster.util.Version.fromString;
-import static org.elasticsearch.test.rest.ObjectPath.createFromResponse;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.notNullValue;
-
-/**
- * Test suite for Lucene indices backward compatibility with N-2 versions. The test suite creates a cluster in N-2 version, then upgrades it
- * to N-1 version and finally upgrades it to the current version. Test methods are executed after each upgrade.
- */
-@TestCaseOrdering(AbstractLuceneIndexCompatibilityTestCase.TestCaseOrdering.class)
-public abstract class AbstractLuceneIndexCompatibilityTestCase extends ESRestTestCase {
-
-    protected static final Version VERSION_MINUS_2 = fromString(System.getProperty("tests.minimum.index.compatible"));
-    protected static final Version VERSION_MINUS_1 = fromString(System.getProperty("tests.minimum.wire.compatible"));
-    protected static final Version VERSION_CURRENT = CURRENT;
-
-    protected static TemporaryFolder REPOSITORY_PATH = new TemporaryFolder();
-
-    protected static LocalClusterConfigProvider clusterConfig = c -> {};
-    private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
-        .distribution(DistributionType.DEFAULT)
-        .version(VERSION_MINUS_2)
-        .nodes(2)
-        .setting("path.repo", () -> REPOSITORY_PATH.getRoot().getPath())
-        .setting("xpack.security.enabled", "false")
-        .setting("xpack.ml.enabled", "false")
-        .setting("path.repo", () -> REPOSITORY_PATH.getRoot().getPath())
-        .apply(() -> clusterConfig)
-        .build();
-
-    @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(REPOSITORY_PATH).around(cluster);
-
-    private static boolean upgradeFailed = false;
-
-    private final Version clusterVersion;
-
-    public AbstractLuceneIndexCompatibilityTestCase(@Name("cluster") Version clusterVersion) {
-        this.clusterVersion = clusterVersion;
-    }
-
-    @ParametersFactory
-    public static Iterable<Object[]> parameters() {
-        return Stream.of(VERSION_MINUS_2, VERSION_MINUS_1, CURRENT).map(v -> new Object[] { v }).toList();
-    }
-
-    @Override
-    protected String getTestRestCluster() {
-        return cluster.getHttpAddresses();
-    }
-
-    @Override
-    protected boolean preserveClusterUponCompletion() {
-        return true;
-    }
-
-    @Before
-    public void maybeUpgrade() throws Exception {
-        // We want to use this test suite for the V9 upgrade, but we are not fully committed to necessarily having N-2 support
-        // in V10, so we add a check here to ensure we'll revisit this decision once V10 exists.
-        assertThat("Explicit check that N-2 version is Elasticsearch 7", VERSION_MINUS_2.getMajor(), equalTo(7));
-
-        var currentVersion = clusterVersion();
-        if (currentVersion.before(clusterVersion)) {
-            try {
-                cluster.upgradeToVersion(clusterVersion);
-                closeClients();
-                initClient();
-            } catch (Exception e) {
-                upgradeFailed = true;
-                throw e;
-            }
-        }
-
-        // Skip remaining tests if upgrade failed
-        assumeFalse("Cluster upgrade failed", upgradeFailed);
-    }
-
-    protected String suffix(String name) {
-        return name + '-' + getTestName().split(" ")[0].toLowerCase(Locale.ROOT);
-    }
-
-    protected static Version clusterVersion() throws Exception {
-        var response = assertOK(client().performRequest(new Request("GET", "/")));
-        var responseBody = createFromResponse(response);
-        var version = Version.fromString(responseBody.evaluate("version.number").toString());
-        assertThat("Failed to retrieve cluster version", version, notNullValue());
-        return version;
-    }
-
-    protected static Version indexLuceneVersion(String indexName) throws Exception {
-        var response = assertOK(client().performRequest(new Request("GET", "/" + indexName + "/_settings")));
-        int id = Integer.parseInt(createFromResponse(response).evaluate(indexName + ".settings.index.version.created"));
-        return new Version((byte) ((id / 1000000) % 100), (byte) ((id / 10000) % 100), (byte) ((id / 100) % 100));
-    }
-
-    /**
-     * Execute the test suite with the parameters provided by the {@link #parameters()} in version order.
-     */
-    public static class TestCaseOrdering implements Comparator<TestMethodAndParams> {
-        @Override
-        public int compare(TestMethodAndParams o1, TestMethodAndParams o2) {
-            var version1 = (Version) o1.getInstanceArguments().get(0);
-            var version2 = (Version) o2.getInstanceArguments().get(0);
-            return version1.compareTo(version2);
-        }
-    }
-}
diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartIndexCompatibilityTestCase.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartIndexCompatibilityTestCase.java
new file mode 100644
index 0000000000000..9ca7132493ae6
--- /dev/null
+++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartIndexCompatibilityTestCase.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.lucene;
+
+import com.carrotsearch.randomizedtesting.TestMethodAndParams;
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
+
+import org.elasticsearch.test.cluster.util.Version;
+
+import java.util.Comparator;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.test.cluster.util.Version.CURRENT;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * Test suite for Lucene indices backward compatibility with N-2 versions after full cluster restart upgrades. The test suite creates a
+ * cluster in N-2 version, then upgrades it to N-1 version and finally upgrades it to the current version. Test methods are executed after
+ * each upgrade.
+ */
+@TestCaseOrdering(FullClusterRestartIndexCompatibilityTestCase.TestCaseOrdering.class)
+public abstract class FullClusterRestartIndexCompatibilityTestCase extends AbstractIndexCompatibilityTestCase {
+
+    private final Version clusterVersion;
+
+    public FullClusterRestartIndexCompatibilityTestCase(@Name("cluster") Version clusterVersion) {
+        this.clusterVersion = clusterVersion;
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        return Stream.of(VERSION_MINUS_2, VERSION_MINUS_1, CURRENT).map(v -> new Object[] { v }).toList();
+    }
+
+    @Override
+    protected void maybeUpgrade() throws Exception {
+        if (nodesVersions().values().stream().anyMatch(version -> version.before(clusterVersion))) {
+            cluster().upgradeToVersion(clusterVersion);
+            closeClients();
+            initClient();
+        }
+        assertThat(isFullyUpgradedTo(clusterVersion), equalTo(true));
+    }
+
+    /**
+     * Execute the test suite with the parameters provided by the {@link #parameters()} in version order.
+     */
+    public static class TestCaseOrdering implements Comparator<TestMethodAndParams> {
+        @Override
+        public int compare(TestMethodAndParams o1, TestMethodAndParams o2) {
+            var version1 = (Version) o1.getInstanceArguments().get(0);
+            var version2 = (Version) o2.getInstanceArguments().get(0);
+            return version1.compareTo(version2);
+        }
+    }
+}
diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartLuceneIndexCompatibilityIT.java
similarity index 58%
rename from qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java
rename to qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartLuceneIndexCompatibilityIT.java
index d6dd949b843d6..15d41cc981cea 100644
--- a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java
+++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartLuceneIndexCompatibilityIT.java
@@ -10,47 +10,42 @@
 package org.elasticsearch.lucene;
 
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.repositories.fs.FsRepository;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.cluster.util.Version;
 
-import java.util.stream.IntStream;
-
-import static org.elasticsearch.test.rest.ObjectPath.createFromResponse;
+import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
 
-public class LuceneCompatibilityIT extends AbstractLuceneIndexCompatibilityTestCase {
+public class FullClusterRestartLuceneIndexCompatibilityIT extends FullClusterRestartIndexCompatibilityTestCase {
 
     static {
         clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial");
     }
 
-    public LuceneCompatibilityIT(Version version) {
+    public FullClusterRestartLuceneIndexCompatibilityIT(Version version) {
         super(version);
     }
 
+    /**
+     * Creates an index and a snapshot on N-2, then restores the snapshot on N.
+     */
     public void testRestoreIndex() throws Exception {
         final String repository = suffix("repository");
         final String snapshot = suffix("snapshot");
         final String index = suffix("index");
         final int numDocs = 1234;
 
-        logger.debug("--> registering repository [{}]", repository);
-        registerRepository(
-            client(),
-            repository,
-            FsRepository.TYPE,
-            true,
-            Settings.builder().put("location", REPOSITORY_PATH.getRoot().getPath()).build()
-        );
+        if (isFullyUpgradedTo(VERSION_MINUS_2)) {
+            logger.debug("--> registering repository [{}]", repository);
+            registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings());
 
-        if (VERSION_MINUS_2.equals(clusterVersion())) {
             logger.debug("--> creating index [{}]", index);
             createIndex(
                 client(),
@@ -63,27 +58,17 @@ public void testRestoreIndex() throws Exception {
             );
 
             logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);
-            final var bulks = new StringBuilder();
-            IntStream.range(0, numDocs).forEach(n -> bulks.append(Strings.format("""
-                {"index":{"_id":"%s","_index":"%s"}}
-                {"test":"test"}
-                """, n, index)));
-
-            var bulkRequest = new Request("POST", "/_bulk");
-            bulkRequest.setJsonEntity(bulks.toString());
-            var bulkResponse = client().performRequest(bulkRequest);
-            assertOK(bulkResponse);
-            assertThat(entityAsMap(bulkResponse).get("errors"), allOf(notNullValue(), is(false)));
+            indexDocs(index, numDocs);
 
             logger.debug("--> creating snapshot [{}]", snapshot);
             createSnapshot(client(), repository, snapshot, true);
             return;
         }
 
-        if (VERSION_MINUS_1.equals(clusterVersion())) {
+        if (isFullyUpgradedTo(VERSION_MINUS_1)) {
             ensureGreen(index);
 
-            assertThat(indexLuceneVersion(index), equalTo(VERSION_MINUS_2));
+            assertThat(indexVersion(index), equalTo(VERSION_MINUS_2));
             assertDocCount(client(), index, numDocs);
 
             logger.debug("--> deleting index [{}]", index);
@@ -91,11 +76,11 @@ public void testRestoreIndex() throws Exception {
             return;
         }
 
-        if (VERSION_CURRENT.equals(clusterVersion())) {
+        if (isFullyUpgradedTo(VERSION_CURRENT)) {
             var restoredIndex = suffix("index-restored");
-            logger.debug("--> restoring index [{}] as archive [{}]", index, restoredIndex);
+            logger.debug("--> restoring index [{}] as [{}]", index, restoredIndex);
 
-            // Restoring the archive will fail as Elasticsearch does not support reading N-2 yet
+            // Restoring the index will fail as Elasticsearch does not support reading N-2 yet
             var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_restore");
             request.addParameter("wait_for_completion", "true");
             request.setJsonEntity(Strings.format("""
@@ -106,9 +91,20 @@ public void testRestoreIndex() throws Exception {
                   "rename_replacement": "%s",
                   "include_aliases": false
                 }""", index, restoredIndex));
-            var responseBody = createFromResponse(client().performRequest(request));
-            assertThat(responseBody.evaluate("snapshot.shards.total"), equalTo((int) responseBody.evaluate("snapshot.shards.failed")));
-            assertThat(responseBody.evaluate("snapshot.shards.successful"), equalTo(0));
+
+            var responseException = expectThrows(ResponseException.class, () -> client().performRequest(request));
+            assertEquals(RestStatus.INTERNAL_SERVER_ERROR.getStatus(), responseException.getResponse().getStatusLine().getStatusCode());
+            assertThat(
+                responseException.getMessage(),
+                allOf(
+                    containsString("cannot restore index [[" + index),
+                    containsString("because it cannot be upgraded"),
+                    containsString("has current compatibility version [" + VERSION_MINUS_2 + '-' + VERSION_MINUS_1.getMajor() + ".0.0]"),
+                    containsString("but the minimum compatible version is [" + VERSION_MINUS_1.getMajor() + ".0.0]."),
+                    containsString("It should be re-indexed in Elasticsearch " + VERSION_MINUS_1.getMajor() + ".x"),
+                    containsString("before upgrading to " + VERSION_CURRENT)
+                )
+            );
         }
     }
 }
diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartSearchableSnapshotIndexCompatibilityIT.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartSearchableSnapshotIndexCompatibilityIT.java
new file mode 100644
index 0000000000000..a7dc5e41fd327
--- /dev/null
+++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartSearchableSnapshotIndexCompatibilityIT.java
@@ -0,0 +1,190 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.lucene;
+
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.repositories.fs.FsRepository;
+import org.elasticsearch.test.cluster.util.Version;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class FullClusterRestartSearchableSnapshotIndexCompatibilityIT extends FullClusterRestartIndexCompatibilityTestCase {
+
+    static {
+        clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial")
+            .setting("xpack.searchable.snapshot.shared_cache.size", "16MB")
+            .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB");
+    }
+
+    public FullClusterRestartSearchableSnapshotIndexCompatibilityIT(Version version) {
+        super(version);
+    }
+
+    /**
+     * Creates an index and a snapshot on N-2, then mounts the snapshot on N.
+     */
+    public void testSearchableSnapshot() throws Exception {
+        final String repository = suffix("repository");
+        final String snapshot = suffix("snapshot");
+        final String index = suffix("index");
+        final int numDocs = 1234;
+
+        if (isFullyUpgradedTo(VERSION_MINUS_2)) {
+            logger.debug("--> registering repository [{}]", repository);
+            registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings());
+
+            logger.debug("--> creating index [{}]", index);
+            createIndex(
+                client(),
+                index,
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
+                    .build()
+            );
+
+            logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);
+            indexDocs(index, numDocs);
+
+            logger.debug("--> creating snapshot [{}]", snapshot);
+            createSnapshot(client(), repository, snapshot, true);
+            return;
+        }
+
+        if (isFullyUpgradedTo(VERSION_MINUS_1)) {
+            ensureGreen(index);
+
+            assertThat(indexVersion(index), equalTo(VERSION_MINUS_2));
+            assertDocCount(client(), index, numDocs);
+
+            logger.debug("--> deleting index [{}]", index);
+            deleteIndex(index);
+            return;
+        }
+
+        if (isFullyUpgradedTo(VERSION_CURRENT)) {
+            var mountedIndex = suffix("index-mounted");
+            logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex);
+            mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex);
+            ensureGreen(mountedIndex);
+
+            assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2));
+            assertDocCount(client(), mountedIndex, numDocs);
+
+            updateRandomIndexSettings(mountedIndex);
+            updateRandomMappings(mountedIndex);
+
+            logger.debug("--> adding replica to test peer-recovery");
+            updateIndexSettings(mountedIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1));
+            ensureGreen(mountedIndex);
+
+            logger.debug("--> closing index [{}]", mountedIndex);
+            closeIndex(mountedIndex);
+            ensureGreen(mountedIndex);
+
+            logger.debug("--> adding replica to test peer-recovery for closed shards");
+            updateIndexSettings(mountedIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2));
+            ensureGreen(mountedIndex);
+
+            logger.debug("--> re-opening index [{}]", mountedIndex);
+            openIndex(mountedIndex);
+            ensureGreen(mountedIndex);
+
+            assertDocCount(client(), mountedIndex, numDocs);
+
+            logger.debug("--> deleting index [{}]", mountedIndex);
+            deleteIndex(mountedIndex);
+        }
+    }
+
+    /**
+     * Creates an index and a snapshot on N-2, mounts the snapshot on N -1 and then upgrades to N.
+     */
+    public void testSearchableSnapshotUpgrade() throws Exception {
+        final String mountedIndex = suffix("index-mounted");
+        final String repository = suffix("repository");
+        final String snapshot = suffix("snapshot");
+        final String index = suffix("index");
+        final int numDocs = 4321;
+
+        if (isFullyUpgradedTo(VERSION_MINUS_2)) {
+            logger.debug("--> registering repository [{}]", repository);
+            registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings());
+
+            logger.debug("--> creating index [{}]", index);
+            createIndex(
+                client(),
+                index,
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
+                    .build()
+            );
+
+            logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);
+            indexDocs(index, numDocs);
+
+            logger.debug("--> creating snapshot [{}]", snapshot);
+            createSnapshot(client(), repository, snapshot, true);
+
+            logger.debug("--> deleting index [{}]", index);
+            deleteIndex(index);
+            return;
+        }
+
+        if (isFullyUpgradedTo(VERSION_MINUS_1)) {
+            logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex);
+            mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex);
+
+            ensureGreen(mountedIndex);
+
+            updateRandomIndexSettings(mountedIndex);
+            updateRandomMappings(mountedIndex);
+
+            assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2));
+            assertDocCount(client(), mountedIndex, numDocs);
+
+            logger.debug("--> adding replica to test replica upgrade");
+            updateIndexSettings(mountedIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1));
+            ensureGreen(mountedIndex);
+
+            if (randomBoolean()) {
+                logger.debug("--> random closing of index [{}] before upgrade", mountedIndex);
+                closeIndex(mountedIndex);
+                ensureGreen(mountedIndex);
+            }
+            return;
+        }
+
+        if (isFullyUpgradedTo(VERSION_CURRENT)) {
+            ensureGreen(mountedIndex);
+
+            if (isIndexClosed(mountedIndex)) {
+                logger.debug("--> re-opening index [{}] after upgrade", mountedIndex);
+                openIndex(mountedIndex);
+                ensureGreen(mountedIndex);
+            }
+
+            assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2));
+            assertDocCount(client(), mountedIndex, numDocs);
+
+            updateRandomIndexSettings(mountedIndex);
+            updateRandomMappings(mountedIndex);
+
+            logger.debug("--> adding replica to test peer-recovery");
+            updateIndexSettings(mountedIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2));
+            ensureGreen(mountedIndex);
+        }
+    }
+}
diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeIndexCompatibilityTestCase.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeIndexCompatibilityTestCase.java
new file mode 100644
index 0000000000000..03b6a9292e355
--- /dev/null
+++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeIndexCompatibilityTestCase.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.lucene;
+
+import com.carrotsearch.randomizedtesting.TestMethodAndParams;
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
+
+import org.elasticsearch.test.cluster.util.Version;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.test.cluster.util.Version.CURRENT;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
+
+/**
+ * Test suite for Lucene indices backward compatibility with N-2 versions during rolling upgrades. The test suite creates a cluster in N-2
+ * version, then upgrades each node sequentially to N-1 version and finally upgrades each node sequentially to the current version. Test
+ * methods are executed after each node upgrade.
+ */
+@TestCaseOrdering(RollingUpgradeIndexCompatibilityTestCase.TestCaseOrdering.class)
+public abstract class RollingUpgradeIndexCompatibilityTestCase extends AbstractIndexCompatibilityTestCase {
+
+    private final List<Version> nodesVersions;
+
+    public RollingUpgradeIndexCompatibilityTestCase(@Name("cluster") List<Version> nodesVersions) {
+        this.nodesVersions = nodesVersions;
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        return Stream.of(
+            // Begin on N-2
+            List.of(VERSION_MINUS_2, VERSION_MINUS_2, VERSION_MINUS_2),
+            // Rolling upgrade to VERSION_MINUS_1
+            List.of(VERSION_MINUS_1, VERSION_MINUS_2, VERSION_MINUS_2),
+            List.of(VERSION_MINUS_1, VERSION_MINUS_1, VERSION_MINUS_2),
+            List.of(VERSION_MINUS_1, VERSION_MINUS_1, VERSION_MINUS_1),
+            // Rolling upgrade to CURRENT
+            List.of(CURRENT, VERSION_MINUS_1, VERSION_MINUS_1),
+            List.of(CURRENT, CURRENT, VERSION_MINUS_1),
+            List.of(CURRENT, CURRENT, CURRENT)
+        ).map(nodesVersion -> new Object[] { nodesVersion }).toList();
+    }
+
+    @Override
+    protected void maybeUpgrade() throws Exception {
+        assertThat(nodesVersions, hasSize(NODES));
+
+        for (int i = 0; i < NODES; i++) {
+            var nodeName = cluster().getName(i);
+
+            var expectedNodeVersion = nodesVersions.get(i);
+            assertThat(expectedNodeVersion, notNullValue());
+
+            var currentNodeVersion = nodesVersions().get(nodeName);
+            assertThat(currentNodeVersion, notNullValue());
+            assertThat(currentNodeVersion.onOrBefore(expectedNodeVersion), equalTo(true));
+
+            if (currentNodeVersion.equals(expectedNodeVersion) == false) {
+                closeClients();
+                cluster().upgradeNodeToVersion(i, expectedNodeVersion);
+                initClient();
+            }
+
+            currentNodeVersion = nodesVersions().get(nodeName);
+            assertThat(currentNodeVersion, equalTo(expectedNodeVersion));
+        }
+    }
+
+    /**
+     * Execute the test suite with the parameters provided by the {@link #parameters()} in nodes versions order.
+     */
+    public static class TestCaseOrdering implements Comparator<TestMethodAndParams> {
+        @Override
+        public int compare(TestMethodAndParams o1, TestMethodAndParams o2) {
+            List<?> nodesVersions1 = asInstanceOf(List.class, o1.getInstanceArguments().get(0));
+            assertThat(nodesVersions1, hasSize(NODES));
+            List<?> nodesVersions2 = asInstanceOf(List.class, o2.getInstanceArguments().get(0));
+            assertThat(nodesVersions2, hasSize(NODES));
+            for (int i = 0; i < NODES; i++) {
+                var nodeVersion1 = asInstanceOf(Version.class, nodesVersions1.get(i));
+                var nodeVersion2 = asInstanceOf(Version.class, nodesVersions2.get(i));
+                var result = nodeVersion1.compareTo(nodeVersion2);
+                if (result != 0) {
+                    return result;
+                }
+            }
+            return 0;
+        }
+    }
+}
diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeSearchableSnapshotIndexCompatibilityIT.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeSearchableSnapshotIndexCompatibilityIT.java
new file mode 100644
index 0000000000000..1117d36024bf0
--- /dev/null
+++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeSearchableSnapshotIndexCompatibilityIT.java
@@ -0,0 +1,165 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.lucene;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.repositories.fs.FsRepository;
+import org.elasticsearch.test.cluster.util.Version;
+
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class RollingUpgradeSearchableSnapshotIndexCompatibilityIT extends RollingUpgradeIndexCompatibilityTestCase {
+
+    static {
+        clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial")
+            .setting("xpack.searchable.snapshot.shared_cache.size", "16MB")
+            .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB");
+    }
+
+    public RollingUpgradeSearchableSnapshotIndexCompatibilityIT(List<Version> nodesVersion) {
+        super(nodesVersion);
+    }
+
+    /**
+     * Creates an index and a snapshot on N-2, then mounts the snapshot during rolling upgrades.
+     */
+    public void testMountSearchableSnapshot() throws Exception {
+        final String repository = suffix("repository");
+        final String snapshot = suffix("snapshot");
+        final String index = suffix("index-rolling-upgrade");
+        final var mountedIndex = suffix("index-rolling-upgrade-mounted");
+        final int numDocs = 3145;
+
+        if (isFullyUpgradedTo(VERSION_MINUS_2)) {
+            logger.debug("--> registering repository [{}]", repository);
+            registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings());
+
+            logger.debug("--> creating index [{}]", index);
+            createIndex(
+                client(),
+                index,
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
+                    .build()
+            );
+
+            logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);
+            indexDocs(index, numDocs);
+
+            logger.debug("--> creating snapshot [{}]", snapshot);
+            createSnapshot(client(), repository, snapshot, true);
+
+            logger.debug("--> deleting index [{}]", index);
+            deleteIndex(index);
+            return;
+        }
+
+        boolean success = false;
+        try {
+            logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex);
+            mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex);
+            ensureGreen(mountedIndex);
+
+            updateRandomIndexSettings(mountedIndex);
+            updateRandomMappings(mountedIndex);
+
+            assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2));
+            assertDocCount(client(), mountedIndex, numDocs);
+
+            logger.debug("--> closing mounted index [{}]", mountedIndex);
+            closeIndex(mountedIndex);
+            ensureGreen(mountedIndex);
+
+            logger.debug("--> re-opening index [{}]", mountedIndex);
+            openIndex(mountedIndex);
+            ensureGreen(mountedIndex);
+
+            logger.debug("--> deleting mounted index [{}]", mountedIndex);
+            deleteIndex(mountedIndex);
+
+            success = true;
+        } finally {
+            if (success == false) {
+                try {
+                    client().performRequest(new Request("DELETE", "/" + mountedIndex));
+                } catch (ResponseException e) {
+                    logger.warn("Failed to delete mounted index [" + mountedIndex + ']', e);
+                }
+            }
+        }
+    }
+
+    /**
+     * Creates an index and a snapshot on N-2, mounts the snapshot and ensures it remains searchable during rolling upgrades.
+     */
+    public void testSearchableSnapshotUpgrade() throws Exception {
+        final String mountedIndex = suffix("index-rolling-upgraded-mounted");
+        final String repository = suffix("repository");
+        final String snapshot = suffix("snapshot");
+        final String index = suffix("index-rolling-upgraded");
+        final int numDocs = 2143;
+
+        if (isFullyUpgradedTo(VERSION_MINUS_2)) {
+            logger.debug("--> registering repository [{}]", repository);
+            registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings());
+
+            logger.debug("--> creating index [{}]", index);
+            createIndex(
+                client(),
+                index,
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
+                    .build()
+            );
+
+            logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);
+            indexDocs(index, numDocs);
+
+            logger.debug("--> creating snapshot [{}]", snapshot);
+            createSnapshot(client(), repository, snapshot, true);
+
+            logger.debug("--> deleting index [{}]", index);
+            deleteIndex(index);
+
+            logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex);
+            mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex);
+        }
+
+        ensureGreen(mountedIndex);
+
+        if (isIndexClosed(mountedIndex)) {
+            logger.debug("--> re-opening index [{}] after upgrade", mountedIndex);
+            openIndex(mountedIndex);
+            ensureGreen(mountedIndex);
+        }
+
+        updateRandomIndexSettings(mountedIndex);
+        updateRandomMappings(mountedIndex);
+
+        assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2));
+        assertDocCount(client(), mountedIndex, numDocs);
+
+        if (randomBoolean()) {
+            logger.debug("--> random closing of index [{}] before upgrade", mountedIndex);
+            closeIndex(mountedIndex);
+            ensureGreen(mountedIndex);
+        }
+    }
+}
diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java
deleted file mode 100644
index 4f348b7fb122f..0000000000000
--- a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.lucene;
-
-import org.elasticsearch.client.Request;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.repositories.fs.FsRepository;
-import org.elasticsearch.test.cluster.util.Version;
-
-import java.util.stream.IntStream;
-
-import static org.elasticsearch.test.rest.ObjectPath.createFromResponse;
-import static org.hamcrest.Matchers.allOf;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-
-public class SearchableSnapshotCompatibilityIT extends AbstractLuceneIndexCompatibilityTestCase {
-
-    static {
-        clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial")
-            .setting("xpack.searchable.snapshot.shared_cache.size", "16MB")
-            .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB");
-    }
-
-    public SearchableSnapshotCompatibilityIT(Version version) {
-        super(version);
-    }
-
-    // TODO Add a test to mount the N-2 index on N-1 and then search it on N
-
-    public void testSearchableSnapshot() throws Exception {
-        final String repository = suffix("repository");
-        final String snapshot = suffix("snapshot");
-        final String index = suffix("index");
-        final int numDocs = 1234;
-
-        logger.debug("--> registering repository [{}]", repository);
-        registerRepository(
-            client(),
-            repository,
-            FsRepository.TYPE,
-            true,
-            Settings.builder().put("location", REPOSITORY_PATH.getRoot().getPath()).build()
-        );
-
-        if (VERSION_MINUS_2.equals(clusterVersion())) {
-            logger.debug("--> creating index [{}]", index);
-            createIndex(
-                client(),
-                index,
-                Settings.builder()
-                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                    .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
-                    .build()
-            );
-
-            logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);
-            final var bulks = new StringBuilder();
-            IntStream.range(0, numDocs).forEach(n -> bulks.append(Strings.format("""
-                {"index":{"_id":"%s","_index":"%s"}}
-                {"test":"test"}
-                """, n, index)));
-
-            var bulkRequest = new Request("POST", "/_bulk");
-            bulkRequest.setJsonEntity(bulks.toString());
-            var bulkResponse = client().performRequest(bulkRequest);
-            assertOK(bulkResponse);
-            assertThat(entityAsMap(bulkResponse).get("errors"), allOf(notNullValue(), is(false)));
-
-            logger.debug("--> creating snapshot [{}]", snapshot);
-            createSnapshot(client(), repository, snapshot, true);
-            return;
-        }
-
-        if (VERSION_MINUS_1.equals(clusterVersion())) {
-            ensureGreen(index);
-
-            assertThat(indexLuceneVersion(index), equalTo(VERSION_MINUS_2));
-            assertDocCount(client(), index, numDocs);
-
-            logger.debug("--> deleting index [{}]", index);
-            deleteIndex(index);
-            return;
-        }
-
-        if (VERSION_CURRENT.equals(clusterVersion())) {
-            var mountedIndex = suffix("index-mounted");
-            logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex);
-
-            // Mounting the index will fail as Elasticsearch does not support reading N-2 yet
-            var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_mount");
-            request.addParameter("wait_for_completion", "true");
-            var storage = randomBoolean() ? "shared_cache" : "full_copy";
-            request.addParameter("storage", storage);
-            request.setJsonEntity(Strings.format("""
-                {
-                  "index": "%s",
-                  "renamed_index": "%s"
-                }""", index, mountedIndex));
-            var responseBody = createFromResponse(client().performRequest(request));
-            assertThat(responseBody.evaluate("snapshot.shards.total"), equalTo((int) responseBody.evaluate("snapshot.shards.failed")));
-            assertThat(responseBody.evaluate("snapshot.shards.successful"), equalTo(0));
-        }
-    }
-}
diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle
index d8f906b23d523..8dd5031c07822 100644
--- a/qa/mixed-cluster/build.gradle
+++ b/qa/mixed-cluster/build.gradle
@@ -67,6 +67,12 @@ excludeList.add('indices.resolve_index/20_resolve_system_index/*')
 // Excluded because the error has changed
 excludeList.add('aggregations/percentiles_hdr_metric/Negative values test')
 
+// sync_id is removed in 9.0
+excludeList.add("cat.shards/10_basic/Help")
+
+// Can't work until auto-expand replicas is 0-1 for synonyms index
+excludeList.add("synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set")
+
 def clusterPath = getPath()
 
 buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java
index 0cd2823080b9b..808aec92fb35d 100644
--- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java
+++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java
@@ -206,13 +206,32 @@ public static void waitForElasticsearchToStart() {
                 ps output:
                 %s
 
-                stdout():
+                Stdout:
                 %s
 
                 Stderr:
+                %s
+
+                Thread dump:
                 %s\
-                """, psOutput, dockerLogs.stdout(), dockerLogs.stderr()));
+                """, psOutput, dockerLogs.stdout(), dockerLogs.stderr(), getThreadDump()));
+        }
+    }
+
+    /**
+     * @return output of jstack for currently running Java process
+     */
+    private static String getThreadDump() {
+        try {
+            String pid = dockerShell.run("/usr/share/elasticsearch/jdk/bin/jps | grep -v 'Jps' | awk '{print $1}'").stdout();
+            if (pid.isEmpty() == false) {
+                return dockerShell.run("/usr/share/elasticsearch/jdk/bin/jstack " + Integer.parseInt(pid)).stdout();
+            }
+        } catch (Exception e) {
+            logger.error("Failed to get thread dump", e);
         }
+
+        return "";
     }
 
     /**
diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java
index 5dc47993072a8..554ae871ea9fa 100644
--- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java
+++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java
@@ -29,6 +29,9 @@
  */
 public class DockerRun {
 
+    // Use less secure entropy source to avoid hanging when generating certificates
+    private static final String DEFAULT_JAVA_OPTS = "-Djava.security.egd=file:/dev/urandom";
+
     private Distribution distribution;
     private final Map<String, String> envVars = new HashMap<>();
     private final Map<Path, Path> volumes = new HashMap<>();
@@ -112,6 +115,11 @@ String build() {
         // Limit container memory
         cmd.add("--memory " + memory);
 
+        // Add default java opts
+        for (String envVar : List.of("CLI_JAVA_OPTS", "ES_JAVA_OPTS")) {
+            this.envVars.put(envVar, this.envVars.getOrDefault(envVar, "") + " " + DEFAULT_JAVA_OPTS);
+        }
+
         this.envVars.forEach((key, value) -> cmd.add("--env " + key + "=\"" + value + "\""));
 
         // Map ports in the container to the host, so that we can send requests
diff --git a/qa/remote-clusters/build.gradle b/qa/remote-clusters/build.gradle
index 8a28ea8fd1c5c..e3d42850375d9 100644
--- a/qa/remote-clusters/build.gradle
+++ b/qa/remote-clusters/build.gradle
@@ -39,14 +39,22 @@ elasticsearch_distributions {
   }
 }
 
+
+interface Injected {
+  @Inject
+  FileSystemOperations getFs()
+}
+
 tasks.named("preProcessFixture").configure {
   dependsOn "copyNodeKeyMaterial", elasticsearch_distributions.docker
+  def injected = project.objects.newInstance(Injected)
+
   doLast {
     // tests expect to have an empty repo
-    project.delete(
-      "${testFixturesDir}/repo",
-      "${testFixturesDir}/oss-repo"
-    )
+    injected.fs.delete {
+      it.delete("${testFixturesDir}/repo")
+      it.delete("${testFixturesDir}/oss-repo")
+    }
     createAndSetWritable(
       "${testFixturesDir}/repo",
       "${testFixturesDir}/oss-repo",
@@ -69,7 +77,8 @@ dockerCompose {
 
 def createAndSetWritable(Object... locations) {
   locations.each { location ->
-    File file = file(location)
+    println "location = $location"
+    File file = new File(location)
     file.mkdirs()
     file.setWritable(true, false)
   }
diff --git a/qa/remote-clusters/output.log b/qa/remote-clusters/output.log
new file mode 100644
index 0000000000000..11eebe6fb8fda
--- /dev/null
+++ b/qa/remote-clusters/output.log
@@ -0,0 +1,125 @@
+Using gradle at '/Users/rene/dev/elastic/elasticsearch/gradlew' to run buildfile '/Users/rene/dev/elastic/elasticsearch/qa/remote-clusters/build.gradle':
+
+Starting a Gradle Daemon (subsequent builds will be faster)
+Calculating task graph as no cached configuration is available for tasks: check
+
+> Task :build-conventions:compileJava
+Note: Some input files use or override a deprecated API.
+Note: Recompile with -Xlint:deprecation for details.
+Note: /Users/rene/dev/elastic/elasticsearch/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java uses unchecked or unsafe operations.
+Note: Recompile with -Xlint:unchecked for details.
+
+> Task :build-conventions:pluginDescriptors
+> Task :build-conventions:processResources
+> Task :build-conventions:classes
+> Task :build-conventions:jar
+> Task :build-tools:reaper:compileJava FROM-CACHE
+> Task :build-tools:reaper:processResources NO-SOURCE
+> Task :build-tools:reaper:classes UP-TO-DATE
+> Task :build-tools-internal:extractPluginRequests FROM-CACHE
+> Task :build-tools:reaper:jar
+> Task :build-tools-internal:generatePluginAdapters FROM-CACHE
+> Task :build-tools-internal:pluginDescriptors
+> Task :build-tools:compileJava FROM-CACHE
+> Task :build-tools:compileGroovy NO-SOURCE
+> Task :build-tools:generateVersionProperties FROM-CACHE
+> Task :build-tools:pluginDescriptors
+> Task :build-tools:processResources
+> Task :build-tools:classes
+> Task :build-tools:jar
+> Task :build-tools-internal:processResources
+
+> Task :build-tools-internal:compileJava
+Note: Some input files use or override a deprecated API.
+Note: Recompile with -Xlint:deprecation for details.
+Note: Some input files use unchecked or unsafe operations.
+Note: Recompile with -Xlint:unchecked for details.
+
+> Task :build-tools-internal:compileGroovy
+> Task :build-tools-internal:compileGroovyPlugins
+> Task :build-tools-internal:classes
+> Task :build-tools-internal:jar
+=======================================
+Elasticsearch Build Hamster says Hello!
+  Gradle Version        : 8.12
+  OS Info               : Mac OS X 15.2 (aarch64)
+  JDK Version           : 21.0.5+9-LTS-239 (Oracle)
+  JAVA_HOME             : /Users/rene/.sdkman/candidates/java/21.0.5-oracle
+  Random Testing Seed   : 8D9A9CFD8E09C560
+  In FIPS 140 mode      : false
+=======================================
+> Task :qa:remote-clusters:processTestResources
+> Task :server:generateModulesList
+> Task :server:generatePluginsList
+> Task :qa:remote-clusters:copyCheckstyleConf
+> Task :qa:remote-clusters:filepermissions
+> Task :client:sniffer:processResources NO-SOURCE
+> Task :libs:core:processResources NO-SOURCE
+> Task :libs:cli:processResources NO-SOURCE
+> Task :libs:entitlement:processResources NO-SOURCE
+> Task :libs:entitlement:bridge:processResources NO-SOURCE
+> Task :libs:entitlement:asm-provider:processResources
+> Task :libs:entitlement:processMain23Resources NO-SOURCE
+> Task :libs:entitlement:bridge:processMain23Resources NO-SOURCE
+> Task :libs:geo:processResources NO-SOURCE
+> Task :libs:logging:processResources NO-SOURCE
+> Task :libs:lz4:processResources NO-SOURCE
+> Task :libs:native:processResources NO-SOURCE
+> Task :libs:native:processMain22Resources NO-SOURCE
+> Task :libs:plugin-analysis-api:processResources NO-SOURCE
+> Task :libs:plugin-api:processResources NO-SOURCE
+> Task :libs:secure-sm:processResources NO-SOURCE
+> Task :libs:simdvec:processResources NO-SOURCE
+> Task :libs:simdvec:processMain21Resources NO-SOURCE
+> Task :libs:simdvec:processMain22Resources NO-SOURCE
+> Task :client:rest:processResources
+> Task :libs:tdigest:processResources NO-SOURCE
+> Task :libs:x-content:impl:processResources
+> Task :libs:x-content:processResources NO-SOURCE
+> Task :modules:transport-netty4:processResources NO-SOURCE
+> Task :server:processResources
+> Task :libs:ssl-config:processResources NO-SOURCE
+> Task :test:yaml-rest-runner:processResources NO-SOURCE
+> Task :qa:remote-clusters:forbiddenApisResources
+> Task :qa:remote-clusters:forbiddenPatterns
+> Task :qa:remote-clusters:licenseHeaders
+> Task :test:logger-usage:processResources NO-SOURCE
+> Task :libs:grok:processResources
+> Task :qa:remote-clusters:testingConventions UP-TO-DATE
+> Task :qa:remote-clusters:copyNodeKeyMaterial FAILED
+> Task :spotlessInternalRegisterDependencies
+> Task :test:framework:processResources
+> Task :libs:plugin-api:compileJava
+> Task :libs:logging:compileJava
+> Task :libs:entitlement:bridge:compileJava
+
+> Task :libs:secure-sm:compileJava
+Note: Some input files use or override a deprecated API that is marked for removal.
+Note: Recompile with -Xlint:removal for details.
+
+> Task :test:logger-usage:compileJava
+> Task :qa:remote-clusters:spotlessJava
+> Task :libs:grok:compileJava
+> Task :libs:geo:compileJava
+> Task :client:rest:compileJava
+> Task :qa:remote-clusters:checkstyleTest
+
+[Incubating] Problems report is available at: file:///Users/rene/dev/elastic/elasticsearch/build/reports/problems/problems-report.html
+
+FAILURE: Build failed with an exception.
+
+* Where:
+Build file '/Users/rene/dev/elastic/elasticsearch/qa/remote-clusters/build.gradle' line: 27
+
+* What went wrong:
+Execution failed for task ':qa:remote-clusters:copyNodeKeyMaterial'.
+> Cannot reference a Gradle script object from a Groovy closure as these are not supported with the configuration cache.
+
+* Try:
+> Run with --stacktrace option to get the stack trace.
+> Run with --info or --debug option to get more log output.
+> Get more help at https://help.gradle.org.
+
+BUILD FAILED in 1m 4s
+45 actionable tasks: 40 executed, 5 from cache
+Configuration cache entry stored.
diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ClusterFeatureMigrationIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ClusterFeatureMigrationIT.java
deleted file mode 100644
index 2b3e10acc15ed..0000000000000
--- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ClusterFeatureMigrationIT.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.upgrades;
-
-import com.carrotsearch.randomizedtesting.annotations.Name;
-
-import org.elasticsearch.client.Request;
-import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.features.FeatureService;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.hasSize;
-
-public class ClusterFeatureMigrationIT extends AbstractRollingUpgradeTestCase {
-
-    @Before
-    public void checkMigrationVersion() {
-        assumeFalse(
-            "This checks migrations from before cluster features were introduced",
-            oldClusterHasFeature(FeatureService.FEATURES_SUPPORTED)
-        );
-    }
-
-    public ClusterFeatureMigrationIT(@Name("upgradedNodes") int upgradedNodes) {
-        super(upgradedNodes);
-    }
-
-    public void testClusterFeatureMigration() throws IOException {
-        if (isUpgradedCluster()) {
-            // check the nodes all have a feature in their cluster state (there should always be features_supported)
-            var response = entityAsMap(adminClient().performRequest(new Request("GET", "/_cluster/state/nodes")));
-            List<?> nodeFeatures = (List<?>) XContentMapValues.extractValue("nodes_features", response);
-            assertThat(nodeFeatures, hasSize(adminClient().getNodes().size()));
-
-            Map<String, List<?>> features = nodeFeatures.stream()
-                .map(o -> (Map<?, ?>) o)
-                .collect(Collectors.toMap(m -> (String) m.get("node_id"), m -> (List<?>) m.get("features")));
-
-            Set<String> missing = features.entrySet()
-                .stream()
-                .filter(e -> e.getValue().contains(FeatureService.FEATURES_SUPPORTED.id()) == false)
-                .map(Map.Entry::getKey)
-                .collect(Collectors.toSet());
-            assertThat(missing + " out of " + features.keySet() + " does not have the required feature", missing, empty());
-        }
-    }
-}
diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java
index d9adec47ff483..30367bf55d8cc 100644
--- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java
+++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java
@@ -11,7 +11,6 @@
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 
-import org.elasticsearch.Build;
 import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesRequest;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.cluster.metadata.DesiredNode;
@@ -84,7 +83,7 @@ private void addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(int ve
                         randomDoubleProcessorCount(),
                         ByteSizeValue.ofGb(randomIntBetween(10, 24)),
                         ByteSizeValue.ofGb(randomIntBetween(128, 256)),
-                        clusterHasFeature(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED) ? null : Build.current().version()
+                        null
                     )
                 )
                 .toList();
@@ -96,7 +95,7 @@ private void addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(int ve
                     new DesiredNode.ProcessorsRange(minProcessors, minProcessors + randomIntBetween(10, 20)),
                     ByteSizeValue.ofGb(randomIntBetween(10, 24)),
                     ByteSizeValue.ofGb(randomIntBetween(128, 256)),
-                    clusterHasFeature(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED) ? null : Build.current().version()
+                    null
                 );
             }).toList();
         }
diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FieldCapsIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FieldCapsIT.java
index a1b3dcc37c45b..c6f4588fb4cd2 100644
--- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FieldCapsIT.java
+++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FieldCapsIT.java
@@ -12,7 +12,6 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 
 import org.apache.http.HttpHost;
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.Build;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.client.Request;
@@ -40,7 +39,6 @@
  * In 8.2 we also added the ability to filter fields by type and metadata, with some post-hoc filtering applied on
  * the co-ordinating node if older nodes were included in the system
  */
-@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103473")
 public class FieldCapsIT extends AbstractRollingUpgradeTestCase {
 
     public FieldCapsIT(@Name("upgradedNodes") int upgradedNodes) {
diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java
index d6254d091a868..c48ae9ba1843b 100644
--- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java
+++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java
@@ -12,9 +12,15 @@
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction;
+import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsAction;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryAction;
+import org.elasticsearch.action.admin.indices.template.get.GetComponentTemplateAction;
+import org.elasticsearch.action.admin.indices.template.get.GetComposableIndexTemplateAction;
+import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesAction;
+import org.elasticsearch.action.admin.indices.template.post.SimulateIndexTemplateAction;
+import org.elasticsearch.action.admin.indices.template.post.SimulateTemplateAction;
 import org.elasticsearch.action.support.CancellableActionTestPlugin;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.RefCountingListener;
@@ -66,6 +72,37 @@ public void testCatAliasesCancellation() {
         runRestActionCancellationTest(new Request(HttpGet.METHOD_NAME, "/_cat/aliases"), GetAliasesAction.NAME);
     }
 
+    public void testGetComponentTemplateCancellation() {
+        runRestActionCancellationTest(new Request(HttpGet.METHOD_NAME, "/_component_template"), GetComponentTemplateAction.NAME);
+    }
+
+    public void testGetIndexTemplateCancellation() {
+        runRestActionCancellationTest(new Request(HttpGet.METHOD_NAME, "/_template"), GetIndexTemplatesAction.NAME);
+    }
+
+    public void testGetComposableTemplateCancellation() {
+        runRestActionCancellationTest(new Request(HttpGet.METHOD_NAME, "/_index_template"), GetComposableIndexTemplateAction.NAME);
+    }
+
+    public void testSimulateTemplateCancellation() {
+        runRestActionCancellationTest(
+            new Request(HttpPost.METHOD_NAME, "/_index_template/_simulate/random_index_template"),
+            SimulateTemplateAction.NAME
+        );
+    }
+
+    public void testSimulateIndexTemplateCancellation() {
+        createIndex("test");
+        runRestActionCancellationTest(
+            new Request(HttpPost.METHOD_NAME, "/_index_template/_simulate_index/test"),
+            SimulateIndexTemplateAction.NAME
+        );
+    }
+
+    public void testClusterGetSettingsCancellation() {
+        runRestActionCancellationTest(new Request(HttpGet.METHOD_NAME, "/_cluster/settings"), ClusterGetSettingsAction.NAME);
+    }
+
     private void runRestActionCancellationTest(Request request, String actionName) {
         final var node = usually() ? internalCluster().getRandomNodeName() : internalCluster().startCoordinatingOnlyNode(Settings.EMPTY);
 
diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchErrorTraceIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchErrorTraceIT.java
new file mode 100644
index 0000000000000..6f9ab8ccdfdec
--- /dev/null
+++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchErrorTraceIT.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.http;
+
+import org.apache.http.entity.ContentType;
+import org.apache.http.nio.entity.NByteArrayEntity;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.search.MultiSearchRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.transport.TransportMessageListener;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xcontent.XContentType;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery;
+
+public class SearchErrorTraceIT extends HttpSmokeTestCase {
+    private AtomicBoolean hasStackTrace;
+
+    @Before
+    private void setupMessageListener() {
+        internalCluster().getDataNodeInstances(TransportService.class).forEach(ts -> {
+            ts.addMessageListener(new TransportMessageListener() {
+                @Override
+                public void onResponseSent(long requestId, String action, Exception error) {
+                    TransportMessageListener.super.onResponseSent(requestId, action, error);
+                    if (action.startsWith("indices:data/read/search")) {
+                        Optional<Throwable> throwable = ExceptionsHelper.unwrapCausesAndSuppressed(
+                            error,
+                            t -> t.getStackTrace().length > 0
+                        );
+                        hasStackTrace.set(throwable.isPresent());
+                    }
+                }
+            });
+        });
+    }
+
+    private void setupIndexWithDocs() {
+        createIndex("test1", "test2");
+        indexRandom(
+            true,
+            prepareIndex("test1").setId("1").setSource("field", "foo"),
+            prepareIndex("test2").setId("10").setSource("field", 5)
+        );
+        refresh();
+    }
+
+    public void testSearchFailingQueryErrorTraceDefault() throws IOException {
+        hasStackTrace = new AtomicBoolean();
+        setupIndexWithDocs();
+
+        Request searchRequest = new Request("POST", "/_search");
+        searchRequest.setJsonEntity("""
+            {
+                "query": {
+                    "simple_query_string" : {
+                        "query": "foo",
+                        "fields": ["field"]
+                    }
+                }
+            }
+            """);
+        getRestClient().performRequest(searchRequest);
+        assertFalse(hasStackTrace.get());
+    }
+
+    public void testSearchFailingQueryErrorTraceTrue() throws IOException {
+        hasStackTrace = new AtomicBoolean();
+        setupIndexWithDocs();
+
+        Request searchRequest = new Request("POST", "/_search");
+        searchRequest.setJsonEntity("""
+            {
+                "query": {
+                    "simple_query_string" : {
+                        "query": "foo",
+                        "fields": ["field"]
+                    }
+                }
+            }
+            """);
+        searchRequest.addParameter("error_trace", "true");
+        getRestClient().performRequest(searchRequest);
+        assertTrue(hasStackTrace.get());
+    }
+
+    public void testSearchFailingQueryErrorTraceFalse() throws IOException {
+        hasStackTrace = new AtomicBoolean();
+        setupIndexWithDocs();
+
+        Request searchRequest = new Request("POST", "/_search");
+        searchRequest.setJsonEntity("""
+            {
+                "query": {
+                    "simple_query_string" : {
+                        "query": "foo",
+                        "fields": ["field"]
+                    }
+                }
+            }
+            """);
+        searchRequest.addParameter("error_trace", "false");
+        getRestClient().performRequest(searchRequest);
+        assertFalse(hasStackTrace.get());
+    }
+
+    public void testMultiSearchFailingQueryErrorTraceDefault() throws IOException {
+        hasStackTrace = new AtomicBoolean();
+        setupIndexWithDocs();
+
+        XContentType contentType = XContentType.JSON;
+        MultiSearchRequest multiSearchRequest = new MultiSearchRequest().add(
+            new SearchRequest("test*").source(new SearchSourceBuilder().query(simpleQueryStringQuery("foo").field("field")))
+        );
+        Request searchRequest = new Request("POST", "/_msearch");
+        byte[] requestBody = MultiSearchRequest.writeMultiLineFormat(multiSearchRequest, contentType.xContent());
+        searchRequest.setEntity(
+            new NByteArrayEntity(requestBody, ContentType.create(contentType.mediaTypeWithoutParameters(), (Charset) null))
+        );
+        getRestClient().performRequest(searchRequest);
+        assertFalse(hasStackTrace.get());
+    }
+
+    public void testMultiSearchFailingQueryErrorTraceTrue() throws IOException {
+        hasStackTrace = new AtomicBoolean();
+        setupIndexWithDocs();
+
+        XContentType contentType = XContentType.JSON;
+        MultiSearchRequest multiSearchRequest = new MultiSearchRequest().add(
+            new SearchRequest("test*").source(new SearchSourceBuilder().query(simpleQueryStringQuery("foo").field("field")))
+        );
+        Request searchRequest = new Request("POST", "/_msearch");
+        byte[] requestBody = MultiSearchRequest.writeMultiLineFormat(multiSearchRequest, contentType.xContent());
+        searchRequest.setEntity(
+            new NByteArrayEntity(requestBody, ContentType.create(contentType.mediaTypeWithoutParameters(), (Charset) null))
+        );
+        searchRequest.addParameter("error_trace", "true");
+        getRestClient().performRequest(searchRequest);
+        assertTrue(hasStackTrace.get());
+    }
+
+    public void testMultiSearchFailingQueryErrorTraceFalse() throws IOException {
+        hasStackTrace = new AtomicBoolean();
+        setupIndexWithDocs();
+
+        XContentType contentType = XContentType.JSON;
+        MultiSearchRequest multiSearchRequest = new MultiSearchRequest().add(
+            new SearchRequest("test*").source(new SearchSourceBuilder().query(simpleQueryStringQuery("foo").field("field")))
+        );
+        Request searchRequest = new Request("POST", "/_msearch");
+        byte[] requestBody = MultiSearchRequest.writeMultiLineFormat(multiSearchRequest, contentType.xContent());
+        searchRequest.setEntity(
+            new NByteArrayEntity(requestBody, ContentType.create(contentType.mediaTypeWithoutParameters(), (Charset) null))
+        );
+        searchRequest.addParameter("error_trace", "false");
+        getRestClient().performRequest(searchRequest);
+
+        assertFalse(hasStackTrace.get());
+    }
+}
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml
index 2d3fa6b568381..4a5ceeb66f661 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml
@@ -222,10 +222,6 @@ setup:
         - headers
         - allowed_warnings
 
-  - requires:
-      cluster_features: ["simulate.mapping.validation.templates"]
-      reason: "ingest simulate index mapping validation added in 8.16"
-
   - do:
       headers:
         Content-Type: application/json
@@ -313,10 +309,6 @@ setup:
         - headers
         - allowed_warnings
 
-  - requires:
-      cluster_features: ["simulate.component.template.substitutions"]
-      reason: "ingest simulate component template substitutions added in 8.16"
-
   - do:
       headers:
         Content-Type: application/json
@@ -494,10 +486,6 @@ setup:
         - headers
         - allowed_warnings
 
-  - requires:
-      cluster_features: ["simulate.component.template.substitutions"]
-      reason: "ingest simulate component template substitutions added in 8.16"
-
   - do:
       headers:
         Content-Type: application/json
@@ -617,10 +605,6 @@ setup:
         - headers
         - allowed_warnings
 
-  - requires:
-      cluster_features: ["simulate.component.template.substitutions"]
-      reason: "ingest simulate component template substitutions added in 8.16"
-
   - do:
       headers:
         Content-Type: application/json
@@ -816,10 +800,6 @@ setup:
         - headers
         - allowed_warnings
 
-  - requires:
-      cluster_features: ["simulate.index.template.substitutions"]
-      reason: "ingest simulate index template substitutions added in 8.16"
-
   - do:
       headers:
         Content-Type: application/json
@@ -1010,10 +990,6 @@ setup:
         - headers
         - allowed_warnings
 
-  - requires:
-      cluster_features: ["simulate.index.template.substitutions"]
-      reason: "ingest simulate component template substitutions added in 8.16"
-
   - do:
       headers:
         Content-Type: application/json
@@ -1227,10 +1203,6 @@ setup:
         - headers
         - allowed_warnings
 
-  - requires:
-      cluster_features: ["simulate.mapping.addition"]
-      reason: "ingest simulate mapping addition added in 8.17"
-
   - do:
       headers:
         Content-Type: application/json
@@ -1463,10 +1435,6 @@ setup:
         - headers
         - allowed_warnings
 
-  - requires:
-      cluster_features: ["simulate.mapping.addition"]
-      reason: "ingest simulate mapping addition added in 8.17"
-
   - do:
       indices.put_template:
         name: my-legacy-template
@@ -1584,10 +1552,6 @@ setup:
         - headers
         - allowed_warnings
 
-  - requires:
-      cluster_features: ["simulate.support.non.template.mapping"]
-      reason: "ingest simulate support for indices with mappings that didn't come from templates added in 8.17"
-
   # A global match-everything legacy template is added to the cluster sometimes (rarely). We have to get rid of this template if it exists
   # because this test is making sure we get correct behavior when an index matches *no* template:
   - do:
@@ -1720,3 +1684,59 @@ setup:
   - match: { docs.0.doc._source.foo: 3 }
   - match: { docs.0.doc._source.bar: "some text value" }
   - not_exists: docs.0.doc.error
+
+---
+"Test ignored_fields":
+  - skip:
+      features:
+        - headers
+        - allowed_warnings
+
+  - requires:
+      cluster_features: ["simulate.ignored.fields"]
+      reason: "ingest simulate ignored fields added in 8.18"
+
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: nonexistent
+        body: >
+          {
+            "docs": [
+              {
+                "_index": "simulate-test",
+                "_id": "y9Es_JIBiw6_GgN-U0qy",
+                "_score": 1,
+                "_source": {
+                  "abc": "sfdsfsfdsfsfdsfsfdsfsfdsfsfdsf"
+                }
+              }
+            ],
+            "index_template_substitutions": {
+              "ind_temp": {
+                "index_patterns": ["simulate-test"],
+                "composed_of": ["simulate-test"]
+              }
+            },
+            "component_template_substitutions": {
+              "simulate-test": {
+                "template": {
+                  "mappings": {
+                    "dynamic": false,
+                    "properties": {
+                      "abc": {
+                        "type": "keyword",
+                        "ignore_above": 1
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "simulate-test" }
+  - match: { docs.0.doc._source.abc: "sfdsfsfdsfsfdsfsfdsfsfdsfsfdsf" }
+  - match: { docs.0.doc.ignored_fields: [ {"field": "abc"} ] }
+  - not_exists: docs.0.doc.error
diff --git a/qa/system-indices/build.gradle b/qa/system-indices/build.gradle
index d035b2f57d55a..c619d4f02e527 100644
--- a/qa/system-indices/build.gradle
+++ b/qa/system-indices/build.gradle
@@ -11,11 +11,11 @@ apply plugin: 'elasticsearch.base-internal-es-plugin'
 apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 esplugin {
-  name 'system-indices-qa'
-  description 'Plugin for performing QA of system indices'
-  classname 'org.elasticsearch.system.indices.SystemIndicesQA'
-  licenseFile rootProject.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
-  noticeFile rootProject.file('NOTICE.txt')
+  name = 'system-indices-qa'
+  description = 'Plugin for performing QA of system indices'
+  classname ='org.elasticsearch.system.indices.SystemIndicesQA'
+  licenseFile = rootProject.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
+  noticeFile = rootProject.file('NOTICE.txt')
 }
 
 testClusters.configureEach {
diff --git a/renovate.json b/renovate.json
index c1637ae651c1c..71c6301f8e0c2 100644
--- a/renovate.json
+++ b/renovate.json
@@ -7,8 +7,8 @@
   "schedule": [
     "after 1pm on tuesday"
   ],
-  "labels": [">non-issue", ":Delivery/Packaging", "Team:Delivery"],
-  "baseBranches": ["main", "8.x"],
+  "labels": [">non-issue", ":Delivery/Packaging", "Team:Delivery", "auto-merge-without-approval"],
+  "baseBranches": ["main", "8.x", "8.17", "8.16"],
   "packageRules": [
     {
       "groupName": "wolfi (versioned)",
diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle
index 7347d9c1312dd..e4b46b98cedda 100644
--- a/rest-api-spec/build.gradle
+++ b/rest-api-spec/build.gradle
@@ -13,7 +13,7 @@ restResources {
 }
 
 // REST API specifications are published under the Apache 2.0 License
-ext.projectLicenses.set(['The Apache Software License, Version 2.0': 'http://www.apache.org/licenses/LICENSE-2.0'])
+ext.projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')])
 licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt'))
 
 configurations {
@@ -60,13 +60,31 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("cat.aliases/10_basic/Deprecated local parameter", "CAT APIs not covered by compatibility policy")
   task.skipTest("cat.shards/10_basic/Help", "sync_id is removed in 9.0")
   task.skipTest("search/500_date_range/from, to, include_lower, include_upper deprecated", "deprecated parameters are removed in 9.0")
-  task.skipTest("tsdb/20_mapping/stored source is supported", "no longer serialize source_mode")
-  task.skipTest("tsdb/20_mapping/Synthetic source", "no longer serialize source_mode")
-  task.skipTest("logsdb/10_settings/create logs index", "no longer serialize source_mode")
-  task.skipTest("logsdb/20_source_mapping/stored _source mode is supported", "no longer serialize source_mode")
-  task.skipTest("logsdb/20_source_mapping/include/exclude is supported with stored _source", "no longer serialize source_mode")
-  task.skipTest("logsdb/20_source_mapping/synthetic _source is default", "no longer serialize source_mode")
-  task.skipTest("search/520_fetch_fields/fetch _seq_no via fields", "error code is changed from 5xx to 400 in 9.0")
+  task.skipTest("logsdb/10_settings/logsdb with default ignore dynamic beyond limit and default sorting", "skip until pr/118968 gets backported")
+  task.skipTest("logsdb/10_settings/logsdb with default ignore dynamic beyond limit and too low limit", "skip until pr/118968 gets backported")
+  task.skipTest("logsdb/10_settings/logsdb with default ignore dynamic beyond limit and subobjects false", "skip until pr/118968 gets backported")
+  task.skipTest("logsdb/10_settings/override sort missing settings", "skip until pr/118968 gets backported")
+  task.skipTest("logsdb/10_settings/override sort order settings", "skip until pr/118968 gets backported")
+  task.skipTest("logsdb/10_settings/override sort mode settings", "skip until pr/118968 gets backported")
   task.skipTest("search.vectors/41_knn_search_bbq_hnsw/Test knn search", "Scoring has changed in latest versions")
   task.skipTest("search.vectors/42_knn_search_bbq_flat/Test knn search", "Scoring has changed in latest versions")
+  task.skipTest("search.vectors/180_update_dense_vector_type/Test create and update dense vector mapping with bulk indexing", "waiting for #118774 backport")
+  task.skipTest("search.vectors/160_knn_query_missing_params/kNN query in a bool clause - missing num_candidates", "waiting for #118774 backport")
+  task.skipTest("search.vectors/110_knn_query_with_filter/Simple knn query", "waiting for #118774 backport")
+  task.skipTest("search.vectors/160_knn_query_missing_params/kNN search used in nested field - missing num_candidates", "waiting for #118774 backport")
+  task.skipTest("search.vectors/180_update_dense_vector_type/Test create and update dense vector mapping to int4 with per-doc indexing and flush", "waiting for #118774 backport")
+  task.skipTest("search.vectors/110_knn_query_with_filter/PRE_FILTER: knn query with internal filter as pre-filter", "waiting for #118774 backport")
+  task.skipTest("search.vectors/180_update_dense_vector_type/Index, update and merge", "waiting for #118774 backport")
+  task.skipTest("search.vectors/160_knn_query_missing_params/kNN query with missing num_candidates param - size provided", "waiting for #118774 backport")
+  task.skipTest("search.vectors/110_knn_query_with_filter/POST_FILTER: knn query with filter from a parent bool query as post-filter", "waiting for #118774 backport")
+  task.skipTest("search.vectors/120_knn_query_multiple_shards/Aggregations with collected number of docs depends on num_candidates", "waiting for #118774 backport")
+  task.skipTest("search.vectors/180_update_dense_vector_type/Test create and update dense vector mapping with per-doc indexing and flush", "waiting for #118774 backport")
+  task.skipTest("search.vectors/110_knn_query_with_filter/PRE_FILTER: knn query with alias filter as pre-filter", "waiting for #118774 backport")
+  task.skipTest("search.vectors/140_knn_query_with_other_queries/Function score query with knn query", "waiting for #118774 backport")
+  task.skipTest("search.vectors/130_knn_query_nested_search/nested kNN search inner_hits size > 1", "waiting for #118774 backport")
+  task.skipTest("search.vectors/110_knn_query_with_filter/PRE_FILTER: pre-filter across multiple aliases", "waiting for #118774 backport")
+  task.skipTest("search.vectors/160_knn_query_missing_params/kNN search in a dis_max query - missing num_candidates", "waiting for #118774 backport")
+  task.skipTest("search.highlight/30_max_analyzed_offset/Plain highlighter with max_analyzed_offset < 0 should FAIL", "semantics of test has changed")
+  task.skipTest("indices.create/10_basic/Create lookup index", "default auto_expand_replicas was removed")
+  task.skipTest("indices.create/10_basic/Create lookup index with one shard", "default auto_expand_replicas was removed")
 })
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.exists_component_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.exists_component_template.json
index 818d034ca8158..b2503659329a3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.exists_component_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.exists_component_template.json
@@ -28,9 +28,10 @@
     "params":{
       "master_timeout":{
         "type":"time",
-        "description":"Explicit operation timeout for connection to master node"
+        "description":"Timeout for waiting for new cluster state in case it is blocked"
       },
       "local":{
+        "deprecated":true,
         "type":"boolean",
         "description":"Return local information, do not retrieve the state from master node (default: false)"
       }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_component_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_component_template.json
index 889e1b817b0fe..def0cc5fb8bb0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_component_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_component_template.json
@@ -34,9 +34,10 @@
     "params":{
       "master_timeout":{
         "type":"time",
-        "description":"Explicit operation timeout for connection to master node"
+        "description":"Timeout for waiting for new cluster state in case it is blocked"
       },
       "local":{
+        "deprecated":true,
         "type":"boolean",
         "description":"Return local information, do not retrieve the state from master node (default: false)"
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json
index 5862804257c67..5004ab8de697d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json
@@ -26,7 +26,7 @@
       },
       "master_timeout":{
         "type":"time",
-        "description":"Explicit operation timeout for connection to master node"
+        "description":"Timeout for waiting for new cluster state in case it is blocked"
       },
       "timeout":{
         "type":"time",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.get.json
index 2645df28c5d1e..670bb4267bdfa 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.get.json
@@ -26,6 +26,13 @@
           }
         }
       ]
+    },
+    "params": {
+      "include_deleted": {
+        "type": "boolean",
+        "default": false,
+        "description": "A flag indicating whether to return connectors that have been soft-deleted."
+      }
     }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json
index 67d2250d3c661..b8c73a09704f1 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json
@@ -47,6 +47,11 @@
       "query": {
         "type": "string",
         "description": "A search string for querying connectors, filtering results by matching against connector names, descriptions, and index names"
+      },
+      "include_deleted": {
+        "type": "boolean",
+        "default": false,
+        "description": "A flag indicating whether to return connectors that have been soft-deleted."
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/eql.search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/eql.search.json
index c854c44d9d761..0f9af508f4c16 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/eql.search.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/eql.search.json
@@ -41,6 +41,16 @@
         "type": "time",
         "description": "Update the time interval in which the results (partial or final) for this search will be available",
         "default": "5d"
+      },
+      "allow_partial_search_results": {
+        "type":"boolean",
+        "description":"Control whether the query should keep running in case of shard failures, and return partial results",
+        "default":false
+      },
+      "allow_partial_sequence_results": {
+        "type":"boolean",
+        "description":"Control whether a sequence query should return partial results or no results at all in case of shard failures. This option has effect only if [allow_partial_search_results] is true.",
+        "default":false
       }
     },
     "body":{
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.async_query_delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.async_query_delete.json
new file mode 100644
index 0000000000000..a6339559afd72
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.async_query_delete.json
@@ -0,0 +1,27 @@
+{
+  "esql.async_query_delete": {
+    "documentation": {
+      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/esql-async-query-delete-api.html",
+      "description": "Delete an async query request given its ID."
+    },
+    "stability": "stable",
+    "visibility": "public",
+    "headers": {
+      "accept": ["application/json"]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/_query/async/{id}",
+          "methods": ["DELETE"],
+          "parts": {
+            "id": {
+              "type": "string",
+              "description": "The async query ID"
+            }
+          }
+        }
+      ]
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.delete_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.delete_lifecycle.json
index 2ff1031ad5c52..cd6397fb61586 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.delete_lifecycle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.delete_lifecycle.json
@@ -25,6 +25,15 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.explain_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.explain_lifecycle.json
index c793ed09281ae..94c37adb802f6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.explain_lifecycle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.explain_lifecycle.json
@@ -33,6 +33,10 @@
       "only_errors": {
         "type": "boolean",
         "description": "filters the indices included in the response to ones in an ILM error state, implies only_managed"
+      },
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.get_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.get_lifecycle.json
index 17bf813093dd6..5abdfac7f5b30 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.get_lifecycle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.get_lifecycle.json
@@ -31,6 +31,15 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.put_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.put_lifecycle.json
index 5a12a778241b3..b7fdbe04a0ffb 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.put_lifecycle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.put_lifecycle.json
@@ -26,7 +26,16 @@
         }
       ]
     },
-    "params":{},
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    },
     "body":{
       "description":"The lifecycle policy definition to register"
     }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.start.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.start.json
index 88b020071ab82..7141673ff9a9d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.start.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.start.json
@@ -19,6 +19,15 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.stop.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.stop.json
index 8401f93badfc4..962fa77263ee4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.stop.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.stop.json
@@ -19,6 +19,15 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json
index 77cb62b022515..a7f272af5b307 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json
@@ -32,9 +32,10 @@
       },
       "master_timeout":{
         "type":"time",
-        "description":"Explicit operation timeout for connection to master node"
+        "description":"Timeout for waiting for new cluster state in case it is blocked"
       },
       "local":{
+        "deprecated":true,
         "type":"boolean",
         "description":"Return local information, do not retrieve the state from master node (default: false)"
       }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle_stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle_stats.json
new file mode 100644
index 0000000000000..8c9e947903402
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle_stats.json
@@ -0,0 +1,21 @@
+{
+  "indices.get_data_lifecycle_stats": {
+    "documentation": {
+      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams-get-lifecycle-stats.html",
+      "description": "Get data stream lifecycle statistics."
+    },
+    "stability": "stable",
+    "visibility": "public",
+    "headers": {
+      "accept": ["application/json"]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/_lifecycle/stats",
+          "methods": ["GET"]
+        }
+      ]
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_index_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_index_template.json
index 4ed82c3bbc5eb..f0351ce8cfe94 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_index_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_index_template.json
@@ -38,9 +38,10 @@
       },
       "master_timeout":{
         "type":"time",
-        "description":"Explicit operation timeout for connection to master node"
+        "description":"Timeout for waiting for new cluster state in case it is blocked"
       },
       "local":{
+        "deprecated":true,
         "type":"boolean",
         "description":"Return local information, do not retrieve the state from master node (default: false)"
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json
index 963d9a4cb670e..74fdd1ef5c8a1 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json
@@ -38,9 +38,10 @@
       },
       "master_timeout":{
         "type":"time",
-        "description":"Explicit operation timeout for connection to master node"
+        "description":"Timeout for waiting for new cluster state in case it is blocked"
       },
       "local":{
+        "deprecated":true,
         "type":"boolean",
         "description":"Return local information, do not retrieve the state from master node (default: false)"
       }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
index 299c24f987d8d..47a1bee665506 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
@@ -63,12 +63,6 @@
         "type":"boolean",
         "default":"false",
         "description":"If set to true, the rollover action will only mark a data stream to signal that it needs to be rolled over at the next write. Only allowed on data streams."
-      },
-      "target_failure_store":{
-        "type":"boolean",
-        "description":"If set to true, the rollover action will be applied on the failure store of the data stream.",
-        "visibility": "feature_flag",
-        "feature_flag": "es.failure_store_feature_flag_enabled"
       }
     },
     "body":{
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.unfreeze.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.unfreeze.json
deleted file mode 100644
index 2327519ff2816..0000000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.unfreeze.json
+++ /dev/null
@@ -1,67 +0,0 @@
-{
-  "indices.unfreeze":{
-    "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/unfreeze-index-api.html",
-      "description":"Unfreezes an index. When a frozen index is unfrozen, the index goes through the normal recovery process and becomes writeable again."
-    },
-    "stability":"stable",
-    "visibility":"public",
-    "headers":{
-      "accept": [ "application/json"]
-    },
-    "url":{
-      "paths":[
-        {
-          "path":"/{index}/_unfreeze",
-          "methods":[
-            "POST"
-          ],
-          "parts":{
-            "index":{
-              "type":"string",
-              "description":"The name of the index to unfreeze"
-            }
-          },
-          "deprecated":{
-            "version":"7.14.0",
-            "description":"Frozen indices are deprecated because they provide no benefit given improvements in heap memory utilization. They will be removed in a future release."
-          }
-        }
-      ]
-    },
-    "params":{
-      "timeout":{
-        "type":"time",
-        "description":"Explicit operation timeout"
-      },
-      "master_timeout":{
-        "type":"time",
-        "description":"Specify timeout for connection to master"
-      },
-      "ignore_unavailable":{
-        "type":"boolean",
-        "description":"Whether specified concrete indices should be ignored when unavailable (missing or closed)"
-      },
-      "allow_no_indices":{
-        "type":"boolean",
-        "description":"Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
-      },
-      "expand_wildcards":{
-        "type":"enum",
-        "options":[
-          "open",
-          "closed",
-          "hidden",
-          "none",
-          "all"
-        ],
-        "default":"closed",
-        "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
-      },
-      "wait_for_active_shards":{
-        "type":"string",
-        "description":"Sets the number of active shards to wait for before the operation returns."
-      }
-    }
-  }
-}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.update.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.update.json
new file mode 100644
index 0000000000000..6c458ce080aa7
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.update.json
@@ -0,0 +1,45 @@
+{
+  "inference.update": {
+    "documentation": {
+      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/update-inference-api.html",
+      "description": "Update inference"
+    },
+    "stability": "stable",
+    "visibility": "public",
+    "headers": {
+      "accept": ["application/json"],
+      "content_type": ["application/json"]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/_inference/{inference_id}/_update",
+          "methods": ["POST"],
+          "parts": {
+            "inference_id": {
+              "type": "string",
+              "description": "The inference Id"
+            }
+          }
+        },
+        {
+          "path": "/_inference/{task_type}/{inference_id}/_update",
+          "methods": ["POST"],
+          "parts": {
+            "task_type": {
+              "type": "string",
+              "description": "The task type"
+            },
+            "inference_id": {
+              "type": "string",
+              "description": "The inference Id"
+            }
+          }
+        }
+      ]
+    },
+    "body": {
+      "description": "The inference endpoint's task and service settings"
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_geoip_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_geoip_database.json
index fe50da720a4da..f76d328836d90 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_geoip_database.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_geoip_database.json
@@ -26,6 +26,14 @@
       ]
     },
     "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
     }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json
index e97d1da276906..341ff5081e270 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json
@@ -26,6 +26,14 @@
       ]
     },
     "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
     }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_geoip_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_geoip_database.json
index 6d088e3f164f4..9c2677d1f7b2f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_geoip_database.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_geoip_database.json
@@ -27,6 +27,14 @@
       ]
     },
     "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
     },
     "body":{
       "description":"The database configuration definition",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json
index 18487969b1a90..782048b98160a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json
@@ -27,6 +27,14 @@
       ]
     },
     "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
     },
     "body":{
       "description":"The database configuration definition",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/license.post_start_trial.json b/rest-api-spec/src/main/resources/rest-api-spec/api/license.post_start_trial.json
index 986040d69cb4f..9fb85807d611f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/license.post_start_trial.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/license.post_start_trial.json
@@ -31,10 +31,6 @@
       "master_timeout": {
         "type": "time",
         "description": "Timeout for processing on master node"
-      },
-      "timeout": {
-        "type": "time",
-        "description": "Timeout for acknowledgement of update from all nodes in cluster"
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.create_from.json b/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.create_from.json
new file mode 100644
index 0000000000000..e17a69a77b252
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.create_from.json
@@ -0,0 +1,37 @@
+{
+  "migrate.create_from":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-stream-reindex.html",
+      "description":"This API creates a destination from a source index. It copies the mappings and settings from the source index while allowing request settings and mappings to override the source values."
+    },
+    "stability":"experimental",
+    "visibility":"private",
+    "headers":{
+      "accept": [ "application/json"],
+      "content_type": ["application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_create_from/{source}/{dest}",
+          "methods":[ "PUT", "POST"],
+          "parts":{
+            "source":{
+              "type":"string",
+              "description":"The source index name"
+            },
+            "dest":{
+              "type":"string",
+              "description":"The destination index name"
+            }
+          }
+        }
+      ]
+    },
+    "body":{
+      "description":"The body contains the fields `mappings_override` and `settings_override`.",
+      "required":false
+    }
+  }
+}
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/security.delegate_pki.json b/rest-api-spec/src/main/resources/rest-api-spec/api/security.delegate_pki.json
new file mode 100644
index 0000000000000..752ea35028b4f
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/security.delegate_pki.json
@@ -0,0 +1,26 @@
+{
+  "security.delegate_pki": {
+    "documentation": {
+      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/security-api-delegate-pki-authentication.html",
+      "description": "Delegate PKI authentication."
+    },
+    "stability": "stable",
+    "visibility": "public",
+    "headers": {
+      "accept": ["application/json"]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/_security/delegate_pki",
+          "methods": ["POST"]
+        }
+      ]
+    },
+    "params": {},
+    "body": {
+      "description":"The X509Certificate chain.",
+      "required":true
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.delete_node.json b/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.delete_node.json
index d990d1da1f144..6f1ec484e94d0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.delete_node.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.delete_node.json
@@ -26,6 +26,15 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.put_node.json b/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.put_node.json
index bf20cf3b70bac..90b19557f5fb2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.put_node.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.put_node.json
@@ -26,7 +26,16 @@
         }
       ]
     },
-    "params":{},
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    },
     "body":{
       "description":"The shutdown type definition to register",
       "required": true
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.delete_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.delete_lifecycle.json
index 12202a7a2a7b1..1d66312f053c7 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.delete_lifecycle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.delete_lifecycle.json
@@ -25,6 +25,15 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_lifecycle.json
index 1395a3d3275ae..71f1727a8638b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_lifecycle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_lifecycle.json
@@ -25,6 +25,15 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_retention.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_retention.json
index f6ce3e75cc379..4166122d5bf1d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_retention.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_retention.json
@@ -19,6 +19,15 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_lifecycle.json
index 94d0772a405da..406fee6015522 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_lifecycle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_lifecycle.json
@@ -31,6 +31,15 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_stats.json
index aa693ad31711c..05281ff46cb8d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_stats.json
@@ -19,6 +19,15 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_status.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_status.json
index 92ba1b4c321e6..404f92f55921f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_status.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_status.json
@@ -19,6 +19,15 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.put_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.put_lifecycle.json
index 7e7babb987c79..621ed870ffdbe 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.put_lifecycle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.put_lifecycle.json
@@ -26,7 +26,16 @@
         }
       ]
     },
-    "params":{},
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    },
     "body":{
       "description":"The snapshot lifecycle policy definition to register"
     }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/xpack.info.json b/rest-api-spec/src/main/resources/rest-api-spec/api/xpack.info.json
index 68b2a5d2c2c8b..35895f0ddb581 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/xpack.info.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/xpack.info.json
@@ -20,6 +20,12 @@
       ]
     },
     "params":{
+      "human":{
+        "type":"boolean",
+        "required":false,
+        "description":"Defines whether additional human-readable information is included in the response. In particular, it adds descriptions and a tag line. The default value is true.",
+        "default":true
+      },
       "categories":{
         "type":"list",
         "description":"Comma-separated list of info categories. Can be any of: build, license, features"
diff --git a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java
index 084e212a913b2..675092bffe8d5 100644
--- a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java
+++ b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java
@@ -14,7 +14,6 @@
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 
 import org.apache.lucene.tests.util.TimeUnits;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
@@ -43,15 +42,9 @@ public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate
         super(testCandidate);
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // remove restCompat check
     @ParametersFactory
     public static Iterable<Object[]> parameters() throws Exception {
-        String restCompatProperty = System.getProperty("tests.restCompat");
-        if ("true".equals(restCompatProperty)) {
-            return createParametersWithLegacyNodeSelectorSupport();
-        } else {
-            return createParameters();
-        }
+        return createParameters();
     }
 
     @Override
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.aliases/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.aliases/10_basic.yml
index 6118453d7805e..0e1dd9fec1b1a 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.aliases/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.aliases/10_basic.yml
@@ -388,8 +388,6 @@
   - do:
       indices.close:
         index: test_index
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       cat.aliases:
@@ -425,8 +423,6 @@
   - do:
       indices.close:
         index: test_index
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       cat.aliases: {}
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.component_templates/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.component_templates/10_basic.yml
new file mode 100644
index 0000000000000..688fdd776450b
--- /dev/null
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.component_templates/10_basic.yml
@@ -0,0 +1,122 @@
+---
+setup:
+
+  - do:
+      cluster.put_component_template:
+        name: captain_america
+        body:
+          template:
+            settings:
+              sort:
+               field: field
+            mappings:
+              properties:
+                field:
+                  type: keyword
+            aliases:
+              steve_rogers: {}
+
+  - do:
+      cluster.put_component_template:
+        name: ms_marvel
+        body:
+          template:
+            settings:
+              default_pipeline: pipeline_a
+              final_pipeline: pipeline_b
+          version: 2
+          _meta:
+            data: {}
+
+  - do:
+      cluster.put_component_template:
+        name: captain_marvel
+        body:
+          version: 3
+          template:
+            mappings:
+              properties:
+                field1:
+                  type: keyword
+                field2:
+                  type: long
+            aliases:
+              carol_danvers: {}
+              monica_rambeau: {}
+
+
+---
+"Retrieve all":
+
+  - do:
+      cat.component_templates: {}
+
+  - match:
+      $body: >
+             /
+               (^|\n)captain_america \s*
+               1 \s*
+               1 \s*
+               1 \s*
+               0 \s*
+               \[\]\s*
+               (\n|$)
+             /
+
+  - match:
+      $body: >
+             /
+               (^|\n)captain_marvel  \s+
+               3   \s+
+               2   \s+
+               2   \s+
+               0   \s+
+               0   \s+
+               \[\]\s*
+               (\n|$)
+             /
+
+  - match:
+      $body: >
+             /
+               (^|\n)ms_marvel  \s+
+               2   \s+
+               0   \s+
+               0   \s+
+               2   \s+
+               1   \s+
+               \[\]\s*
+               (\n|$)
+             /
+
+---
+"Retrieve by name (verbose/headers)":
+
+  - do:
+      cat.component_templates:
+         name: ms_marvel
+         v: true
+
+  - match:
+      $body: >
+             /
+               ^name \s+version \s*alias_count \s*mapping_count \s*settings_count \s*metadata_count \s*included_in\n
+                ms_marvel \s* 2 \s*          0 \s*            0 \s*             2 \s*             1 \s*\[\]\s*$
+             /
+
+
+---
+"Retrieve by wildcard (sorted)":
+
+  - do:
+      cat.component_templates:
+         name: captain_*
+         s: name
+
+  - match:
+      $body: >
+             /
+               ^captain_america \s*  \s*1 \s*1 \s*1 \s*0 \s*\[\]\s*\n
+                captain_marvel  \s*3 \s*2 \s*2 \s*0 \s*0 \s*\[\]\s*$
+             /
+
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/10_basic.yml
index d687462df5872..ce0ad990a2c21 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/10_basic.yml
@@ -87,8 +87,6 @@
   - do:
       indices.close:
         index: index-2
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
 
   - do:
@@ -132,8 +130,6 @@
   - do:
       indices.close:
         index: index-2
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
 
   - do:
@@ -278,8 +274,6 @@
   - do:
       indices.close:
         index: bar
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       cat.indices:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.recovery/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.recovery/10_basic.yml
index 21b307c42398d..55f7457b429ab 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.recovery/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.recovery/10_basic.yml
@@ -97,8 +97,6 @@
   - do:
       indices.close:
         index: index2
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
 
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.segments/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.segments/10_basic.yml
index 646530214bf09..94052cb4fe967 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.segments/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.segments/10_basic.yml
@@ -97,8 +97,6 @@
   - do:
       indices.close:
         index: index1
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       catch: bad_request
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.shards/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.shards/10_basic.yml
index 03d8b2068d23e..45f381eab80b1 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.shards/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.shards/10_basic.yml
@@ -45,6 +45,7 @@
              indexing.index_time              .+   \n
              indexing.index_total             .+   \n
              indexing.index_failed            .+   \n
+             indexing.index_failed_due_to_version_conflict .+   \n
              merges.current                   .+   \n
              merges.current_docs              .+   \n
              merges.current_size              .+   \n
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yml
index d045775d695b4..caf4c55e00e7e 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yml
@@ -69,8 +69,6 @@
   - do:
       indices.close:
         index: test_closed
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - match: { acknowledged: true }
 
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
index f698d3399f27d..800dec2a795a4 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
@@ -171,3 +171,33 @@
   - match: {component_templates.0.component_template.template.lifecycle.enabled: true}
   - match: {component_templates.0.component_template.template.lifecycle.data_retention: "10d"}
   - is_true: component_templates.0.component_template.template.lifecycle.rollover
+
+---
+"Deprecated local parameter":
+  - requires:
+      capabilities:
+        - method: GET
+          path: /_component_template
+          capabilities: ["local_param_deprecated"]
+      test_runner_features: ["capabilities", "warnings"]
+      reason: Deprecation was implemented with capability
+
+  - do:
+      cluster.get_component_template:
+        local: true
+      warnings:
+        - "the [?local] query parameter to this API has no effect, is now deprecated, and will be removed in a future version"
+
+---
+"Deprecated local parameter works in v8 compat mode":
+  - requires:
+      test_runner_features: ["headers"]
+
+  - do:
+      headers:
+        Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
+        Accept: "application/vnd.elasticsearch+json;compatible-with=8"
+      cluster.get_component_template:
+        local: true
+
+  - exists: component_templates
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/10_basic.yml
index c1c9741eb5fa3..0ffbd7fa557c3 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/10_basic.yml
@@ -219,8 +219,6 @@
   - do:
       indices.close:
         index: index-2
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
 
   # closing the index-2 turns the cluster health back to green
@@ -297,8 +295,6 @@
   - do:
       indices.close:
         index: index-2
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
 
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/30_indices_options.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/30_indices_options.yml
index 8756b35569135..feef05e3733f6 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/30_indices_options.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/30_indices_options.yml
@@ -25,8 +25,6 @@ setup:
   - do:
       indices.close:
         index: index-2
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       cluster.health:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.state/30_expand_wildcards.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.state/30_expand_wildcards.yml
index 25520b26d8302..b44e6aee33eaa 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.state/30_expand_wildcards.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.state/30_expand_wildcards.yml
@@ -27,8 +27,6 @@ setup:
   - do:
       indices.close:
         index: test_close_index
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
 ---
 "Test expand_wildcards parameter on closed, open indices and both":
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/70_index_mode.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/70_index_mode.yml
index 9da6d2c5f086e..ce3f7f0198399 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/70_index_mode.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/70_index_mode.yml
@@ -1,9 +1,5 @@
 ---
 setup:
-  - requires:
-      cluster_features: "mapper.query_index_mode"
-      reason: "require index_mode"
-
   - do:
       indices.create:
         index: test_metrics
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml
index 13f6ca58ea295..a0061272a2c23 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml
@@ -1014,10 +1014,6 @@ flattened field:
 
 ---
 flattened field with ignore_above:
-  - requires:
-      cluster_features: ["mapper.track_ignored_source"]
-      reason: requires tracking ignored source
-
   - do:
       indices.create:
         index: test
@@ -1070,10 +1066,6 @@ flattened field with ignore_above:
 
 ---
 flattened field with ignore_above and arrays:
-  - requires:
-      cluster_features: ["mapper.flattened.ignore_above_with_arrays_support"]
-      reason: requires support of ignore_above synthetic source with arrays
-
   - do:
       indices.create:
         index: test
@@ -1127,10 +1119,6 @@ flattened field with ignore_above and arrays:
 
 ---
 completion:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_fallback"]
-      reason: introduced in 8.15.0
-
   - do:
       indices.create:
         index: test
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml
index 414c24cfffd7d..7b8f785a2cb93 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml
@@ -2,7 +2,6 @@
 "Metrics object indexing":
   - requires:
       test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ]
-      cluster_features: ["mapper.subobjects_auto"]
       reason: requires supporting subobjects auto setting
 
   - do:
@@ -69,7 +68,6 @@
 "Root with metrics":
   - requires:
       test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ]
-      cluster_features: ["mapper.subobjects_auto"]
       reason: requires supporting subobjects auto setting
 
   - do:
@@ -131,7 +129,6 @@
 "Metrics object indexing with synthetic source":
   - requires:
       test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ]
-      cluster_features: ["mapper.subobjects_auto"]
       reason: added in 8.4.0
 
   - do:
@@ -201,7 +198,6 @@
 "Root without subobjects with synthetic source":
   - requires:
       test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ]
-      cluster_features: ["mapper.subobjects_auto"]
       reason: added in 8.4.0
 
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml
index d0e1759073e1b..8645c91a51ad3 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml
@@ -171,7 +171,6 @@
         index: test_lookup
 
   - match: { test_lookup.settings.index.number_of_shards: "1"}
-  - match: { test_lookup.settings.index.auto_expand_replicas: "0-all"}
 
 ---
 "Create lookup index with one shard":
@@ -196,7 +195,6 @@
         index: test_lookup
 
   - match: { test_lookup.settings.index.number_of_shards: "1"}
-  - match: { test_lookup.settings.index.auto_expand_replicas: "0-all"}
 
 ---
 "Create lookup index with two shards":
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
index af3d88fb35734..72dddcf8052cc 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
@@ -6,7 +6,7 @@ setup:
 ---
 object with unmapped fields:
   - requires:
-      cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"]
+      cluster_features: ["mapper.bwc_workaround_9_0"]
       reason: requires tracking ignored source
 
   - do:
@@ -56,10 +56,6 @@ object with unmapped fields:
 
 ---
 unmapped arrays:
-  - requires:
-      cluster_features: ["mapper.track_ignored_source"]
-      reason: requires tracking ignored source
-
   - do:
       indices.create:
         index: test
@@ -103,7 +99,7 @@ unmapped arrays:
 ---
 nested object with unmapped fields:
   - requires:
-      cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"]
+      cluster_features: ["mapper.bwc_workaround_9_0"]
       reason: requires tracking ignored source
 
   - do:
@@ -154,7 +150,7 @@ nested object with unmapped fields:
 ---
 empty object with unmapped fields:
   - requires:
-      cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"]
+      cluster_features: ["mapper.bwc_workaround_9_0"]
       reason: requires tracking ignored source
 
   - do:
@@ -314,10 +310,6 @@ disabled object contains array:
 
 ---
 disabled subobject:
-  - requires:
-      cluster_features: ["mapper.track_ignored_source"]
-      reason: requires tracking ignored source
-
   - do:
       indices.create:
         index: test
@@ -353,10 +345,6 @@ disabled subobject:
 
 ---
 disabled subobject with array:
-  - requires:
-      cluster_features: ["mapper.track_ignored_source"]
-      reason: requires tracking ignored source
-
   - do:
       indices.create:
         index: test
@@ -393,10 +381,6 @@ disabled subobject with array:
 
 ---
 mixed disabled and enabled objects:
-  - requires:
-      cluster_features: ["mapper.track_ignored_source"]
-      reason: requires tracking ignored source
-
   - do:
       indices.create:
         index: test
@@ -488,10 +472,6 @@ object with dynamic override:
 
 ---
 subobject with dynamic override:
-  - requires:
-      cluster_features: ["mapper.track_ignored_source"]
-      reason: requires tracking ignored source
-
   - do:
       indices.create:
         index: test
@@ -537,10 +517,6 @@ subobject with dynamic override:
 
 ---
 object array in object with dynamic override:
-  - requires:
-      cluster_features: ["mapper.synthetic_source_keep"]
-      reason: requires tracking ignored source
-
   - do:
       indices.create:
         index: test
@@ -592,10 +568,6 @@ object array in object with dynamic override:
 
 ---
 value array in object with dynamic override:
-  - requires:
-      cluster_features: ["mapper.track_ignored_source"]
-      reason: requires tracking ignored source
-
   - do:
       indices.create:
         index: test
@@ -636,10 +608,6 @@ value array in object with dynamic override:
 
 ---
 nested object:
-  - requires:
-      cluster_features: ["mapper.track_ignored_source"]
-      reason: requires tracking ignored source
-
   - do:
       indices.create:
         index: test
@@ -682,10 +650,6 @@ nested object:
 
 ---
 nested object next to regular:
-  - requires:
-      cluster_features: ["mapper.track_ignored_source"]
-      reason: requires tracking ignored source
-
   - do:
       indices.create:
         index: test
@@ -729,10 +693,6 @@ nested object next to regular:
 
 ---
 nested object with disabled:
-  - requires:
-      cluster_features: ["mapper.track_ignored_source"]
-      reason: requires tracking ignored source
-
   - do:
       indices.create:
         index: test
@@ -818,10 +778,6 @@ nested object with disabled:
 
 ---
 doubly nested object:
-  - requires:
-      cluster_features: ["mapper.track_ignored_source"]
-      reason: requires tracking ignored source
-
   - do:
       indices.create:
         index: test
@@ -915,7 +871,7 @@ doubly nested object:
 ---
 subobjects auto:
   - requires:
-      cluster_features: ["mapper.subobjects_auto", "mapper.bwc_workaround_9_0"]
+      cluster_features: ["mapper.bwc_workaround_9_0"]
       reason: requires tracking ignored source and supporting subobjects auto setting
 
   - do:
@@ -1003,10 +959,6 @@ subobjects auto:
 
 ---
 synthetic_source with copy_to:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_copy_to_fix"]
-      reason: requires copy_to support in synthetic source
-
   - do:
       indices.create:
         index: test
@@ -1140,10 +1092,6 @@ synthetic_source with copy_to:
 
 ---
 synthetic_source with disabled doc_values:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"]
-      reason: requires disabled doc_values support in synthetic source
-
   - do:
       indices.create:
         index: test
@@ -1224,10 +1172,6 @@ synthetic_source with disabled doc_values:
 
 ---
 fallback synthetic_source for text field:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"]
-      reason: requires disabled doc_values support in synthetic source
-
   - do:
       indices.create:
         index: test
@@ -1259,10 +1203,6 @@ fallback synthetic_source for text field:
 
 ---
 synthetic_source with copy_to and ignored values:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_copy_to_fix"]
-      reason: requires copy_to support in synthetic source
-
   - do:
       indices.create:
         index: test
@@ -1328,10 +1268,6 @@ synthetic_source with copy_to and ignored values:
 
 ---
 synthetic_source with copy_to field having values in source:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_copy_to_fix"]
-      reason: requires copy_to support in synthetic source
-
   - do:
       indices.create:
         index: test
@@ -1392,10 +1328,6 @@ synthetic_source with copy_to field having values in source:
 
 ---
 synthetic_source with ignored source field using copy_to:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_copy_to_fix"]
-      reason: requires copy_to support in synthetic source
-
   - do:
       indices.create:
         index: test
@@ -1457,10 +1389,6 @@ synthetic_source with ignored source field using copy_to:
 
 ---
 synthetic_source with copy_to field from dynamic template having values in source:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_copy_to_fix"]
-      reason: requires copy_to support in synthetic source
-
   - do:
       indices.create:
         index: test
@@ -1555,7 +1483,6 @@ synthetic_source with copy_to field from dynamic template having values in sourc
 ---
 synthetic_source with copy_to and invalid values for copy:
   - requires:
-      cluster_features: ["mapper.source.synthetic_source_copy_to_fix"]
       reason: requires copy_to support in synthetic source
       test_runner_features: "contains"
 
@@ -1592,10 +1519,6 @@ synthetic_source with copy_to and invalid values for copy:
 
 ---
 synthetic_source with copy_to pointing inside object:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"]
-      reason: requires copy_to support in synthetic source
-
   - do:
       indices.create:
         index: test
@@ -1697,10 +1620,6 @@ synthetic_source with copy_to pointing inside object:
 
 ---
 synthetic_source with copy_to pointing to ambiguous field:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"]
-      reason: requires copy_to support in synthetic source
-
   - do:
       indices.create:
         index: test
@@ -1745,10 +1664,6 @@ synthetic_source with copy_to pointing to ambiguous field:
 
 ---
 synthetic_source with copy_to pointing to ambiguous field and subobjects false:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"]
-      reason: requires copy_to support in synthetic source
-
   - do:
       indices.create:
         index: test
@@ -1794,10 +1709,6 @@ synthetic_source with copy_to pointing to ambiguous field and subobjects false:
 
 ---
 synthetic_source with copy_to pointing to ambiguous field and subobjects auto:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"]
-      reason: requires copy_to support in synthetic source
-
   - do:
       indices.create:
         index: test
@@ -1845,7 +1756,6 @@ synthetic_source with copy_to pointing to ambiguous field and subobjects auto:
 synthetic_source with copy_to pointing at dynamic field:
   - requires:
       test_runner_features: contains
-      cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"]
       reason: requires copy_to support in synthetic source
 
   - do:
@@ -1931,10 +1841,6 @@ synthetic_source with copy_to pointing at dynamic field:
 
 ---
 synthetic_source with copy_to pointing inside dynamic object:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"]
-      reason: requires copy_to support in synthetic source
-
   - do:
       indices.create:
         index: test
@@ -2012,3 +1918,41 @@ synthetic_source with copy_to pointing inside dynamic object:
       hits.hits.2.fields:
         c.copy.keyword: [ "hello", "zap" ]
 
+---
+create index with use_synthetic_source:
+  - requires:
+      cluster_features: ["mapper.synthetic_recovery_source"]
+      reason: requires synthetic recovery source
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              recovery:
+                use_synthetic_source: true
+              mapping:
+                source:
+                  mode: synthetic
+
+  - do:
+      indices.get_settings: {}
+  - match: { test.settings.index.mapping.source.mode: synthetic}
+  - is_true: test.settings.index.recovery.use_synthetic_source
+
+  - do:
+      index:
+        index: test
+        id: 1
+        refresh: true
+        body: { foo: bar }
+  - match: { _version: 1 }
+
+  - do:
+      indices.disk_usage:
+        index: test
+        run_expensive_tasks: true
+        flush: false
+  - gt: { test.store_size_in_bytes: 0 }
+  - is_false: test.fields._recovery_source
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml
index 095665e9337b1..803b8a7d0062f 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml
@@ -6,7 +6,7 @@ setup:
 ---
 object param - store complex object:
   - requires:
-      cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"]
+      cluster_features: ["mapper.bwc_workaround_9_0"]
       reason: requires tracking ignored source
 
   - do:
@@ -71,7 +71,7 @@ object param - store complex object:
 ---
 object param - object array:
   - requires:
-      cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"]
+      cluster_features: ["mapper.bwc_workaround_9_0"]
       reason: requires tracking ignored source
 
   - do:
@@ -136,7 +136,7 @@ object param - object array:
 ---
 object param - object array within array:
   - requires:
-      cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"]
+      cluster_features: ["mapper.bwc_workaround_9_0"]
       reason: requires tracking ignored source
 
   - do:
@@ -180,7 +180,7 @@ object param - object array within array:
 ---
 object param - no object array:
   - requires:
-      cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"]
+      cluster_features: ["mapper.bwc_workaround_9_0"]
       reason: requires tracking ignored source
 
   - do:
@@ -223,7 +223,7 @@ object param - no object array:
 ---
 object param - field ordering in object array:
   - requires:
-      cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"]
+      cluster_features: ["mapper.bwc_workaround_9_0"]
       reason: requires tracking ignored source
 
   - do:
@@ -273,7 +273,7 @@ object param - field ordering in object array:
 ---
 object param - nested object array next to other fields:
   - requires:
-      cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"]
+      cluster_features: ["mapper.bwc_workaround_9_0"]
       reason: requires tracking ignored source
 
   - do:
@@ -380,7 +380,7 @@ object param - nested object with stored array:
 ---
 index param - nested array within array:
   - requires:
-      cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"]
+      cluster_features: ["mapper.bwc_workaround_9_0"]
       reason: requires tracking ignored source
 
   - do:
@@ -428,7 +428,7 @@ index param - nested array within array:
 # 112156
 stored field under object with store_array_source:
   - requires:
-      cluster_features: ["mapper.source.synthetic_source_stored_fields_advance_fix", "mapper.bwc_workaround_9_0"]
+      cluster_features: ["mapper.bwc_workaround_9_0"]
       reason: requires bug fix to be implemented
 
   - do:
@@ -477,10 +477,6 @@ stored field under object with store_array_source:
 
 ---
 field param - keep root array:
-  - requires:
-      cluster_features: ["mapper.synthetic_source_keep"]
-      reason: requires keeping array source
-
   - do:
       indices.create:
         index: test
@@ -535,10 +531,6 @@ field param - keep root array:
 
 ---
 field param - keep nested array:
-  - requires:
-      cluster_features: ["mapper.synthetic_source_keep"]
-      reason: requires keeping array source
-
   - do:
       indices.create:
         index: test
@@ -605,7 +597,6 @@ field param - keep nested array:
 field param - keep root singleton fields:
   - requires:
       test_runner_features: close_to
-      cluster_features: ["mapper.synthetic_source_keep"]
       reason: requires keeping singleton source
 
   - do:
@@ -695,7 +686,6 @@ field param - keep root singleton fields:
 field param - keep nested singleton fields:
   - requires:
       test_runner_features: close_to
-      cluster_features: ["mapper.synthetic_source_keep"]
       reason: requires keeping singleton source
 
   - do:
@@ -776,10 +766,6 @@ field param - keep nested singleton fields:
 
 ---
 field param - nested array within array:
-  - requires:
-      cluster_features: ["mapper.synthetic_source_keep"]
-      reason: requires tracking ignored source
-
   - do:
       indices.create:
         index: test
@@ -821,7 +807,7 @@ field param - nested array within array:
 ---
 index param - root arrays:
   - requires:
-      cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"]
+      cluster_features: ["mapper.bwc_workaround_9_0"]
       reason: requires keeping array source
 
   - do:
@@ -900,10 +886,6 @@ index param - root arrays:
 
 ---
 index param - dynamic root arrays:
-  - requires:
-      cluster_features: ["mapper.synthetic_source_keep"]
-      reason: requires keeping array source
-
   - do:
       indices.create:
         index: test
@@ -952,10 +934,6 @@ index param - dynamic root arrays:
 
 ---
 index param - object array within array:
-  - requires:
-      cluster_features: ["mapper.synthetic_source_keep"]
-      reason: requires keeping array source
-
   - do:
       indices.create:
         index: test
@@ -1001,10 +979,6 @@ index param - object array within array:
 
 ---
 index param - no object array:
-  - requires:
-      cluster_features: ["mapper.synthetic_source_keep"]
-      reason: requires keeping array source
-
   - do:
       indices.create:
         index: test
@@ -1045,10 +1019,6 @@ index param - no object array:
 
 ---
 index param - field ordering:
-  - requires:
-      cluster_features: ["mapper.synthetic_source_keep"]
-      reason: requires keeping array source
-
   - do:
       indices.create:
         index: test
@@ -1095,10 +1065,6 @@ index param - field ordering:
 
 ---
 index param - nested arrays:
-  - requires:
-      cluster_features: ["mapper.synthetic_source_keep"]
-      reason: requires keeping array source
-
   - do:
       indices.create:
         index: test
@@ -1162,10 +1128,6 @@ index param - nested arrays:
 
 ---
 index param - nested object with stored array:
-  - requires:
-      cluster_features: ["mapper.synthetic_source_keep"]
-      reason: requires keeping array source
-
   - do:
       indices.create:
         index: test
@@ -1214,7 +1176,7 @@ index param - nested object with stored array:
 ---
 index param - flattened fields:
   - requires:
-      cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"]
+      cluster_features: ["mapper.bwc_workaround_9_0"]
       reason: requires keeping array source
 
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.exists_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.exists_template/10_basic.yml
index 67592a013e8f1..4031423209830 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.exists_template/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.exists_template/10_basic.yml
@@ -31,10 +31,15 @@ setup:
 
 ---
 "Test indices.exists_template with local flag":
+  - requires:
+      test_runner_features: ["allowed_warnings"]
+
   - do:
       indices.exists_template:
         name: test
         local: true
+      allowed_warnings:
+        - "the [?local] query parameter to this API has no effect, is now deprecated, and will be removed in a future version"
 
   - is_false: ''
 
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get/10_basic.yml
index e2fd28e745f0e..c72f10aaa66d4 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get/10_basic.yml
@@ -40,8 +40,6 @@ setup:
   - do:
       indices.close:
         index: test_index_3
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       cluster.health:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_alias/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_alias/10_basic.yml
index 63ab40f3bf578..d454fde15c824 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_alias/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_alias/10_basic.yml
@@ -297,8 +297,6 @@ setup:
   - do:
       indices.close:
         index: test_index_2
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       indices.get_alias:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml
index c47df413df9e7..afb3b0d1ff83c 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml
@@ -82,14 +82,48 @@ setup:
 
 ---
 "Get index template with local flag":
+  - requires:
+      test_runner_features: ["allowed_warnings"]
 
   - do:
       indices.get_index_template:
         name: test
         local: true
+      allowed_warnings:
+        - "the [?local] query parameter to this API has no effect, is now deprecated, and will be removed in a future version"
 
   - match: {index_templates.0.name: test}
 
+---
+"Deprecated local parameter":
+  - requires:
+      capabilities:
+        - method: GET
+          path: /_get_index_template
+          capabilities: ["local_param_deprecated"]
+      test_runner_features: ["capabilities", "warnings"]
+      reason: Deprecation was implemented with capability
+
+  - do:
+      indices.get_index_template:
+        local: true
+      warnings:
+        - "the [?local] query parameter to this API has no effect, is now deprecated, and will be removed in a future version"
+
+---
+"Deprecated local parameter works in v8 compat mode":
+  - requires:
+      test_runner_features: ["headers"]
+
+  - do:
+      headers:
+        Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
+        Accept: "application/vnd.elasticsearch+json;compatible-with=8"
+      indices.get_index_template:
+        local: true
+
+  - exists: index_templates
+
 ---
 "Add data stream lifecycle":
   - requires:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_mapping/50_wildcard_expansion.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_mapping/50_wildcard_expansion.yml
index 853837f1375ff..0eb717f7f6c1f 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_mapping/50_wildcard_expansion.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_mapping/50_wildcard_expansion.yml
@@ -55,8 +55,6 @@ setup:
   - do:
       indices.close:
         index: test-xyy
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
         cluster.health:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_template/10_basic.yml
index 9becbd54a3773..c602d913502d5 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_template/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_template/10_basic.yml
@@ -62,14 +62,50 @@ setup:
 
 ---
 "Get template with local flag":
+  - requires:
+      test_runner_features: ["allowed_warnings"]
 
   - do:
       indices.get_template:
         name: test
         local: true
+      allowed_warnings:
+        - "the [?local] query parameter to this API has no effect, is now deprecated, and will be removed in a future version"
 
   - is_true: test
 
+---
+"Deprecated local parameter":
+  - requires:
+      capabilities:
+        - method: GET
+          path: /_template
+          capabilities: ["local_param_deprecated"]
+      test_runner_features: ["capabilities", "warnings"]
+      reason: Deprecation was implemented with capability
+
+  - do:
+      indices.get_template:
+        name: test
+        local: true
+      warnings:
+        - "the [?local] query parameter to this API has no effect, is now deprecated, and will be removed in a future version"
+
+---
+"Deprecated local parameter works in v8 compat mode":
+  - requires:
+      test_runner_features: ["headers"]
+
+  - do:
+      headers:
+        Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
+        Accept: "application/vnd.elasticsearch+json;compatible-with=8"
+      indices.get_template:
+        name: test
+        local: true
+
+  - exists: test.index_patterns
+
 ---
 "Get template with flat settings and master timeout":
 
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.open/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.open/10_basic.yml
index 9101475fc9055..d1614df7f9037 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.open/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.open/10_basic.yml
@@ -15,8 +15,6 @@
   - do:
       indices.close:
         index: test_index
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
 
   - do:
@@ -55,8 +53,6 @@
   - do:
       indices.close:
         index: test_index
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
 
   - do:
@@ -113,8 +109,6 @@
   - do:
       indices.close:
         index: "index_*"
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - match: { acknowledged: true }
   - match: { shards_acknowledged: true }
@@ -123,11 +117,14 @@
   - match: { indices.index_3.closed: true }
 
 ---
-"?wait_for_active_shards=index-setting is deprecated":
+"?wait_for_active_shards=index-setting is removed":
   - requires:
-      cluster_features: ["gte_v8.0.0"]
-      reason: "required deprecation warning is only emitted in 8.0 and later"
-      test_runner_features: ["warnings"]
+      reason: "Parameter value 'index-setting' of 'wait-for-active-shards' is rejected with specialised error."
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /{index}/_close
+          capabilities: [ wait-for-active-shards-index-setting-removed ]
 
   - do:
       indices.create:
@@ -137,8 +134,7 @@
             number_of_replicas: 0
 
   - do:
+      catch: /The 'index-setting' value for parameter 'wait_for_active_shards' is the default behaviour and this configuration value is not supported anymore. Please remove 'wait_for_active_shards=index-setting'/
       indices.close:
         index: "index_*"
         wait_for_active_shards: index-setting
-      warnings:
-        - "?wait_for_active_shards=index-setting is now the default behaviour; the 'index-setting' value for this parameter should no longer be used since it will become unsupported in version 9"
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.open/20_multiple_indices.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.open/20_multiple_indices.yml
index ec71423bdc24b..222731c294597 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.open/20_multiple_indices.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.open/20_multiple_indices.yml
@@ -24,8 +24,6 @@ setup:
   - do:
       indices.close:
         index: _all
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
 
   - do:
@@ -57,8 +55,6 @@ setup:
   - do:
       indices.close:
         index: test_*
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
 
   - do:
@@ -90,8 +86,6 @@ setup:
   - do:
       indices.close:
         index: '*'
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
 
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml
index 3d82539944a97..89816be5ca8e7 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml
@@ -453,8 +453,6 @@
 ---
 "Composable index templates that include subobjects: auto at root":
   - requires:
-      cluster_features: ["mapper.subobjects_auto"]
-      reason: "https://github.com/elastic/elasticsearch/issues/96768 fixed at 8.11.0"
       test_runner_features: "allowed_warnings"
 
   - do:
@@ -504,8 +502,6 @@
 ---
 "Composable index templates that include subobjects: auto on arbitrary field":
   - requires:
-      cluster_features: ["mapper.subobjects_auto"]
-      reason: "https://github.com/elastic/elasticsearch/issues/96768 fixed at 8.11.0"
       test_runner_features: "allowed_warnings"
 
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_settings/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_settings/10_basic.yml
index 00cdd287f06df..0772017bcb24b 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_settings/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_settings/10_basic.yml
@@ -92,8 +92,6 @@ setup:
   - do:
       indices.close:
         index: test-index
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       indices.put_settings:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.recovery/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.recovery/10_basic.yml
index 06865d2f620e3..8ad06910ebe4d 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.recovery/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.recovery/10_basic.yml
@@ -59,8 +59,6 @@
   - do:
       indices.close:
         index: test_2
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
 
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.recovery/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.recovery/20_synthetic_source.yml
new file mode 100644
index 0000000000000..493b834fc5a90
--- /dev/null
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.recovery/20_synthetic_source.yml
@@ -0,0 +1,33 @@
+---
+test recovery empty index with use_synthetic_source:
+  - requires:
+      cluster_features: ["mapper.synthetic_recovery_source"]
+      reason: requires synthetic recovery source
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              number_of_replicas: 0
+              recovery:
+                use_synthetic_source: true
+              mapping:
+                source:
+                  mode: synthetic
+
+  - do:
+      indices.get_settings: {}
+  - match: { test.settings.index.mapping.source.mode: synthetic}
+  - is_true: test.settings.index.recovery.use_synthetic_source
+
+  - do:
+      indices.put_settings:
+        index: test
+        body:
+          index.number_of_replicas: 1
+
+  - do:
+      cluster.health:
+        wait_for_events: languid
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/10_basic_resolve_cluster.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/10_basic_resolve_cluster.yml
index ba341e0d220e1..46bd0b8099e4a 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/10_basic_resolve_cluster.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/10_basic_resolve_cluster.yml
@@ -20,8 +20,6 @@ setup:
   - do:
       indices.close:
         index: index2
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
 ---
 "Resolve cluster with indices and aliases":
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml
index f7252d0a9a89c..b689cfba43b86 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml
@@ -24,8 +24,6 @@ setup:
   - do:
       indices.close:
         index: test_index2
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       indices.create:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.segments/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.segments/10_basic.yml
index 0235d5219e47e..f0cfc4e585cc6 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.segments/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.segments/10_basic.yml
@@ -105,8 +105,6 @@
   - do:
       indices.close:
         index: index1
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       catch: bad_request
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/20_nested.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/20_nested.yml
index c88d638199dba..d07d03cb7146c 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/20_nested.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/20_nested.yml
@@ -1,8 +1,5 @@
 ---
 sort doc with nested object:
-  - requires:
-      cluster_features: ["mapper.index_sorting_on_nested"]
-      reason: uses index sorting on nested fields
   - do:
       indices.create:
         index: test
@@ -66,9 +63,6 @@ sort doc with nested object:
 
 ---
 sort doc on nested field:
-  - requires:
-      cluster_features: [ "mapper.index_sorting_on_nested" ]
-      reason: uses index sorting on nested fields
   - do:
       catch: /cannot apply index sort to field \[nested_field\.foo\] under nested object \[nested_field\]/
       indices.create:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
index 463df7d2ab1bb..2a31b3bd387c4 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
@@ -10,14 +10,6 @@ setup:
 
 ---
 create logs index:
-  - requires:
-      test_runner_features: [ capabilities ]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [ logsdb_index_mode ]
-      reason: "Support for 'logsdb' index mode capability required"
-
   - do:
       indices.create:
         index: test
@@ -78,14 +70,6 @@ create logs index:
 
 ---
 using default timestamp field mapping:
-  - requires:
-      test_runner_features: [ capabilities ]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [ logsdb_index_mode ]
-      reason: "Support for 'logsdb' index mode capability required"
-
   - do:
       indices.create:
         index: test-timestamp-missing
@@ -110,14 +94,6 @@ using default timestamp field mapping:
 
 ---
 missing hostname field:
-  - requires:
-      test_runner_features: [ capabilities ]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [ logsdb_index_mode ]
-      reason: "Support for 'logsdb' index mode capability required"
-
   - do:
       indices.create:
         index: test-hostname-missing
@@ -149,14 +125,6 @@ missing hostname field:
 
 ---
 missing sort field:
-  - requires:
-      test_runner_features: [ capabilities ]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [ logsdb_index_mode ]
-      reason: "Support for 'logsdb' index mode capability required"
-
   - do:
       catch: bad_request
       indices.create:
@@ -190,14 +158,6 @@ missing sort field:
 
 ---
 non-default sort settings:
-  - requires:
-      test_runner_features: [ capabilities ]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [ logsdb_index_mode ]
-      reason: "Support for 'logsdb' index mode capability required"
-
   - do:
       indices.create:
         index: test-sort
@@ -244,14 +204,10 @@ non-default sort settings:
 ---
 override sort order settings:
   - requires:
-      test_runner_features: [ capabilities ]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [ logsdb_index_mode ]
-      reason: "Support for 'logsdb' index mode capability required"
-
+      cluster_features: [ "index.logsdb_no_host_name_field" ]
+      reason: "Change in default sort config for logsdb"
   - do:
+      catch: bad_request
       indices.create:
         index: test-sort-order
         body:
@@ -278,28 +234,16 @@ override sort order settings:
               message:
                 type: text
 
-  - do:
-      indices.get_settings:
-        index: test-sort-order
-
-  - is_true: test-sort-order
-  - match: { test-sort-order.settings.index.mode: "logsdb" }
-  - match: { test-sort-order.settings.index.sort.field.0: null }
-  - match: { test-sort-order.settings.index.sort.field.1: null }
-  - match: { test-sort-order.settings.index.sort.order.0: "asc" }
-  - match: { test-sort-order.settings.index.sort.order.1: "asc" }
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "index.sort.fields:[] index.sort.order:[asc, asc], size mismatch" }
 
 ---
 override sort missing settings:
   - requires:
-      test_runner_features: [ capabilities ]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [ logsdb_index_mode ]
-      reason: "Support for 'logsdb' index mode capability required"
-
+      cluster_features: [ "index.logsdb_no_host_name_field" ]
+      reason: "Change in default sort config for logsdb"
   - do:
+      catch: bad_request
       indices.create:
         index: test-sort-missing
         body:
@@ -326,28 +270,16 @@ override sort missing settings:
               message:
                 type: text
 
-  - do:
-      indices.get_settings:
-        index: test-sort-missing
-
-  - is_true: test-sort-missing
-  - match: { test-sort-missing.settings.index.mode: "logsdb" }
-  - match: { test-sort-missing.settings.index.sort.field.0: null }
-  - match: { test-sort-missing.settings.index.sort.field.1: null }
-  - match: { test-sort-missing.settings.index.sort.missing.0: "_last" }
-  - match: { test-sort-missing.settings.index.sort.missing.1: "_first" }
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "index.sort.fields:[] index.sort.missing:[_last, _first], size mismatch" }
 
 ---
 override sort mode settings:
   - requires:
-      test_runner_features: [ capabilities ]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [ logsdb_index_mode ]
-      reason: "Support for 'logsdb' index mode capability required"
-
+      cluster_features: [ "index.logsdb_no_host_name_field" ]
+      reason: "Change in default sort config for logsdb"
   - do:
+      catch: bad_request
       indices.create:
         index: test-sort-mode
         body:
@@ -374,21 +306,12 @@ override sort mode settings:
               message:
                 type: text
 
-  - do:
-      indices.get_settings:
-        index: test-sort-mode
-
-  - is_true: test-sort-mode
-  - match: { test-sort-mode.settings.index.mode: "logsdb" }
-  - match: { test-sort-mode.settings.index.sort.field.0: null }
-  - match: { test-sort-mode.settings.index.sort.field.1: null }
-  - match: { test-sort-mode.settings.index.sort.mode.0: "max" }
-  - match: { test-sort-mode.settings.index.sort.mode.1: "max" }
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "index.sort.fields:[] index.sort.mode:[MAX, MAX], size mismatch" }
 
 ---
 override sort field using nested field type in sorting:
   - requires:
-      cluster_features: ["mapper.index_sorting_on_nested"]
       test_runner_features: [ capabilities ]
       capabilities:
         - method: PUT
@@ -434,14 +357,6 @@ override sort field using nested field type in sorting:
 
 ---
 override sort field using nested field type:
-  - requires:
-      cluster_features: ["mapper.index_sorting_on_nested"]
-      test_runner_features: [ capabilities ]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [ logsdb_index_mode ]
-      reason: "Support for 'logsdb' index mode capability required"
 
   - do:
       indices.create:
@@ -475,14 +390,6 @@ override sort field using nested field type:
 
 ---
 routing path not allowed in logs mode:
-  - requires:
-      test_runner_features: [ capabilities ]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [ logsdb_index_mode ]
-      reason: "Support for 'logsdb' index mode capability required"
-
   - do:
       catch: bad_request
       indices.create:
@@ -514,15 +421,49 @@ routing path not allowed in logs mode:
   - match: { error.reason: "[index.routing_path] requires [index.mode=time_series]" }
 
 ---
-start time not allowed in logs mode:
+routing path allowed in logs mode with routing on sort fields:
   - requires:
-      test_runner_features: [ capabilities ]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [ logsdb_index_mode ]
-      reason: "Support for 'logsdb' index mode capability required"
+      cluster_features: [ "routing.logsb_route_on_sort_fields" ]
+      reason: introduction of route on index sorting fields
 
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mode: logsdb
+              number_of_replicas: 0
+              number_of_shards: 2
+              routing_path: [ host.name, agent_id ]
+              logsdb:
+                route_on_sort_fields: true
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              host.name:
+                type: keyword
+              agent_id:
+                type: keyword
+              process_id:
+                type: integer
+              http_method:
+                type: keyword
+              message:
+                type: text
+
+  - do:
+      indices.get_settings:
+        index: test
+
+  - is_true: test
+  - match: { test.settings.index.mode: logsdb }
+  - match: { test.settings.index.logsdb.route_on_sort_fields: "true" }
+  - match: { test.settings.index.routing_path: [ host.name, agent_id ] }
+
+---
+start time not allowed in logs mode:
   - do:
       catch: bad_request
       indices.create:
@@ -556,14 +497,6 @@ start time not allowed in logs mode:
 
 ---
 end time not allowed in logs mode:
-  - requires:
-      test_runner_features: [ capabilities ]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [ logsdb_index_mode ]
-      reason: "Support for 'logsdb' index mode capability required"
-
   - do:
       catch: bad_request
       indices.create:
@@ -643,10 +576,10 @@ ignore dynamic beyond limit logsdb override value:
   - match: { test-ignore-dynamic-override.settings.index.mapping.total_fields.ignore_dynamic_beyond_limit: "false" }
 
 ---
-logsdb with default ignore dynamic beyond limit and default sorting:
+default ignore dynamic beyond limit and default sorting:
   - requires:
-      cluster_features: ["mapper.logsdb_default_ignore_dynamic_beyond_limit"]
-      reason: requires default value for ignore_dynamic_beyond_limit
+      cluster_features: [ "index.logsdb_no_host_name_field" ]
+      reason: "No host.name field injection"
 
   - do:
       indices.create:
@@ -656,19 +589,8 @@ logsdb with default ignore dynamic beyond limit and default sorting:
             index:
               mode: logsdb
               mapping:
-                # NOTE: When the index mode is set to `logsdb`, the `host.name` field is automatically injected if
-                # sort settings are not overridden.
-                # With `subobjects` set to `true` (default), this creates a `host` object field and a nested `name`
-                # keyword field (`host.name`).
-                #
-                # As a result, there are always at least 4 statically mapped fields (`@timestamp`, `host`, `host.name`
-                # and `name`). We cannot use a field limit lower than 4 because these fields are always present.
-                #
-                # Indeed, if `index.mapping.total_fields.ignore_dynamic_beyond_limit` is `true`, any dynamically
-                # mapped fields beyond the limit `index.mapping.total_fields.limit` are ignored, but the statically
-                # mapped fields are always counted.
                 total_fields:
-                  limit: 4
+                  limit: 2
           mappings:
             properties:
               "@timestamp":
@@ -688,9 +610,9 @@ logsdb with default ignore dynamic beyond limit and default sorting:
         refresh: true
         body:
           - '{ "index": { } }'
-          - '{ "@timestamp": "2024-08-13T12:30:00Z", "name": "foo", "host.name": "92f4a67c", "value": 10, "message": "the quick brown fox", "region": "us-west", "pid": 153462 }'
+          - '{ "@timestamp": "2024-08-13T12:30:00Z", "name": "foo", "value": 10, "message": "the quick brown fox", "region": "us-west", "pid": 153462 }'
           - '{ "index": { } }'
-          - '{ "@timestamp": "2024-08-13T12:01:00Z", "name": "bar", "host.name": "24eea278", "value": 20, "message": "jumps over the lazy dog", "region": "us-central", "pid": 674972 }'
+          - '{ "@timestamp": "2024-08-13T12:01:00Z", "name": "bar", "value": 20, "message": "jumps over the lazy dog", "region": "us-central", "pid": 674972 }'
   - match: { errors: false }
 
   - do:
@@ -712,132 +634,82 @@ logsdb with default ignore dynamic beyond limit and default sorting:
   - match: { hits.hits.1._ignored: [ "message", "pid", "region", "value" ] }
 
 ---
-logsdb with default ignore dynamic beyond limit and non-default sorting:
+default ignore dynamic beyond limit and default sorting with hostname:
   - requires:
-      cluster_features: ["mapper.logsdb_default_ignore_dynamic_beyond_limit"]
-      reason: requires default value for ignore_dynamic_beyond_limit
+      cluster_features: [ "index.logsdb_no_host_name_field" ]
+      reason: "No host.name field injection"
 
   - do:
       indices.create:
-        index: test-logsdb-non-default-sort
+        index: test-logsdb-default-sort
         body:
           settings:
             index:
-              sort.field: [ "name" ]
-              sort.order: [ "desc" ]
               mode: logsdb
               mapping:
-                # NOTE: Here sort settings are overridden and we do not have any additional statically mapped field other
-                # than `name` and `timestamp`. As a result, there are only 2 statically mapped fields.
                 total_fields:
-                  limit: 2
+                  limit: 3
           mappings:
             properties:
               "@timestamp":
                 type: date
-              name:
+              host.name:
                 type: keyword
 
   - do:
       indices.get_settings:
-        index: test-logsdb-non-default-sort
+        index: test-logsdb-default-sort
 
-  - match: { test-logsdb-non-default-sort.settings.index.mode: "logsdb" }
+  - match: { test-logsdb-default-sort.settings.index.mode: "logsdb" }
 
   - do:
       bulk:
-        index: test-logsdb-non-default-sort
+        index: test-logsdb-default-sort
         refresh: true
         body:
           - '{ "index": { } }'
-          - '{ "@timestamp": "2024-08-13T12:30:00Z", "name": "foo", "host.name": "92f4a67c", "value": 10, "message": "the quick brown fox", "region": "us-west", "pid": 153462 }'
+          - '{ "@timestamp": "2024-08-13T12:30:00Z", "host.name": "foo", "value": 10, "message": "the quick brown fox", "region": "us-west", "pid": 153462 }'
           - '{ "index": { } }'
-          - '{ "@timestamp": "2024-08-13T12:01:00Z", "name": "bar", "host.name": "24eea278", "value": 20, "message": "jumps over the lazy dog", "region": "us-central", "pid": 674972 }'
+          - '{ "@timestamp": "2024-08-13T12:01:00Z", "host.name": "bar", "value": 20, "message": "jumps over the lazy dog", "region": "us-central", "pid": 674972 }'
   - match: { errors: false }
 
   - do:
       search:
-        index: test-logsdb-non-default-sort
+        index: test-logsdb-default-sort
         body:
           query:
             match_all: {}
           sort: "@timestamp"
 
   - match: { hits.total.value: 2 }
-  - match: { hits.hits.0._source.name: "bar" }
+  - match: { hits.hits.0._source.host.name: "bar" }
   - match: { hits.hits.0._source.value: 20 }
   - match: { hits.hits.0._source.message: "jumps over the lazy dog" }
-  - match: { hits.hits.0._ignored: [ "host", "message", "pid", "region", "value" ] }
-  - match: { hits.hits.1._source.name: "foo" }
+  - match: { hits.hits.0._ignored: [ "message", "pid", "region", "value" ] }
+  - match: { hits.hits.1._source.host.name: "foo" }
   - match: { hits.hits.1._source.value: 10 }
   - match: { hits.hits.1._source.message: "the quick brown fox" }
-  - match: { hits.hits.1._ignored: [ "host", "message", "pid", "region", "value" ] }
-
----
-logsdb with default ignore dynamic beyond limit and too low limit:
-  - requires:
-      cluster_features: ["mapper.logsdb_default_ignore_dynamic_beyond_limit"]
-      reason: requires default value for ignore_dynamic_beyond_limit
-
-  - do:
-      catch: bad_request
-      indices.create:
-        index: test-logsdb-low-limit
-        body:
-          settings:
-            index:
-              mode: logsdb
-              mapping:
-                # NOTE: When the index mode is set to `logsdb`, the `host.name` field is automatically injected if
-                # sort settings are not overridden.
-                # With `subobjects` set to `true` (default), this creates a `host` object field and a nested `name`
-                # keyword field (`host.name`).
-                #
-                # As a result, there are always at least 4 statically mapped fields (`@timestamp`, `host`, `host.name`
-                # and `name`). We cannot use a field limit lower than 4 because these fields are always present.
-                #
-                # Indeed, if `index.mapping.total_fields.ignore_dynamic_beyond_limit` is `true`, any dynamically
-                # mapped fields beyond the limit `index.mapping.total_fields.limit` are ignored, but the statically
-                # mapped fields are always counted.
-                total_fields:
-                  limit: 3
-          mappings:
-            properties:
-              "@timestamp":
-                type: date
-              name:
-                type: keyword
-  - match: { error.type: "illegal_argument_exception" }
-  - match: { error.reason: "Limit of total fields [3] has been exceeded" }
+  - match: { hits.hits.1._ignored: [ "message", "pid", "region", "value" ] }
 
 ---
-logsdb with default ignore dynamic beyond limit and subobjects false:
+default ignore dynamic beyond limit and non-default sorting:
   - requires:
-      cluster_features: ["mapper.logsdb_default_ignore_dynamic_beyond_limit"]
-      reason: requires default value for ignore_dynamic_beyond_limit
+      cluster_features: [ "index.logsdb_no_host_name_field" ]
+      reason: "No host.name field injection"
 
   - do:
       indices.create:
-        index: test-logsdb-subobjects-false
+        index: test-logsdb-non-default-sort
         body:
           settings:
             index:
+              sort.field: [ "name" ]
+              sort.order: [ "desc" ]
               mode: logsdb
               mapping:
-                # NOTE: When the index mode is set to `logsdb`, the `host.name` field is automatically injected if
-                # sort settings are not overridden.
-                # With `subobjects` set to `false` anyway, a single `host.name` keyword field is automatically mapped.
-                #
-                # As a result, there are just 3 statically mapped fields (`@timestamp`, `host.name` and `name`).
-                # We cannot use a field limit lower than 3 because these fields are always present.
-                #
-                # Indeed, if `index.mapping.total_fields.ignore_dynamic_beyond_limit` is `true`, any dynamically
-                # mapped fields beyond the limit `index.mapping.total_fields.limit` are ignored, but the statically
-                # mapped fields are always counted.
                 total_fields:
-                  limit: 3
+                  limit: 2
           mappings:
-            subobjects: false
             properties:
               "@timestamp":
                 type: date
@@ -846,13 +718,13 @@ logsdb with default ignore dynamic beyond limit and subobjects false:
 
   - do:
       indices.get_settings:
-        index: test-logsdb-subobjects-false
+        index: test-logsdb-non-default-sort
 
-  - match: { test-logsdb-subobjects-false.settings.index.mode: "logsdb" }
+  - match: { test-logsdb-non-default-sort.settings.index.mode: "logsdb" }
 
   - do:
       bulk:
-        index: test-logsdb-subobjects-false
+        index: test-logsdb-non-default-sort
         refresh: true
         body:
           - '{ "index": { } }'
@@ -863,7 +735,7 @@ logsdb with default ignore dynamic beyond limit and subobjects false:
 
   - do:
       search:
-        index: test-logsdb-subobjects-false
+        index: test-logsdb-non-default-sort
         body:
           query:
             match_all: {}
@@ -873,8 +745,8 @@ logsdb with default ignore dynamic beyond limit and subobjects false:
   - match: { hits.hits.0._source.name: "bar" }
   - match: { hits.hits.0._source.value: 20 }
   - match: { hits.hits.0._source.message: "jumps over the lazy dog" }
-  - match: { hits.hits.0._ignored: [ "message", "pid", "region", "value" ] }
+  - match: { hits.hits.0._ignored: [ "host", "message", "pid", "region", "value" ] }
   - match: { hits.hits.1._source.name: "foo" }
   - match: { hits.hits.1._source.value: 10 }
   - match: { hits.hits.1._source.message: "the quick brown fox" }
-  - match: { hits.hits.1._ignored: [ "message", "pid", "region", "value" ] }
+  - match: { hits.hits.1._ignored: [ "host", "message", "pid", "region", "value" ] }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml
index 084f104932d99..8485aba0ecc6a 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml
@@ -55,9 +55,6 @@ keyword:
 
 ---
 keyword with normalizer:
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
   - do:
       indices.create:
         index: test-keyword-with-normalizer
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml
index 9d6e8da8c1e1e..2a14c291d5d31 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml
@@ -417,7 +417,6 @@
 
   - requires:
       test_runner_features: [arbitrary_key]
-      cluster_features: ["mapper.query_index_mode"]
       reason:  "_ignored_source added to mappings"
 
   - do:
@@ -511,10 +510,6 @@
 ---
 "Lucene segment level fields stats":
 
-  - requires:
-      cluster_features: ["mapper.segment_level_fields_stats"]
-      reason: "segment level fields stats"
-
   - do:
       indices.create:
         index: index1
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/90_fs_watermark_stats.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/90_fs_watermark_stats.yml
index 3ec854e93d82c..20e9d92a36088 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/90_fs_watermark_stats.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/90_fs_watermark_stats.yml
@@ -1,8 +1,6 @@
 ---
 "Allocation stats":
   - requires:
-      cluster_features: ["stats.include_disk_thresholds"]
-      reason: "fs watermark stats was added in 8.15.0"
       test_runner_features: [arbitrary_key]
 
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/10_basic.yml
index 3432a1e34c018..6ca17cc9cdce9 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/10_basic.yml
@@ -520,10 +520,6 @@ setup:
 ---
 "Null bounds":
 
-  - requires:
-      cluster_features: ["mapper.range.null_values_off_by_one_fix"]
-      reason: fixed in 8.15.0
-
   - do:
       index:
         index: test
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml
index d732fb084db3d..9b3291e19c5fd 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml
@@ -115,12 +115,70 @@ setup:
   - match: {hits.hits.0.highlight.field2.0: "The quick brown <em>fox</em> went to the forest and saw another fox."}
 
 ---
-"Plain highlighter with max_analyzed_offset < 0 should FAIL":
+"Plain highlighter on a field WITH OFFSETS exceeding index.highlight.max_analyzed_offset with max_analyzed_offset=0 should FAIL":
+
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: GET
+          path: /_search
+          capabilities: [ highlight_max_analyzed_offset_default ]
+      reason: Behavior of max_analyzed_offset query param changed in 8.18.
+
+  - do:
+      catch: bad_request
+      search:
+        rest_total_hits_as_int: true
+        index: test1
+        body: {"query" : {"match" : {"field2" : "fox"}}, "highlight" : {"type" : "plain", "fields" : {"field2" : {}}, "max_analyzed_offset": 0}}
+  - match: { status: 400 }
+  - match: { error.root_cause.0.type: "x_content_parse_exception" }
+  - match: { error.caused_by.type: "illegal_argument_exception" }
+  - match: { error.caused_by.reason: "[max_analyzed_offset] must be a positive integer, or -1" }
+
+---
+"Plain highlighter on a field WITH OFFSETS exceeding index.highlight.max_analyzed_offset with max_analyzed_offset=1 should SUCCEED":
 
   - requires:
       cluster_features: ["gte_v7.12.0"]
       reason: max_analyzed_offset query param added in 7.12.0
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test1
+        body: {"query" : {"match" : {"field2" : "fox"}}, "highlight" : {"type" : "plain", "fields" : {"field2" : {}}, "max_analyzed_offset": 1}}
+  - match: { hits.hits.0.highlight: null }
+
+---
+"Plain highlighter with max_analyzed_offset = -1 default to index analyze offset should SUCCEED":
+
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: GET
+          path: /_search
+          capabilities: [ highlight_max_analyzed_offset_default ]
+      reason: Behavior of max_analyzed_offset query param changed in 8.18.
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test1
+        body: {"query" : {"match" : {"field2" : "fox"}}, "highlight" : {"type" : "plain", "fields" : {"field2" : {}}, "max_analyzed_offset": -1}}
+  - match: {hits.hits.0.highlight.field2.0: "The quick brown <em>fox</em> went to the forest and saw another fox."}
+
+---
+"Plain highlighter with max_analyzed_offset < -1 should FAIL":
+
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: GET
+          path: /_search
+          capabilities: [ highlight_max_analyzed_offset_default ]
+      reason: Behavior of max_analyzed_offset query param changed in 8.18.
+
   - do:
       catch: bad_request
       search:
@@ -130,4 +188,4 @@ setup:
   - match: { status: 400 }
   - match: { error.root_cause.0.type: "x_content_parse_exception" }
   - match: { error.caused_by.type: "illegal_argument_exception" }
-  - match: { error.caused_by.reason: "[max_analyzed_offset] must be a positive integer" }
+  - match: { error.caused_by.reason: "[max_analyzed_offset] must be a positive integer, or -1" }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/60_unified_matched_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/60_unified_matched_fields.yml
index bd14fb182ac5a..94db54d152941 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/60_unified_matched_fields.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/60_unified_matched_fields.yml
@@ -1,8 +1,4 @@
 setup:
-  - requires:
-      cluster_features: 'unified_highlighter_matched_fields'
-      reason: 'test requires unified highlighter to support matched_fields'
-
   - do:
       indices.create:
           index: index1
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml
index e49f0634a4887..1f07884c9fadf 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml
@@ -82,7 +82,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -100,7 +100,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Get rescoring scores - hit ordering may change depending on how things are distributed
   - match: { hits.total: 3 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/30_rescorer_retriever.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/30_rescorer_retriever.yml
new file mode 100644
index 0000000000000..2c16de61c6b15
--- /dev/null
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/30_rescorer_retriever.yml
@@ -0,0 +1,225 @@
+setup:
+  - requires:
+      cluster_features: [ "search.retriever.rescorer.enabled" ]
+      reason: "Support for rescorer retriever"
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              available:
+                type: boolean
+              features:
+                type: rank_features
+
+  - do:
+      bulk:
+        refresh: true
+        index: test
+        body:
+          - '{"index": {"_id": 1 }}'
+          - '{"features": { "first_stage": 1, "second_stage": 10}, "available": true, "group": 1}'
+          - '{"index": {"_id": 2 }}'
+          - '{"features": { "first_stage": 2, "second_stage": 9}, "available": false, "group": 1}'
+          - '{"index": {"_id": 3 }}'
+          - '{"features": { "first_stage": 3, "second_stage": 8}, "available": false, "group": 3}'
+          - '{"index": {"_id": 4 }}'
+          - '{"features": { "first_stage": 4, "second_stage": 7}, "available": true, "group": 1}'
+          - '{"index": {"_id": 5 }}'
+          - '{"features": { "first_stage": 5, "second_stage": 6}, "available": true, "group": 3}'
+          - '{"index": {"_id": 6 }}'
+          - '{"features": { "first_stage": 6, "second_stage": 5}, "available": false, "group": 2}'
+          - '{"index": {"_id": 7 }}'
+          - '{"features": { "first_stage": 7, "second_stage": 4}, "available": true, "group": 3}'
+          - '{"index": {"_id": 8 }}'
+          - '{"features": { "first_stage": 8, "second_stage": 3}, "available": true, "group": 1}'
+          - '{"index": {"_id": 9 }}'
+          - '{"features": { "first_stage": 9, "second_stage": 2}, "available": true, "group": 2}'
+          - '{"index": {"_id": 10 }}'
+          - '{"features": { "first_stage": 10, "second_stage": 1}, "available": false, "group": 1}'
+
+---
+"Rescorer retriever basic":
+  - do:
+      search:
+        index: test
+        body:
+          retriever:
+            rescorer:
+              rescore:
+                window_size: 10
+                query:
+                  rescore_query:
+                    rank_feature:
+                      field: "features.second_stage"
+                      linear: { }
+                  query_weight: 0
+              retriever:
+                standard:
+                  query:
+                    rank_feature:
+                      field: "features.first_stage"
+                      linear: { }
+          size: 2
+
+  - match: { hits.total.value: 10 }
+  - match: { hits.hits.0._id: "1" }
+  - match: { hits.hits.0._score: 10.0 }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.1._score: 9.0 }
+
+  - do:
+      search:
+        index: test
+        body:
+          retriever:
+            rescorer:
+              rescore:
+                window_size: 3
+                query:
+                  rescore_query:
+                    rank_feature:
+                      field: "features.second_stage"
+                      linear: {}
+                  query_weight: 0
+              retriever:
+                standard:
+                  query:
+                    rank_feature:
+                      field: "features.first_stage"
+                      linear: {}
+          size: 2
+
+  - match: {hits.total.value: 10}
+  - match: {hits.hits.0._id: "8"}
+  - match: { hits.hits.0._score: 3.0 }
+  - match: {hits.hits.1._id: "9"}
+  - match: { hits.hits.1._score: 2.0 }
+
+---
+"Rescorer retriever with pre-filters":
+  - do:
+      search:
+        index: test
+        body:
+          retriever:
+            rescorer:
+              filter:
+                match:
+                  available: true
+              rescore:
+                window_size: 10
+                query:
+                  rescore_query:
+                    rank_feature:
+                      field: "features.second_stage"
+                      linear: { }
+                  query_weight: 0
+              retriever:
+                standard:
+                  query:
+                    rank_feature:
+                      field: "features.first_stage"
+                      linear: { }
+          size: 2
+
+  - match: { hits.total.value: 6 }
+  - match: { hits.hits.0._id: "1" }
+  - match: { hits.hits.0._score: 10.0 }
+  - match: { hits.hits.1._id: "4" }
+  - match: { hits.hits.1._score: 7.0 }
+
+  - do:
+      search:
+        index: test
+        body:
+          retriever:
+            rescorer:
+              rescore:
+                window_size: 4
+                query:
+                  rescore_query:
+                    rank_feature:
+                      field: "features.second_stage"
+                      linear: { }
+                  query_weight: 0
+              retriever:
+                standard:
+                  filter:
+                    match:
+                      available: true
+                  query:
+                    rank_feature:
+                      field: "features.first_stage"
+                      linear: { }
+          size: 2
+
+  - match: { hits.total.value: 6 }
+  - match: { hits.hits.0._id: "5" }
+  - match: { hits.hits.0._score: 6.0 }
+  - match: { hits.hits.1._id: "7" }
+  - match: { hits.hits.1._score: 4.0 }
+
+---
+"Rescorer retriever and collapsing":
+  - do:
+      search:
+        index: test
+        body:
+          retriever:
+            rescorer:
+              rescore:
+                window_size: 10
+                query:
+                  rescore_query:
+                    rank_feature:
+                      field: "features.second_stage"
+                      linear: { }
+                  query_weight: 0
+              retriever:
+                standard:
+                  query:
+                    rank_feature:
+                      field: "features.first_stage"
+                      linear: { }
+          collapse:
+            field: group
+          size: 3
+
+  - match: { hits.total.value: 10 }
+  - match: { hits.hits.0._id: "1" }
+  - match: { hits.hits.0._score: 10.0 }
+  - match: { hits.hits.1._id: "3" }
+  - match: { hits.hits.1._score: 8.0 }
+  - match: { hits.hits.2._id: "6" }
+  - match: { hits.hits.2._score: 5.0 }
+
+---
+"Rescorer retriever and invalid window size":
+  - do:
+      catch: "/\\[rescorer\\] requires \\[window_size: 5\\] be greater than or equal to \\[size: 10\\]/"
+      search:
+        index: test
+        body:
+          retriever:
+            rescorer:
+              rescore:
+                window_size: 5
+                query:
+                  rescore_query:
+                    rank_feature:
+                      field: "features.second_stage"
+                      linear: { }
+                  query_weight: 0
+              retriever:
+                standard:
+                  query:
+                    rank_feature:
+                      field: "features.first_stage"
+                      linear: { }
+          size: 10
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/101_knn_nested_search_bits.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/101_knn_nested_search_bits.yml
index a3d920d903ae8..bc4e262ea53c6 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/101_knn_nested_search_bits.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/101_knn_nested_search_bits.yml
@@ -1,8 +1,6 @@
 setup:
   - requires:
-      cluster_features: "mapper.vectors.bit_vectors"
       test_runner_features: close_to
-      reason: 'bit vectors added in 8.15'
   - do:
       indices.create:
         index: test
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/110_knn_query_with_filter.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/110_knn_query_with_filter.yml
index 618951711cffd..cffc12a8d24ae 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/110_knn_query_with_filter.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/110_knn_query_with_filter.yml
@@ -59,7 +59,6 @@ setup:
 
 ---
 "Simple knn query":
-
   - do:
       search:
         index: my_index
@@ -71,8 +70,9 @@ setup:
               field: my_vector
               query_vector: [1, 1, 1, 1]
               num_candidates: 5
+              k: 5
 
-  - match: { hits.total.value: 5 } # collector sees num_candidates docs
+  - match: { hits.total.value: 5 }
   - length: {hits.hits: 3}
   - match: { hits.hits.0._id: "1" }
   - match: { hits.hits.0.fields.my_name.0: v1 }
@@ -93,8 +93,9 @@ setup:
               field: my_vector
               query_vector: [1, 1, 1, 1]
               num_candidates: 5
+              k: 5
 
-  - match: { hits.total.value: 5 } # collector sees num_candidates docs
+  - match: { hits.total.value: 5 }
   - length: {hits.hits: 3}
   - match: { hits.hits.0._id: "2" }
   - match: { hits.hits.0.fields.my_name.0: v2 }
@@ -140,6 +141,7 @@ setup:
                     field: my_vector
                     query_vector: [1, 1, 1, 1]
                     num_candidates: 5
+                    k: 5
 
   - match: { hits.total.value: 5 }
   - length: { hits.hits: 3 }
@@ -184,6 +186,7 @@ setup:
               field: my_vector
               query_vector: [1, 1, 1, 1]
               num_candidates: 100
+              k: 100
 
   - match: { hits.total.value: 10 } # 5 docs from each alias
   - length: {hits.hits: 6}
@@ -213,6 +216,7 @@ setup:
               field: my_vector
               query_vector: [1, 1, 1, 1]
               num_candidates: 5
+              k: 5
               filter:
                 term:
                   my_name: v2
@@ -243,9 +247,10 @@ setup:
                     field: my_vector
                     query_vector: [1, 1, 1, 1]
                     num_candidates: 5
+                    k: 5
 
   - match: { hits.total.value: 2 }
-  - length: {hits.hits: 2} # knn query returns top 5 docs, but they are post-filtered to 2 docs
+  - length: {hits.hits: 2} # knn query returns top 3 docs, but they are post-filtered to 2 docs
   - match: { hits.hits.0._id: "2" }
   - match: { hits.hits.0.fields.my_name.0: v2 }
   - match: { hits.hits.1._id: "4" }
@@ -271,4 +276,4 @@ setup:
                         my_name: v1
 
   - match: { hits.total.value:  0}
-  - length: { hits.hits: 0 } # knn query returns top 5 docs, but they are post-filtered to 0 docs
+  - length: { hits.hits: 0 } # knn query returns top 3 docs, but they are post-filtered to 0 docs
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/120_knn_query_multiple_shards.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/120_knn_query_multiple_shards.yml
index c6f3e187f7953..c68565e6629f5 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/120_knn_query_multiple_shards.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/120_knn_query_multiple_shards.yml
@@ -166,55 +166,3 @@ setup:
   - close_to: { hits.hits.2._score: { value: 120, error: 0.00001 } }
   - close_to: { hits.hits.2.matched_queries.bm25_query: { value: 100.0, error: 0.00001 } }
   - close_to: { hits.hits.2.matched_queries.knn_query: { value: 20.0, error: 0.00001 } }
-
----
-"Aggregations with collected number of docs depends on num_candidates":
-  - do:
-      search:
-        index: my_index
-        body:
-          size: 2
-          query:
-            knn:
-              field: my_vector
-              query_vector: [1, 1, 1, 1]
-              num_candidates: 100 # collect up to 100 candidates from each shard
-          aggs:
-            my_agg:
-              terms:
-                field: my_name
-                order:
-                  _key: asc
-
-  - length: {hits.hits: 2}
-  - match: {hits.total.value: 12}
-  - match: {aggregations.my_agg.buckets.0.key: 'v1'}
-  - match: {aggregations.my_agg.buckets.1.key: 'v2'}
-  - match: {aggregations.my_agg.buckets.0.doc_count: 6}
-  - match: {aggregations.my_agg.buckets.1.doc_count: 6}
-
-  - do:
-      search:
-        index: my_index
-        body:
-          size: 2
-          query:
-            knn:
-              field: my_vector
-              query_vector: [ 1, 1, 1, 1 ]
-              num_candidates: 3 # collect 3 candidates from each shard
-          aggs:
-            my_agg2:
-              terms:
-                field: my_name
-                order:
-                  _key: asc
-            my_sum_buckets:
-              sum_bucket:
-                buckets_path: "my_agg2>_count"
-
-  - length: { hits.hits: 2 }
-  - match: { hits.total.value: 6 }
-  - match: { aggregations.my_agg2.buckets.0.key: 'v1' }
-  - match: { aggregations.my_agg2.buckets.1.key: 'v2' }
-  - match: { aggregations.my_sum_buckets.value: 6.0 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/130_knn_query_nested_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/130_knn_query_nested_search.yml
index 79ff3f61742f8..bf07144975650 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/130_knn_query_nested_search.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/130_knn_query_nested_search.yml
@@ -273,6 +273,7 @@ setup:
                 knn:
                   field: nested.vector
                   query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
+                  k: 5
                   num_candidates: 5
               inner_hits: { size: 2, "fields": [ "nested.paragraph_id" ], _source: false }
 
@@ -295,6 +296,7 @@ setup:
                 knn:
                   field: nested.vector
                   query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
+                  k: 5
                   num_candidates: 5
               inner_hits: { size: 2, "fields": [ "nested.paragraph_id" ], _source: false }
 
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/140_knn_query_with_other_queries.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/140_knn_query_with_other_queries.yml
index d52a5daf22344..1e54e497f286f 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/140_knn_query_with_other_queries.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/140_knn_query_with_other_queries.yml
@@ -69,6 +69,7 @@ setup:
                   field: my_vector
                   query_vector: [ 1, 1, 1, 1 ]
                   num_candidates: 5
+                  k: 5
               functions:
                 - filter: { match: { my_name: v1 } }
                   weight: 10
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/160_knn_query_missing_params.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/160_knn_query_missing_params.yml
index 02962e049e267..26c52060dfb22 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/160_knn_query_missing_params.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/160_knn_query_missing_params.yml
@@ -100,8 +100,9 @@ setup:
             knn:
               field: vector
               query_vector: [1, 1, 1]
+              k: 2
           size: 1
-  - match: { hits.total: 2 }  # due to num_candidates defined as round(1.5 * size), so we only see 2 results
+  - match: { hits.total: 2 }  # k defaults to size
   - length: { hits.hits: 1 }  # one result is only returned though
 
 ---
@@ -117,6 +118,7 @@ setup:
               field: vector
               query_vector: [-1, -1, -1]
               num_candidates: 1
+              k: 1
           size: 10
 
   - match: { hits.total: 1 }
@@ -137,9 +139,10 @@ setup:
                 - knn:
                     field: vector
                     query_vector: [ 1, 1, 0]
+                    k: 1
           size: 1
 
-  - match: { hits.total: 2 }  # due to num_candidates defined as round(1.5 * size), so we only see 2 results from cat:A
+  - match: { hits.total: 1 }
   - length: { hits.hits: 1 }
 
 ---
@@ -154,6 +157,7 @@ setup:
                 - knn:
                     field: vector
                     query_vector: [1, 1, 0]
+                    k: 2
                 - match:
                     category: B
               tie_breaker: 0.8
@@ -175,6 +179,7 @@ setup:
                 knn:
                   field: nested.vector
                   query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
+                  k: 2
               inner_hits: { size: 1, "fields": [ "nested.paragraph_id" ], _source: false }
           size: 1
 
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml
index 855daeaa7f163..99943ef2671bb 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml
@@ -109,6 +109,7 @@ setup:
               field: embedding
               query_vector: [1, 1, 1, 1]
               num_candidates: 10
+              k: 10
 
   - match: { hits.total.value: 10 }
   - length: {hits.hits: 3}
@@ -215,6 +216,7 @@ setup:
               field: embedding
               query_vector: [ 1, 1, 1, 1 ]
               num_candidates: 20
+              k: 20
 
   - match: { hits.total.value: 20 }
   - length: { hits.hits: 3 }
@@ -322,6 +324,7 @@ setup:
               field: embedding
               query_vector: [ 1, 1, 1, 1 ]
               num_candidates: 30
+              k: 30
 
   - match: { hits.total.value: 30 }
   - length: { hits.hits: 4 }
@@ -430,6 +433,7 @@ setup:
               field: embedding
               query_vector: [ 1, 1, 1, 1 ]
               num_candidates: 40
+              k: 40
 
   - match: { hits.total.value: 40 }
   - length: { hits.hits: 5 }
@@ -499,6 +503,7 @@ setup:
               field: embedding
               query_vector: [1, 1, 1, 1]
               num_candidates: 10
+              k: 10
 
   - match: { hits.total.value: 10 }
   - length: {hits.hits: 3}
@@ -559,6 +564,7 @@ setup:
               field: embedding
               query_vector: [ 1, 1, 1, 1 ]
               num_candidates: 20
+              k: 20
 
   - match: { hits.total.value: 20 }
   - length: { hits.hits: 3 }
@@ -620,6 +626,7 @@ setup:
               field: embedding
               query_vector: [ 1, 1, 1, 1 ]
               num_candidates: 30
+              k: 30
 
   - match: { hits.total.value: 30 }
   - length: { hits.hits: 4 }
@@ -682,6 +689,7 @@ setup:
               field: embedding
               query_vector: [ 1, 1, 1, 1 ]
               num_candidates: 40
+              k: 40
 
   - match: { hits.total.value: 40 }
   - length: { hits.hits: 5 }
@@ -751,6 +759,7 @@ setup:
               field: embedding
               query_vector: [ 1, 1, 1, 1 ]
               num_candidates: 10
+              k: 10
 
   - match: { hits.total.value: 10 }
   - length: { hits.hits: 3 }
@@ -791,6 +800,7 @@ setup:
               field: embedding
               query_vector: [ 1, 1, 1, 1 ]
               num_candidates: 10
+              k: 10
 
   - match: { hits.total.value: 10 }
   - length: { hits.hits: 3 }
@@ -833,6 +843,7 @@ setup:
               field: embedding
               query_vector: [ 1, 1, 1, 1 ]
               num_candidates: 20
+              k: 20
 
   - match: { hits.total.value: 20 }
   - length: { hits.hits: 3 }
@@ -869,6 +880,7 @@ setup:
               field: embedding
               query_vector: [ 1, 1, 1, 1 ]
               num_candidates: 20
+              k: 20
 
   - match: { hits.total.value: 20 }
   - length: { hits.hits: 3 }
@@ -911,6 +923,7 @@ setup:
               field: embedding
               query_vector: [ 1, 1, 1, 1 ]
               num_candidates: 30
+              k: 30
 
   - match: { hits.total.value: 30 }
   - length: { hits.hits: 4 }
@@ -933,6 +946,7 @@ setup:
             knn:
               field: embedding
               query_vector: [ 1, 1, 1, 1 ]
+              k: 30
               num_candidates: 30
 
   - match: { hits.total.value: 30 }
@@ -1769,6 +1783,7 @@ setup:
               field: embedding
               query_vector: [1, 1, 1, 1]
               num_candidates: 10
+              k: 10
 
   - match: { hits.total.value: 10 }
   - length: {hits.hits: 3}
@@ -1875,6 +1890,7 @@ setup:
               field: embedding
               query_vector: [ 1, 1, 1, 1 ]
               num_candidates: 20
+              k: 20
 
   - match: { hits.total.value: 20 }
   - length: { hits.hits: 3 }
@@ -1982,6 +1998,7 @@ setup:
               field: embedding
               query_vector: [ 1, 1, 1, 1 ]
               num_candidates: 30
+              k: 30
 
   - match: { hits.total.value: 30 }
   - length: { hits.hits: 4 }
@@ -2090,6 +2107,7 @@ setup:
               field: embedding
               query_vector: [ 1, 1, 1, 1 ]
               num_candidates: 40
+              k: 40
 
   - match: { hits.total.value: 40 }
   - length: { hits.hits: 5 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/190_knn_query-with-k-param.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/190_knn_query-with-k-param.yml
index f6538b573809a..c92c88df91641 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/190_knn_query-with-k-param.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/190_knn_query-with-k-param.yml
@@ -1,8 +1,6 @@
 # test how knn query interact with other queries
 setup:
   - requires:
-      cluster_features: "search.vectors.k_param_supported"
-      reason: 'k param for knn as query is required'
       test_runner_features: close_to
 
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/210_knn_search_profile.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/210_knn_search_profile.yml
index d4bf5e7e9807f..be35dcde2eff3 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/210_knn_search_profile.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/210_knn_search_profile.yml
@@ -106,9 +106,9 @@ setup:
             k: 3
             num_candidates: 3
             "rescore_vector":
-              "num_candidates_factor": 2.0
+              "oversample": 2.0
 
-  # We expect the knn search ops + rescoring num_cnaidates (for rescoring) per shard
+  # We expect the knn search ops + rescoring k * oversample (for rescoring) per shard
   - match: { profile.shards.0.dfs.knn.0.vector_operations_count: 6 }
 
   # Search with similarity to check number of operations are propagated correctly
@@ -131,7 +131,7 @@ setup:
             num_candidates: 3
             similarity: 100000
             "rescore_vector":
-              "num_candidates_factor": 2.0
+              "oversample": 2.0
 
-  # We expect the knn search ops + rescoring num_cnaidates (for rescoring) per shard
+  # We expect the knn search ops + rescoring k * oversample (for rescoring) per shard
   - match: { profile.shards.0.dfs.knn.0.vector_operations_count: 6 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml
index 7d4690204acc7..8f846dd76721d 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml
@@ -558,7 +558,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -598,7 +598,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Compare scores as hit IDs may change depending on how things are distributed
   - match: { hits.total: 3 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml
index 2567a4ac597d9..abde3e86dd05b 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml
@@ -1,7 +1,4 @@
 setup:
-  - requires:
-      cluster_features: "mapper.vectors.bbq"
-      reason: 'kNN float to better-binary quantization is required'
   - do:
       indices.create:
         index: bbq_hnsw
@@ -115,7 +112,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -140,7 +137,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Get rescoring scores - hit ordering may change depending on how things are distributed
   - match: { hits.total: 3 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml
index b1e35789e8737..229d705bc317c 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml
@@ -378,7 +378,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -398,7 +398,7 @@ setup:
               field: vector
               query_vector: [0.5, 111.3, -13.0, 14.8, -156.0]
               rescore_vector:
-                num_candidates_factor: 1.5
+                oversample: 1.5
 
   # Get rescoring scores - hit ordering may change depending on how things are distributed
   - match: { hits.total: 3 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml
index 54e9eadf42e0b..9b27aea4b1db7 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml
@@ -1,7 +1,4 @@
 setup:
-  - requires:
-      cluster_features: "mapper.vectors.int4_quantization"
-      reason: 'kNN float to half-byte quantization is required'
   - do:
       indices.create:
         index: hnsw_byte_quantized
@@ -556,7 +553,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -575,7 +572,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Get rescoring scores - hit ordering may change depending on how things are distributed
   - match: { hits.total: 3 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml
index a3cd624ef0ab8..2541de7023bf0 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml
@@ -1,7 +1,4 @@
 setup:
-  - requires:
-      cluster_features: "mapper.vectors.bbq"
-      reason: 'kNN float to better-binary quantization is required'
   - do:
       indices.create:
         index: bbq_flat
@@ -114,7 +111,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -139,7 +136,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Get rescoring scores - hit ordering may change depending on how things are distributed
   - match: { hits.total: 3 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_flat.yml
index a59aedceff3d3..358ff547036e6 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_flat.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_flat.yml
@@ -264,7 +264,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -304,7 +304,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Compare scores as hit IDs may change depending on how things are distributed
   - match: { hits.total: 3 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml
index 6796a92122f9a..f9f8d56e1d9c9 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml
@@ -1,7 +1,4 @@
 setup:
-  - requires:
-      cluster_features: "mapper.vectors.int4_quantization"
-      reason: 'kNN float to half-byte quantization is required'
   - do:
       indices.create:
         index: int4_flat
@@ -352,7 +349,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -371,7 +368,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Get rescoring scores - hit ordering may change depending on how things are distributed
   - match: { hits.total: 3 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml
index d1d312449cb70..6b59b8f641ee9 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml
@@ -269,7 +269,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -288,7 +288,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Get rescoring scores - hit ordering may change depending on how things are distributed
   - match: { hits.total: 3 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml
index effa3fff61525..ef2ae3ba7ee0a 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml
@@ -1,8 +1,4 @@
 setup:
-  - requires:
-      cluster_features: "mapper.vectors.bit_vectors"
-      reason: 'mapper.vectors.bit_vectors'
-
   - do:
       indices.create:
         index: test
@@ -414,7 +410,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -454,7 +450,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Compare scores as hit IDs may change depending on how things are distributed
   - match: { hits.total: 3 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit_flat.yml
index cdc1d9c64763e..07261e6a30c77 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit_flat.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit_flat.yml
@@ -1,8 +1,4 @@
 setup:
-  - requires:
-      cluster_features: "mapper.vectors.bit_vectors"
-      reason: 'mapper.vectors.bit_vectors'
-
   - do:
       indices.create:
         index: test
@@ -230,7 +226,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -270,7 +266,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Compare scores as hit IDs may change depending on how things are distributed
   - match: { hits.total: 3 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml
index 213b571a0b4be..6559b8d969cb9 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml
@@ -263,7 +263,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -303,7 +303,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Compare scores as hit IDs may change depending on how things are distributed
   - match: { hits.total: 3 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml
index 44d966b76f34e..8915325c3a67b 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml
@@ -1128,10 +1128,6 @@ fetch geo_point:
 
 ---
 "Test with subobjects: auto":
-  - requires:
-      cluster_features: "mapper.subobjects_auto"
-      reason: requires support for subobjects auto setting
-
   - do:
       indices.create:
         index: test
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/530_ignore_above_stored_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/530_ignore_above_stored_source.yml
index 1730a49f743d9..7e00cbb01c589 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/530_ignore_above_stored_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/530_ignore_above_stored_source.yml
@@ -1,8 +1,5 @@
 ---
 ignore_above mapping level setting:
-  - requires:
-      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
-      reason: introduce ignore_above index level setting
   - do:
       indices.create:
         index:  test
@@ -42,9 +39,6 @@ ignore_above mapping level setting:
 
 ---
 ignore_above mapping level setting on arrays:
-  - requires:
-      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
-      reason: introduce ignore_above index level setting
   - do:
       indices.create:
         index:  test
@@ -84,9 +78,6 @@ ignore_above mapping level setting on arrays:
 
 ---
 ignore_above mapping overrides setting:
-  - requires:
-      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
-      reason: introduce ignore_above index level setting
   - do:
       indices.create:
         index:  test
@@ -128,9 +119,6 @@ ignore_above mapping overrides setting:
 
 ---
 ignore_above mapping overrides setting on arrays:
-  - requires:
-      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
-      reason: introduce ignore_above index level setting
   - do:
       indices.create:
         index: test
@@ -172,9 +160,6 @@ ignore_above mapping overrides setting on arrays:
 
 ---
 date ignore_above index level setting:
-  - requires:
-      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
-      reason: introduce ignore_above index level setting
   - do:
       indices.create:
         index:  test
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml
index 772c3c24170cd..045f757b08302 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml
@@ -5,9 +5,6 @@ setup:
 
 ---
 ignore_above mapping level setting:
-  - requires:
-      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
-      reason: introduce ignore_above index level setting
   - do:
       indices.create:
         index:  test
@@ -47,9 +44,6 @@ ignore_above mapping level setting:
 
 ---
 ignore_above mapping level setting on arrays:
-  - requires:
-      cluster_features: [ "mapper.flattened.ignore_above_with_arrays_support" ]
-      reason: requires support of ignore_above with arrays for flattened fields
   - do:
       indices.create:
         index:  test
@@ -90,9 +84,6 @@ ignore_above mapping level setting on arrays:
 
 ---
 ignore_above mapping overrides setting:
-  - requires:
-      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
-      reason: introduce ignore_above index level setting
   - do:
       indices.create:
         index:  test
@@ -135,9 +126,6 @@ ignore_above mapping overrides setting:
 
 ---
 ignore_above mapping overrides setting on arrays:
-  - requires:
-      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
-      reason: introduce ignore_above index level setting
   - do:
       indices.create:
         index: test
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/550_ignore_above_invalid.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/550_ignore_above_invalid.yml
index 3c29845871fe7..6e711ee143b06 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/550_ignore_above_invalid.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/550_ignore_above_invalid.yml
@@ -16,9 +16,6 @@ ignore_above index setting negative value:
 
 ---
 keyword ignore_above mapping setting negative value:
-  - requires:
-      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
-      reason: introduce ignore_above index level setting
   - do:
       catch: bad_request
       indices.create:
@@ -32,9 +29,6 @@ keyword ignore_above mapping setting negative value:
 
 ---
 flattened ignore_above mapping setting negative value:
-  - requires:
-      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
-      reason: introduce ignore_above index level setting
   - do:
       catch: bad_request
       indices.create:
@@ -48,9 +42,6 @@ flattened ignore_above mapping setting negative value:
 
 ---
 wildcard ignore_above mapping setting negative value:
-  - requires:
-      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
-      reason: introduce ignore_above index level setting
   - do:
       catch: bad_request
       indices.create:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml
index a4a9b1aaecb22..71e0c2d147c1e 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml
@@ -1,8 +1,5 @@
 ---
 flattened ignore_above single-value field:
-  - requires:
-      cluster_features: [ "flattened.ignore_above_support" ]
-      reason: introduce ignore_above support in flattened fields
   - do:
       indices.create:
         index: test
@@ -65,9 +62,6 @@ flattened ignore_above single-value field:
 
 ---
 flattened ignore_above multi-value field:
-  - requires:
-      cluster_features: [ "flattened.ignore_above_support" ]
-      reason: introduce ignore_above support in flattened fields
   - do:
       indices.create:
         index: test
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/80_indices_options.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/80_indices_options.yml
index 2022389a69660..1af6816d902ae 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/80_indices_options.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/80_indices_options.yml
@@ -37,8 +37,6 @@
   - do:
       indices.close:
           index:  index_closed
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       catch:  /index_closed_exception/
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/90_search_after.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/90_search_after.yml
index 1fefc8bffffa1..d3b2b5a412717 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/90_search_after.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/90_search_after.yml
@@ -218,31 +218,6 @@
   - match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828740" }
   - match: {hits.hits.0.sort: [1571617804828740000] }
 
-
----
-"_shard_doc sort":
-  - requires:
-      cluster_features: ["gte_v7.12.0"]
-      reason: _shard_doc sort was added in 7.12
-
-  - do:
-      indices.create:
-        index:  test
-  - do:
-      index:
-        index:  test
-        id:     "1"
-        body:   { id: 1, foo: bar, age: 18 }
-
-  - do:
-      catch:      /\[_shard_doc\] sort field cannot be used without \[point in time\]/
-      search:
-        index: test
-        body:
-          size: 1
-          sort: ["_shard_doc"]
-          search_after: [ 0L ]
-
 ---
 "Format sort values":
   - requires:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml
index da0f00d960534..70a3b0253c78f 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml
@@ -119,10 +119,6 @@ setup:
   - skip:
       features: headers
 
-  - requires:
-      cluster_features: ["simulate.mapping.validation"]
-      reason: "ingest simulate index mapping validation added in 8.16"
-
   - do:
       headers:
         Content-Type: application/json
@@ -265,10 +261,6 @@ setup:
         - headers
         - allowed_warnings
 
-  - requires:
-      cluster_features: ["simulate.mapping.validation.templates"]
-      reason: "ingest simulate index mapping validation added in 8.16"
-
   - do:
       indices.put_template:
         name: v1_template
@@ -401,10 +393,6 @@ setup:
        - headers
        - allowed_warnings
 
-  - requires:
-      cluster_features: ["simulate.mapping.validation.templates"]
-      reason: "ingest simulate index mapping validation added in 8.16"
-
   - do:
       allowed_warnings:
         - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.restore/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.restore/10_basic.yml
index ecc6330a73123..133b556ce4c91 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.restore/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.restore/10_basic.yml
@@ -46,8 +46,6 @@ setup:
   - do:
       indices.close:
         index : test_index
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       snapshot.restore:
@@ -92,8 +90,6 @@ setup:
   - do:
       indices.close:
         index : test_index
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       snapshot.restore:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml
index d6c98673253fb..4e6bd83f07955 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml
@@ -1,8 +1,8 @@
----
-"Reload analyzers for specific synonym set":
+setup:
   - requires:
       cluster_features: ["gte_v8.10.0"]
       reason: Reloading analyzers for specific synonym set is introduced in 8.10.0
+
   # Create synonyms_set1
   - do:
       synonyms.put_synonym:
@@ -100,7 +100,12 @@
           - '{"index": {"_index": "my_index2", "_id": "2"}}'
           - '{"my_field": "goodbye"}'
 
-  # An update of synonyms_set1 must trigger auto-reloading of analyzers only for synonyms_set1
+---
+"Reload analyzers for specific synonym set":
+# These specific tests can't succeed in BwC, as synonyms auto-expand replicas are 0-all. Replicas can't be associated to
+#  upgraded nodes, and thus we are not able to guarantee that the shards are not failed.
+# This test is skipped for BwC until synonyms index has auto-exapnd replicas set to 0-1.
+
   - do:
       synonyms.put_synonym:
         id: synonyms_set1
@@ -108,13 +113,12 @@
           synonyms_set:
             - synonyms: "hello, salute"
             - synonyms: "ciao => goodbye"
+
   - match: { result: "updated" }
   - gt: { reload_analyzers_details._shards.total: 0 }
   - gt: { reload_analyzers_details._shards.successful: 0 }
   - match: { reload_analyzers_details._shards.failed: 0 }
-  - length: { reload_analyzers_details.reload_details: 1 } # reload details contain only a single index
-  - match: { reload_analyzers_details.reload_details.0.index: "my_index1" }
-  - match: { reload_analyzers_details.reload_details.0.reloaded_analyzers.0: "my_analyzer1" }
+
 
   # Confirm that the index analyzers are reloaded for my_index1
   - do:
@@ -127,6 +131,23 @@
                 query: salute
   - match: { hits.total.value: 1 }
 
+---
+"Check analyzer reloaded and non failed shards for bwc tests":
+
+  - do:
+      synonyms.put_synonym:
+        id: synonyms_set1
+        body:
+          synonyms_set:
+            - synonyms: "hello, salute"
+            - synonyms: "ciao => goodbye"
+  - match: { result: "updated" }
+  - gt: { reload_analyzers_details._shards.total: 0 }
+  - gt: { reload_analyzers_details._shards.successful: 0 }
+  - length: { reload_analyzers_details.reload_details: 1 } # reload details contain only a single index
+  - match: { reload_analyzers_details.reload_details.0.index: "my_index1" }
+  - match: { reload_analyzers_details.reload_details.0.reloaded_analyzers.0: "my_analyzer1" }
+
   # Confirm that the index analyzers are still the same for my_index2
   - do:
       search:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml
index 616afd3cf67ad..1e841c8893fc6 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml
@@ -122,8 +122,6 @@ missing dimension on routing path field:
 multi-value routing path field succeeds:
   - requires:
       test_runner_features: close_to
-      cluster_features: ["routing.multi_value_routing_path"]
-      reason: support for multi-value dimensions
 
   - do:
       indices.create:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml
index 9fe3f5e0b7272..f25601fc2e228 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml
@@ -127,7 +127,7 @@ exact match object type:
       reason: routing_path error message updated in 8.14.0
 
   - do:
-      catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[dim\] was \[object\]./'
+      catch: '/All fields that match routing_path must be .*flattened fields.* \[dim\] was \[object\]./'
       indices.create:
         index: tsdb_index
         body:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml
index 4faa0424adb43..5a5ae03ab938f 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml
@@ -65,9 +65,6 @@ setup:
 
 ---
 generates a consistent id:
-  - requires:
-        cluster_features: "tsdb.ts_routing_hash_doc_value_parse_byte_ref"
-        reason: _tsid routing hash doc value parsing has been fixed
   - do:
       bulk:
         refresh: true
@@ -427,7 +424,7 @@ delete over _bulk:
   - match: {items.0.delete.result: deleted}
   - match: {items.1.delete.result: deleted}
   - match: {items.2.delete.status: 404}
-  - match: {items.2.delete.error.reason: "invalid id [not found ++ not found] for index [id_generation_test] in time series mode"}
+  - match: {items.2.delete.error.reason: '/invalid\ id\ \[not\ found\ \+\+\ not\ found\]\ for\ index\ \[id_generation_test\]\ in\ time.series\ mode/'}
 
 ---
 routing_path matches deep object:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml
index b0e1595f3d0e3..2f177d275c099 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml
@@ -120,8 +120,6 @@ teardown:
   - do:
       indices.close:
         index : test_index
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   # Restore index
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml
index dae50704dd0d0..a8d256bbc097e 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml
@@ -340,9 +340,6 @@ sort by tsid:
 
 ---
 aggs by index_mode:
-  - requires:
-      cluster_features: ["mapper.query_index_mode"]
-      reason: require _index_mode metadata field
   - do:
       search:
         index: test
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml
index c32d3c50b0784..c71555dd073d6 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml
@@ -95,7 +95,7 @@ split:
       reason: tsdb indexing changed in 8.2.0
 
   - do:
-      catch: /index-split is not supported because the destination index \[test\] is in time series mode/
+      catch: /index-split is not supported because the destination index \[test\] is in time.series mode/
       indices.split:
         index: test
         target: test_split
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml
index 54b2bf59c8ddc..142d1281ad12b 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml
@@ -75,7 +75,7 @@ index with routing:
       reason: tsdb indexing changed in 8.2.0
 
   - do:
-      catch: /specifying routing is not supported because the destination index \[test\] is in time series mode/
+      catch: /specifying routing is not supported because the destination index \[test\] is in time.series mode/
       index:
         index:   test
         routing: foo
@@ -104,7 +104,7 @@ index with routing over _bulk:
         body:
           - '{"index": {"routing": "foo"}}'
           - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
-  - match: {items.0.index.error.reason: "specifying routing is not supported because the destination index [test] is in time series mode"}
+  - match: {items.0.index.error.reason: '/specifying\ routing\ is\ not\ supported\ because\ the\ destination\ index\ \[test\]\ is\ in\ time.series\ mode/'}
 
 ---
 noop update:
@@ -120,7 +120,7 @@ noop update:
   - length: {hits.hits: 1}
 
   - do:
-      catch: /update is not supported because the destination index \[test\] is in time series mode/
+      catch: /update is not supported because the destination index \[test\] is in time.series mode/
       update:
         index:   test
         id:      "1"
@@ -136,7 +136,7 @@ regular update:
 
   # We fail even though the document isn't found.
   - do:
-      catch: /update is not supported because the destination index \[test\] is in time series mode/
+      catch: /update is not supported because the destination index \[test\] is in time.series mode/
       update:
         index:   test
         id:      "1"
@@ -165,7 +165,7 @@ update over _bulk:
         body:
           - '{"update": {"_id": 1}}'
           - '{"doc":{"@timestamp": "2021-04-28T18:03:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}}'
-  - match: {items.0.update.error.reason: "update is not supported because the destination index [test] is in time series mode"}
+  - match: {items.0.update.error.reason: '/update\ is\ not\ supported\ because\ the\ destination\ index\ \[test\]\ is\ in\ time.series\ mode/'}
 
 ---
 search with routing:
@@ -175,7 +175,7 @@ search with routing:
 
   # We fail even though the document isn't found.
   - do:
-      catch: /searching with a specified routing is not supported because the destination index \[test\] is in time series mode/
+      catch: /searching with a specified routing is not supported because the destination index \[test\] is in time.series mode/
       search:
         index:   test
         routing: rrrr
diff --git a/server/build.gradle b/server/build.gradle
index 0bd807751ecbb..b186999c36ca3 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -15,7 +15,7 @@ apply plugin: 'elasticsearch.internal-test-artifact'
 publishing {
   publications {
     elastic {
-      artifactId 'elasticsearch'
+      artifactId = 'elasticsearch'
     }
   }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java
index f5860cedcd989..68e65b16aa3a2 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.action.admin.indices.cache.clear.TransportClearIndicesCacheAction;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
@@ -157,9 +158,11 @@ public void setup() {
         for (int i = 0; i < numIndices; i++) {
             indices.add("test" + i);
         }
-        for (String index : indices) {
-            assertAcked(prepareCreate(index).addAlias(new Alias(index + "-alias")));
-        }
+        assertAcked(
+            indices.stream()
+                .map(index -> prepareCreate(index).addAlias(new Alias(index + "-alias")))
+                .toArray(CreateIndexRequestBuilder[]::new)
+        );
         ensureGreen();
     }
 
@@ -516,7 +519,7 @@ public void testDeleteIndex() {
     public void testGetMappings() {
         interceptTransportActions(GetMappingsAction.NAME);
 
-        GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(randomIndicesOrAliases());
+        GetMappingsRequest getMappingsRequest = new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(randomIndicesOrAliases());
         internalCluster().coordOnlyNodeClient().admin().indices().getMappings(getMappingsRequest).actionGet();
 
         clearInterceptedActions();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java
index a954325c65a97..22bba47b0c5f9 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java
@@ -252,11 +252,13 @@ public void testDoNotWaitForCompletion() throws Exception {
         if (waitForCompletion) {
             assertFalse(cancelFuture.isDone());
         } else {
-            assertBusy(() -> assertTrue(cancelFuture.isDone()));
+            cancelFuture.get();
         }
         allowEntireRequest(rootRequest);
         waitForRootTask(mainTaskFuture, false);
-        cancelFuture.actionGet();
+        if (waitForCompletion) {
+            cancelFuture.actionGet();
+        }
         ensureBansAndCancellationsConsistency();
     }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
index 1e16357a24412..8afdbc5906491 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
@@ -779,17 +779,13 @@ public void testTaskStoringSuccessfulResult() throws Exception {
         assertNoFailures(indicesAdmin().prepareRefresh(TaskResultsService.TASK_INDEX).get());
 
         assertHitCount(
+            1L,
             prepareSearch(TaskResultsService.TASK_INDEX).setSource(
                 SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("task.action", taskInfo.action()))
             ),
-            1L
-        );
-
-        assertHitCount(
             prepareSearch(TaskResultsService.TASK_INDEX).setSource(
                 SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("task.node", taskInfo.taskId().getNodeId()))
-            ),
-            1L
+            )
         );
 
         GetTaskResponse getResponse = expectFinishedTask(taskId);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/alias/ValidateIndicesAliasesRequestIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/alias/ValidateIndicesAliasesRequestIT.java
index 5a49322666768..b742661a32c1a 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/alias/ValidateIndicesAliasesRequestIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/alias/ValidateIndicesAliasesRequestIT.java
@@ -81,7 +81,10 @@ public void testAllowed() {
         final IndicesAliasesRequest request = new IndicesAliasesRequest().origin("allowed");
         request.addAliasAction(IndicesAliasesRequest.AliasActions.add().index("index").alias("alias"));
         assertAcked(client().admin().indices().aliases(request).actionGet());
-        final GetAliasesResponse response = client().admin().indices().getAliases(new GetAliasesRequest("alias")).actionGet();
+        final GetAliasesResponse response = client().admin()
+            .indices()
+            .getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "alias"))
+            .actionGet();
         assertThat(response.getAliases().keySet().size(), equalTo(1));
         assertThat(response.getAliases().keySet().iterator().next(), equalTo("index"));
         final List<AliasMetadata> aliasMetadata = response.getAliases().get("index");
@@ -117,6 +120,8 @@ public void testSomeAllowed() {
         final Exception e = expectThrows(IllegalStateException.class, client().admin().indices().aliases(request));
         final String index = "foo_allowed".equals(origin) ? "bar" : "foo";
         assertThat(e, hasToString(containsString("origin [" + origin + "] not allowed for index [" + index + "]")));
-        assertTrue(client().admin().indices().getAliases(new GetAliasesRequest("alias")).actionGet().getAliases().isEmpty());
+        assertTrue(
+            client().admin().indices().getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "alias")).actionGet().getAliases().isEmpty()
+        );
     }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java
index b10e17b3f4e0f..ea7cec710e31e 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java
@@ -273,7 +273,7 @@ public void testAutoCreateSystemAliasViaComposableTemplateAllowsTemplates() thro
     private void assertAliasesHidden(String nonPrimaryIndex, Set<String> aliasNames, int aliasCount) throws InterruptedException,
         ExecutionException {
         final GetAliasesResponse getAliasesResponse = indicesAdmin().getAliases(
-            new GetAliasesRequest().indicesOptions(IndicesOptions.strictExpandHidden())
+            new GetAliasesRequest(TEST_REQUEST_TIMEOUT).indicesOptions(IndicesOptions.strictExpandHidden())
         ).get();
 
         assertThat(getAliasesResponse.getAliases().size(), equalTo(1));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java
index 47f96aebacd7d..fa2b053ead348 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java
@@ -39,7 +39,7 @@ protected boolean forbidPrivateIndexSettings() {
     }
 
     public void testCreateCloneIndex() {
-        IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random());
+        IndexVersion version = IndexVersionUtils.randomCompatibleWriteVersion(random());
         int numPrimaryShards = randomIntBetween(1, 5);
         prepareCreate("source").setSettings(
             Settings.builder().put(indexSettings()).put("number_of_shards", numPrimaryShards).put("index.version.created", version)
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
index e8160a311bedb..b14bf38f3cbcc 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
@@ -120,7 +120,7 @@ public void testNonNestedMappings() throws Exception {
             )
         );
 
-        GetMappingsResponse response = indicesAdmin().prepareGetMappings("test").get();
+        GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get();
 
         MappingMetadata mappings = response.mappings().get("test");
         assertNotNull(mappings);
@@ -130,7 +130,7 @@ public void testNonNestedMappings() throws Exception {
     public void testEmptyNestedMappings() throws Exception {
         assertAcked(prepareCreate("test").setMapping(XContentFactory.jsonBuilder().startObject().endObject()));
 
-        GetMappingsResponse response = indicesAdmin().prepareGetMappings("test").get();
+        GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get();
 
         MappingMetadata mappings = response.mappings().get("test");
         assertNotNull(mappings);
@@ -150,7 +150,7 @@ public void testEmptyMappings() throws Exception {
             prepareCreate("test").setMapping(XContentFactory.jsonBuilder().startObject().startObject("_doc").endObject().endObject())
         );
 
-        GetMappingsResponse response = indicesAdmin().prepareGetMappings("test").get();
+        GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get();
 
         MappingMetadata mappings = response.mappings().get("test");
         assertNotNull(mappings);
@@ -158,8 +158,7 @@ public void testEmptyMappings() throws Exception {
     }
 
     public void testTwoEmptyEqualMappings() throws Exception {
-        assertAcked(prepareCreate("test1"));
-        assertAcked(prepareCreate("test2").setMapping(XContentFactory.jsonBuilder().startObject().endObject()));
+        assertAcked(prepareCreate("test1"), prepareCreate("test2").setMapping(XContentFactory.jsonBuilder().startObject().endObject()));
         FieldCapabilitiesRequest fieldCapsReq1 = new FieldCapabilitiesRequest();
         fieldCapsReq1.indices("test1");
         fieldCapsReq1.fields("*");
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java
index 112ed994385ef..b9dadf86c3345 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java
@@ -115,7 +115,7 @@ public void testNonPrimarySystemIndexIsAutoCreatedViaConcreteName() throws Excep
 
         // Check that a non-primary system index is not assigned as the write index for the alias
         final GetAliasesResponse getAliasesResponse = indicesAdmin().getAliases(
-            new GetAliasesRequest().indicesOptions(IndicesOptions.strictExpandHidden())
+            new GetAliasesRequest(TEST_REQUEST_TIMEOUT).indicesOptions(IndicesOptions.strictExpandHidden())
         ).actionGet();
 
         assertThat(getAliasesResponse.getAliases().size(), equalTo(1));
@@ -322,7 +322,7 @@ public void onFailure(Exception e) {
      */
     private void assertAliases(String concreteIndex) {
         final GetAliasesResponse getAliasesResponse = indicesAdmin().getAliases(
-            new GetAliasesRequest().indicesOptions(IndicesOptions.strictExpandHidden())
+            new GetAliasesRequest(TEST_REQUEST_TIMEOUT).indicesOptions(IndicesOptions.strictExpandHidden())
         ).actionGet();
 
         assertThat(getAliasesResponse.getAliases().size(), equalTo(1));
@@ -334,7 +334,7 @@ private void assertAliases(String concreteIndex) {
     private void assertHasAliases(Set<String> aliasNames, String name, String primaryName, int aliasCount) throws InterruptedException,
         java.util.concurrent.ExecutionException {
         final GetAliasesResponse getAliasesResponse = indicesAdmin().getAliases(
-            new GetAliasesRequest().indicesOptions(IndicesOptions.strictExpandHidden())
+            new GetAliasesRequest(TEST_REQUEST_TIMEOUT).indicesOptions(IndicesOptions.strictExpandHidden())
         ).get();
 
         assertThat(getAliasesResponse.getAliases().size(), equalTo(1));
@@ -357,8 +357,9 @@ private void assertHasAliases(Set<String> aliasNames, String name, String primar
      * Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values.
      */
     private void assertMappingsAndSettings(String expectedMappings, String concreteIndex) {
-        final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(INDEX_NAME))
-            .actionGet();
+        final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings(
+            new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(INDEX_NAME)
+        ).actionGet();
 
         final Map<String, MappingMetadata> mappings = getMappingsResponse.getMappings();
         assertThat(
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java
index 4f6d24b419595..4e9304a7979d6 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java
@@ -113,8 +113,11 @@ public void testCreateShrinkIndexToN() {
                 .get();
         }
         flushAndRefresh();
-        assertHitCount(prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), 20);
-        assertHitCount(prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), 20);
+        assertHitCount(
+            20,
+            prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")),
+            prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar"))
+        );
 
         // relocate all shards to one node such that we can merge it.
         updateIndexSettings(
@@ -145,9 +148,12 @@ public void testCreateShrinkIndexToN() {
                 .get();
         }
         flushAndRefresh();
-        assertHitCount(prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), 20);
-        assertHitCount(prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), 20);
-        assertHitCount(prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), 20);
+        assertHitCount(
+            20,
+            prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")),
+            prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")),
+            prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar"))
+        );
 
         assertNoResizeSourceIndexSettings("first_shrink");
         assertNoResizeSourceIndexSettings("second_shrink");
@@ -236,7 +242,7 @@ private static IndexMetadata indexMetadata(final Client client, final String ind
 
     public void testCreateShrinkIndex() {
         internalCluster().ensureAtLeastNumDataNodes(2);
-        IndexVersion version = IndexVersionUtils.randomVersion(random());
+        IndexVersion version = IndexVersionUtils.randomWriteVersion();
         prepareCreate("source").setSettings(
             Settings.builder().put(indexSettings()).put("number_of_shards", randomIntBetween(2, 7)).put("index.version.created", version)
         ).get();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java
index 8391ab270b1d1..980e94fd5dfce 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java
@@ -236,9 +236,12 @@ private void splitToN(int sourceShards, int firstSplitShards, int secondSplitSha
             GetResponse getResponse = client().prepareGet("second_split", Integer.toString(i)).setRouting(routingValue[i]).get();
             assertTrue(getResponse.isExists());
         }
-        assertHitCount(prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), numDocs);
-        assertHitCount(prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), numDocs);
-        assertHitCount(prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), numDocs);
+        assertHitCount(
+            numDocs,
+            prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")),
+            prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")),
+            prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar"))
+        );
         if (useNested) {
             assertNested("source", numDocs);
             assertNested("first_split", numDocs);
@@ -341,8 +344,8 @@ private static IndexMetadata indexMetadata(final Client client, final String ind
         return clusterStateResponse.getState().metadata().index(index);
     }
 
-    public void testCreateSplitIndex() throws Exception {
-        IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random());
+    public void testCreateSplitIndex() {
+        IndexVersion version = IndexVersionUtils.randomCompatibleWriteVersion(random());
         prepareCreate("source").setSettings(
             Settings.builder().put(indexSettings()).put("number_of_shards", 1).put("index.version.created", version)
         ).get();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
index 46b4675adc8a1..746f35992e721 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
@@ -649,9 +649,11 @@ public void testRolloverWithClosedIndexInAlias() {
         final String openNonwriteIndex = "open-index-nonwrite";
         final String closedIndex = "closed-index-nonwrite";
         final String writeIndexPrefix = "write-index-";
-        assertAcked(prepareCreate(openNonwriteIndex).addAlias(new Alias(aliasName)).get());
-        assertAcked(prepareCreate(closedIndex).addAlias(new Alias(aliasName)).get());
-        assertAcked(prepareCreate(writeIndexPrefix + "000001").addAlias(new Alias(aliasName).writeIndex(true)).get());
+        assertAcked(
+            prepareCreate(openNonwriteIndex).addAlias(new Alias(aliasName)),
+            prepareCreate(closedIndex).addAlias(new Alias(aliasName)),
+            prepareCreate(writeIndexPrefix + "000001").addAlias(new Alias(aliasName).writeIndex(true))
+        );
         ensureGreen();
 
         index(closedIndex, null, "{\"foo\": \"bar\"}");
@@ -674,17 +676,18 @@ public void testRolloverWithClosedWriteIndex() throws Exception {
         final String openNonwriteIndex = "open-index-nonwrite";
         final String closedIndex = "closed-index-nonwrite";
         final String writeIndexPrefix = "write-index-";
-        assertAcked(prepareCreate(openNonwriteIndex).addAlias(new Alias(aliasName)).get());
-        assertAcked(prepareCreate(closedIndex).addAlias(new Alias(aliasName)).get());
-        assertAcked(prepareCreate(writeIndexPrefix + "000001").addAlias(new Alias(aliasName).writeIndex(true)).get());
+        assertAcked(
+            prepareCreate(openNonwriteIndex).addAlias(new Alias(aliasName)),
+            prepareCreate(closedIndex).addAlias(new Alias(aliasName)),
+            prepareCreate(writeIndexPrefix + "000001").addAlias(new Alias(aliasName).writeIndex(true))
+        );
         ensureGreen(openNonwriteIndex, closedIndex, writeIndexPrefix + "000001");
         index(closedIndex, null, "{\"foo\": \"bar\"}");
         index(aliasName, null, "{\"foo\": \"bar\"}");
         index(aliasName, null, "{\"foo\": \"bar\"}");
         refresh(aliasName);
 
-        assertAcked(indicesAdmin().prepareClose(closedIndex).get());
-        assertAcked(indicesAdmin().prepareClose(writeIndexPrefix + "000001").get());
+        assertAcked(indicesAdmin().prepareClose(closedIndex, writeIndexPrefix + "000001").get());
         ensureGreen(aliasName);
 
         RolloverResponse rolloverResponse = indicesAdmin().prepareRolloverIndex(aliasName)
@@ -863,7 +866,7 @@ public void testRolloverConcurrently() throws Exception {
 
         for (int i = 0; i < numOfThreads; i++) {
             var aliasName = "test-" + i;
-            var response = indicesAdmin().getAliases(new GetAliasesRequest(aliasName)).get();
+            var response = indicesAdmin().getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT, aliasName)).get();
             List<Map.Entry<String, List<AliasMetadata>>> actual = response.getAliases().entrySet().stream().toList();
             List<Map.Entry<String, List<AliasMetadata>>> expected = new ArrayList<>(numberOfRolloversPerThread);
             int numOfIndices = numberOfRolloversPerThread + 1;
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java
index e45555b1dec19..2cd319d148321 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java
@@ -55,7 +55,7 @@ public void testBulkIndexCreatesMapping() throws Exception {
         bulkBuilder.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, XContentType.JSON);
         bulkBuilder.get();
         assertBusy(() -> {
-            GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings().get();
+            GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get();
             assertTrue(mappingsResponse.getMappings().containsKey("logstash-2014.03.30"));
         });
     }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java
index 67576059de1e0..5db2651c703d2 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java
@@ -307,11 +307,15 @@ public void testFinalReduce() throws ExecutionException, InterruptedException {
 
     public void testWaitForRefreshIndexValidation() throws Exception {
         int numberOfShards = randomIntBetween(3, 10);
-        assertAcked(prepareCreate("test1").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards)));
-        assertAcked(prepareCreate("test2").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards)));
-        assertAcked(prepareCreate("test3").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards)));
-        indicesAdmin().prepareAliases().addAlias("test1", "testAlias").get();
-        indicesAdmin().prepareAliases().addAlias(new String[] { "test2", "test3" }, "testFailedAlias").get();
+        assertAcked(
+            prepareCreate("test1").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards)),
+            prepareCreate("test2").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards)),
+            prepareCreate("test3").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards))
+        );
+        assertAcked(
+            indicesAdmin().prepareAliases().addAlias("test1", "testAlias"),
+            indicesAdmin().prepareAliases().addAlias(new String[] { "test2", "test3" }, "testFailedAlias")
+        );
 
         long[] validCheckpoints = new long[numberOfShards];
         Arrays.fill(validCheckpoints, SequenceNumbers.UNASSIGNED_SEQ_NO);
@@ -376,8 +380,10 @@ public void testShardCountLimit() throws Exception {
         try {
             final int numPrimaries1 = randomIntBetween(2, 10);
             final int numPrimaries2 = randomIntBetween(1, 10);
-            assertAcked(prepareCreate("test1").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numPrimaries1)));
-            assertAcked(prepareCreate("test2").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numPrimaries2)));
+            assertAcked(
+                prepareCreate("test1").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numPrimaries1)),
+                prepareCreate("test2").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numPrimaries2))
+            );
 
             // no exception
             prepareSearch("test1").get().decRef();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
index cba1d9362c91e..d16fbf9298552 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
@@ -531,8 +531,10 @@ private void checkBrownFoxTermVector(Fields fields, String fieldName, boolean wi
     public void testDuelWithAndWithoutTermVectors() throws IOException, ExecutionException, InterruptedException {
         // setup indices
         String[] indexNames = new String[] { "with_tv", "without_tv" };
-        assertAcked(prepareCreate(indexNames[0]).setMapping("field1", "type=text,term_vector=with_positions_offsets,analyzer=keyword"));
-        assertAcked(prepareCreate(indexNames[1]).setMapping("field1", "type=text,term_vector=no,analyzer=keyword"));
+        assertAcked(
+            prepareCreate(indexNames[1]).setMapping("field1", "type=text,term_vector=no,analyzer=keyword"),
+            prepareCreate(indexNames[0]).setMapping("field1", "type=text,term_vector=with_positions_offsets,analyzer=keyword")
+        );
         ensureGreen();
 
         // index documents with and without term vectors
@@ -1074,9 +1076,7 @@ public void testWithKeywordAndNormalizer() throws IOException, ExecutionExceptio
                     "type=text,term_vector=with_positions_offsets,analyzer=my_analyzer",
                     "field2",
                     "type=text,term_vector=with_positions_offsets,analyzer=keyword"
-                )
-        );
-        assertAcked(
+                ),
             prepareCreate(indexNames[1]).setSettings(builder.build())
                 .setMapping("field1", "type=keyword,normalizer=my_normalizer", "field2", "type=keyword")
         );
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java
index 309bf69f00be0..ad17d8dbaa4ad 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java
@@ -323,10 +323,8 @@ public void testSearchingFilteringAliasesSingleIndex() throws Exception {
     }
 
     public void testSearchingFilteringAliasesTwoIndices() throws Exception {
-        logger.info("--> creating index [test1]");
-        assertAcked(prepareCreate("test1").setMapping("name", "type=text"));
-        logger.info("--> creating index [test2]");
-        assertAcked(prepareCreate("test2").setMapping("name", "type=text"));
+        logger.info("--> creating indices [test1, test2]");
+        assertAcked(prepareCreate("test1").setMapping("name", "type=text"), prepareCreate("test2").setMapping("name", "type=text"));
         ensureGreen();
 
         logger.info("--> adding filtering aliases to index [test1]");
@@ -525,8 +523,7 @@ public void testSearchingFilteringAliasesMultipleIndices() throws Exception {
 
     public void testDeletingByQueryFilteringAliases() throws Exception {
         logger.info("--> creating index [test1] and [test2");
-        assertAcked(prepareCreate("test1").setMapping("name", "type=text"));
-        assertAcked(prepareCreate("test2").setMapping("name", "type=text"));
+        assertAcked(prepareCreate("test1").setMapping("name", "type=text"), prepareCreate("test2").setMapping("name", "type=text"));
         ensureGreen();
 
         logger.info("--> adding filtering aliases to index [test1]");
@@ -580,8 +577,7 @@ public void testDeletingByQueryFilteringAliases() throws Exception {
 
     public void testDeleteAliases() throws Exception {
         logger.info("--> creating index [test1] and [test2]");
-        assertAcked(prepareCreate("test1").setMapping("name", "type=text"));
-        assertAcked(prepareCreate("test2").setMapping("name", "type=text"));
+        assertAcked(prepareCreate("test1").setMapping("name", "type=text"), prepareCreate("test2").setMapping("name", "type=text"));
         ensureGreen();
 
         logger.info("--> adding filtering aliases to index [test1]");
@@ -615,12 +611,11 @@ public void testDeleteAliases() throws Exception {
         assertAliasesVersionIncreases(indices, () -> indicesAdmin().prepareAliases().removeAlias(indices, aliases).get());
 
         for (String alias : aliases) {
-            assertTrue(indicesAdmin().prepareGetAliases(alias).get().getAliases().isEmpty());
+            assertTrue(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, alias).get().getAliases().isEmpty());
         }
 
         logger.info("--> creating index [foo_foo] and [bar_bar]");
-        assertAcked(prepareCreate("foo_foo"));
-        assertAcked(prepareCreate("bar_bar"));
+        assertAcked(prepareCreate("foo_foo"), prepareCreate("bar_bar"));
         ensureGreen();
 
         logger.info("--> adding [foo] alias to [foo_foo] and [bar_bar]");
@@ -632,9 +627,9 @@ public void testDeleteAliases() throws Exception {
             () -> assertAcked(indicesAdmin().prepareAliases().addAliasAction(AliasActions.remove().index("foo*").alias("foo")))
         );
 
-        assertFalse(indicesAdmin().prepareGetAliases("foo").get().getAliases().isEmpty());
-        assertTrue(indicesAdmin().prepareGetAliases("foo").setIndices("foo_foo").get().getAliases().isEmpty());
-        assertFalse(indicesAdmin().prepareGetAliases("foo").setIndices("bar_bar").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foo").get().getAliases().isEmpty());
+        assertTrue(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foo").setIndices("foo_foo").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foo").setIndices("bar_bar").get().getAliases().isEmpty());
         IllegalArgumentException iae = expectThrows(
             IllegalArgumentException.class,
             indicesAdmin().prepareAliases().addAliasAction(AliasActions.remove().index("foo").alias("foo"))
@@ -788,7 +783,7 @@ public void testIndicesGetAliases() throws Exception {
         );
 
         logger.info("--> getting alias1");
-        GetAliasesResponse getResponse = indicesAdmin().prepareGetAliases("alias1").get();
+        GetAliasesResponse getResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias1").get();
         assertThat(getResponse, notNullValue());
         assertThat(getResponse.getAliases().size(), equalTo(1));
         assertThat(getResponse.getAliases().get("foobar").size(), equalTo(1));
@@ -797,10 +792,10 @@ public void testIndicesGetAliases() throws Exception {
         assertThat(getResponse.getAliases().get("foobar").get(0).getFilter(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getIndexRouting(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getSearchRouting(), nullValue());
-        assertFalse(indicesAdmin().prepareGetAliases("alias1").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias1").get().getAliases().isEmpty());
 
         logger.info("--> getting all aliases that start with alias*");
-        getResponse = indicesAdmin().prepareGetAliases("alias*").get();
+        getResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias*").get();
         assertThat(getResponse, notNullValue());
         assertThat(getResponse.getAliases().size(), equalTo(1));
         assertThat(getResponse.getAliases().get("foobar").size(), equalTo(2));
@@ -814,7 +809,7 @@ public void testIndicesGetAliases() throws Exception {
         assertThat(getResponse.getAliases().get("foobar").get(1).getFilter(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(1).getIndexRouting(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(1).getSearchRouting(), nullValue());
-        assertFalse(indicesAdmin().prepareGetAliases("alias*").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias*").get().getAliases().isEmpty());
 
         logger.info("--> creating aliases [bar, baz, foo]");
         assertAliasesVersionIncreases(
@@ -835,7 +830,7 @@ public void testIndicesGetAliases() throws Exception {
         );
 
         logger.info("--> getting bar and baz for index bazbar");
-        getResponse = indicesAdmin().prepareGetAliases("bar", "bac").setIndices("bazbar").get();
+        getResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "bar", "bac").setIndices("bazbar").get();
         assertThat(getResponse, notNullValue());
         assertThat(getResponse.getAliases().size(), equalTo(1));
         assertThat(getResponse.getAliases().get("bazbar").size(), equalTo(2));
@@ -851,13 +846,13 @@ public void testIndicesGetAliases() throws Exception {
         assertThat(getResponse.getAliases().get("bazbar").get(1).getFilter(), nullValue());
         assertThat(getResponse.getAliases().get("bazbar").get(1).getIndexRouting(), nullValue());
         assertThat(getResponse.getAliases().get("bazbar").get(1).getSearchRouting(), nullValue());
-        assertFalse(indicesAdmin().prepareGetAliases("bar").get().getAliases().isEmpty());
-        assertFalse(indicesAdmin().prepareGetAliases("bac").get().getAliases().isEmpty());
-        assertFalse(indicesAdmin().prepareGetAliases("bar").setIndices("bazbar").get().getAliases().isEmpty());
-        assertFalse(indicesAdmin().prepareGetAliases("bac").setIndices("bazbar").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "bar").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "bac").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "bar").setIndices("bazbar").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "bac").setIndices("bazbar").get().getAliases().isEmpty());
 
         logger.info("--> getting *b* for index baz*");
-        getResponse = indicesAdmin().prepareGetAliases("*b*").setIndices("baz*").get();
+        getResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "*b*").setIndices("baz*").get();
         assertThat(getResponse, notNullValue());
         assertThat(getResponse.getAliases().size(), equalTo(1));
         assertThat(getResponse.getAliases().get("bazbar").size(), equalTo(2));
@@ -873,10 +868,10 @@ public void testIndicesGetAliases() throws Exception {
         assertThat(getResponse.getAliases().get("bazbar").get(1).getFilter(), nullValue());
         assertThat(getResponse.getAliases().get("bazbar").get(1).getIndexRouting(), nullValue());
         assertThat(getResponse.getAliases().get("bazbar").get(1).getSearchRouting(), nullValue());
-        assertFalse(indicesAdmin().prepareGetAliases("*b*").setIndices("baz*").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "*b*").setIndices("baz*").get().getAliases().isEmpty());
 
         logger.info("--> getting *b* for index *bar");
-        getResponse = indicesAdmin().prepareGetAliases("b*").setIndices("*bar").get();
+        getResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "b*").setIndices("*bar").get();
         assertThat(getResponse, notNullValue());
         assertThat(getResponse.getAliases().size(), equalTo(2));
         assertThat(getResponse.getAliases().get("bazbar").size(), equalTo(2));
@@ -897,10 +892,10 @@ public void testIndicesGetAliases() throws Exception {
         assertThat(getResponse.getAliases().get("foobar").get(0).getFilter(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getIndexRouting(), equalTo("bla"));
         assertThat(getResponse.getAliases().get("foobar").get(0).getSearchRouting(), equalTo("bla"));
-        assertFalse(indicesAdmin().prepareGetAliases("b*").setIndices("*bar").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "b*").setIndices("*bar").get().getAliases().isEmpty());
 
         logger.info("--> getting f* for index *bar");
-        getResponse = indicesAdmin().prepareGetAliases("f*").setIndices("*bar").get();
+        getResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "f*").setIndices("*bar").get();
         assertThat(getResponse, notNullValue());
         assertThat(getResponse.getAliases().size(), equalTo(1));
         assertThat(getResponse.getAliases().get("foobar").get(0), notNullValue());
@@ -908,11 +903,11 @@ public void testIndicesGetAliases() throws Exception {
         assertThat(getResponse.getAliases().get("foobar").get(0).getFilter(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getIndexRouting(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getSearchRouting(), nullValue());
-        assertFalse(indicesAdmin().prepareGetAliases("f*").setIndices("*bar").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "f*").setIndices("*bar").get().getAliases().isEmpty());
 
         // alias at work
         logger.info("--> getting f* for index *bac");
-        getResponse = indicesAdmin().prepareGetAliases("foo").setIndices("*bac").get();
+        getResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foo").setIndices("*bac").get();
         assertThat(getResponse, notNullValue());
         assertThat(getResponse.getAliases().size(), equalTo(1));
         assertThat(getResponse.getAliases().get("foobar").size(), equalTo(1));
@@ -921,10 +916,10 @@ public void testIndicesGetAliases() throws Exception {
         assertThat(getResponse.getAliases().get("foobar").get(0).getFilter(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getIndexRouting(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getSearchRouting(), nullValue());
-        assertFalse(indicesAdmin().prepareGetAliases("foo").setIndices("*bac").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foo").setIndices("*bac").get().getAliases().isEmpty());
 
         logger.info("--> getting foo for index foobar");
-        getResponse = indicesAdmin().prepareGetAliases("foo").setIndices("foobar").get();
+        getResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foo").setIndices("foobar").get();
         assertThat(getResponse, notNullValue());
         assertThat(getResponse.getAliases().size(), equalTo(1));
         assertThat(getResponse.getAliases().get("foobar").get(0), notNullValue());
@@ -932,13 +927,13 @@ public void testIndicesGetAliases() throws Exception {
         assertThat(getResponse.getAliases().get("foobar").get(0).getFilter(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getIndexRouting(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getSearchRouting(), nullValue());
-        assertFalse(indicesAdmin().prepareGetAliases("foo").setIndices("foobar").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foo").setIndices("foobar").get().getAliases().isEmpty());
 
         for (String aliasName : new String[] { null, "_all", "*" }) {
             logger.info("--> getting {} alias for index foobar", aliasName);
             getResponse = aliasName != null
-                ? indicesAdmin().prepareGetAliases(aliasName).setIndices("foobar").get()
-                : indicesAdmin().prepareGetAliases().setIndices("foobar").get();
+                ? indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, aliasName).setIndices("foobar").get()
+                : indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT).setIndices("foobar").get();
             assertThat(getResponse, notNullValue());
             assertThat(getResponse.getAliases().size(), equalTo(1));
             assertThat(getResponse.getAliases().get("foobar").size(), equalTo(4));
@@ -950,20 +945,20 @@ public void testIndicesGetAliases() throws Exception {
 
         // alias at work again
         logger.info("--> getting * for index *bac");
-        getResponse = indicesAdmin().prepareGetAliases("*").setIndices("*bac").get();
+        getResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "*").setIndices("*bac").get();
         assertThat(getResponse, notNullValue());
         assertThat(getResponse.getAliases().size(), equalTo(2));
         assertThat(getResponse.getAliases().get("foobar").size(), equalTo(4));
         assertThat(getResponse.getAliases().get("bazbar").size(), equalTo(2));
-        assertFalse(indicesAdmin().prepareGetAliases("*").setIndices("*bac").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "*").setIndices("*bac").get().getAliases().isEmpty());
 
         assertAcked(indicesAdmin().prepareAliases().removeAlias("foobar", "foo"));
 
-        getResponse = indicesAdmin().prepareGetAliases("foo").setIndices("foobar").get();
+        getResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foo").setIndices("foobar").get();
         for (final Map.Entry<String, List<AliasMetadata>> entry : getResponse.getAliases().entrySet()) {
             assertTrue(entry.getValue().isEmpty());
         }
-        assertTrue(indicesAdmin().prepareGetAliases("foo").setIndices("foobar").get().getAliases().isEmpty());
+        assertTrue(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foo").setIndices("foobar").get().getAliases().isEmpty());
     }
 
     public void testGetAllAliasesWorks() {
@@ -975,7 +970,7 @@ public void testGetAllAliasesWorks() {
             () -> assertAcked(indicesAdmin().prepareAliases().addAlias("index1", "alias1").addAlias("index2", "alias2"))
         );
 
-        GetAliasesResponse response = indicesAdmin().prepareGetAliases().get();
+        GetAliasesResponse response = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT).get();
         assertThat(response.getAliases(), hasKey("index1"));
         assertThat(response.getAliases(), hasKey("index1"));
     }
@@ -1119,8 +1114,11 @@ public void testAliasesWithBlocks() {
                     () -> assertAcked(indicesAdmin().prepareAliases().addAlias("test", "alias1").addAlias("test", "alias2"))
                 );
                 assertAliasesVersionIncreases("test", () -> assertAcked(indicesAdmin().prepareAliases().removeAlias("test", "alias1")));
-                assertThat(indicesAdmin().prepareGetAliases("alias2").get().getAliases().get("test").size(), equalTo(1));
-                assertFalse(indicesAdmin().prepareGetAliases("alias2").get().getAliases().isEmpty());
+                assertThat(
+                    indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias2").get().getAliases().get("test").size(),
+                    equalTo(1)
+                );
+                assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias2").get().getAliases().isEmpty());
             } finally {
                 disableIndexBlock("test", block);
             }
@@ -1137,8 +1135,8 @@ public void testAliasesWithBlocks() {
                 "test",
                 () -> assertBlocked(indicesAdmin().prepareAliases().removeAlias("test", "alias2"), INDEX_READ_ONLY_BLOCK)
             );
-            assertThat(indicesAdmin().prepareGetAliases("alias2").get().getAliases().get("test").size(), equalTo(1));
-            assertFalse(indicesAdmin().prepareGetAliases("alias2").get().getAliases().isEmpty());
+            assertThat(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias2").get().getAliases().get("test").size(), equalTo(1));
+            assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias2").get().getAliases().isEmpty());
 
         } finally {
             disableIndexBlock("test", SETTING_READ_ONLY);
@@ -1155,21 +1153,23 @@ public void testAliasesWithBlocks() {
                 "test",
                 () -> assertBlocked(indicesAdmin().prepareAliases().removeAlias("test", "alias2"), INDEX_METADATA_BLOCK)
             );
-            assertBlocked(indicesAdmin().prepareGetAliases("alias2"), INDEX_METADATA_BLOCK);
-            assertBlocked(indicesAdmin().prepareGetAliases("alias2"), INDEX_METADATA_BLOCK);
+            assertBlocked(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias2"), INDEX_METADATA_BLOCK);
+            assertBlocked(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias2"), INDEX_METADATA_BLOCK);
 
         } finally {
             disableIndexBlock("test", SETTING_BLOCKS_METADATA);
         }
     }
 
-    public void testAliasActionRemoveIndex() throws InterruptedException, ExecutionException {
-        assertAcked(prepareCreate("foo_foo"));
-        assertAcked(prepareCreate("bar_bar"));
-        assertAliasesVersionIncreases(new String[] { "foo_foo", "bar_bar" }, () -> {
-            assertAcked(indicesAdmin().prepareAliases().addAlias("foo_foo", "foo"));
-            assertAcked(indicesAdmin().prepareAliases().addAlias("bar_bar", "foo"));
-        });
+    public void testAliasActionRemoveIndex() {
+        assertAcked(prepareCreate("foo_foo"), prepareCreate("bar_bar"));
+        assertAliasesVersionIncreases(
+            new String[] { "foo_foo", "bar_bar" },
+            () -> assertAcked(
+                indicesAdmin().prepareAliases().addAlias("bar_bar", "foo"),
+                indicesAdmin().prepareAliases().addAlias("foo_foo", "foo")
+            )
+        );
 
         IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, indicesAdmin().prepareAliases().removeIndex("foo"));
         assertEquals(
@@ -1179,12 +1179,12 @@ public void testAliasActionRemoveIndex() throws InterruptedException, ExecutionE
 
         assertAcked(indicesAdmin().prepareAliases().removeIndex("foo*"));
         assertFalse(indexExists("foo_foo"));
-        assertFalse(indicesAdmin().prepareGetAliases("foo").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foo").get().getAliases().isEmpty());
         assertTrue(indexExists("bar_bar"));
-        assertFalse(indicesAdmin().prepareGetAliases("foo").setIndices("bar_bar").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foo").setIndices("bar_bar").get().getAliases().isEmpty());
 
         assertAcked(indicesAdmin().prepareAliases().removeIndex("bar_bar"));
-        assertTrue(indicesAdmin().prepareGetAliases("foo").get().getAliases().isEmpty());
+        assertTrue(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foo").get().getAliases().isEmpty());
         assertFalse(indexExists("bar_bar"));
     }
 
@@ -1323,7 +1323,7 @@ public void testGetAliasAndAliasExistsForHiddenAliases() {
     }
 
     private void checkAliases() {
-        GetAliasesResponse getAliasesResponse = indicesAdmin().prepareGetAliases("alias1").get();
+        GetAliasesResponse getAliasesResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias1").get();
         assertThat(getAliasesResponse.getAliases().get("test").size(), equalTo(1));
         AliasMetadata aliasMetadata = getAliasesResponse.getAliases().get("test").get(0);
         assertThat(aliasMetadata.alias(), equalTo("alias1"));
@@ -1332,7 +1332,7 @@ private void checkAliases() {
         assertThat(aliasMetadata.searchRouting(), nullValue());
         assertThat(aliasMetadata.isHidden(), nullValue());
 
-        getAliasesResponse = indicesAdmin().prepareGetAliases("alias2").get();
+        getAliasesResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias2").get();
         assertThat(getAliasesResponse.getAliases().get("test").size(), equalTo(1));
         aliasMetadata = getAliasesResponse.getAliases().get("test").get(0);
         assertThat(aliasMetadata.alias(), equalTo("alias2"));
@@ -1341,7 +1341,7 @@ private void checkAliases() {
         assertThat(aliasMetadata.searchRouting(), nullValue());
         assertThat(aliasMetadata.isHidden(), nullValue());
 
-        getAliasesResponse = indicesAdmin().prepareGetAliases("alias3").get();
+        getAliasesResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias3").get();
         assertThat(getAliasesResponse.getAliases().get("test").size(), equalTo(1));
         aliasMetadata = getAliasesResponse.getAliases().get("test").get(0);
         assertThat(aliasMetadata.alias(), equalTo("alias3"));
@@ -1350,7 +1350,7 @@ private void checkAliases() {
         assertThat(aliasMetadata.searchRouting(), equalTo("search"));
         assertThat(aliasMetadata.isHidden(), nullValue());
 
-        getAliasesResponse = indicesAdmin().prepareGetAliases("alias4").get();
+        getAliasesResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias4").get();
         assertThat(getAliasesResponse.getAliases().get("test").size(), equalTo(1));
         aliasMetadata = getAliasesResponse.getAliases().get("test").get(0);
         assertThat(aliasMetadata.alias(), equalTo("alias4"));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java b/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java
index cf54bc50398c4..f0ee06336b493 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java
@@ -49,7 +49,7 @@ public void testGetAliasWithNetNewSystemIndices() throws Exception {
         }
         ensureGreen();
 
-        GetAliasesRequest getAliasesRequest = new GetAliasesRequest();
+        GetAliasesRequest getAliasesRequest = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         GetAliasesResponse aliasResponse = indicesAdmin().getAliases(getAliasesRequest).get();
         assertThat(aliasResponse.getAliases().size(), is(0));
     }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/aliases/SystemIndexAliasIT.java b/server/src/internalClusterTest/java/org/elasticsearch/aliases/SystemIndexAliasIT.java
index fbade0ac2c871..38696d75b93da 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/aliases/SystemIndexAliasIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/aliases/SystemIndexAliasIT.java
@@ -41,7 +41,7 @@ public void testCreateAliasForSystemIndex() throws Exception {
         assertAcked(indicesAdmin().prepareAliases().addAlias(PRIMARY_INDEX_NAME, INDEX_NAME + "-system-alias"));
 
         final GetAliasesResponse getAliasesResponse = indicesAdmin().getAliases(
-            new GetAliasesRequest().indicesOptions(IndicesOptions.strictExpandHidden())
+            new GetAliasesRequest(TEST_REQUEST_TIMEOUT).indicesOptions(IndicesOptions.strictExpandHidden())
         ).get();
 
         assertThat(getAliasesResponse.getAliases().size(), equalTo(1));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
index 58b9af7724aaa..15b4a557b2b8b 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
@@ -9,7 +9,6 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlocks;
@@ -71,6 +70,7 @@
 import static org.elasticsearch.test.XContentTestUtils.convertToMap;
 import static org.elasticsearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder;
 import static org.elasticsearch.test.index.IndexVersionUtils.randomVersion;
+import static org.elasticsearch.test.index.IndexVersionUtils.randomWriteVersion;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
@@ -218,7 +218,7 @@ private ClusterState.Builder randomCoordinationMetadata(ClusterState clusterStat
 
     private DiscoveryNode randomNode(String nodeId) {
         Version nodeVersion = VersionUtils.randomVersion(random());
-        IndexVersion indexVersion = randomVersion(random());
+        IndexVersion indexVersion = randomVersion();
         return DiscoveryNodeUtils.builder(nodeId)
             .roles(emptySet())
             .version(nodeVersion, IndexVersion.fromId(indexVersion.id() - 1_000_000), indexVersion)
@@ -561,10 +561,10 @@ public IndexMetadata randomCreate(String name) {
                 IndexMetadata.Builder builder = IndexMetadata.builder(name);
                 Settings.Builder settingsBuilder = Settings.builder();
                 setRandomIndexSettings(random(), settingsBuilder);
-                settingsBuilder.put(randomSettings(Settings.EMPTY)).put(IndexMetadata.SETTING_VERSION_CREATED, randomVersion(random()));
+                settingsBuilder.put(randomSettings(Settings.EMPTY)).put(IndexMetadata.SETTING_VERSION_CREATED, randomWriteVersion());
                 builder.settings(settingsBuilder);
                 builder.numberOfShards(randomIntBetween(1, 10)).numberOfReplicas(randomInt(10));
-                builder.eventIngestedRange(IndexLongFieldRange.UNKNOWN, TransportVersion.current());
+                builder.eventIngestedRange(IndexLongFieldRange.UNKNOWN);
                 int aliasCount = randomInt(10);
                 for (int i = 0; i < aliasCount; i++) {
                     builder.putAlias(randomAlias());
@@ -736,7 +736,7 @@ public ClusterState.Custom randomCreate(String name) {
                             ImmutableOpenMap.of(),
                             null,
                             SnapshotInfoTestUtils.randomUserMetadata(),
-                            randomVersion(random())
+                            randomVersion()
                         )
                     );
                     case 1 -> new RestoreInProgress.Builder().add(
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
index def125e5c99c3..9eed1f757b5b1 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
@@ -183,7 +183,8 @@ public void testIndexTemplates() throws Exception {
         ClusterStateResponse clusterStateResponseUnfiltered = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get();
         assertThat(clusterStateResponseUnfiltered.getState().metadata().templates().size(), is(greaterThanOrEqualTo(2)));
 
-        GetIndexTemplatesResponse getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates("foo_template").get();
+        GetIndexTemplatesResponse getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT, "foo_template")
+            .get();
         assertIndexTemplateExists(getIndexTemplatesResponse, "foo_template");
     }
 
@@ -257,11 +258,14 @@ public void testLargeClusterStatePublishing() throws Exception {
                 .setTimeout(TimeValue.timeValueMinutes(1))
         );
         ensureGreen(); // wait for green state, so its both green, and there are no more pending events
-        MappingMetadata masterMappingMetadata = indicesAdmin().prepareGetMappings("test").get().getMappings().get("test");
+        MappingMetadata masterMappingMetadata = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test")
+            .get()
+            .getMappings()
+            .get("test");
         for (Client client : clients()) {
             MappingMetadata mappingMetadata = client.admin()
                 .indices()
-                .prepareGetMappings("test")
+                .prepareGetMappings(TEST_REQUEST_TIMEOUT, "test")
                 .setLocal(true)
                 .get()
                 .getMappings()
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java
index 5e51c83d54795..b5c3861633350 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java
@@ -95,7 +95,9 @@ public void testTemplateUpdate() throws Exception {
             // the updates only happen on cluster state updates, so we need to make sure that the cluster state updates are happening
             // so we need to simulate updates to make sure the template upgrade kicks in
             updateClusterSettings(Settings.builder().put(TestPlugin.UPDATE_TEMPLATE_DUMMY_SETTING.getKey(), updateCount.incrementAndGet()));
-            List<IndexTemplateMetadata> templates = indicesAdmin().prepareGetTemplates("test_*").get().getIndexTemplates();
+            List<IndexTemplateMetadata> templates = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT, "test_*")
+                .get()
+                .getIndexTemplates();
             assertThat(templates, hasSize(3));
             boolean addedFound = false;
             boolean changedFound = false;
@@ -139,7 +141,9 @@ private void assertTemplates() throws Exception {
             // so we need to simulate updates to make sure the template upgrade kicks in
             updateClusterSettings(Settings.builder().put(TestPlugin.UPDATE_TEMPLATE_DUMMY_SETTING.getKey(), updateCount.incrementAndGet()));
 
-            List<IndexTemplateMetadata> templates = indicesAdmin().prepareGetTemplates("test_*").get().getIndexTemplates();
+            List<IndexTemplateMetadata> templates = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT, "test_*")
+                .get()
+                .getIndexTemplates();
             assertThat(templates, hasSize(2));
             boolean addedFound = false;
             boolean changedFound = false;
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/UpdateShardAllocationSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/UpdateShardAllocationSettingsIT.java
index 198e27a70c0bc..7fb763b5b91a3 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/UpdateShardAllocationSettingsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/UpdateShardAllocationSettingsIT.java
@@ -39,8 +39,10 @@ public void testEnableRebalance() {
         // we test with 2 shards since otherwise it's pretty fragile if there are difference in the num or shards such that
         // all shards are relocated to the second node which is not what we want here. It's solely a test for the settings to take effect
         final int numShards = 2;
-        assertAcked(prepareCreate("test").setSettings(indexSettings(numShards, 0)));
-        assertAcked(prepareCreate("test_1").setSettings(indexSettings(numShards, 0)));
+        assertAcked(
+            prepareCreate("test").setSettings(indexSettings(numShards, 0)),
+            prepareCreate("test_1").setSettings(indexSettings(numShards, 0))
+        );
         ensureGreen();
         assertAllShardsOnNodes("test", firstNode);
         assertAllShardsOnNodes("test_1", firstNode);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/features/ClusterFeaturesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/features/ClusterFeaturesIT.java
index 74fd945ed3779..b756b4ca770c4 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/features/ClusterFeaturesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/features/ClusterFeaturesIT.java
@@ -30,7 +30,6 @@ public void testClusterHasFeatures() {
 
         FeatureService service = internalCluster().getCurrentMasterNodeInstance(FeatureService.class);
 
-        assertThat(service.getNodeFeatures(), hasKey(FeatureService.FEATURES_SUPPORTED.id()));
         assertThat(service.getNodeFeatures(), hasKey(FeatureService.TEST_FEATURES_ENABLED.id()));
 
         // check the nodes all have a feature in their cluster state (there should always be features_supported)
@@ -38,7 +37,7 @@ public void testClusterHasFeatures() {
         var features = response.getState().clusterFeatures().nodeFeatures();
         Set<String> missing = features.entrySet()
             .stream()
-            .filter(e -> e.getValue().contains(FeatureService.FEATURES_SUPPORTED.id()) == false)
+            .filter(e -> e.getValue().contains(FeatureService.TEST_FEATURES_ENABLED.id()) == false)
             .map(Map.Entry::getKey)
             .collect(Collectors.toSet());
         assertThat(missing + " out of " + features.keySet() + " does not have the required feature", missing, empty());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java
index d6ccdf3dc0399..256566045c59a 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java
@@ -115,7 +115,7 @@ public void testMetaWrittenWhenIndexIsClosedAndMetaUpdated() throws Exception {
             )
             .get();
 
-        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(index).get();
+        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, index).get();
         assertNotNull(
             ((Map<String, ?>) (getMappingsResponse.getMappings().get(index).getSourceAsMap().get("properties"))).get("integer_field")
         );
@@ -146,7 +146,7 @@ public void testMetaWrittenWhenIndexIsClosedAndMetaUpdated() throws Exception {
             )
             .get();
 
-        getMappingsResponse = indicesAdmin().prepareGetMappings(index).get();
+        getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, index).get();
         assertNotNull(
             ((Map<String, ?>) (getMappingsResponse.getMappings().get(index).getSourceAsMap().get("properties"))).get("float_field")
         );
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java
index d116c091fbe4a..24af560f608d3 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java
@@ -97,7 +97,7 @@ public void testGlobalTemplatesDoNotApply() {
 
         assertAcked(indicesAdmin().prepareCreate("a_hidden_index").setSettings(Settings.builder().put("index.hidden", true).build()));
 
-        GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings("a_hidden_index").get();
+        GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "a_hidden_index").get();
         assertThat(mappingsResponse.mappings().size(), is(1));
         MappingMetadata mappingMetadata = mappingsResponse.mappings().get("a_hidden_index");
         assertNotNull(mappingMetadata);
@@ -150,13 +150,13 @@ public void testAliasesForHiddenIndices() {
         );
 
         // The index should be returned here when queried by name or by wildcard because the alias is visible
-        final GetAliasesRequestBuilder req = indicesAdmin().prepareGetAliases(visibleAlias);
+        final GetAliasesRequestBuilder req = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, visibleAlias);
         GetAliasesResponse response = req.get();
         assertThat(response.getAliases().get(hiddenIndex), hasSize(1));
         assertThat(response.getAliases().get(hiddenIndex).get(0).alias(), equalTo(visibleAlias));
         assertThat(response.getAliases().get(hiddenIndex).get(0).isHidden(), nullValue());
 
-        response = indicesAdmin().prepareGetAliases("alias*").get();
+        response = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias*").get();
         assertThat(response.getAliases().get(hiddenIndex), hasSize(1));
         assertThat(response.getAliases().get(hiddenIndex).get(0).alias(), equalTo(visibleAlias));
         assertThat(response.getAliases().get(hiddenIndex).get(0).isHidden(), nullValue());
@@ -169,19 +169,21 @@ public void testAliasesForHiddenIndices() {
         );
 
         // Querying by name directly should get the right result
-        response = indicesAdmin().prepareGetAliases(hiddenAlias).get();
+        response = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, hiddenAlias).get();
         assertThat(response.getAliases().get(hiddenIndex), hasSize(1));
         assertThat(response.getAliases().get(hiddenIndex).get(0).alias(), equalTo(hiddenAlias));
         assertThat(response.getAliases().get(hiddenIndex).get(0).isHidden(), equalTo(true));
 
         // querying by wildcard should get the right result because the indices options include hidden by default
-        response = indicesAdmin().prepareGetAliases("alias*").get();
+        response = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias*").get();
         assertThat(response.getAliases().get(hiddenIndex), hasSize(1));
         assertThat(response.getAliases().get(hiddenIndex).get(0).alias(), equalTo(hiddenAlias));
         assertThat(response.getAliases().get(hiddenIndex).get(0).isHidden(), equalTo(true));
 
         // But we should get no results if we specify indices options that don't include hidden
-        response = indicesAdmin().prepareGetAliases("alias*").setIndicesOptions(IndicesOptions.strictExpandOpen()).get();
+        response = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias*")
+            .setIndicesOptions(IndicesOptions.strictExpandOpen())
+            .get();
         assertThat(response.getAliases().get(hiddenIndex), nullValue());
 
         // Now try with a hidden alias that starts with a dot
@@ -192,7 +194,7 @@ public void testAliasesForHiddenIndices() {
         );
 
         // Check that querying by dot-prefixed pattern returns the alias
-        response = indicesAdmin().prepareGetAliases(".alias*").get();
+        response = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, ".alias*").get();
         assertThat(response.getAliases().get(hiddenIndex), hasSize(1));
         assertThat(response.getAliases().get(hiddenIndex).get(0).alias(), equalTo(dotHiddenAlias));
         assertThat(response.getAliases().get(hiddenIndex).get(0).isHidden(), equalTo(true));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java
index 960ee2fd7ca60..6bca87ebd6e3d 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java
@@ -49,7 +49,7 @@ public void testBasic() {
         assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createRequest));
         Settings settings = client().admin().indices().prepareGetSettings("hosts").get().getIndexToSettings().get("hosts");
         assertThat(settings.get("index.mode"), equalTo("lookup"));
-        assertThat(settings.get("index.auto_expand_replicas"), equalTo("0-all"));
+        assertNull(settings.get("index.auto_expand_replicas"));
         Map<String, String> allHosts = Map.of(
             "192.168.1.2",
             "Windows",
@@ -141,7 +141,6 @@ public void testResizeLookupIndex() {
         Settings settings = client().admin().indices().prepareGetSettings("lookup-2").get().getIndexToSettings().get("lookup-2");
         assertThat(settings.get("index.mode"), equalTo("lookup"));
         assertThat(settings.get("index.number_of_shards"), equalTo("1"));
-        assertThat(settings.get("index.auto_expand_replicas"), equalTo("0-all"));
 
         ResizeRequest split = new ResizeRequest("lookup-3", "lookup-1");
         split.setResizeType(ResizeType.SPLIT);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java
index f7bf775bc4f8b..06561bc6d4c97 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java
@@ -96,7 +96,11 @@ public void testSimpleDynamicMappingsSuccessful() {
         client().prepareIndex("index").setId("1").setSource("a.x", 1).get();
         client().prepareIndex("index").setId("2").setSource("a.y", 2).get();
 
-        Map<String, Object> mappings = indicesAdmin().prepareGetMappings("index").get().mappings().get("index").sourceAsMap();
+        Map<String, Object> mappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "index")
+            .get()
+            .mappings()
+            .get("index")
+            .sourceAsMap();
         assertTrue(new WriteField("properties.a", () -> mappings).exists());
         assertTrue(new WriteField("properties.a.properties.x", () -> mappings).exists());
     }
@@ -183,7 +187,7 @@ private Map<String, Object> indexConcurrently(int numberOfFieldsToCreate, Settin
         for (int i = 0; i < numberOfFieldsToCreate; ++i) {
             assertTrue(client().prepareGet("index", Integer.toString(i)).get().isExists());
         }
-        GetMappingsResponse mappings = indicesAdmin().prepareGetMappings("index").get();
+        GetMappingsResponse mappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "index").get();
         MappingMetadata indexMappings = mappings.getMappings().get("index");
         assertNotNull(indexMappings);
         Map<String, Object> typeMappingsMap = indexMappings.getSourceAsMap();
@@ -214,7 +218,11 @@ public void testConcurrentDynamicMappingsWithConflictingType() throws Throwable
         for (int i = 0; i < numberOfDocsToCreate; ++i) {
             assertTrue(client().prepareGet("index", Integer.toString(i)).get().isExists());
         }
-        Map<String, Object> index = indicesAdmin().prepareGetMappings("index").get().getMappings().get("index").getSourceAsMap();
+        Map<String, Object> index = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "index")
+            .get()
+            .getMappings()
+            .get("index")
+            .getSourceAsMap();
         for (int i = 0, j = 1; i < numberOfDocsToCreate; i++, j++) {
             assertThat(new WriteField("properties.field" + i + ".type", () -> index).get(null), is(oneOf("long", "float")));
             assertThat(new WriteField("properties.field" + j + ".type", () -> index).get(null), is(oneOf("long", "float")));
@@ -683,9 +691,12 @@ public void testDynamicRuntimeNoConflicts() {
         BulkResponse bulkItemResponses = client().bulk(bulkRequest).actionGet();
         assertFalse(bulkItemResponses.buildFailureMessage(), bulkItemResponses.hasFailures());
 
-        assertHitCount(prepareSearch("test").setQuery(new MatchQueryBuilder("one", "one")), 1);
-        assertHitCount(prepareSearch("test").setQuery(new MatchQueryBuilder("one.two", 3.5)), 1);
-        assertHitCount(prepareSearch("test").setQuery(new MatchQueryBuilder("one.two.three", "1")), 1);
+        assertHitCount(
+            1,
+            prepareSearch("test").setQuery(new MatchQueryBuilder("one", "one")),
+            prepareSearch("test").setQuery(new MatchQueryBuilder("one.two", 3.5)),
+            prepareSearch("test").setQuery(new MatchQueryBuilder("one.two.three", "1"))
+        );
     }
 
     public void testDynamicRuntimeObjectFields() {
@@ -722,10 +733,13 @@ public void testDynamicRuntimeObjectFields() {
         BulkResponse bulkItemResponses = client().bulk(bulkRequest).actionGet();
         assertFalse(bulkItemResponses.buildFailureMessage(), bulkItemResponses.hasFailures());
 
-        assertHitCount(prepareSearch("test").setQuery(new MatchQueryBuilder("obj.one", 1)), 1);
-        assertHitCount(prepareSearch("test").setQuery(new MatchQueryBuilder("anything", "anything")), 1);
-        assertHitCount(prepareSearch("test").setQuery(new MatchQueryBuilder("obj.runtime.one", "one")), 1);
-        assertHitCount(prepareSearch("test").setQuery(new MatchQueryBuilder("obj.runtime.one.two", "1")), 1);
+        assertHitCount(
+            1,
+            prepareSearch("test").setQuery(new MatchQueryBuilder("obj.one", 1)),
+            prepareSearch("test").setQuery(new MatchQueryBuilder("anything", "anything")),
+            prepareSearch("test").setQuery(new MatchQueryBuilder("obj.runtime.one", "one")),
+            prepareSearch("test").setQuery(new MatchQueryBuilder("obj.runtime.one.two", "1"))
+        );
 
         Exception exception = expectThrows(DocumentParsingException.class, prepareIndex("test").setSource("obj.runtime", "value"));
         assertThat(
@@ -800,7 +814,11 @@ public void testSubobjectsFalseAtRoot() throws Exception {
         assertEquals(RestStatus.CREATED, indexResponse.status());
 
         assertBusy(() -> {
-            Map<String, Object> mappings = indicesAdmin().prepareGetMappings("test").get().mappings().get("test").sourceAsMap();
+            Map<String, Object> mappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test")
+                .get()
+                .mappings()
+                .get("test")
+                .sourceAsMap();
             @SuppressWarnings("unchecked")
             Map<String, Object> properties = (Map<String, Object>) mappings.get("properties");
             assertEquals(4, properties.size());
@@ -845,7 +863,11 @@ public void testSubobjectsFalse() throws Exception {
         assertEquals(RestStatus.CREATED, indexResponse.status());
 
         assertBusy(() -> {
-            Map<String, Object> mappings = indicesAdmin().prepareGetMappings("test").get().mappings().get("test").sourceAsMap();
+            Map<String, Object> mappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test")
+                .get()
+                .mappings()
+                .get("test")
+                .sourceAsMap();
             Map<String, Object> properties = (Map<String, Object>) mappings.get("properties");
             Map<String, Object> foo = (Map<String, Object>) properties.get("foo");
             properties = (Map<String, Object>) foo.get("properties");
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java
index fea5d256b1993..1a51fc12fed8e 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java
@@ -37,7 +37,7 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase {
     public void testMultiFields() throws Exception {
         assertAcked(indicesAdmin().prepareCreate("my-index").setMapping(createTypeSource()));
 
-        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get();
+        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get();
         MappingMetadata mappingMetadata = getMappingsResponse.mappings().get("my-index");
         assertThat(mappingMetadata, not(nullValue()));
         Map<String, Object> mappingSource = mappingMetadata.sourceAsMap();
@@ -53,7 +53,7 @@ public void testMultiFields() throws Exception {
 
         assertAcked(indicesAdmin().preparePutMapping("my-index").setSource(createPutMappingSource()));
 
-        getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get();
+        getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get();
         mappingMetadata = getMappingsResponse.mappings().get("my-index");
         assertThat(mappingMetadata, not(nullValue()));
         mappingSource = mappingMetadata.sourceAsMap();
@@ -74,7 +74,7 @@ public void testMultiFields() throws Exception {
     public void testGeoPointMultiField() throws Exception {
         assertAcked(indicesAdmin().prepareCreate("my-index").setMapping(createMappingSource("geo_point")));
 
-        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get();
+        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get();
         MappingMetadata mappingMetadata = getMappingsResponse.mappings().get("my-index");
         assertThat(mappingMetadata, not(nullValue()));
         Map<String, Object> mappingSource = mappingMetadata.sourceAsMap();
@@ -91,18 +91,18 @@ public void testGeoPointMultiField() throws Exception {
         GeoPoint point = new GeoPoint(51, 19);
         prepareIndex("my-index").setId("1").setSource("a", point.toString()).setRefreshPolicy(IMMEDIATE).get();
         assertHitCount(
+            1L,
             prepareSearch("my-index").setSize(0)
                 .setQuery(constantScoreQuery(geoDistanceQuery("a").point(51, 19).distance(50, DistanceUnit.KILOMETERS))),
-            1L
+            prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", point.geohash()))
         );
-        assertHitCount(prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", point.geohash())), 1L);
     }
 
     @SuppressWarnings("unchecked")
     public void testCompletionMultiField() throws Exception {
         assertAcked(indicesAdmin().prepareCreate("my-index").setMapping(createMappingSource("completion")));
 
-        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get();
+        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get();
         MappingMetadata mappingMetadata = getMappingsResponse.mappings().get("my-index");
         assertThat(mappingMetadata, not(nullValue()));
         Map<String, Object> mappingSource = mappingMetadata.sourceAsMap();
@@ -123,7 +123,7 @@ public void testCompletionMultiField() throws Exception {
     public void testIpMultiField() throws Exception {
         assertAcked(indicesAdmin().prepareCreate("my-index").setMapping(createMappingSource("ip")));
 
-        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get();
+        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get();
         MappingMetadata mappingMetadata = getMappingsResponse.mappings().get("my-index");
         assertThat(mappingMetadata, not(nullValue()));
         Map<String, Object> mappingSource = mappingMetadata.sourceAsMap();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/PeerRecoveryRetentionLeaseCreationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/PeerRecoveryRetentionLeaseCreationIT.java
index 07f9d9ee7b6c3..92e5eb8e046bc 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/PeerRecoveryRetentionLeaseCreationIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/PeerRecoveryRetentionLeaseCreationIT.java
@@ -48,7 +48,7 @@ public void testCanRecoverFromStoreWithoutPeerRecoveryRetentionLease() throws Ex
                 Settings.builder()
                     .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
                     .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
-                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomCompatibleVersion(random()))
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomCompatibleWriteVersion(random()))
             )
         );
         ensureGreen(INDEX_NAME);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
index 545ed83bb79c8..c53cf3b56f65a 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
@@ -497,7 +497,9 @@ public void testDeleteIndexWildcard() throws Exception {
     public void testPutAlias() throws Exception {
         createIndex("foobar");
         verify(indicesAdmin().prepareAliases().addAlias("foobar", "foobar_alias"), false);
-        assertFalse(indicesAdmin().prepareGetAliases("foobar_alias").setIndices("foobar").get().getAliases().isEmpty());
+        assertFalse(
+            indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foobar_alias").setIndices("foobar").get().getAliases().isEmpty()
+        );
 
     }
 
@@ -505,16 +507,24 @@ public void testPutAliasWildcard() throws Exception {
         createIndex("foo", "foobar", "bar", "barbaz");
 
         verify(indicesAdmin().prepareAliases().addAlias("foo*", "foobar_alias"), false);
-        assertFalse(indicesAdmin().prepareGetAliases("foobar_alias").setIndices("foo").get().getAliases().isEmpty());
-        assertFalse(indicesAdmin().prepareGetAliases("foobar_alias").setIndices("foobar").get().getAliases().isEmpty());
-        assertTrue(indicesAdmin().prepareGetAliases("foobar_alias").setIndices("bar").get().getAliases().isEmpty());
-        assertTrue(indicesAdmin().prepareGetAliases("foobar_alias").setIndices("barbaz").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foobar_alias").setIndices("foo").get().getAliases().isEmpty());
+        assertFalse(
+            indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foobar_alias").setIndices("foobar").get().getAliases().isEmpty()
+        );
+        assertTrue(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foobar_alias").setIndices("bar").get().getAliases().isEmpty());
+        assertTrue(
+            indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foobar_alias").setIndices("barbaz").get().getAliases().isEmpty()
+        );
 
         verify(indicesAdmin().prepareAliases().addAlias("*", "foobar_alias"), false);
-        assertFalse(indicesAdmin().prepareGetAliases("foobar_alias").setIndices("foo").get().getAliases().isEmpty());
-        assertFalse(indicesAdmin().prepareGetAliases("foobar_alias").setIndices("foobar").get().getAliases().isEmpty());
-        assertFalse(indicesAdmin().prepareGetAliases("foobar_alias").setIndices("bar").get().getAliases().isEmpty());
-        assertFalse(indicesAdmin().prepareGetAliases("foobar_alias").setIndices("barbaz").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foobar_alias").setIndices("foo").get().getAliases().isEmpty());
+        assertFalse(
+            indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foobar_alias").setIndices("foobar").get().getAliases().isEmpty()
+        );
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foobar_alias").setIndices("bar").get().getAliases().isEmpty());
+        assertFalse(
+            indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "foobar_alias").setIndices("barbaz").get().getAliases().isEmpty()
+        );
 
     }
 
@@ -527,26 +537,26 @@ public void testPutMapping() throws Exception {
         }
 
         verify(indicesAdmin().preparePutMapping("foo").setSource("field", "type=text"), false);
-        assertThat(indicesAdmin().prepareGetMappings("foo").get().mappings().get("foo"), notNullValue());
+        assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foo").get().mappings().get("foo"), notNullValue());
         verify(indicesAdmin().preparePutMapping("b*").setSource("field", "type=text"), false);
-        assertThat(indicesAdmin().prepareGetMappings("bar").get().mappings().get("bar"), notNullValue());
-        assertThat(indicesAdmin().prepareGetMappings("barbaz").get().mappings().get("barbaz"), notNullValue());
+        assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "bar").get().mappings().get("bar"), notNullValue());
+        assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "barbaz").get().mappings().get("barbaz"), notNullValue());
         verify(indicesAdmin().preparePutMapping("_all").setSource("field", "type=text"), false);
-        assertThat(indicesAdmin().prepareGetMappings("foo").get().mappings().get("foo"), notNullValue());
-        assertThat(indicesAdmin().prepareGetMappings("foobar").get().mappings().get("foobar"), notNullValue());
-        assertThat(indicesAdmin().prepareGetMappings("bar").get().mappings().get("bar"), notNullValue());
-        assertThat(indicesAdmin().prepareGetMappings("barbaz").get().mappings().get("barbaz"), notNullValue());
+        assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foo").get().mappings().get("foo"), notNullValue());
+        assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foobar").get().mappings().get("foobar"), notNullValue());
+        assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "bar").get().mappings().get("bar"), notNullValue());
+        assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "barbaz").get().mappings().get("barbaz"), notNullValue());
         verify(indicesAdmin().preparePutMapping().setSource("field", "type=text"), false);
-        assertThat(indicesAdmin().prepareGetMappings("foo").get().mappings().get("foo"), notNullValue());
-        assertThat(indicesAdmin().prepareGetMappings("foobar").get().mappings().get("foobar"), notNullValue());
-        assertThat(indicesAdmin().prepareGetMappings("bar").get().mappings().get("bar"), notNullValue());
-        assertThat(indicesAdmin().prepareGetMappings("barbaz").get().mappings().get("barbaz"), notNullValue());
+        assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foo").get().mappings().get("foo"), notNullValue());
+        assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foobar").get().mappings().get("foobar"), notNullValue());
+        assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "bar").get().mappings().get("bar"), notNullValue());
+        assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "barbaz").get().mappings().get("barbaz"), notNullValue());
 
         verify(indicesAdmin().preparePutMapping("c*").setSource("field", "type=text"), true);
 
         assertAcked(indicesAdmin().prepareClose("barbaz").get());
         verify(indicesAdmin().preparePutMapping("barbaz").setSource("field", "type=text"), false);
-        assertThat(indicesAdmin().prepareGetMappings("barbaz").get().mappings().get("barbaz"), notNullValue());
+        assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "barbaz").get().mappings().get("barbaz"), notNullValue());
     }
 
     public static final class TestPlugin extends Plugin {
@@ -646,7 +656,7 @@ static ValidateQueryRequestBuilder validateQuery(String... indices) {
     }
 
     static GetAliasesRequestBuilder getAliases(String... indices) {
-        return indicesAdmin().prepareGetAliases("dummy").setIndices(indices);
+        return indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "dummy").setIndices(indices);
     }
 
     static GetFieldMappingsRequestBuilder getFieldMapping(String... indices) {
@@ -654,7 +664,7 @@ static GetFieldMappingsRequestBuilder getFieldMapping(String... indices) {
     }
 
     static GetMappingsRequestBuilder getMapping(String... indices) {
-        return indicesAdmin().prepareGetMappings(indices);
+        return indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, indices);
     }
 
     static GetSettingsRequestBuilder getSettings(String... indices) {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
index 52492ba7ce657..03fb3d43793cf 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
@@ -321,9 +321,11 @@ public void testQueryRewriteDates() throws Exception {
     public void testQueryRewriteDatesWithNow() throws Exception {
         Client client = client();
         Settings settings = indexSettings(1, 0).put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true).build();
-        assertAcked(indicesAdmin().prepareCreate("index-1").setMapping("d", "type=date").setSettings(settings).get());
-        assertAcked(indicesAdmin().prepareCreate("index-2").setMapping("d", "type=date").setSettings(settings).get());
-        assertAcked(indicesAdmin().prepareCreate("index-3").setMapping("d", "type=date").setSettings(settings).get());
+        assertAcked(
+            indicesAdmin().prepareCreate("index-1").setMapping("d", "type=date").setSettings(settings),
+            indicesAdmin().prepareCreate("index-2").setMapping("d", "type=date").setSettings(settings),
+            indicesAdmin().prepareCreate("index-3").setMapping("d", "type=date").setSettings(settings)
+        );
         ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
         DateFormatter formatter = DateFormatter.forPattern("strict_date_optional_time");
         indexRandom(
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java
index de565605ff58a..7264585337fc7 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java
@@ -123,8 +123,9 @@ private void triggerClusterStateUpdates() {
      * Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values.
      */
     private void assertMappingsAndSettings(String expectedMappings) {
-        final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(INDEX_NAME))
-            .actionGet();
+        final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings(
+            new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(INDEX_NAME)
+        ).actionGet();
 
         final Map<String, MappingMetadata> mappings = getMappingsResponse.getMappings();
         assertThat(
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java
index 48aef0d348045..891b0319f880d 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java
@@ -47,7 +47,7 @@ public void testThatPreBuiltAnalyzersAreNotClosedOnIndexClose() throws Exception
             PreBuiltAnalyzers preBuiltAnalyzer = PreBuiltAnalyzers.values()[randomInt];
             String name = preBuiltAnalyzer.name().toLowerCase(Locale.ROOT);
 
-            IndexVersion randomVersion = IndexVersionUtils.randomVersion(random());
+            IndexVersion randomVersion = IndexVersionUtils.randomWriteVersion();
             if (loadedAnalyzers.containsKey(preBuiltAnalyzer) == false) {
                 loadedAnalyzers.put(preBuiltAnalyzer, new ArrayList<>());
             }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/MalformedDynamicTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/MalformedDynamicTemplateIT.java
deleted file mode 100644
index 9fd657b809f27..0000000000000
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/MalformedDynamicTemplateIT.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.indices.mapping;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.UpdateForV9;
-import org.elasticsearch.index.IndexVersion;
-import org.elasticsearch.index.IndexVersions;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.index.IndexVersionUtils;
-import org.elasticsearch.xcontent.XContentType;
-
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.hamcrest.Matchers.containsString;
-
-public class MalformedDynamicTemplateIT extends ESIntegTestCase {
-
-    @Override
-    protected boolean forbidPrivateIndexSettings() {
-        return false;
-    }
-
-    /**
-     * Check that we can index a document into an 7.x index with a matching dynamic template that
-     * contains unknown parameters. We were able to create those templates in 7.x still, so we need
-     * to be able to index new documents into them. Indexing should issue a deprecation warning though.
-     */
-    @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT)
-    @AwaitsFix(bugUrl = "this is testing 7.x specific compatibility which may be n/a now after 9.0 bump")
-    public void testBWCMalformedDynamicTemplate() {
-        // this parameter is not supported by "keyword" field type
-        String mapping = """
-            { "dynamic_templates": [
-                  {
-                    "my_template": {
-                      "mapping": {
-                        "ignore_malformed": true,
-                        "type": "keyword"
-                      },
-                      "path_match": "*"
-                    }
-                  }
-                ]
-              }
-            }}""";
-        String indexName = "malformed_dynamic_template";
-        assertAcked(
-            prepareCreate(indexName).setSettings(
-                Settings.builder()
-                    .put(indexSettings())
-                    .put("number_of_shards", 1)
-                    .put("index.version.created", IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.V_8_0_0))
-            ).setMapping(mapping)
-        );
-        prepareIndex(indexName).setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get();
-        assertNoFailures((indicesAdmin().prepareRefresh(indexName)).get());
-        assertHitCount(prepareSearch(indexName), 1);
-
-        MapperParsingException ex = expectThrows(
-            MapperParsingException.class,
-            prepareCreate("malformed_dynamic_template_8.0").setSettings(
-                Settings.builder().put(indexSettings()).put("number_of_shards", 1).put("index.version.created", IndexVersion.current())
-            ).setMapping(mapping)
-        );
-        assertThat(ex.getMessage(), containsString("dynamic template [my_template] has invalid content"));
-    }
-
-}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java
index e6927aefbc5cf..e3092bda185fe 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java
@@ -81,9 +81,10 @@ private XContentBuilder getMappingForType() throws IOException {
     }
 
     public void testGetFieldMappings() throws Exception {
-
-        assertAcked(prepareCreate("indexa").setMapping(getMappingForType()));
-        assertAcked(indicesAdmin().prepareCreate("indexb").setMapping(getMappingForType()));
+        assertAcked(
+            prepareCreate("indexa").setMapping(getMappingForType()),
+            indicesAdmin().prepareCreate("indexb").setMapping(getMappingForType())
+        );
 
         // Get mappings by full name
         GetFieldMappingsResponse response = indicesAdmin().prepareGetFieldMappings("indexa").setFields("field1", "obj.subfield").get();
@@ -197,7 +198,7 @@ public void testGetFieldMappingsWithBlocks() throws Exception {
 
         try {
             enableIndexBlock("test", SETTING_BLOCKS_METADATA);
-            assertBlocked(indicesAdmin().prepareGetMappings(), INDEX_METADATA_BLOCK);
+            assertBlocked(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT), INDEX_METADATA_BLOCK);
         } finally {
             disableIndexBlock("test", SETTING_BLOCKS_METADATA);
         }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java
index 20e59fab3bd0c..023aa402b7337 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java
@@ -42,7 +42,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
 
     public void testGetMappingsWhereThereAreNone() {
         createIndex("index");
-        GetMappingsResponse response = indicesAdmin().prepareGetMappings().get();
+        GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get();
         assertThat(response.mappings().containsKey("index"), equalTo(true));
         assertEquals(MappingMetadata.EMPTY_MAPPINGS, response.mappings().get("index"));
     }
@@ -70,19 +70,19 @@ public void testSimpleGetMappings() throws Exception {
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
 
         // Get all mappings
-        GetMappingsResponse response = indicesAdmin().prepareGetMappings().get();
+        GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get();
         assertThat(response.mappings().size(), equalTo(2));
         assertThat(response.mappings().get("indexa"), notNullValue());
         assertThat(response.mappings().get("indexb"), notNullValue());
 
         // Get all mappings, via wildcard support
-        response = indicesAdmin().prepareGetMappings("*").get();
+        response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "*").get();
         assertThat(response.mappings().size(), equalTo(2));
         assertThat(response.mappings().get("indexa"), notNullValue());
         assertThat(response.mappings().get("indexb"), notNullValue());
 
         // Get mappings in indexa
-        response = indicesAdmin().prepareGetMappings("indexa").get();
+        response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "indexa").get();
         assertThat(response.mappings().size(), equalTo(1));
         assertThat(response.mappings().get("indexa"), notNullValue());
     }
@@ -94,7 +94,7 @@ public void testGetMappingsWithBlocks() throws IOException {
         for (String block : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY)) {
             try {
                 enableIndexBlock("test", block);
-                GetMappingsResponse response = indicesAdmin().prepareGetMappings().get();
+                GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get();
                 assertThat(response.mappings().size(), equalTo(1));
                 assertNotNull(response.mappings().get("test"));
             } finally {
@@ -104,7 +104,7 @@ public void testGetMappingsWithBlocks() throws IOException {
 
         try {
             enableIndexBlock("test", SETTING_BLOCKS_METADATA);
-            assertBlocked(indicesAdmin().prepareGetMappings(), INDEX_METADATA_BLOCK);
+            assertBlocked(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT), INDEX_METADATA_BLOCK);
         } finally {
             disableIndexBlock("test", SETTING_BLOCKS_METADATA);
         }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java
index 6f6e488d46b23..fa2598348a1ce 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java
@@ -109,7 +109,7 @@ public void testUpdateMappingWithoutType() {
 
         assertThat(putMappingResponse.isAcknowledged(), equalTo(true));
 
-        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("test").get();
+        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get();
         assertThat(getMappingsResponse.mappings().get("test").source().toString(), equalTo("""
             {"_doc":{"properties":{"body":{"type":"text"},"date":{"type":"integer"}}}}"""));
     }
@@ -123,7 +123,7 @@ public void testUpdateMappingWithoutTypeMultiObjects() {
 
         assertThat(putMappingResponse.isAcknowledged(), equalTo(true));
 
-        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("test").get();
+        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get();
         assertThat(getMappingsResponse.mappings().get("test").source().toString(), equalTo("""
             {"_doc":{"properties":{"date":{"type":"integer"}}}}"""));
     }
@@ -215,7 +215,10 @@ public void testUpdateMappingConcurrently() throws Throwable {
                         .get();
 
                     assertThat(response.isAcknowledged(), equalTo(true));
-                    GetMappingsResponse getMappingResponse = client2.admin().indices().prepareGetMappings(indexName).get();
+                    GetMappingsResponse getMappingResponse = client2.admin()
+                        .indices()
+                        .prepareGetMappings(TEST_REQUEST_TIMEOUT, indexName)
+                        .get();
                     MappingMetadata mappings = getMappingResponse.getMappings().get(indexName);
                     @SuppressWarnings("unchecked")
                     Map<String, Object> properties = (Map<String, Object>) mappings.getSourceAsMap().get("properties");
@@ -284,7 +287,7 @@ private void assertConcreteMappingsOnAll(final String index, final String... fie
      * Waits for the given mapping type to exists on the master node.
      */
     private void assertMappingOnMaster(final String index, final String... fieldNames) {
-        GetMappingsResponse response = indicesAdmin().prepareGetMappings(index).get();
+        GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, index).get();
         MappingMetadata mappings = response.getMappings().get(index);
         assertThat(mappings, notNullValue());
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
index 7d4269550bb88..fa1348c82d71a 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
@@ -2050,7 +2050,7 @@ public void testPostRecoveryMergeDisabledOnOlderIndices() throws Exception {
                     IndexMetadata.SETTING_VERSION_CREATED,
                     IndexVersionUtils.randomVersionBetween(
                         random(),
-                        IndexVersionUtils.getFirstVersion(),
+                        IndexVersionUtils.getLowestWriteCompatibleVersion(),
                         IndexVersionUtils.getPreviousVersion(IndexVersions.MERGE_ON_RECOVERY_VERSION)
                     )
                 )
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/IndexTemplateBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/IndexTemplateBlocksIT.java
index 2aa834b98987a..a5d3dcaf104ad 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/IndexTemplateBlocksIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/IndexTemplateBlocksIT.java
@@ -50,7 +50,7 @@ public void testIndexTemplatesWithBlocks() throws IOException {
         try {
             setClusterReadOnly(true);
 
-            GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates("template_blocks").get();
+            GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT, "template_blocks").get();
             assertThat(response.getIndexTemplates(), hasSize(1));
 
             assertBlocked(
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
index 8496180e85d4e..ac3fc28aab307 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
@@ -79,7 +79,7 @@ public void testSimpleIndexTemplateTests() throws Exception {
         indicesAdmin().prepareDeleteTemplate("*").get();
 
         // check get all templates on an empty index.
-        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates().get();
+        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
         assertThat(response.getIndexTemplates(), empty());
 
         indicesAdmin().preparePutTemplate("template_1")
@@ -145,7 +145,7 @@ public void testSimpleIndexTemplateTests() throws Exception {
             );
         expectThrows(IllegalArgumentException.class, builder);
 
-        response = indicesAdmin().prepareGetTemplates().get();
+        response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
         assertThat(response.getIndexTemplates(), hasSize(2));
 
         // index something into test_index, will match on both templates
@@ -292,7 +292,7 @@ public void testThatGetIndexTemplatesWorks() throws Exception {
             .get();
 
         logger.info("--> get template template_1");
-        GetIndexTemplatesResponse getTemplate1Response = indicesAdmin().prepareGetTemplates("template_1").get();
+        GetIndexTemplatesResponse getTemplate1Response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT, "template_1").get();
         assertThat(getTemplate1Response.getIndexTemplates(), hasSize(1));
         assertThat(getTemplate1Response.getIndexTemplates().get(0), is(notNullValue()));
         assertThat(getTemplate1Response.getIndexTemplates().get(0).patterns(), is(Collections.singletonList("te*")));
@@ -300,7 +300,8 @@ public void testThatGetIndexTemplatesWorks() throws Exception {
         assertThat(getTemplate1Response.getIndexTemplates().get(0).getVersion(), is(123));
 
         logger.info("--> get non-existing-template");
-        GetIndexTemplatesResponse getTemplate2Response = indicesAdmin().prepareGetTemplates("non-existing-template").get();
+        GetIndexTemplatesResponse getTemplate2Response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT, "non-existing-template")
+            .get();
         assertThat(getTemplate2Response.getIndexTemplates(), hasSize(0));
     }
 
@@ -375,7 +376,7 @@ public void testThatGetIndexTemplatesWithSimpleRegexWorks() throws Exception {
             .get();
 
         logger.info("--> get template template_*");
-        GetIndexTemplatesResponse getTemplate1Response = indicesAdmin().prepareGetTemplates("template_*").get();
+        GetIndexTemplatesResponse getTemplate1Response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT, "template_*").get();
         assertThat(getTemplate1Response.getIndexTemplates(), hasSize(2));
 
         List<String> templateNames = new ArrayList<>();
@@ -384,7 +385,7 @@ public void testThatGetIndexTemplatesWithSimpleRegexWorks() throws Exception {
         assertThat(templateNames, containsInAnyOrder("template_1", "template_2"));
 
         logger.info("--> get all templates");
-        getTemplate1Response = indicesAdmin().prepareGetTemplates("template*").get();
+        getTemplate1Response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT, "template*").get();
         assertThat(getTemplate1Response.getIndexTemplates(), hasSize(3));
 
         templateNames = new ArrayList<>();
@@ -394,7 +395,7 @@ public void testThatGetIndexTemplatesWithSimpleRegexWorks() throws Exception {
         assertThat(templateNames, containsInAnyOrder("template_1", "template_2", "template3"));
 
         logger.info("--> get templates template_1 and template_2");
-        getTemplate1Response = indicesAdmin().prepareGetTemplates("template_1", "template_2").get();
+        getTemplate1Response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT, "template_1", "template_2").get();
         assertThat(getTemplate1Response.getIndexTemplates(), hasSize(2));
 
         templateNames = new ArrayList<>();
@@ -419,7 +420,7 @@ public void testThatInvalidGetIndexTemplatesFails() throws Exception {
 
     private void testExpectActionRequestValidationException(String... names) {
         assertRequestBuilderThrows(
-            indicesAdmin().prepareGetTemplates(names),
+            indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT, names),
             ActionRequestValidationException.class,
             "get template with " + Arrays.toString(names)
         );
@@ -430,7 +431,7 @@ public void testBrokenMapping() throws Exception {
         indicesAdmin().prepareDeleteTemplate("*").get();
 
         // check get all templates on an empty index.
-        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates().get();
+        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
         assertThat(response.getIndexTemplates(), empty());
 
         MapperParsingException e = expectThrows(
@@ -441,7 +442,7 @@ public void testBrokenMapping() throws Exception {
         );
         assertThat(e.getMessage(), containsString("Failed to parse mapping"));
 
-        response = indicesAdmin().prepareGetTemplates().get();
+        response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
         assertThat(response.getIndexTemplates(), hasSize(0));
     }
 
@@ -450,7 +451,7 @@ public void testInvalidSettings() throws Exception {
         indicesAdmin().prepareDeleteTemplate("*").get();
 
         // check get all templates on an empty index.
-        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates().get();
+        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
         assertThat(response.getIndexTemplates(), empty());
 
         IllegalArgumentException e = expectThrows(
@@ -465,7 +466,7 @@ public void testInvalidSettings() throws Exception {
             e.getMessage()
         );
 
-        response = indicesAdmin().prepareGetTemplates().get();
+        response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
         assertEquals(0, response.getIndexTemplates().size());
 
         createIndex("test");
@@ -494,7 +495,7 @@ public void testIndexTemplateWithAliases() throws Exception {
         prepareIndex("test_index").setId("4").setSource("type", "typeY", "field", "D value").get();
         prepareIndex("test_index").setId("5").setSource("type", "typeZ", "field", "E value").get();
 
-        GetAliasesResponse getAliasesResponse = indicesAdmin().prepareGetAliases().setIndices("test_index").get();
+        GetAliasesResponse getAliasesResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT).setIndices("test_index").get();
         assertThat(getAliasesResponse.getAliases().size(), equalTo(1));
         assertThat(getAliasesResponse.getAliases().get("test_index").size(), equalTo(4));
 
@@ -538,7 +539,7 @@ public void testIndexTemplateWithAliasesInSource() {
         assertAcked(prepareCreate("test_index"));
         ensureGreen();
 
-        GetAliasesResponse getAliasesResponse = indicesAdmin().prepareGetAliases().setIndices("test_index").get();
+        GetAliasesResponse getAliasesResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT).setIndices("test_index").get();
         assertThat(getAliasesResponse.getAliases().size(), equalTo(1));
         assertThat(getAliasesResponse.getAliases().get("test_index").size(), equalTo(1));
 
@@ -574,7 +575,7 @@ public void testIndexTemplateWithAliasesSource() {
         assertAcked(prepareCreate("test_index"));
         ensureGreen();
 
-        GetAliasesResponse getAliasesResponse = indicesAdmin().prepareGetAliases().setIndices("test_index").get();
+        GetAliasesResponse getAliasesResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT).setIndices("test_index").get();
         assertThat(getAliasesResponse.getAliases().size(), equalTo(1));
         assertThat(getAliasesResponse.getAliases().get("test_index").size(), equalTo(3));
 
@@ -597,7 +598,7 @@ public void testDuplicateAlias() throws Exception {
             .addAlias(new Alias("my_alias").filter(termQuery("field", "value2")))
             .get();
 
-        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates("template_1").get();
+        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT, "template_1").get();
         assertThat(response.getIndexTemplates().size(), equalTo(1));
         assertThat(response.getIndexTemplates().get(0).getAliases().size(), equalTo(1));
         assertThat(response.getIndexTemplates().get(0).getAliases().get("my_alias").filter().string(), containsString("\"value1\""));
@@ -610,7 +611,7 @@ public void testAliasInvalidFilterValidJson() throws Exception {
             .addAlias(new Alias("invalid_alias").filter("{ \"invalid\": {} }"))
             .get();
 
-        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates("template_1").get();
+        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT, "template_1").get();
         assertThat(response.getIndexTemplates().size(), equalTo(1));
         assertThat(response.getIndexTemplates().get(0).getAliases().size(), equalTo(1));
         assertThat(response.getIndexTemplates().get(0).getAliases().get("invalid_alias").filter().string(), equalTo("{\"invalid\":{}}"));
@@ -631,7 +632,7 @@ public void testAliasInvalidFilterInvalidJson() throws Exception {
         );
         assertThat(e.getMessage(), equalTo("failed to parse filter for alias [invalid_alias]"));
 
-        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates("template_1").get();
+        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT, "template_1").get();
         assertThat(response.getIndexTemplates().size(), equalTo(0));
     }
 
@@ -685,7 +686,7 @@ public void testMultipleAliasesPrecedence() throws Exception {
 
         ensureGreen();
 
-        GetAliasesResponse getAliasesResponse = indicesAdmin().prepareGetAliases().setIndices("test").get();
+        GetAliasesResponse getAliasesResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT).setIndices("test").get();
         assertThat(getAliasesResponse.getAliases().get("test").size(), equalTo(4));
 
         for (AliasMetadata aliasMetadata : getAliasesResponse.getAliases().get("test")) {
@@ -747,7 +748,7 @@ public void testCombineTemplates() throws Exception {
         indicesAdmin().prepareDeleteTemplate("*").get();
 
         // check get all templates on an empty index.
-        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates().get();
+        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
         assertThat(response.getIndexTemplates(), empty());
 
         // Now, a complete mapping with two separated templates is error
@@ -789,7 +790,7 @@ public void testCombineTemplates() throws Exception {
         );
         assertThat(e.getMessage(), containsString("analyzer [custom_1] has not been configured in mappings"));
 
-        response = indicesAdmin().prepareGetTemplates().get();
+        response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
         assertThat(response.getIndexTemplates(), hasSize(1));
 
     }
@@ -806,7 +807,7 @@ public void testOrderAndVersion() {
                 .setMapping("field", "type=text")
         );
 
-        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates("versioned_template").get();
+        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT, "versioned_template").get();
         assertThat(response.getIndexTemplates().size(), equalTo(1));
         assertThat(response.getIndexTemplates().get(0).getVersion(), equalTo(version));
         assertThat(response.getIndexTemplates().get(0).getOrder(), equalTo(order));
@@ -856,7 +857,7 @@ public void testPartitionedTemplate() throws Exception {
         indicesAdmin().prepareDeleteTemplate("*").get();
 
         // check get all templates on an empty index.
-        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates().get();
+        GetIndexTemplatesResponse response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
         assertThat(response.getIndexTemplates(), empty());
 
         // provide more partitions than shards
@@ -882,7 +883,7 @@ public void testPartitionedTemplate() throws Exception {
         assertThat(eBadMapping.getMessage(), containsString("must have routing required for partitioned index"));
 
         // no templates yet
-        response = indicesAdmin().prepareGetTemplates().get();
+        response = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
         assertEquals(0, response.getIndexTemplates().size());
 
         // a valid configuration that only provides the partition size
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java
index a8028e8671450..fee2c0494365e 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java
@@ -16,28 +16,23 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexMode;
-import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
-import org.elasticsearch.plugins.ScriptPlugin;
-import org.elasticsearch.script.LongFieldScript;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptEngine;
-import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.telemetry.Measurement;
 import org.elasticsearch.telemetry.TestTelemetryPlugin;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.FailingFieldPlugin;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.json.JsonXContent;
 import org.hamcrest.Matcher;
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import static org.elasticsearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER;
 import static org.hamcrest.Matchers.equalTo;
@@ -82,6 +77,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
     static final String STANDARD_INDEXING_COUNT = "es.indices.standard.indexing.total";
     static final String STANDARD_INDEXING_TIME = "es.indices.standard.indexing.time";
     static final String STANDARD_INDEXING_FAILURE = "es.indices.standard.indexing.failure.total";
+    static final String STANDARD_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT = "es.indices.standard.indexing.failure.version_conflict.total";
 
     static final String TIME_SERIES_INDEX_COUNT = "es.indices.time_series.total";
     static final String TIME_SERIES_BYTES_SIZE = "es.indices.time_series.size";
@@ -95,6 +91,8 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
     static final String TIME_SERIES_INDEXING_COUNT = "es.indices.time_series.indexing.total";
     static final String TIME_SERIES_INDEXING_TIME = "es.indices.time_series.indexing.time";
     static final String TIME_SERIES_INDEXING_FAILURE = "es.indices.time_series.indexing.failure.total";
+    static final String TIME_SERIES_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT =
+        "es.indices.time_series.indexing.failure.version_conflict.total";
 
     static final String LOGSDB_INDEX_COUNT = "es.indices.logsdb.total";
     static final String LOGSDB_BYTES_SIZE = "es.indices.logsdb.size";
@@ -108,6 +106,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
     static final String LOGSDB_INDEXING_COUNT = "es.indices.logsdb.indexing.total";
     static final String LOGSDB_INDEXING_TIME = "es.indices.logsdb.indexing.time";
     static final String LOGSDB_INDEXING_FAILURE = "es.indices.logsdb.indexing.failure.total";
+    static final String LOGSDB_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT = "es.indices.logsdb.indexing.failure.version_conflict.total";
 
     public void testIndicesMetrics() {
         String indexNode = internalCluster().startNode();
@@ -138,7 +137,9 @@ public void testIndicesMetrics() {
                 STANDARD_INDEXING_TIME,
                 greaterThanOrEqualTo(0L),
                 STANDARD_INDEXING_FAILURE,
-                equalTo(indexing1.getIndexFailedCount() - indexing0.getIndexCount())
+                equalTo(indexing1.getIndexFailedCount() - indexing0.getIndexFailedCount()),
+                STANDARD_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT,
+                equalTo(indexing1.getIndexFailedDueToVersionConflictCount() - indexing0.getIndexFailedDueToVersionConflictCount())
             )
         );
 
@@ -161,7 +162,9 @@ public void testIndicesMetrics() {
                 TIME_SERIES_INDEXING_TIME,
                 greaterThanOrEqualTo(0L),
                 TIME_SERIES_INDEXING_FAILURE,
-                equalTo(indexing2.getIndexFailedCount() - indexing1.getIndexFailedCount())
+                equalTo(indexing1.getIndexFailedCount() - indexing0.getIndexFailedCount()),
+                TIME_SERIES_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT,
+                equalTo(indexing1.getIndexFailedDueToVersionConflictCount() - indexing0.getIndexFailedDueToVersionConflictCount())
             )
         );
 
@@ -183,13 +186,14 @@ public void testIndicesMetrics() {
                 LOGSDB_INDEXING_TIME,
                 greaterThanOrEqualTo(0L),
                 LOGSDB_INDEXING_FAILURE,
-                equalTo(indexing3.getIndexFailedCount() - indexing2.getIndexFailedCount())
+                equalTo(indexing3.getIndexFailedCount() - indexing2.getIndexFailedCount()),
+                LOGSDB_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT,
+                equalTo(indexing3.getIndexFailedDueToVersionConflictCount() - indexing2.getIndexFailedDueToVersionConflictCount())
             )
         );
         // already collected indexing stats
-        collectThenAssertMetrics(
-            telemetry,
-            4,
+        Map<String, Matcher<Long>> zeroMatchers = new HashMap<>();
+        zeroMatchers.putAll(
             Map.of(
                 STANDARD_INDEXING_COUNT,
                 equalTo(0L),
@@ -197,22 +201,35 @@ public void testIndicesMetrics() {
                 equalTo(0L),
                 STANDARD_INDEXING_FAILURE,
                 equalTo(0L),
-
+                STANDARD_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT,
+                equalTo(0L)
+            )
+        );
+        zeroMatchers.putAll(
+            Map.of(
                 TIME_SERIES_INDEXING_COUNT,
                 equalTo(0L),
                 TIME_SERIES_INDEXING_TIME,
                 equalTo(0L),
                 TIME_SERIES_INDEXING_FAILURE,
                 equalTo(0L),
-
+                TIME_SERIES_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT,
+                equalTo(0L)
+            )
+        );
+        zeroMatchers.putAll(
+            Map.of(
                 LOGSDB_INDEXING_COUNT,
                 equalTo(0L),
                 LOGSDB_INDEXING_TIME,
                 equalTo(0L),
                 LOGSDB_INDEXING_FAILURE,
+                equalTo(0L),
+                LOGSDB_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT,
                 equalTo(0L)
             )
         );
+        collectThenAssertMetrics(telemetry, 4, zeroMatchers);
         String searchNode = internalCluster().startDataOnlyNode();
         indicesService = internalCluster().getInstance(IndicesService.class, searchNode);
         telemetry = internalCluster().getInstance(PluginsService.class, searchNode)
@@ -455,48 +472,4 @@ private Map<String, Object> parseMapping(String mapping) throws IOException {
             return parser.map();
         }
     }
-
-    public static class FailingFieldPlugin extends Plugin implements ScriptPlugin {
-
-        @Override
-        public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<?>> contexts) {
-            return new ScriptEngine() {
-                @Override
-                public String getType() {
-                    return "failing_field";
-                }
-
-                @Override
-                @SuppressWarnings("unchecked")
-                public <FactoryType> FactoryType compile(
-                    String name,
-                    String code,
-                    ScriptContext<FactoryType> context,
-                    Map<String, String> params
-                ) {
-                    return (FactoryType) new LongFieldScript.Factory() {
-                        @Override
-                        public LongFieldScript.LeafFactory newFactory(
-                            String fieldName,
-                            Map<String, Object> params,
-                            SearchLookup searchLookup,
-                            OnScriptError onScriptError
-                        ) {
-                            return ctx -> new LongFieldScript(fieldName, params, searchLookup, onScriptError, ctx) {
-                                @Override
-                                public void execute() {
-                                    throw new IllegalStateException("Accessing failing field");
-                                }
-                            };
-                        }
-                    };
-                }
-
-                @Override
-                public Set<ScriptContext<?>> getSupportedContexts() {
-                    return Set.of(LongFieldScript.CONTEXT);
-                }
-            };
-        }
-    }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java
index e4d44212f2854..04130d176b9e5 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java
@@ -9,14 +9,17 @@
 
 package org.elasticsearch.monitor.metrics;
 
+import org.apache.lucene.analysis.TokenStream;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.bulk.IncrementalBulkService;
 import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -25,6 +28,13 @@
 import org.elasticsearch.core.AbstractRefCounted;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexingPressure;
+import org.elasticsearch.index.VersionType;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
+import org.elasticsearch.index.engine.VersionConflictEngineException;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.rest.RestStatus;
@@ -43,13 +53,16 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Function;
 
+import static java.util.Collections.singletonMap;
 import static org.elasticsearch.index.IndexingPressure.MAX_COORDINATING_BYTES;
 import static org.elasticsearch.index.IndexingPressure.MAX_PRIMARY_BYTES;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThan;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
@@ -66,7 +79,7 @@ public List<Setting<?>> getSettings() {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return List.of(TestTelemetryPlugin.class, TestAPMInternalSettings.class);
+        return List.of(TestTelemetryPlugin.class, TestAPMInternalSettings.class, TestAnalysisPlugin.class);
     }
 
     @Override
@@ -77,6 +90,197 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
             .build();
     }
 
+    public void testZeroMetricsForVersionConflictsForNonIndexingOperations() {
+        final String dataNode = internalCluster().startNode();
+        ensureStableCluster(1);
+
+        final TestTelemetryPlugin plugin = internalCluster().getInstance(PluginsService.class, dataNode)
+            .filterPlugins(TestTelemetryPlugin.class)
+            .findFirst()
+            .orElseThrow();
+        plugin.resetMeter();
+
+        assertAcked(prepareCreate("index_no_refresh", Settings.builder().put("index.refresh_interval", "-1")));
+        assertAcked(prepareCreate("index_with_default_refresh"));
+
+        for (String indexName : List.of("index_no_refresh", "index_with_default_refresh")) {
+            String docId = randomUUID();
+            client(dataNode).index(new IndexRequest(indexName).id(docId).source(Map.of())).actionGet();
+            // test version conflicts are counted when getting from the translog
+            if (randomBoolean()) {
+                // this get has the side effect of tracking translog location in the live version map,
+                // which potentially influences the engine conflict exception path
+                client(dataNode).get(new GetRequest(indexName, docId).realtime(randomBoolean())).actionGet();
+            }
+            {
+                var e = expectThrows(
+                    VersionConflictEngineException.class,
+                    () -> client(dataNode).get(
+                        new GetRequest(indexName, docId).version(10).versionType(randomFrom(VersionType.EXTERNAL, VersionType.EXTERNAL_GTE))
+                    ).actionGet()
+                );
+                assertThat(e.getMessage(), containsString("version conflict"));
+                assertThat(e.status(), is(RestStatus.CONFLICT));
+            }
+            if (randomBoolean()) {
+                client(dataNode).get(new GetRequest(indexName, docId).realtime(false)).actionGet();
+            }
+            client(dataNode).admin().indices().prepareRefresh(indexName).get();
+            {
+                var e = expectThrows(
+                    VersionConflictEngineException.class,
+                    () -> client(dataNode).get(
+                        new GetRequest(indexName, docId).version(5)
+                            .versionType(randomFrom(VersionType.EXTERNAL, VersionType.EXTERNAL_GTE))
+                            .realtime(false)
+                    ).actionGet()
+                );
+                assertThat(e.getMessage(), containsString("version conflict"));
+                assertThat(e.status(), is(RestStatus.CONFLICT));
+            }
+            // updates
+            {
+                var e = expectThrows(
+                    VersionConflictEngineException.class,
+                    () -> client(dataNode).update(
+                        new UpdateRequest(indexName, docId).setIfPrimaryTerm(1)
+                            .setIfSeqNo(randomIntBetween(2, 5))
+                            .doc(Map.of(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)))
+                    ).actionGet()
+                );
+                assertThat(e.getMessage(), containsString("version conflict"));
+                assertThat(e.status(), is(RestStatus.CONFLICT));
+            }
+            // deletes
+            {
+                var e = expectThrows(
+                    VersionConflictEngineException.class,
+                    () -> client(dataNode).delete(
+                        new DeleteRequest(indexName, docId).setIfPrimaryTerm(randomIntBetween(2, 5)).setIfSeqNo(0)
+                    ).actionGet()
+                );
+                assertThat(e.getMessage(), containsString("version conflict"));
+                assertThat(e.status(), is(RestStatus.CONFLICT));
+            }
+        }
+
+        // simulate async apm `polling` call for metrics
+        plugin.collect();
+
+        // there are no indexing (version conflict) failures reported because only gets/updates/deletes generated the conflicts
+        // and those are not "indexing" operations
+        var indexingFailedTotal = getSingleRecordedMetric(plugin::getLongAsyncCounterMeasurement, "es.indexing.indexing.failed.total");
+        assertThat(indexingFailedTotal.getLong(), equalTo(0L));
+        var indexingFailedDueToVersionConflictTotal = getSingleRecordedMetric(
+            plugin::getLongAsyncCounterMeasurement,
+            "es.indexing.indexing.failed.version_conflict.total"
+        );
+        assertThat(indexingFailedDueToVersionConflictTotal.getLong(), equalTo(0L));
+    }
+
+    public void testMetricsForIndexingVersionConflicts() {
+        final String dataNode = internalCluster().startNode();
+        ensureStableCluster(1);
+
+        final TestTelemetryPlugin plugin = internalCluster().getInstance(PluginsService.class, dataNode)
+            .filterPlugins(TestTelemetryPlugin.class)
+            .findFirst()
+            .orElseThrow();
+        plugin.resetMeter();
+
+        assertAcked(
+            prepareCreate(
+                "test",
+                Settings.builder()
+                    .put("index.refresh_interval", "-1")
+                    .put("index.analysis.analyzer.test_analyzer.type", "custom")
+                    .put("index.analysis.analyzer.test_analyzer.tokenizer", "standard")
+                    .putList("index.analysis.analyzer.test_analyzer.filter", "test_token_filter")
+            ).setMapping(Map.of("properties", Map.of("test_field", Map.of("type", "text", "analyzer", "test_analyzer")))).get()
+        );
+
+        String docId = randomUUID();
+        // successful index (with version)
+        client(dataNode).index(
+            new IndexRequest("test").id(docId)
+                .version(10)
+                .versionType(randomFrom(VersionType.EXTERNAL, VersionType.EXTERNAL_GTE))
+                .source(Map.of())
+        ).actionGet();
+        // if_primary_term conflict
+        {
+            var e = expectThrows(
+                VersionConflictEngineException.class,
+                () -> client(dataNode).index(new IndexRequest("test").id(docId).source(Map.of()).setIfSeqNo(0).setIfPrimaryTerm(2))
+                    .actionGet()
+            );
+            assertThat(e.getMessage(), containsString("version conflict"));
+            assertThat(e.status(), is(RestStatus.CONFLICT));
+        }
+        // if_seq_no conflict
+        {
+            var e = expectThrows(
+                VersionConflictEngineException.class,
+                () -> client(dataNode).index(new IndexRequest("test").id(docId).source(Map.of()).setIfSeqNo(1).setIfPrimaryTerm(1))
+                    .actionGet()
+            );
+            assertThat(e.getMessage(), containsString("version conflict"));
+            assertThat(e.status(), is(RestStatus.CONFLICT));
+        }
+        // version conflict
+        {
+            var e = expectThrows(
+                VersionConflictEngineException.class,
+                () -> client(dataNode).index(
+                    new IndexRequest("test").id(docId)
+                        .source(Map.of())
+                        .version(3)
+                        .versionType(randomFrom(VersionType.EXTERNAL, VersionType.EXTERNAL_GTE))
+                ).actionGet()
+            );
+            assertThat(e.getMessage(), containsString("version conflict"));
+            assertThat(e.status(), is(RestStatus.CONFLICT));
+        }
+        // indexing failure that is NOT a version conflict
+        PluginsService pluginService = internalCluster().getInstance(PluginsService.class, dataNode);
+        pluginService.filterPlugins(TestAnalysisPlugin.class).forEach(p -> p.throwParsingError.set(true));
+        {
+            var e = expectThrows(
+                MapperParsingException.class,
+                () -> client(dataNode).index(new IndexRequest("test").id(docId + "other").source(Map.of("test_field", "this will error")))
+                    .actionGet()
+            );
+            assertThat(e.status(), is(RestStatus.BAD_REQUEST));
+        }
+
+        plugin.collect();
+
+        var indexingFailedTotal = getSingleRecordedMetric(plugin::getLongAsyncCounterMeasurement, "es.indexing.indexing.failed.total");
+        assertThat(indexingFailedTotal.getLong(), equalTo(4L));
+        var indexingFailedDueToVersionConflictTotal = getSingleRecordedMetric(
+            plugin::getLongAsyncCounterMeasurement,
+            "es.indexing.indexing.failed.version_conflict.total"
+        );
+        assertThat(indexingFailedDueToVersionConflictTotal.getLong(), equalTo(3L));
+    }
+
+    public static final class TestAnalysisPlugin extends Plugin implements AnalysisPlugin {
+        final AtomicBoolean throwParsingError = new AtomicBoolean(false);
+
+        @Override
+        public Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
+            return singletonMap("test_token_filter", (indexSettings, environment, name, settings) -> new AbstractTokenFilterFactory(name) {
+                @Override
+                public TokenStream create(TokenStream tokenStream) {
+                    if (throwParsingError.get()) {
+                        throw new MapperParsingException("simulate mapping parsing error");
+                    }
+                    return tokenStream;
+                }
+            });
+        }
+    }
+
     public void testNodeIndexingMetricsArePublishing() {
 
         final String dataNode = internalCluster().startNode();
@@ -116,6 +320,11 @@ public void testNodeIndexingMetricsArePublishing() {
 
         var indexingFailedTotal = getSingleRecordedMetric(plugin::getLongAsyncCounterMeasurement, "es.indexing.indexing.failed.total");
         assertThat(indexingFailedTotal.getLong(), equalTo(0L));
+        var indexingFailedDueToVersionConflictTotal = getSingleRecordedMetric(
+            plugin::getLongAsyncCounterMeasurement,
+            "es.indexing.indexing.failed.version_conflict.total"
+        );
+        assertThat(indexingFailedDueToVersionConflictTotal.getLong(), equalTo(0L));
 
         var deletionTotal = getSingleRecordedMetric(plugin::getLongAsyncCounterMeasurement, "es.indexing.deletion.docs.total");
         assertThat(deletionTotal.getLong(), equalTo((long) deletesCount));
@@ -336,8 +545,10 @@ public void testPrimaryDocumentRejectionMetricsArePublishing() {
         plugin.resetMeter();
 
         final int numberOfShards = randomIntBetween(1, 5);
-        assertAcked(prepareCreate("test-one", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards)).get());
-        assertAcked(prepareCreate("test-two", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)).get());
+        assertAcked(
+            prepareCreate("test-one", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards)),
+            prepareCreate("test-two", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1))
+        );
 
         final BulkRequest bulkRequestOne = new BulkRequest();
         final int batchCountOne = randomIntBetween(50, 100);
@@ -397,8 +608,10 @@ public void testPrimaryDocumentRejectionMetricsFluctuatingOverTime() throws Exce
         ensureStableCluster(2);
 
         // for simplicity do not mix small and big documents in single index/shard
-        assertAcked(prepareCreate("test-index-one", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)).get());
-        assertAcked(prepareCreate("test-index-two", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)).get());
+        assertAcked(
+            prepareCreate("test-index-one", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)),
+            prepareCreate("test-index-two", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1))
+        );
 
         final TestTelemetryPlugin plugin = internalCluster().getInstance(PluginsService.class, dataNode)
             .filterPlugins(TestTelemetryPlugin.class)
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
index 77c4f8a26f478..43bca39c02ce5 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.OperationRouting;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.util.CollectionUtils;
@@ -365,11 +365,10 @@ private void iterateAssertCount(final int numberOfShards, final int iterations,
                 );
             }
 
-            ClusterService clusterService = clusterService();
-            final ClusterState state = clusterService.state();
+            final ClusterState state = clusterService().state();
             for (int shard = 0; shard < numberOfShards; shard++) {
                 for (String id : ids) {
-                    ShardId docShard = clusterService.operationRouting().shardId(state, "test", id, null);
+                    ShardId docShard = OperationRouting.shardId(state, "test", id, null);
                     if (docShard.id() == shard) {
                         final IndexShardRoutingTable indexShardRoutingTable = state.routingTable().shardRoutingTable("test", shard);
                         for (int copy = 0; copy < indexShardRoutingTable.size(); copy++) {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/IndexSnapshotsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/IndexSnapshotsServiceIT.java
index 4460bd2e0c302..c10217b93c868 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/IndexSnapshotsServiceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/IndexSnapshotsServiceIT.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.core.Strings;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
@@ -109,8 +108,6 @@ public void testGetShardSnapshotOnEmptyRepositoriesListThrowsAnError() {
         expectThrows(IllegalArgumentException.class, () -> getLatestSnapshotForShardFuture(Collections.emptyList(), "idx", 0, false));
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION)
-    // below we were selecting an index version between current and 7.5.0, this has been updated to 8.0.0 now but that might need to change
     public void testGetShardSnapshotReturnsTheLatestSuccessfulSnapshot() throws Exception {
         final String repoName = "repo-name";
         final Path repoPath = randomRepoPath();
@@ -118,7 +115,7 @@ public void testGetShardSnapshotReturnsTheLatestSuccessfulSnapshot() throws Exce
 
         final boolean useBwCFormat = randomBoolean();
         if (useBwCFormat) {
-            final IndexVersion version = randomVersionBetween(random(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current());
+            final IndexVersion version = randomVersionBetween(random(), IndexVersions.V_7_5_0, IndexVersion.current());
             initWithSnapshotVersion(repoName, repoPath, version);
         }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java
index 8e0dee2396411..c4c19dd8748a6 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java
@@ -456,14 +456,14 @@ private void assertComponentAndIndexTemplateDelete(CountDownLatch savedClusterSt
 
         final var response = client().execute(
             GetComposableIndexTemplateAction.INSTANCE,
-            new GetComposableIndexTemplateAction.Request("template*")
+            new GetComposableIndexTemplateAction.Request(TEST_REQUEST_TIMEOUT, "template*")
         ).get();
 
         assertThat(response.indexTemplates().keySet().stream().collect(Collectors.toSet()), containsInAnyOrder("template_1", "template_2"));
 
         final var componentResponse = client().execute(
             GetComponentTemplateAction.INSTANCE,
-            new GetComponentTemplateAction.Request("other*")
+            new GetComponentTemplateAction.Request(TEST_REQUEST_TIMEOUT, "other*")
         ).get();
 
         assertTrue(componentResponse.getComponentTemplates().isEmpty());
@@ -594,7 +594,7 @@ private void assertClusterStateNotSaved(CountDownLatch savedClusterState, Atomic
 
         final var response = client().execute(
             GetComposableIndexTemplateAction.INSTANCE,
-            new GetComposableIndexTemplateAction.Request("err*")
+            new GetComposableIndexTemplateAction.Request(TEST_REQUEST_TIMEOUT, "err*")
         ).get();
 
         assertTrue(response.indexTemplates().isEmpty());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasRoutingIT.java
index 199c9a9fb4c8c..e14c7c8368917 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasRoutingIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasRoutingIT.java
@@ -123,19 +123,25 @@ public void testAliasSearchRouting() throws Exception {
 
         logger.info("--> search with wrong routing, should not find");
         for (int i = 0; i < 5; i++) {
-            assertHitCount(prepareSearch().setRouting("1").setQuery(QueryBuilders.matchAllQuery()), 0);
-            assertHitCount(prepareSearch().setSize(0).setRouting("1").setQuery(QueryBuilders.matchAllQuery()), 0);
-            assertHitCount(prepareSearch("alias1").setQuery(QueryBuilders.matchAllQuery()), 0);
-            assertHitCount(prepareSearch("alias1").setSize(0).setQuery(QueryBuilders.matchAllQuery()), 0);
+            assertHitCount(
+                0,
+                prepareSearch().setRouting("1").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch().setSize(0).setRouting("1").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias1").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias1").setSize(0).setQuery(QueryBuilders.matchAllQuery())
+            );
         }
 
         logger.info("--> search with correct routing, should find");
         for (int i = 0; i < 5; i++) {
 
-            assertHitCount(prepareSearch().setRouting("0").setQuery(QueryBuilders.matchAllQuery()), 1);
-            assertHitCount(prepareSearch().setSize(0).setRouting("0").setQuery(QueryBuilders.matchAllQuery()), 1);
-            assertHitCount(prepareSearch("alias0").setQuery(QueryBuilders.matchAllQuery()), 1);
-            assertHitCount(prepareSearch("alias0").setSize(0).setQuery(QueryBuilders.matchAllQuery()), 1);
+            assertHitCount(
+                1,
+                prepareSearch().setRouting("0").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch().setSize(0).setRouting("0").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias0").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias0").setSize(0).setQuery(QueryBuilders.matchAllQuery())
+            );
         }
 
         logger.info("--> indexing with id [2], and routing [1] using alias");
@@ -143,50 +149,71 @@ public void testAliasSearchRouting() throws Exception {
 
         logger.info("--> search with no routing, should fine two");
         for (int i = 0; i < 5; i++) {
-            assertHitCount(prepareSearch().setQuery(QueryBuilders.matchAllQuery()), 2);
-            assertHitCount(prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()), 2);
+            assertHitCount(
+                2,
+                prepareSearch().setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery())
+            );
         }
 
         logger.info("--> search with 0 routing, should find one");
         for (int i = 0; i < 5; i++) {
-            assertHitCount(prepareSearch().setRouting("0").setQuery(QueryBuilders.matchAllQuery()), 1);
-            assertHitCount(prepareSearch().setSize(0).setRouting("0").setQuery(QueryBuilders.matchAllQuery()), 1);
-            assertHitCount(prepareSearch("alias0").setQuery(QueryBuilders.matchAllQuery()), 1);
-            assertHitCount(prepareSearch("alias0").setSize(0).setQuery(QueryBuilders.matchAllQuery()), 1);
+            assertHitCount(
+                1,
+                prepareSearch().setRouting("0").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch().setSize(0).setRouting("0").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias0").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias0").setSize(0).setQuery(QueryBuilders.matchAllQuery())
+            );
         }
 
         logger.info("--> search with 1 routing, should find one");
         for (int i = 0; i < 5; i++) {
-            assertHitCount(prepareSearch().setRouting("1").setQuery(QueryBuilders.matchAllQuery()), 1);
-            assertHitCount(prepareSearch().setSize(0).setRouting("1").setQuery(QueryBuilders.matchAllQuery()), 1);
-            assertHitCount(prepareSearch("alias1").setQuery(QueryBuilders.matchAllQuery()), 1);
-            assertHitCount(prepareSearch("alias1").setSize(0).setQuery(QueryBuilders.matchAllQuery()), 1);
+            assertHitCount(
+                1,
+                prepareSearch().setRouting("1").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch().setSize(0).setRouting("1").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias1").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias1").setSize(0).setQuery(QueryBuilders.matchAllQuery())
+            );
         }
 
         logger.info("--> search with 0,1 indexRoutings , should find two");
         for (int i = 0; i < 5; i++) {
-            assertHitCount(prepareSearch().setRouting("0", "1").setQuery(QueryBuilders.matchAllQuery()), 2);
-            assertHitCount(prepareSearch().setSize(0).setRouting("0", "1").setQuery(QueryBuilders.matchAllQuery()), 2);
-            assertHitCount(prepareSearch("alias01").setQuery(QueryBuilders.matchAllQuery()), 2);
-            assertHitCount(prepareSearch("alias01").setSize(0).setQuery(QueryBuilders.matchAllQuery()), 2);
+            assertHitCount(
+                2,
+                prepareSearch().setRouting("0", "1").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch().setSize(0).setRouting("0", "1").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias01").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias01").setSize(0).setQuery(QueryBuilders.matchAllQuery())
+            );
         }
 
         logger.info("--> search with two routing aliases , should find two");
         for (int i = 0; i < 5; i++) {
-            assertHitCount(prepareSearch("alias0", "alias1").setQuery(QueryBuilders.matchAllQuery()), 2);
-            assertHitCount(prepareSearch("alias0", "alias1").setSize(0).setQuery(QueryBuilders.matchAllQuery()), 2);
+            assertHitCount(
+                2,
+                prepareSearch("alias0", "alias1").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias0", "alias1").setSize(0).setQuery(QueryBuilders.matchAllQuery())
+            );
         }
 
         logger.info("--> search with alias0, alias1 and alias01, should find two");
         for (int i = 0; i < 5; i++) {
-            assertHitCount(prepareSearch("alias0", "alias1", "alias01").setQuery(QueryBuilders.matchAllQuery()), 2);
-            assertHitCount(prepareSearch("alias0", "alias1", "alias01").setSize(0).setQuery(QueryBuilders.matchAllQuery()), 2);
+            assertHitCount(
+                2,
+                prepareSearch("alias0", "alias1", "alias01").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias0", "alias1", "alias01").setSize(0).setQuery(QueryBuilders.matchAllQuery())
+            );
         }
 
         logger.info("--> search with test, alias0 and alias1, should find two");
         for (int i = 0; i < 5; i++) {
-            assertHitCount(prepareSearch("test", "alias0", "alias1").setQuery(QueryBuilders.matchAllQuery()), 2);
-            assertHitCount(prepareSearch("test", "alias0", "alias1").setSize(0).setQuery(QueryBuilders.matchAllQuery()), 2);
+            assertHitCount(
+                2,
+                prepareSearch("test", "alias0", "alias1").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("test", "alias0", "alias1").setSize(0).setQuery(QueryBuilders.matchAllQuery())
+            );
         }
 
     }
@@ -236,20 +263,29 @@ public void testAliasSearchRoutingWithTwoIndices() throws Exception {
 
         logger.info("--> search with alias-a1,alias-b0, should not find");
         for (int i = 0; i < 5; i++) {
-            assertHitCount(prepareSearch("alias-a1", "alias-b0").setQuery(QueryBuilders.matchAllQuery()), 0);
-            assertHitCount(prepareSearch("alias-a1", "alias-b0").setSize(0).setQuery(QueryBuilders.matchAllQuery()), 0);
+            assertHitCount(
+                0,
+                prepareSearch("alias-a1", "alias-b0").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias-a1", "alias-b0").setSize(0).setQuery(QueryBuilders.matchAllQuery())
+            );
         }
 
         logger.info("--> search with alias-ab, should find two");
         for (int i = 0; i < 5; i++) {
-            assertHitCount(prepareSearch("alias-ab").setQuery(QueryBuilders.matchAllQuery()), 2);
-            assertHitCount(prepareSearch("alias-ab").setSize(0).setQuery(QueryBuilders.matchAllQuery()), 2);
+            assertHitCount(
+                2,
+                prepareSearch("alias-ab").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias-ab").setSize(0).setQuery(QueryBuilders.matchAllQuery())
+            );
         }
 
         logger.info("--> search with alias-a0,alias-b1 should find two");
         for (int i = 0; i < 5; i++) {
-            assertHitCount(prepareSearch("alias-a0", "alias-b1").setQuery(QueryBuilders.matchAllQuery()), 2);
-            assertHitCount(prepareSearch("alias-a0", "alias-b1").setSize(0).setQuery(QueryBuilders.matchAllQuery()), 2);
+            assertHitCount(
+                2,
+                prepareSearch("alias-a0", "alias-b1").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias-a0", "alias-b1").setSize(0).setQuery(QueryBuilders.matchAllQuery())
+            );
         }
     }
 
@@ -317,8 +353,11 @@ public void testIndexingAliasesOverTime() throws Exception {
         logger.info("--> verifying get and search with routing, should find");
         for (int i = 0; i < 5; i++) {
             assertThat(client().prepareGet("test", "0").setRouting("3").get().isExists(), equalTo(true));
-            assertHitCount(prepareSearch("alias").setQuery(QueryBuilders.matchAllQuery()), 1);
-            assertHitCount(prepareSearch("alias").setSize(0).setQuery(QueryBuilders.matchAllQuery()), 1);
+            assertHitCount(
+                1,
+                prepareSearch("alias").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias").setSize(0).setQuery(QueryBuilders.matchAllQuery())
+            );
         }
 
         logger.info("--> creating alias with routing [4]");
@@ -326,8 +365,11 @@ public void testIndexingAliasesOverTime() throws Exception {
 
         logger.info("--> verifying search with wrong routing should not find");
         for (int i = 0; i < 5; i++) {
-            assertHitCount(prepareSearch("alias").setQuery(QueryBuilders.matchAllQuery()), 0);
-            assertHitCount(prepareSearch("alias").setSize(0).setQuery(QueryBuilders.matchAllQuery()), 0);
+            assertHitCount(
+                0,
+                prepareSearch("alias").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias").setSize(0).setQuery(QueryBuilders.matchAllQuery())
+            );
         }
 
         logger.info("--> creating alias with search routing [3,4] and index routing 4");
@@ -344,8 +386,11 @@ public void testIndexingAliasesOverTime() throws Exception {
         for (int i = 0; i < 5; i++) {
             assertThat(client().prepareGet("test", "0").setRouting("3").get().isExists(), equalTo(true));
             assertThat(client().prepareGet("test", "1").setRouting("4").get().isExists(), equalTo(true));
-            assertHitCount(prepareSearch("alias").setQuery(QueryBuilders.matchAllQuery()), 2);
-            assertHitCount(prepareSearch("alias").setSize(0).setQuery(QueryBuilders.matchAllQuery()), 2);
+            assertHitCount(
+                2,
+                prepareSearch("alias").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch("alias").setSize(0).setQuery(QueryBuilders.matchAllQuery())
+            );
         }
     }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/SimpleRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/SimpleRoutingIT.java
index 246138fa5573d..06d850e0d9abc 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/routing/SimpleRoutingIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/SimpleRoutingIT.java
@@ -141,14 +141,20 @@ public void testSimpleSearchRouting() {
 
         logger.info("--> search with wrong routing, should not find");
         for (int i = 0; i < 5; i++) {
-            assertHitCount(prepareSearch().setRouting("1").setQuery(QueryBuilders.matchAllQuery()), 0);
-            assertHitCount(prepareSearch().setSize(0).setRouting("1").setQuery(QueryBuilders.matchAllQuery()), 0);
+            assertHitCount(
+                0,
+                prepareSearch().setRouting("1").setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch().setSize(0).setRouting("1").setQuery(QueryBuilders.matchAllQuery())
+            );
         }
 
         logger.info("--> search with correct routing, should find");
         for (int i = 0; i < 5; i++) {
-            assertHitCount(prepareSearch().setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery()), 1);
-            assertHitCount(prepareSearch().setSize(0).setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery()), 1);
+            assertHitCount(
+                1,
+                prepareSearch().setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery()),
+                prepareSearch().setSize(0).setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery())
+            );
         }
 
         String secondRoutingValue = "1";
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java
index 907f943e68422..6c67bd2a98606 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java
@@ -56,7 +56,7 @@ protected boolean forbidPrivateIndexSettings() {
         return false;
     }
 
-    private final IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random());
+    private final IndexVersion version = IndexVersionUtils.randomCompatibleWriteVersion(random());
 
     private IndexRequestBuilder indexCity(String idx, String name, String... latLons) throws Exception {
         XContentBuilder source = jsonBuilder().startObject().field("city", name);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java
index 1ad7d1a11bea7..1de51d6df8197 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java
@@ -49,7 +49,7 @@ protected boolean forbidPrivateIndexSettings() {
         return false;
     }
 
-    private final IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random());
+    private final IndexVersion version = IndexVersionUtils.randomCompatibleWriteVersion(random());
 
     static Map<String, Integer> expectedDocCountsForGeoHash = null;
     static Map<String, Integer> multiValuedExpectedDocCountsForGeoHash = null;
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
index 80c47d6180db0..affa371d92aa9 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
@@ -119,10 +119,10 @@ public static String randomExecutionHint() {
 
     @Override
     public void setupSuiteScopeCluster() throws Exception {
-        assertAcked(prepareCreate("idx").setMapping(TERMS_AGGS_FIELD, "type=keyword", "text", "type=text,store=true"));
-        assertAcked(prepareCreate("field-collapsing").setMapping("group", "type=keyword"));
-        createIndex("empty");
         assertAcked(
+            prepareCreate("idx").setMapping(TERMS_AGGS_FIELD, "type=keyword", "text", "type=text,store=true"),
+            prepareCreate("field-collapsing").setMapping("group", "type=keyword"),
+            prepareCreate("empty"),
             prepareCreate("articles").setMapping(
                 jsonBuilder().startObject()
                     .startObject("_doc")
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/QueryRewriteContextIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/QueryRewriteContextIT.java
index f4aa34c9b47b7..965fd0d8011c0 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/QueryRewriteContextIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/QueryRewriteContextIT.java
@@ -154,8 +154,10 @@ public void testResolvedIndices_TransportSearchAction() {
     public void testResolvedIndices_TransportExplainAction() {
         final String[] indices = { "test1", "test2" };
         createIndex(indices);
-        assertAcked(indicesAdmin().prepareAliases().addAlias("test1", "alias1"));
-        assertAcked(indicesAdmin().prepareAliases().addAlias(indices, "alias2"));
+        assertAcked(
+            indicesAdmin().prepareAliases().addAlias("test1", "alias1"),
+            indicesAdmin().prepareAliases().addAlias(indices, "alias2")
+        );
 
         assertResolvedIndices(client().prepareExplain("test1", "1"), Set.of("test1"), Set.of("test1"), r -> {});
         assertResolvedIndices(client().prepareExplain("alias1", "1"), Set.of("alias1"), Set.of("test1"), r -> {});
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java
index 9c1daccd2cc9e..ab79fd7ba1813 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java
@@ -40,18 +40,12 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.AbstractMultiClustersTestCase;
 import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.test.SkipUnavailableRule;
+import org.elasticsearch.test.SkipUnavailableRule.NotSkipped;
 import org.elasticsearch.usage.UsageService;
 import org.junit.Assert;
 import org.junit.Rule;
-import org.junit.rules.TestRule;
-import org.junit.runner.Description;
-import org.junit.runners.model.Statement;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-import java.util.Arrays;
+
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -59,8 +53,6 @@
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
-import java.util.stream.Collectors;
 
 import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.ASYNC_FEATURE;
 import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.MRT_FEATURE;
@@ -498,7 +490,7 @@ public void testRemoteOnlyTimesOut() throws Exception {
         assertThat(perCluster.get(REMOTE2), equalTo(null));
     }
 
-    @SkipOverride(aliases = { REMOTE1 })
+    @NotSkipped(aliases = { REMOTE1 })
     public void testRemoteTimesOutFailure() throws Exception {
         Map<String, Object> testClusterInfo = setupClusters();
         String remoteIndex = (String) testClusterInfo.get("remote.index");
@@ -528,7 +520,7 @@ public void testRemoteTimesOutFailure() throws Exception {
     /**
     * Search when all the remotes failed and not skipped
     */
-    @SkipOverride(aliases = { REMOTE1, REMOTE2 })
+    @NotSkipped(aliases = { REMOTE1, REMOTE2 })
     public void testFailedAllRemotesSearch() throws Exception {
         Map<String, Object> testClusterInfo = setupClusters();
         String localIndex = (String) testClusterInfo.get("local.index");
@@ -577,7 +569,7 @@ public void testRemoteHasNoIndex() throws Exception {
     /**
      * Test that we're still counting remote search even if remote cluster has no such index
      */
-    @SkipOverride(aliases = { REMOTE1 })
+    @NotSkipped(aliases = { REMOTE1 })
     public void testRemoteHasNoIndexFailure() throws Exception {
         SearchRequest searchRequest = makeSearchRequest(REMOTE1 + ":no_such_index");
         CCSTelemetrySnapshot telemetry = getTelemetryFromFailedSearch(searchRequest);
@@ -695,40 +687,4 @@ private void indexDocs(Client client, String index, ActionListener<Void> listene
         bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).execute(listener.safeMap(r -> null));
     }
 
-    /**
-     * Annotation to mark specific cluster in a test as not to be skipped when unavailable
-     */
-    @Retention(RetentionPolicy.RUNTIME)
-    @Target(ElementType.METHOD)
-    @interface SkipOverride {
-        String[] aliases();
-    }
-
-    /**
-     * Test rule to process skip annotations
-     */
-    static class SkipUnavailableRule implements TestRule {
-        private final Map<String, Boolean> skipMap;
-
-        SkipUnavailableRule(String... clusterAliases) {
-            this.skipMap = Arrays.stream(clusterAliases).collect(Collectors.toMap(Function.identity(), alias -> true));
-        }
-
-        public Map<String, Boolean> getMap() {
-            return skipMap;
-        }
-
-        @Override
-        public Statement apply(Statement base, Description description) {
-            // Check for annotation named "SkipOverride" and set the overrides accordingly
-            var aliases = description.getAnnotation(SkipOverride.class);
-            if (aliases != null) {
-                for (String alias : aliases.aliases()) {
-                    skipMap.put(alias, false);
-                }
-            }
-            return base;
-        }
-
-    }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterIT.java
index 57a9f8131ac2d..21dca6a35659a 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterIT.java
@@ -183,7 +183,11 @@ public void testProxyConnectionDisconnect() throws Exception {
                     }
                 }
             });
-            assertBusy(() -> assertTrue(future.isDone()));
+            try {
+                future.get();
+            } catch (ExecutionException e) {
+                // ignored
+            }
             configureAndConnectsToRemoteClusters();
         } finally {
             SearchListenerPlugin.allowQueryPhase();
@@ -298,20 +302,21 @@ public void testCancel() throws Exception {
         }
 
         SearchListenerPlugin.allowQueryPhase();
-        assertBusy(() -> assertTrue(queryFuture.isDone()));
-        assertBusy(() -> assertTrue(cancelFuture.isDone()));
+        try {
+            queryFuture.get();
+            fail("query should have failed");
+        } catch (ExecutionException e) {
+            assertNotNull(e.getCause());
+            Throwable t = ExceptionsHelper.unwrap(e, TaskCancelledException.class);
+            assertNotNull(t);
+        }
+        cancelFuture.get();
         assertBusy(() -> {
             final Iterable<TransportService> transportServices = cluster("cluster_a").getInstances(TransportService.class);
             for (TransportService transportService : transportServices) {
                 assertThat(transportService.getTaskManager().getBannedTaskIds(), Matchers.empty());
             }
         });
-
-        ExecutionException e = expectThrows(ExecutionException.class, () -> queryFuture.result());
-        assertNotNull(e);
-        assertNotNull(e.getCause());
-        Throwable t = ExceptionsHelper.unwrap(e, TaskCancelledException.class);
-        assertNotNull(t);
     }
 
     /**
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java
index 823d3198bc7a2..d4f60a868dcd4 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java
@@ -379,11 +379,9 @@ public void testClusterDetailsAfterCCSWithFailuresOnRemoteClusterOnly() throws E
             r.incRef();
             l.onResponse(r);
         }));
-        assertBusy(() -> assertTrue(queryFuture.isDone()));
-
         // dfs=true overrides the minimize_roundtrips=true setting and does not minimize roundtrips
         if (skipUnavailable == false && minimizeRoundtrips && dfs == false) {
-            ExecutionException ee = expectThrows(ExecutionException.class, () -> queryFuture.get());
+            ExecutionException ee = expectThrows(ExecutionException.class, queryFuture::get);
             assertNotNull(ee.getCause());
             assertThat(ee.getCause(), instanceOf(RemoteTransportException.class));
             Throwable rootCause = ExceptionsHelper.unwrap(ee.getCause(), IllegalStateException.class);
@@ -622,10 +620,8 @@ public void testRemoteClusterOnlyCCSWithFailuresOnAllShards() throws Exception {
             r.incRef();
             l.onResponse(r);
         }));
-        assertBusy(() -> assertTrue(queryFuture.isDone()));
-
         if (skipUnavailable == false || minimizeRoundtrips == false) {
-            ExecutionException ee = expectThrows(ExecutionException.class, () -> queryFuture.get());
+            ExecutionException ee = expectThrows(ExecutionException.class, queryFuture::get);
             assertNotNull(ee.getCause());
             Throwable rootCause = ExceptionsHelper.unwrap(ee, IllegalStateException.class);
             assertThat(rootCause.getMessage(), containsString("index corrupted"));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
index fc105d3d4fcd2..ee04ae624b16f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
@@ -2632,6 +2632,41 @@ public void testPostingsHighlighterOrderByScore() throws Exception {
         });
     }
 
+    public void testMaxQueryOffsetDefault() throws Exception {
+        assertAcked(
+            prepareCreate("test").setMapping(type1PostingsffsetsMapping())
+                .setSettings(Settings.builder().put("index.highlight.max_analyzed_offset", "10").build())
+        );
+        ensureGreen();
+
+        prepareIndex("test").setSource(
+            "field1",
+            new String[] {
+                "This sentence contains one match, not that short. This sentence contains zero sentence matches. "
+                    + "This one contains no matches.",
+                "This is the second value's first sentence. This one contains no matches. "
+                    + "This sentence contains three sentence occurrences (sentence).",
+                "One sentence match here and scored lower since the text is quite long, not that appealing. "
+                    + "This one contains no matches." }
+        ).get();
+        refresh();
+
+        // Specific for this test: by passing "-1" as "maxAnalyzedOffset", the index highlight setting above will be used.
+        SearchSourceBuilder source = searchSource().query(termQuery("field1", "sentence"))
+            .highlighter(highlight().field("field1").order("score").maxAnalyzedOffset(-1));
+
+        assertResponse(client().search(new SearchRequest("test").source(source)), response -> {
+            Map<String, HighlightField> highlightFieldMap = response.getHits().getAt(0).getHighlightFields();
+            assertThat(highlightFieldMap.size(), equalTo(1));
+            HighlightField field1 = highlightFieldMap.get("field1");
+            assertThat(field1.fragments().length, equalTo(1));
+            assertThat(
+                field1.fragments()[0].string(),
+                equalTo("This <em>sentence</em> contains one match, not that short. This sentence contains zero sentence matches.")
+            );
+        });
+    }
+
     public void testPostingsHighlighterEscapeHtml() throws Exception {
         assertAcked(prepareCreate("test").setMapping("title", "type=text," + randomStoreField() + "index_options=offsets"));
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java
index 6cca07bfacc9e..fe49ce57d0400 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.fieldcaps.FieldCapabilities;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
@@ -380,8 +381,10 @@ public void testTwoIndicesOneClosedIgnoreUnavailable() {
     }
 
     public void testWithIndexFilter() throws InterruptedException {
-        assertAcked(prepareCreate("index-1").setMapping("timestamp", "type=date", "field1", "type=keyword"));
-        assertAcked(prepareCreate("index-2").setMapping("timestamp", "type=date", "field1", "type=long"));
+        assertAcked(
+            prepareCreate("index-1").setMapping("timestamp", "type=date", "field1", "type=keyword"),
+            prepareCreate("index-2").setMapping("timestamp", "type=date", "field1", "type=long")
+        );
 
         List<IndexRequestBuilder> reqs = new ArrayList<>();
         reqs.add(prepareIndex("index-1").setSource("timestamp", "2015-07-08"));
@@ -474,8 +477,7 @@ public void testFieldMetricsAndDimensions() {
 
     public void testFailures() throws InterruptedException {
         // in addition to the existing "old_index" and "new_index", create two where the test query throws an error on rewrite
-        assertAcked(prepareCreate("index1-error"));
-        assertAcked(prepareCreate("index2-error"));
+        assertAcked(prepareCreate("index1-error"), prepareCreate("index2-error"));
         ensureGreen("index1-error", "index2-error");
         FieldCapabilitiesResponse response = client().prepareFieldCaps()
             .setFields("*")
@@ -503,9 +505,7 @@ private void populateTimeRangeIndices() throws Exception {
         internalCluster().ensureAtLeastNumDataNodes(2);
         assertAcked(
             prepareCreate("log-index-1").setSettings(indexSettings(between(1, 5), 1))
-                .setMapping("timestamp", "type=date", "field1", "type=keyword")
-        );
-        assertAcked(
+                .setMapping("timestamp", "type=date", "field1", "type=keyword"),
             prepareCreate("log-index-2").setSettings(indexSettings(between(1, 5), 1))
                 .setMapping("timestamp", "type=date", "field1", "type=long")
         );
@@ -666,9 +666,11 @@ public void testManyIndicesWithSameMapping() {
              }
             """;
         String[] indices = IntStream.range(0, between(1, 9)).mapToObj(n -> "test_many_index_" + n).toArray(String[]::new);
-        for (String index : indices) {
-            assertAcked(indicesAdmin().prepareCreate(index).setMapping(mapping).get());
-        }
+        assertAcked(
+            Arrays.stream(indices)
+                .map(index -> indicesAdmin().prepareCreate(index).setMapping(mapping))
+                .toArray(CreateIndexRequestBuilder[]::new)
+        );
         FieldCapabilitiesRequest request = new FieldCapabilitiesRequest();
         request.indices("test_many_index_*");
         request.fields("*");
@@ -787,9 +789,11 @@ public void testIndexMode() throws Exception {
             Settings settings = Settings.builder().put("mode", "time_series").putList("routing_path", List.of("hostname")).build();
             int numIndices = between(1, 5);
             for (int i = 0; i < numIndices; i++) {
-                assertAcked(indicesAdmin().prepareCreate("test_metrics_" + i).setSettings(settings).setMapping(metricsMapping).get());
+                assertAcked(
+                    indicesAdmin().prepareCreate("test_metrics_" + i).setSettings(settings).setMapping(metricsMapping),
+                    indicesAdmin().prepareCreate("test_old_metrics_" + i).setMapping(metricsMapping)
+                );
                 indexModes.put("test_metrics_" + i, IndexMode.TIME_SERIES);
-                assertAcked(indicesAdmin().prepareCreate("test_old_metrics_" + i).setMapping(metricsMapping).get());
                 indexModes.put("test_old_metrics_" + i, IndexMode.STANDARD);
             }
         }
@@ -808,9 +812,11 @@ public void testIndexMode() throws Exception {
             Settings settings = Settings.builder().put("mode", "logsdb").build();
             int numIndices = between(1, 5);
             for (int i = 0; i < numIndices; i++) {
-                assertAcked(indicesAdmin().prepareCreate("test_logs_" + i).setSettings(settings).setMapping(logsMapping).get());
+                assertAcked(
+                    indicesAdmin().prepareCreate("test_logs_" + i).setSettings(settings).setMapping(logsMapping),
+                    indicesAdmin().prepareCreate("test_old_logs_" + i).setMapping(logsMapping)
+                );
                 indexModes.put("test_logs_" + i, IndexMode.LOGSDB);
-                assertAcked(indicesAdmin().prepareCreate("test_old_logs_" + i).setMapping(logsMapping).get());
                 indexModes.put("test_old_logs_" + i, IndexMode.STANDARD);
             }
         }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
index 9988624f6a677..a55edf3782bcc 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
@@ -748,7 +748,7 @@ public void testDateWithoutOrigin() throws Exception {
     }
 
     public void testManyDocsLin() throws Exception {
-        IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random());
+        IndexVersion version = IndexVersionUtils.randomCompatibleWriteVersion(random());
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
         XContentBuilder xContentBuilder = jsonBuilder().startObject()
             .startObject("_doc")
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
index a7efb2fe0e68b..fbdcfe26d28ee 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
@@ -38,6 +38,7 @@
 import org.elasticsearch.search.collapse.CollapseBuilder;
 import org.elasticsearch.search.rescore.QueryRescoreMode;
 import org.elasticsearch.search.rescore.QueryRescorerBuilder;
+import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xcontent.ParseField;
@@ -840,6 +841,20 @@ public void testRescorePhaseWithInvalidSort() throws Exception {
                 }
             }
         );
+
+        assertResponse(
+            prepareSearch().addSort(SortBuilders.scoreSort())
+                .addSort(new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME))
+                .setTrackScores(true)
+                .addRescorer(new QueryRescorerBuilder(matchAllQuery()).setRescoreQueryWeight(100.0f), 50),
+            response -> {
+                assertThat(response.getHits().getTotalHits().value(), equalTo(5L));
+                assertThat(response.getHits().getHits().length, equalTo(5));
+                for (SearchHit hit : response.getHits().getHits()) {
+                    assertThat(hit.getScore(), equalTo(101f));
+                }
+            }
+        );
     }
 
     record GroupDoc(String id, String group, float firstPassScore, float secondPassScore, boolean shouldFilter) {}
@@ -879,6 +894,10 @@ public void testRescoreAfterCollapse() throws Exception {
             .setQuery(fieldValueScoreQuery("firstPassScore"))
             .addRescorer(new QueryRescorerBuilder(fieldValueScoreQuery("secondPassScore")))
             .setCollapse(new CollapseBuilder("group"));
+        if (randomBoolean()) {
+            request.addSort(SortBuilders.scoreSort());
+            request.addSort(new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME));
+        }
         assertResponse(request, resp -> {
             assertThat(resp.getHits().getTotalHits().value(), equalTo(5L));
             assertThat(resp.getHits().getHits().length, equalTo(3));
@@ -958,6 +977,10 @@ public void testRescoreAfterCollapseRandom() throws Exception {
             .addRescorer(new QueryRescorerBuilder(fieldValueScoreQuery("secondPassScore")).setQueryWeight(0f).windowSize(numGroups))
             .setCollapse(new CollapseBuilder("group"))
             .setSize(Math.min(numGroups, 10));
+        if (randomBoolean()) {
+            request.addSort(SortBuilders.scoreSort());
+            request.addSort(new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME));
+        }
         long expectedNumHits = numHits;
         assertResponse(request, resp -> {
             assertThat(resp.getHits().getTotalHits().value(), equalTo(expectedNumHits));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoPointIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoPointIT.java
index 2489889be19e5..8104e4ed7a825 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoPointIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoPointIT.java
@@ -32,6 +32,6 @@ public XContentBuilder getMapping() throws IOException {
 
     @Override
     public IndexVersion randomSupportedVersion() {
-        return IndexVersionUtils.randomCompatibleVersion(random());
+        return IndexVersionUtils.randomCompatibleWriteVersion(random());
     }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoDistanceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoDistanceIT.java
index 9b4e28055a988..a309fa81f6dc1 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoDistanceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoDistanceIT.java
@@ -96,7 +96,7 @@ protected boolean forbidPrivateIndexSettings() {
 
     @Before
     public void setupTestIndex() throws IOException {
-        IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random());
+        IndexVersion version = IndexVersionUtils.randomCompatibleWriteVersion(random());
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder()
             .startObject()
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPolygonIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPolygonIT.java
index 4b8f29f3cc9a5..aadefd9bd8018 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPolygonIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPolygonIT.java
@@ -39,7 +39,7 @@ protected boolean forbidPrivateIndexSettings() {
 
     @Override
     protected void setupSuiteScopeCluster() throws Exception {
-        IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random());
+        IndexVersion version = IndexVersionUtils.randomCompatibleWriteVersion(random());
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
 
         assertAcked(
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java
index f790cf30e1c0e..6d1f2784a739c 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -164,22 +164,19 @@ public void testIssue3177() {
         forceMerge();
         refresh();
         assertHitCount(
+            3L,
             prepareSearch().setQuery(matchAllQuery())
                 .setPostFilter(
                     boolQuery().must(matchAllQuery())
                         .must(boolQuery().mustNot(boolQuery().must(termQuery("field1", "value1")).must(termQuery("field1", "value2"))))
                 ),
-            3L
-        );
-        assertHitCount(
             prepareSearch().setQuery(
                 boolQuery().must(
                     boolQuery().should(termQuery("field1", "value1"))
                         .should(termQuery("field1", "value2"))
                         .should(termQuery("field1", "value3"))
                 ).filter(boolQuery().mustNot(boolQuery().must(termQuery("field1", "value1")).must(termQuery("field1", "value2"))))
-            ),
-            3L
+            )
         );
         assertHitCount(prepareSearch().setQuery(matchAllQuery()).setPostFilter(boolQuery().mustNot(termQuery("field1", "value3"))), 2L);
     }
@@ -309,11 +306,14 @@ public void testQueryStringAnalyzedWildcard() throws Exception {
         prepareIndex("test").setId("1").setSource("field1", "value_1", "field2", "value_2").get();
         refresh();
 
-        assertHitCount(prepareSearch().setQuery(queryStringQuery("value*")), 1L);
-        assertHitCount(prepareSearch().setQuery(queryStringQuery("*ue*")), 1L);
-        assertHitCount(prepareSearch().setQuery(queryStringQuery("*ue_1")), 1L);
-        assertHitCount(prepareSearch().setQuery(queryStringQuery("val*e_1")), 1L);
-        assertHitCount(prepareSearch().setQuery(queryStringQuery("v?l*e?1")), 1L);
+        assertHitCount(
+            1L,
+            prepareSearch().setQuery(queryStringQuery("value*")),
+            prepareSearch().setQuery(queryStringQuery("*ue*")),
+            prepareSearch().setQuery(queryStringQuery("*ue_1")),
+            prepareSearch().setQuery(queryStringQuery("val*e_1")),
+            prepareSearch().setQuery(queryStringQuery("v?l*e?1"))
+        );
     }
 
     public void testLowercaseExpandedTerms() {
@@ -322,10 +322,13 @@ public void testLowercaseExpandedTerms() {
         prepareIndex("test").setId("1").setSource("field1", "value_1", "field2", "value_2").get();
         refresh();
 
-        assertHitCount(prepareSearch().setQuery(queryStringQuery("VALUE_3~1")), 1L);
-        assertHitCount(prepareSearch().setQuery(queryStringQuery("ValUE_*")), 1L);
-        assertHitCount(prepareSearch().setQuery(queryStringQuery("vAl*E_1")), 1L);
-        assertHitCount(prepareSearch().setQuery(queryStringQuery("[VALUE_1 TO VALUE_3]")), 1L);
+        assertHitCount(
+            1L,
+            prepareSearch().setQuery(queryStringQuery("VALUE_3~1")),
+            prepareSearch().setQuery(queryStringQuery("ValUE_*")),
+            prepareSearch().setQuery(queryStringQuery("vAl*E_1")),
+            prepareSearch().setQuery(queryStringQuery("[VALUE_1 TO VALUE_3]"))
+        );
     }
 
     // Issue #3540
@@ -340,8 +343,11 @@ public void testDateRangeInQueryString() {
         prepareIndex("test").setId("1").setSource("past", aMonthAgo, "future", aMonthFromNow).get();
         refresh();
 
-        assertHitCount(prepareSearch().setQuery(queryStringQuery("past:[now-2M/d TO now/d]")), 1L);
-        assertHitCount(prepareSearch().setQuery(queryStringQuery("future:[now/d TO now+2M/d]")), 1L);
+        assertHitCount(
+            1L,
+            prepareSearch().setQuery(queryStringQuery("past:[now-2M/d TO now/d]")),
+            prepareSearch().setQuery(queryStringQuery("future:[now/d TO now+2M/d]"))
+        );
 
         SearchPhaseExecutionException e = expectThrows(
             SearchPhaseExecutionException.class,
@@ -377,21 +383,17 @@ public void testDateRangeInQueryStringWithTimeZone_10477() {
         refresh();
 
         // Timezone set with dates
-        assertHitCount(prepareSearch().setQuery(queryStringQuery("past:[2015-04-06T00:00:00+0200 TO 2015-04-06T23:00:00+0200]")), 2L);
-
-        // Same timezone set with time_zone
         assertHitCount(
-            prepareSearch().setQuery(queryStringQuery("past:[2015-04-06T00:00:00 TO 2015-04-06T23:00:00]").timeZone("+0200")),
-            2L
+            2L,
+            prepareSearch().setQuery(queryStringQuery("past:[2015-04-06T00:00:00+0200 TO 2015-04-06T23:00:00+0200]")),
+            prepareSearch().setQuery(queryStringQuery("past:[2015-04-06T00:00:00 TO 2015-04-06T23:00:00]").timeZone("+0200"))
         );
 
         // We set a timezone which will give no result
-        assertHitCount(prepareSearch().setQuery(queryStringQuery("past:[2015-04-06T00:00:00-0200 TO 2015-04-06T23:00:00-0200]")), 0L);
-
-        // Same timezone set with time_zone but another timezone is set directly within dates which has the precedence
         assertHitCount(
-            prepareSearch().setQuery(queryStringQuery("past:[2015-04-06T00:00:00-0200 TO 2015-04-06T23:00:00-0200]").timeZone("+0200")),
-            0L
+            0L,
+            prepareSearch().setQuery(queryStringQuery("past:[2015-04-06T00:00:00-0200 TO 2015-04-06T23:00:00-0200]")),
+            prepareSearch().setQuery(queryStringQuery("past:[2015-04-06T00:00:00-0200 TO 2015-04-06T23:00:00-0200]").timeZone("+0200"))
         );
     }
 
@@ -502,13 +504,12 @@ public void testPassQueryOrFilterAsJSONString() throws Exception {
         prepareIndex("test").setId("1").setSource("field1", "value1_1", "field2", "value2_1").setRefreshPolicy(IMMEDIATE).get();
 
         WrapperQueryBuilder wrapper = new WrapperQueryBuilder("{ \"term\" : { \"field1\" : \"value1_1\" } }");
-        assertHitCount(prepareSearch().setQuery(wrapper), 1L);
-
-        BoolQueryBuilder bool = boolQuery().must(wrapper).must(new TermQueryBuilder("field2", "value2_1"));
-        assertHitCount(prepareSearch().setQuery(bool), 1L);
-
-        WrapperQueryBuilder wrapperFilter = wrapperQuery("{ \"term\" : { \"field1\" : \"value1_1\" } }");
-        assertHitCount(prepareSearch().setPostFilter(wrapperFilter), 1L);
+        assertHitCount(
+            1L,
+            prepareSearch().setQuery(wrapper),
+            prepareSearch().setQuery(boolQuery().must(wrapper).must(new TermQueryBuilder("field2", "value2_1"))),
+            prepareSearch().setPostFilter(wrapperQuery("{ \"term\" : { \"field1\" : \"value1_1\" } }"))
+        );
     }
 
     public void testFiltersWithCustomCacheKey() throws Exception {
@@ -516,10 +517,13 @@ public void testFiltersWithCustomCacheKey() throws Exception {
 
         prepareIndex("test").setId("1").setSource("field1", "value1").get();
         refresh();
-        assertHitCount(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("field1", "value1"))), 1L);
-        assertHitCount(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("field1", "value1"))), 1L);
-        assertHitCount(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("field1", "value1"))), 1L);
-        assertHitCount(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("field1", "value1"))), 1L);
+        assertHitCount(
+            1L,
+            prepareSearch("test").setQuery(constantScoreQuery(termsQuery("field1", "value1"))),
+            prepareSearch("test").setQuery(constantScoreQuery(termsQuery("field1", "value1"))),
+            prepareSearch("test").setQuery(constantScoreQuery(termsQuery("field1", "value1"))),
+            prepareSearch("test").setQuery(constantScoreQuery(termsQuery("field1", "value1")))
+        );
     }
 
     public void testMatchQueryNumeric() throws Exception {
@@ -994,13 +998,14 @@ public void testTermsLookupFilter() throws Exception {
             "4"
         );
 
-        assertHitCount(prepareSearch("test").setQuery(termsLookupQuery("not_exists", new TermsLookup("lookup2", "3", "arr.term"))), 0L);
-
-        // index "lookup" id "missing" document does not exist: ignore the lookup terms
-        assertHitCount(prepareSearch("test").setQuery(termsLookupQuery("term", new TermsLookup("lookup", "missing", "terms"))), 0L);
-
-        // index "lookup3" has the source disabled: ignore the lookup terms
-        assertHitCount(prepareSearch("test").setQuery(termsLookupQuery("term", new TermsLookup("lookup3", "1", "terms"))), 0L);
+        assertHitCount(
+            0L,
+            prepareSearch("test").setQuery(termsLookupQuery("not_exists", new TermsLookup("lookup2", "3", "arr.term"))),
+            // index "lookup" id "missing" document does not exist: ignore the lookup terms
+            prepareSearch("test").setQuery(termsLookupQuery("term", new TermsLookup("lookup", "missing", "terms"))),
+            // index "lookup3" has the source disabled: ignore the lookup terms
+            prepareSearch("test").setQuery(termsLookupQuery("term", new TermsLookup("lookup3", "1", "terms")))
+        );
     }
 
     public void testBasicQueryById() throws Exception {
@@ -1120,18 +1125,14 @@ public void testNumericRangeFilter_2826() throws Exception {
         refresh();
 
         assertHitCount(
+            4L,
             prepareSearch("test").setPostFilter(
                 boolQuery().should(rangeQuery("num_long").from(1).to(2)).should(rangeQuery("num_long").from(3).to(4))
             ),
-            4L
-        );
-
-        // This made 2826 fail! (only with bit based filters)
-        assertHitCount(
+            // This made 2826 fail! (only with bit based filters)
             prepareSearch("test").setPostFilter(
                 boolQuery().should(rangeQuery("num_long").from(1).to(2)).should(rangeQuery("num_long").from(3).to(4))
-            ),
-            4L
+            )
         );
 
         // This made #2979 fail!
@@ -1697,11 +1698,11 @@ public void testWildcardQueryNormalizationOnKeywordField() {
         refresh();
 
         {
-            WildcardQueryBuilder wildCardQuery = wildcardQuery("field1", "Bb*");
-            assertHitCount(prepareSearch().setQuery(wildCardQuery), 1L);
-
-            wildCardQuery = wildcardQuery("field1", "bb*");
-            assertHitCount(prepareSearch().setQuery(wildCardQuery), 1L);
+            assertHitCount(
+                1L,
+                prepareSearch().setQuery(wildcardQuery("field1", "Bb*")),
+                prepareSearch().setQuery(wildcardQuery("field1", "bb*"))
+            );
         }
     }
 
@@ -1725,12 +1726,12 @@ public void testWildcardQueryNormalizationOnTextField() {
             WildcardQueryBuilder wildCardQuery = wildcardQuery("field1", "Bb*");
             assertHitCount(prepareSearch().setQuery(wildCardQuery), 0L);
 
-            // the following works not because of normalization but because of the `case_insensitive` parameter
-            wildCardQuery = wildcardQuery("field1", "Bb*").caseInsensitive(true);
-            assertHitCount(prepareSearch().setQuery(wildCardQuery), 1L);
-
-            wildCardQuery = wildcardQuery("field1", "bb*");
-            assertHitCount(prepareSearch().setQuery(wildCardQuery), 1L);
+            assertHitCount(
+                1L,
+                // the following works not because of normalization but because of the `case_insensitive` parameter
+                prepareSearch().setQuery(wildcardQuery("field1", "Bb*").caseInsensitive(true)),
+                prepareSearch().setQuery(wildcardQuery("field1", "bb*"))
+            );
         }
     }
 
@@ -1751,11 +1752,11 @@ public void testWildcardQueryNormalizationKeywordSpecialCharacters() {
         prepareIndex("test").setId("1").setSource("field", "label-1").get();
         refresh();
 
-        WildcardQueryBuilder wildCardQuery = wildcardQuery("field", "la*");
-        assertHitCount(prepareSearch().setQuery(wildCardQuery), 1L);
-
-        wildCardQuery = wildcardQuery("field", "la*el-?");
-        assertHitCount(prepareSearch().setQuery(wildCardQuery), 1L);
+        assertHitCount(
+            1L,
+            prepareSearch().setQuery(wildcardQuery("field", "la*")),
+            prepareSearch().setQuery(wildcardQuery("field", "la*el-?"))
+        );
     }
 
     public static class MockAnalysisPlugin extends Plugin implements AnalysisPlugin {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/slice/SearchSliceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/slice/SearchSliceIT.java
index e079994003751..37e299c816562 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/slice/SearchSliceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/slice/SearchSliceIT.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.builder.PointInTimeBuilder;
 import org.elasticsearch.search.sort.ShardDocSortField;
@@ -97,14 +96,14 @@ public void testSearchSort() throws Exception {
             int fetchSize = randomIntBetween(10, 100);
             // test _doc sort
             SearchRequestBuilder request = prepareSearch("test").setQuery(matchAllQuery())
-                .setScroll(new Scroll(TimeValue.timeValueSeconds(10)))
+                .setScroll(TimeValue.timeValueSeconds(10))
                 .setSize(fetchSize)
                 .addSort(SortBuilders.fieldSort("_doc"));
             assertSearchSlicesWithScroll(request, field, max, numDocs);
 
             // test numeric sort
             request = prepareSearch("test").setQuery(matchAllQuery())
-                .setScroll(new Scroll(TimeValue.timeValueSeconds(10)))
+                .setScroll(TimeValue.timeValueSeconds(10))
                 .addSort(SortBuilders.fieldSort("random_int"))
                 .setSize(fetchSize);
             assertSearchSlicesWithScroll(request, field, max, numDocs);
@@ -121,7 +120,7 @@ public void testWithPreferenceAndRoutings() throws Exception {
             int max = randomIntBetween(2, numShards * 3);
             int fetchSize = randomIntBetween(10, 100);
             SearchRequestBuilder request = prepareSearch("test").setQuery(matchAllQuery())
-                .setScroll(new Scroll(TimeValue.timeValueSeconds(10)))
+                .setScroll(TimeValue.timeValueSeconds(10))
                 .setSize(fetchSize)
                 .setPreference("_shards:1,4")
                 .addSort(SortBuilders.fieldSort("_doc"));
@@ -133,7 +132,7 @@ public void testWithPreferenceAndRoutings() throws Exception {
             int max = randomIntBetween(2, numShards * 3);
             int fetchSize = randomIntBetween(10, 100);
             SearchRequestBuilder request = prepareSearch("test").setQuery(matchAllQuery())
-                .setScroll(new Scroll(TimeValue.timeValueSeconds(10)))
+                .setScroll(TimeValue.timeValueSeconds(10))
                 .setSize(fetchSize)
                 .setRouting("foo", "bar")
                 .addSort(SortBuilders.fieldSort("_doc"));
@@ -151,7 +150,7 @@ public void testWithPreferenceAndRoutings() throws Exception {
             int max = randomIntBetween(2, numShards * 3);
             int fetchSize = randomIntBetween(10, 100);
             SearchRequestBuilder request = prepareSearch("alias1", "alias3").setQuery(matchAllQuery())
-                .setScroll(new Scroll(TimeValue.timeValueSeconds(10)))
+                .setScroll(TimeValue.timeValueSeconds(10))
                 .setSize(fetchSize)
                 .addSort(SortBuilders.fieldSort("_doc"));
             assertSearchSlicesWithScroll(request, "_id", max, numDocs);
@@ -176,7 +175,7 @@ private void assertSearchSlicesWithScroll(SearchRequestBuilder request, String f
                     searchResponse.decRef();
                     searchResponse = client().prepareSearchScroll("test")
                         .setScrollId(scrollId)
-                        .setScroll(new Scroll(TimeValue.timeValueSeconds(10)))
+                        .setScroll(TimeValue.timeValueSeconds(10))
                         .get();
                     scrollId = searchResponse.getScrollId();
                     totalResults += searchResponse.getHits().getHits().length;
@@ -271,7 +270,7 @@ public void testInvalidFields() throws Exception {
         SearchPhaseExecutionException exc = expectThrows(
             SearchPhaseExecutionException.class,
             prepareSearch("test").setQuery(matchAllQuery())
-                .setScroll(new Scroll(TimeValue.timeValueSeconds(10)))
+                .setScroll(TimeValue.timeValueSeconds(10))
                 .slice(new SliceBuilder("invalid_random_int", 0, 10))
 
         );
@@ -282,7 +281,7 @@ public void testInvalidFields() throws Exception {
         exc = expectThrows(
             SearchPhaseExecutionException.class,
             prepareSearch("test").setQuery(matchAllQuery())
-                .setScroll(new Scroll(TimeValue.timeValueSeconds(10)))
+                .setScroll(TimeValue.timeValueSeconds(10))
                 .slice(new SliceBuilder("invalid_random_kw", 0, 10))
         );
         rootCause = findRootCause(exc);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java
index bf7a315040caa..f407c14c48c52 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java
@@ -2003,10 +2003,12 @@ public void testLongSortOptimizationCorrectResults() {
     }
 
     public void testSortMixedFieldTypes() {
-        assertAcked(prepareCreate("index_long").setMapping("foo", "type=long").get());
-        assertAcked(prepareCreate("index_integer").setMapping("foo", "type=integer").get());
-        assertAcked(prepareCreate("index_double").setMapping("foo", "type=double").get());
-        assertAcked(prepareCreate("index_keyword").setMapping("foo", "type=keyword").get());
+        assertAcked(
+            prepareCreate("index_long").setMapping("foo", "type=long"),
+            prepareCreate("index_integer").setMapping("foo", "type=integer"),
+            prepareCreate("index_double").setMapping("foo", "type=double"),
+            prepareCreate("index_keyword").setMapping("foo", "type=keyword")
+        );
 
         prepareIndex("index_long").setId("1").setSource("foo", "123").get();
         prepareIndex("index_integer").setId("1").setSource("foo", "123").get();
@@ -2038,9 +2040,11 @@ public void testSortMixedFieldTypes() {
     }
 
     public void testSortMixedFieldTypesWithNoDocsForOneType() {
-        assertAcked(prepareCreate("index_long").setMapping("foo", "type=long").get());
-        assertAcked(prepareCreate("index_other").setMapping("bar", "type=keyword").get());
-        assertAcked(prepareCreate("index_double").setMapping("foo", "type=double").get());
+        assertAcked(
+            prepareCreate("index_long").setMapping("foo", "type=long"),
+            prepareCreate("index_other").setMapping("bar", "type=keyword"),
+            prepareCreate("index_double").setMapping("foo", "type=double")
+        );
 
         prepareIndex("index_long").setId("1").setSource("foo", "123").get();
         prepareIndex("index_long").setId("2").setSource("foo", "124").get();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceIT.java
index e80678c4f5fc6..f55d4505f3f58 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceIT.java
@@ -45,7 +45,7 @@ protected boolean forbidPrivateIndexSettings() {
     }
 
     public void testDistanceSortingMVFields() throws Exception {
-        IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random());
+        IndexVersion version = IndexVersionUtils.randomCompatibleWriteVersion(random());
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder()
             .startObject()
@@ -237,7 +237,7 @@ public void testDistanceSortingMVFields() throws Exception {
     // Regression bug:
     // https://github.com/elastic/elasticsearch/issues/2851
     public void testDistanceSortingWithMissingGeoPoint() throws Exception {
-        IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random());
+        IndexVersion version = IndexVersionUtils.randomCompatibleWriteVersion(random());
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder()
             .startObject()
@@ -299,7 +299,7 @@ public void testDistanceSortingWithMissingGeoPoint() throws Exception {
     }
 
     public void testDistanceSortingNestedFields() throws Exception {
-        IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random());
+        IndexVersion version = IndexVersionUtils.randomCompatibleWriteVersion(random());
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder()
             .startObject()
@@ -551,7 +551,7 @@ public void testDistanceSortingNestedFields() throws Exception {
      * Issue 3073
      */
     public void testGeoDistanceFilter() throws IOException {
-        IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random());
+        IndexVersion version = IndexVersionUtils.randomCompatibleWriteVersion(random());
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
         double lat = 40.720611;
         double lon = -73.998776;
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
index aabca1b9333f8..d53c90a5d1e28 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
@@ -60,7 +60,7 @@ public void testManyToManyGeoPoints() throws ExecutionException, InterruptedExce
          * |___________________________
          * 1   2   3   4   5   6   7
          */
-        IndexVersion version = randomBoolean() ? IndexVersion.current() : IndexVersionUtils.randomCompatibleVersion(random());
+        IndexVersion version = randomBoolean() ? IndexVersion.current() : IndexVersionUtils.randomCompatibleWriteVersion(random());
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
         assertAcked(prepareCreate("index").setSettings(settings).setMapping(LOCATION_FIELD, "type=geo_point"));
         XContentBuilder d1Builder = jsonBuilder();
@@ -152,7 +152,7 @@ public void testSingeToManyAvgMedian() throws ExecutionException, InterruptedExc
          * d1 = (0, 1), (0, 4), (0, 10); so avg. distance is 5, median distance is 4
          * d2 = (0, 1), (0, 5), (0, 6); so avg. distance is 4, median distance is 5
          */
-        IndexVersion version = randomBoolean() ? IndexVersion.current() : IndexVersionUtils.randomCompatibleVersion(random());
+        IndexVersion version = randomBoolean() ? IndexVersion.current() : IndexVersionUtils.randomCompatibleWriteVersion(random());
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
         assertAcked(prepareCreate("index").setSettings(settings).setMapping(LOCATION_FIELD, "type=geo_point"));
         XContentBuilder d1Builder = jsonBuilder();
@@ -225,7 +225,7 @@ public void testManyToManyGeoPointsWithDifferentFormats() throws ExecutionExcept
          * |______________________
          * 1   2   3   4   5   6
          */
-        IndexVersion version = randomBoolean() ? IndexVersion.current() : IndexVersionUtils.randomCompatibleVersion(random());
+        IndexVersion version = randomBoolean() ? IndexVersion.current() : IndexVersionUtils.randomCompatibleWriteVersion(random());
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
         assertAcked(prepareCreate("index").setSettings(settings).setMapping(LOCATION_FIELD, "type=geo_point"));
         XContentBuilder d1Builder = jsonBuilder();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java
index 9f40b1928dce6..2530dd35946fe 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java
@@ -19,22 +19,17 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.search.stats.SearchStats.Stats;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.ScriptPlugin;
-import org.elasticsearch.script.LongFieldScript;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptEngine;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
-import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.FailingFieldPlugin;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
 
@@ -80,50 +75,6 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
         }
     }
 
-    public static class FailingFieldPlugin extends Plugin implements ScriptPlugin {
-
-        @Override
-        public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<?>> contexts) {
-            return new ScriptEngine() {
-                @Override
-                public String getType() {
-                    return "failing_field";
-                }
-
-                @Override
-                @SuppressWarnings("unchecked")
-                public <FactoryType> FactoryType compile(
-                    String name,
-                    String code,
-                    ScriptContext<FactoryType> context,
-                    Map<String, String> params
-                ) {
-                    return (FactoryType) new LongFieldScript.Factory() {
-                        @Override
-                        public LongFieldScript.LeafFactory newFactory(
-                            String fieldName,
-                            Map<String, Object> params,
-                            SearchLookup searchLookup,
-                            OnScriptError onScriptError
-                        ) {
-                            return ctx -> new LongFieldScript(fieldName, params, searchLookup, onScriptError, ctx) {
-                                @Override
-                                public void execute() {
-                                    throw new IllegalArgumentException("Accessing failing field");
-                                }
-                            };
-                        }
-                    };
-                }
-
-                @Override
-                public Set<ScriptContext<?>> getSupportedContexts() {
-                    return Set.of(LongFieldScript.CONTEXT);
-                }
-            };
-        }
-    }
-
     @Override
     protected int numberOfReplicas() {
         return 0;
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CloneSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CloneSnapshotIT.java
index 209195301a659..caf7f852cc15e 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CloneSnapshotIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CloneSnapshotIT.java
@@ -565,9 +565,7 @@ public void testSnapshotQueuedAfterCloneFromBrokenSourceSnapshot() throws Except
         assertSuccessful(sn1);
         assertSuccessful(sn2);
         assertSuccessful(sn3);
-        assertAcked(clone1.get());
-        assertAcked(clone2.get());
-        assertAcked(clone3.get());
+        assertAcked(clone1, clone2, clone3);
     }
 
     public void testStartSnapshotWithSuccessfulShardClonePendingFinalization() throws Exception {
@@ -624,8 +622,7 @@ public void testStartCloneWithSuccessfulShardClonePendingFinalization() throws E
         unblockNode(repoName, masterName);
         awaitNoMoreRunningOperations(masterName);
         awaitMasterFinishRepoOperations();
-        assertAcked(blockedClone.get());
-        assertAcked(otherClone.get());
+        assertAcked(blockedClone, otherClone);
         assertEquals(getSnapshot(repoName, cloneName).state(), SnapshotState.SUCCESS);
         assertEquals(getSnapshot(repoName, otherCloneName).state(), SnapshotState.SUCCESS);
     }
@@ -732,8 +729,7 @@ public void testManyConcurrentClonesStartOutOfOrder() throws Exception {
         awaitClusterState(state -> SnapshotsInProgress.get(state).forRepo(repoName).stream().anyMatch(entry -> entry.state().completed()));
         repo.unblock();
 
-        assertAcked(clone1.get());
-        assertAcked(clone2.get());
+        assertAcked(clone1, clone2);
     }
 
     public void testRemoveFailedCloneFromCSWithoutIO() throws Exception {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java
index de62c0152817a..e16aff56527c3 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java
@@ -478,8 +478,7 @@ public void testCascadedAborts() throws Exception {
         assertThat(thirdSnapshotResponse.get().getSnapshotInfo().state(), is(SnapshotState.FAILED));
 
         logger.info("--> verify both deletes have completed");
-        assertAcked(deleteSnapshotsResponse.get());
-        assertAcked(allDeletedResponse.get());
+        assertAcked(deleteSnapshotsResponse, allDeletedResponse);
 
         logger.info("--> verify that all snapshots are gone");
         assertThat(clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, repoName).get().getSnapshots(), empty());
@@ -715,8 +714,7 @@ public void testQueuedOperationsOnMasterDisconnect() throws Exception {
         networkDisruption.stopDisrupting();
 
         logger.info("--> make sure all failing requests get a response");
-        assertAcked(firstDeleteFuture.get());
-        assertAcked(secondDeleteFuture.get());
+        assertAcked(firstDeleteFuture, secondDeleteFuture);
         expectThrows(SnapshotException.class, createSnapshot);
         awaitNoMoreRunningOperations();
     }
@@ -1014,8 +1012,7 @@ public void testBackToBackQueuedDeletes() throws Exception {
         awaitNDeletionsInProgress(2);
 
         unblockNode(repoName, masterName);
-        assertAcked(deleteSnapshotOne.get());
-        assertAcked(deleteSnapshotTwo.get());
+        assertAcked(deleteSnapshotOne, deleteSnapshotTwo);
 
         final RepositoryData repositoryData = getRepositoryData(repoName);
         assertThat(repositoryData.getSnapshotIds(), empty());
@@ -1361,9 +1358,12 @@ public void testConcurrentOperationsLimit() throws Exception {
         if (deleteAndAbortAll) {
             awaitNumberOfSnapshotsInProgress(0);
             for (ActionFuture<CreateSnapshotResponse> snapshotFuture : snapshotFutures) {
-                // just check that the futures resolve, whether or not things worked out with the snapshot actually finalizing or failing
-                // due to the abort does not matter
-                assertBusy(() -> assertTrue(snapshotFuture.isDone()));
+                try {
+                    snapshotFuture.get();
+                } catch (ExecutionException e) {
+                    // just check that the futures resolve, whether or not things worked out with the snapshot actually finalizing or
+                    // failing due to the abort does not matter
+                }
             }
             assertThat(getRepositoryData(repoName).getSnapshotIds(), empty());
         } else {
@@ -1890,8 +1890,7 @@ public void testIndexDeletedWhileSnapshotAndCloneQueuedAfterClone() throws Excep
         assertSuccessful(snapshot3);
         unblockNode(repository, master);
 
-        assertAcked(cloneSnapshot.get());
-        assertAcked(cloneSnapshot2.get());
+        assertAcked(cloneSnapshot, cloneSnapshot2);
         assertAcked(startDeleteSnapshot(repository, cloneTarget).get());
 
         assertThat(
@@ -2031,8 +2030,7 @@ public void testCloneQueuedAfterMissingShard() throws Exception {
         awaitNumberOfSnapshotsInProgress(2);
 
         unblockNode(repository, master);
-        assertAcked(deleteFuture.get());
-        assertAcked(cloneFuture.get());
+        assertAcked(deleteFuture, cloneFuture);
         awaitNoMoreRunningOperations();
         assertThat(snapshot1.get().getSnapshotInfo().state(), is(SnapshotState.PARTIAL));
     }
@@ -2109,8 +2107,7 @@ public void testSnapshotAndCloneQueuedAfterMissingShard() throws Exception {
         awaitNumberOfSnapshotsInProgress(3);
 
         unblockNode(repository, master);
-        assertAcked(deleteFuture.get());
-        assertAcked(cloneFuture.get());
+        assertAcked(deleteFuture, cloneFuture);
         awaitNoMoreRunningOperations();
         assertThat(snapshot1.get().getSnapshotInfo().state(), is(SnapshotState.PARTIAL));
         assertThat(snapshot2.get().getSnapshotInfo().state(), is(SnapshotState.PARTIAL));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
index 588753f570c97..0d359300bbdc1 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
@@ -664,8 +664,7 @@ public void testRestoreShrinkIndex() throws Exception {
         createSnapshot(repo, snapshot, Collections.singletonList(shrunkIdx));
 
         logger.info("--> delete index and stop the data node");
-        assertAcked(indicesAdmin().prepareDelete(sourceIdx).get());
-        assertAcked(indicesAdmin().prepareDelete(shrunkIdx).get());
+        assertAcked(indicesAdmin().prepareDelete(sourceIdx), indicesAdmin().prepareDelete(shrunkIdx));
         internalCluster().stopRandomDataNode();
         clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT).setTimeout(TimeValue.timeValueSeconds(30)).setWaitForNodes("1");
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java
index 4ba06a34ca3a7..b2bb38c5364a5 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java
@@ -340,14 +340,14 @@ public void testRestoreAliases() throws Exception {
                 .addAlias("test-idx-1", "alias-1")
         );
 
-        assertFalse(indicesAdmin().prepareGetAliases("alias-123").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias-123").get().getAliases().isEmpty());
 
         createSnapshot("test-repo", "test-snap", Collections.emptyList());
 
         logger.info("-->  delete all indices");
         cluster().wipeIndices("test-idx-1", "test-idx-2", "test-idx-3");
-        assertTrue(indicesAdmin().prepareGetAliases("alias-123").get().getAliases().isEmpty());
-        assertTrue(indicesAdmin().prepareGetAliases("alias-1").get().getAliases().isEmpty());
+        assertTrue(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias-123").get().getAliases().isEmpty());
+        assertTrue(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias-1").get().getAliases().isEmpty());
 
         logger.info("--> restore snapshot with aliases");
         RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(
@@ -362,8 +362,8 @@ public void testRestoreAliases() throws Exception {
         );
 
         logger.info("--> check that aliases are restored");
-        assertFalse(indicesAdmin().prepareGetAliases("alias-123").get().getAliases().isEmpty());
-        assertFalse(indicesAdmin().prepareGetAliases("alias-1").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias-123").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias-1").get().getAliases().isEmpty());
 
         logger.info("-->  update aliases");
         assertAcked(indicesAdmin().prepareAliases().removeAlias("test-idx-3", "alias-123"));
@@ -372,8 +372,8 @@ public void testRestoreAliases() throws Exception {
         logger.info("-->  delete and close indices");
         cluster().wipeIndices("test-idx-1", "test-idx-2");
         assertAcked(indicesAdmin().prepareClose("test-idx-3"));
-        assertTrue(indicesAdmin().prepareGetAliases("alias-123").get().getAliases().isEmpty());
-        assertTrue(indicesAdmin().prepareGetAliases("alias-1").get().getAliases().isEmpty());
+        assertTrue(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias-123").get().getAliases().isEmpty());
+        assertTrue(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias-1").get().getAliases().isEmpty());
 
         logger.info("--> restore snapshot without aliases");
         restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, "test-repo", "test-snap")
@@ -388,9 +388,9 @@ public void testRestoreAliases() throws Exception {
         );
 
         logger.info("--> check that aliases are not restored and existing aliases still exist");
-        assertTrue(indicesAdmin().prepareGetAliases("alias-123").get().getAliases().isEmpty());
-        assertTrue(indicesAdmin().prepareGetAliases("alias-1").get().getAliases().isEmpty());
-        assertFalse(indicesAdmin().prepareGetAliases("alias-3").get().getAliases().isEmpty());
+        assertTrue(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias-123").get().getAliases().isEmpty());
+        assertTrue(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias-1").get().getAliases().isEmpty());
+        assertFalse(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias-3").get().getAliases().isEmpty());
     }
 
     public void testRestoreTemplates() throws Exception {
@@ -424,7 +424,7 @@ public void testRestoreTemplates() throws Exception {
 
         logger.info("-->  delete test template");
         assertThat(indicesAdmin().prepareDeleteTemplate("test-template").get().isAcknowledged(), equalTo(true));
-        GetIndexTemplatesResponse getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates().get();
+        GetIndexTemplatesResponse getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
         assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template");
 
         logger.info("--> restore cluster state");
@@ -437,7 +437,7 @@ public void testRestoreTemplates() throws Exception {
         assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(0));
 
         logger.info("--> check that template is restored");
-        getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates().get();
+        getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
         assertIndexTemplateExists(getIndexTemplatesResponse, "test-template");
     }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotCustomPluginStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotCustomPluginStateIT.java
index 8adac063dcc5b..99cfafe48bc4f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotCustomPluginStateIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotCustomPluginStateIT.java
@@ -133,7 +133,7 @@ public void testIncludeGlobalState() throws Exception {
         if (testTemplate) {
             logger.info("-->  delete test template");
             cluster().wipeTemplates("test-template");
-            GetIndexTemplatesResponse getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates().get();
+            GetIndexTemplatesResponse getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
             assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template");
         }
 
@@ -161,7 +161,7 @@ public void testIncludeGlobalState() throws Exception {
         assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(0));
 
         logger.info("--> check that template wasn't restored");
-        GetIndexTemplatesResponse getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates().get();
+        GetIndexTemplatesResponse getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
         assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template");
 
         logger.info("--> restore cluster state");
@@ -173,7 +173,7 @@ public void testIncludeGlobalState() throws Exception {
 
         if (testTemplate) {
             logger.info("--> check that template is restored");
-            getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates().get();
+            getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
             assertIndexTemplateExists(getIndexTemplatesResponse, "test-template");
         }
 
@@ -225,7 +225,7 @@ public void testIncludeGlobalState() throws Exception {
             );
         }
 
-        getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates().get();
+        getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
         assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template");
 
         logger.info("--> try restoring index and cluster state from snapshot without global state");
@@ -238,7 +238,7 @@ public void testIncludeGlobalState() throws Exception {
         assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
 
         logger.info("--> check that global state wasn't restored but index was");
-        getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates().get();
+        getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
         assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template");
         assertFalse(getPipelines("barbaz").isFound());
         assertNull(safeExecute(GetStoredScriptAction.INSTANCE, new GetStoredScriptRequest(TEST_REQUEST_TIMEOUT, "foobar")).getSource());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java
index 755ee960be73e..aeac8959df617 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java
@@ -753,11 +753,16 @@ private static void putShutdownMetadata(
         clusterService.submitUnbatchedStateUpdateTask("mark node for removal", new ClusterStateUpdateTask() {
             @Override
             public ClusterState execute(ClusterState currentState) {
-                final var nodeId = currentState.nodes().resolveNode(nodeName).getId();
+                final var node = currentState.nodes().resolveNode(nodeName);
                 return currentState.copyAndUpdateMetadata(
                     mdb -> mdb.putCustom(
                         NodesShutdownMetadata.TYPE,
-                        new NodesShutdownMetadata(Map.of(nodeId, shutdownMetadataBuilder.setNodeId(nodeId).build()))
+                        new NodesShutdownMetadata(
+                            Map.of(
+                                node.getId(),
+                                shutdownMetadataBuilder.setNodeId(node.getId()).setNodeEphemeralId(node.getEphemeralId()).build()
+                            )
+                        )
                     )
                 );
             }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java
index 15c5e3379734a..ac322868989bb 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java
@@ -1235,15 +1235,16 @@ public ClusterState execute(ClusterState currentState) {
                                             Strings.toString(currentState),
                                             currentState.metadata().nodeShutdowns().getAll().isEmpty()
                                         );
-                                        final var nodeId = currentState.nodes().resolveNode(node.nodeName).getId();
+                                        final var discoveryNode = currentState.nodes().resolveNode(node.nodeName);
                                         return currentState.copyAndUpdateMetadata(
                                             mdb -> mdb.putCustom(
                                                 NodesShutdownMetadata.TYPE,
                                                 new NodesShutdownMetadata(
                                                     Map.of(
-                                                        nodeId,
+                                                        discoveryNode.getId(),
                                                         SingleNodeShutdownMetadata.builder()
-                                                            .setNodeId(nodeId)
+                                                            .setNodeId(discoveryNode.getId())
+                                                            .setNodeEphemeralId(discoveryNode.getEphemeralId())
                                                             .setType(SingleNodeShutdownMetadata.Type.REMOVE)
                                                             .setStartedAtMillis(clusterService.threadPool().absoluteTimeInMillis())
                                                             .setReason("test")
diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java
index 51896fb80a62c..2a68b65bcdccb 100644
--- a/server/src/main/java/module-info.java
+++ b/server/src/main/java/module-info.java
@@ -424,21 +424,14 @@
 
     provides org.elasticsearch.features.FeatureSpecification
         with
-            org.elasticsearch.action.admin.indices.stats.IndicesStatsFeatures,
             org.elasticsearch.action.bulk.BulkFeatures,
             org.elasticsearch.features.FeatureInfrastructureFeatures,
-            org.elasticsearch.health.HealthFeatures,
-            org.elasticsearch.cluster.metadata.MetadataFeatures,
-            org.elasticsearch.rest.RestFeatures,
-            org.elasticsearch.repositories.RepositoriesFeatures,
-            org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures,
             org.elasticsearch.rest.action.admin.cluster.ClusterRerouteFeatures,
             org.elasticsearch.index.mapper.MapperFeatures,
-            org.elasticsearch.ingest.IngestGeoIpFeatures,
+            org.elasticsearch.index.IndexFeatures,
             org.elasticsearch.search.SearchFeatures,
             org.elasticsearch.script.ScriptFeatures,
             org.elasticsearch.search.retriever.RetrieversFeatures,
-            org.elasticsearch.reservedstate.service.FileSettingsFeatures,
             org.elasticsearch.action.admin.cluster.stats.ClusterStatsFeatures;
 
     uses org.elasticsearch.plugins.internal.SettingsExtension;
@@ -480,4 +473,5 @@
     exports org.elasticsearch.inference.configuration;
     exports org.elasticsearch.monitor.metrics;
     exports org.elasticsearch.plugins.internal.rewriter to org.elasticsearch.inference;
+    exports org.elasticsearch.lucene.util.automaton;
 }
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
index d6e1d31d9b7f7..b2300e1640104 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersions.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -52,7 +52,10 @@ static TransportVersion def(int id) {
     @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // remove the transport versions with which v9 will not need to interact
     public static final TransportVersion ZERO = def(0);
     public static final TransportVersion V_7_0_0 = def(7_00_00_99);
+    public static final TransportVersion V_7_1_0 = def(7_01_00_99);
+    public static final TransportVersion V_7_2_0 = def(7_02_00_99);
     public static final TransportVersion V_7_3_0 = def(7_03_00_99);
+    public static final TransportVersion V_7_3_2 = def(7_03_02_99);
     public static final TransportVersion V_7_4_0 = def(7_04_00_99);
     public static final TransportVersion V_7_6_0 = def(7_06_00_99);
     public static final TransportVersion V_7_8_0 = def(7_08_00_99);
@@ -137,6 +140,21 @@ static TransportVersion def(int id) {
     public static final TransportVersion KNN_QUERY_RESCORE_OVERSAMPLE = def(8_806_00_0);
     public static final TransportVersion SEMANTIC_QUERY_LENIENT = def(8_807_00_0);
     public static final TransportVersion ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS = def(8_808_00_0);
+    public static final TransportVersion EQL_ALLOW_PARTIAL_SEARCH_RESULTS = def(8_809_00_0);
+    public static final TransportVersion NODE_VERSION_INFORMATION_WITH_MIN_READ_ONLY_INDEX_VERSION = def(8_810_00_0);
+    public static final TransportVersion ERROR_TRACE_IN_TRANSPORT_HEADER = def(8_811_00_0);
+    public static final TransportVersion FAILURE_STORE_ENABLED_BY_CLUSTER_SETTING = def(8_812_00_0);
+    public static final TransportVersion SIMULATE_IGNORED_FIELDS = def(8_813_00_0);
+    public static final TransportVersion TRANSFORMS_UPGRADE_MODE = def(8_814_00_0);
+    public static final TransportVersion NODE_SHUTDOWN_EPHEMERAL_ID_ADDED = def(8_815_00_0);
+    public static final TransportVersion ESQL_CCS_TELEMETRY_STATS = def(8_816_00_0);
+    public static final TransportVersion TEXT_EMBEDDING_QUERY_VECTOR_BUILDER_INFER_MODEL_ID = def(8_817_00_0);
+    public static final TransportVersion ESQL_ENABLE_NODE_LEVEL_REDUCTION = def(8_818_00_0);
+    public static final TransportVersion JINA_AI_INTEGRATION_ADDED = def(8_819_00_0);
+    public static final TransportVersion TRACK_INDEX_FAILED_DUE_TO_VERSION_CONFLICT_METRIC = def(8_820_00_0);
+    public static final TransportVersion REPLACE_FAILURE_STORE_OPTIONS_WITH_SELECTOR_SYNTAX = def(8_821_00_0);
+    public static final TransportVersion ELASTIC_INFERENCE_SERVICE_UNIFIED_CHAT_COMPLETIONS_INTEGRATION = def(8_822_00_0);
+    public static final TransportVersion KQL_QUERY_TECH_PREVIEW = def(8_823_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java
index 47c43eadcfb03..8873c9b0e281e 100644
--- a/server/src/main/java/org/elasticsearch/Version.java
+++ b/server/src/main/java/org/elasticsearch/Version.java
@@ -191,6 +191,7 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_8_16_0 = new Version(8_16_00_99);
     public static final Version V_8_16_1 = new Version(8_16_01_99);
     public static final Version V_8_16_2 = new Version(8_16_02_99);
+    public static final Version V_8_16_3 = new Version(8_16_03_99);
     public static final Version V_8_17_0 = new Version(8_17_00_99);
     public static final Version V_8_17_1 = new Version(8_17_01_99);
     public static final Version V_8_18_0 = new Version(8_18_00_99);
diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index 98d6284fd91d2..ec393b7af5cdf 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -1004,7 +1004,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster, Predicate<
 
         // Desired nodes
         registerHandler.accept(new RestGetDesiredNodesAction());
-        registerHandler.accept(new RestUpdateDesiredNodesAction(clusterSupportsFeature));
+        registerHandler.accept(new RestUpdateDesiredNodesAction());
         registerHandler.accept(new RestDeleteDesiredNodesAction());
 
         for (ActionPlugin plugin : actionPlugins) {
diff --git a/server/src/main/java/org/elasticsearch/action/ResolvedIndices.java b/server/src/main/java/org/elasticsearch/action/ResolvedIndices.java
index f149603f12d8b..16f37c9573a8e 100644
--- a/server/src/main/java/org/elasticsearch/action/ResolvedIndices.java
+++ b/server/src/main/java/org/elasticsearch/action/ResolvedIndices.java
@@ -150,10 +150,26 @@ public static ResolvedIndices resolveWithIndicesRequest(
         RemoteClusterService remoteClusterService,
         long startTimeInMillis
     ) {
-        final Map<String, OriginalIndices> remoteClusterIndices = remoteClusterService.groupIndices(
+        return resolveWithIndexNamesAndOptions(
+            request.indices(),
             request.indicesOptions(),
-            request.indices()
+            clusterState,
+            indexNameExpressionResolver,
+            remoteClusterService,
+            startTimeInMillis
         );
+    }
+
+    public static ResolvedIndices resolveWithIndexNamesAndOptions(
+        String[] indexNames,
+        IndicesOptions indicesOptions,
+        ClusterState clusterState,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        RemoteClusterService remoteClusterService,
+        long startTimeInMillis
+    ) {
+        final Map<String, OriginalIndices> remoteClusterIndices = remoteClusterService.groupIndices(indicesOptions, indexNames);
+
         final OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
 
         Index[] concreteLocalIndices = localIndices == null
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/AllocationStatsFeatures.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/AllocationStatsFeatures.java
deleted file mode 100644
index d754d652c8d88..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/AllocationStatsFeatures.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.action.admin.cluster.allocation;
-
-import org.elasticsearch.features.FeatureSpecification;
-import org.elasticsearch.features.NodeFeature;
-
-import java.util.Set;
-
-public class AllocationStatsFeatures implements FeatureSpecification {
-    public static final NodeFeature INCLUDE_DISK_THRESHOLD_SETTINGS = new NodeFeature("stats.include_disk_thresholds");
-
-    @Override
-    public Set<NodeFeature> getFeatures() {
-        return Set.of(INCLUDE_DISK_THRESHOLD_SETTINGS);
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
index 23f88540c2666..3b6a161c3db8a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
@@ -37,7 +37,7 @@
 import java.util.Locale;
 
 import static org.elasticsearch.cluster.routing.allocation.AbstractAllocationDecision.discoveryNodeToXContent;
-import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.singleChunk;
+import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.chunk;
 
 /**
  * A {@code ClusterAllocationExplanation} is an explanation of why a shard is unassigned,
@@ -169,7 +169,7 @@ public ShardAllocationDecision getShardAllocationDecision() {
     }
 
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return Iterators.concat(singleChunk((builder, p) -> {
+        return Iterators.concat(chunk((builder, p) -> {
             builder.startObject();
 
             if (isSpecificShard() == false) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponse.java
index 2f22b63f78bc6..2a35851f0017b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponse.java
@@ -33,8 +33,8 @@
 import java.util.Objects;
 import java.util.Set;
 
+import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.chunk;
 import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.endObject;
-import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.singleChunk;
 import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.startObject;
 
 public class DesiredBalanceResponse extends ActionResponse implements ChunkedToXContentObject {
@@ -88,7 +88,7 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         return Iterators.concat(
-            singleChunk(
+            chunk(
                 (builder, p) -> builder.startObject()
                     .field("stats", stats)
                     .field("cluster_balance_stats", clusterBalanceStats)
@@ -101,16 +101,16 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
                     Iterators.flatMap(
                         indexEntry.getValue().entrySet().iterator(),
                         shardEntry -> Iterators.concat(
-                            singleChunk((builder, p) -> builder.field(String.valueOf(shardEntry.getKey()))),
+                            chunk((builder, p) -> builder.field(String.valueOf(shardEntry.getKey()))),
                             shardEntry.getValue().toXContentChunked(params)
                         )
                     ),
                     endObject()
                 )
             ),
-            singleChunk((builder, p) -> builder.endObject().startObject("cluster_info")),
+            chunk((builder, p) -> builder.endObject().startObject("cluster_info")),
             clusterInfo.toXContentChunked(params),
-            singleChunk((builder, p) -> builder.endObject().endObject())
+            chunk((builder, p) -> builder.endObject().endObject())
         );
     }
 
@@ -173,9 +173,9 @@ public void writeTo(StreamOutput out) throws IOException {
         @Override
         public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
             return Iterators.concat(
-                singleChunk((builder, p) -> builder.startObject().startArray("current")),
+                chunk((builder, p) -> builder.startObject().startArray("current")),
                 current().iterator(),
-                singleChunk((builder, p) -> builder.endArray().field("desired").value(desired, p).endObject())
+                chunk((builder, p) -> builder.endArray().field("desired").value(desired, p).endObject())
             );
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java
index d929fb457d5d1..23bf22e08985e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
@@ -49,7 +48,6 @@ public class TransportGetAllocationStatsAction extends TransportMasterNodeReadAc
 
     private final AllocationStatsService allocationStatsService;
     private final DiskThresholdSettings diskThresholdSettings;
-    private final FeatureService featureService;
 
     @Inject
     public TransportGetAllocationStatsAction(
@@ -58,8 +56,7 @@ public TransportGetAllocationStatsAction(
         ThreadPool threadPool,
         ActionFilters actionFilters,
         IndexNameExpressionResolver indexNameExpressionResolver,
-        AllocationStatsService allocationStatsService,
-        FeatureService featureService
+        AllocationStatsService allocationStatsService
     ) {
         super(
             TYPE.name(),
@@ -74,7 +71,6 @@ public TransportGetAllocationStatsAction(
         );
         this.allocationStatsService = allocationStatsService;
         this.diskThresholdSettings = new DiskThresholdSettings(clusterService.getSettings(), clusterService.getClusterSettings());
-        this.featureService = featureService;
     }
 
     @Override
@@ -92,10 +88,7 @@ protected void masterOperation(Task task, Request request, ClusterState state, A
         listener.onResponse(
             new Response(
                 request.metrics().contains(Metric.ALLOCATIONS) ? allocationStatsService.stats() : Map.of(),
-                request.metrics().contains(Metric.FS)
-                    && featureService.clusterHasFeature(clusterService.state(), AllocationStatsFeatures.INCLUDE_DISK_THRESHOLD_SETTINGS)
-                        ? diskThresholdSettings
-                        : null
+                request.metrics().contains(Metric.FS) ? diskThresholdSettings : null
             )
         );
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java
index 9fede2ebb5be6..beb0e1f927de2 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java
@@ -9,7 +9,6 @@
 
 package org.elasticsearch.action.admin.cluster.node.capabilities;
 
-import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
@@ -20,11 +19,9 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.UpdateForV9;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.admin.cluster.RestNodesCapabilitiesAction;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequest;
@@ -32,7 +29,6 @@
 
 import java.io.IOException;
 import java.util.List;
-import java.util.Optional;
 import java.util.Set;
 
 public class TransportNodesCapabilitiesAction extends TransportNodesAction<
@@ -45,7 +41,6 @@ public class TransportNodesCapabilitiesAction extends TransportNodesAction<
     public static final ActionType<NodesCapabilitiesResponse> TYPE = new ActionType<>("cluster:monitor/nodes/capabilities");
 
     private final RestController restController;
-    private final FeatureService featureService;
 
     @Inject
     public TransportNodesCapabilitiesAction(
@@ -53,8 +48,7 @@ public TransportNodesCapabilitiesAction(
         ClusterService clusterService,
         TransportService transportService,
         ActionFilters actionFilters,
-        RestController restController,
-        FeatureService featureService
+        RestController restController
     ) {
         super(
             TYPE.name(),
@@ -65,23 +59,6 @@ public TransportNodesCapabilitiesAction(
             threadPool.executor(ThreadPool.Names.MANAGEMENT)
         );
         this.restController = restController;
-        this.featureService = featureService;
-    }
-
-    @Override
-    protected void doExecute(Task task, NodesCapabilitiesRequest request, ActionListener<NodesCapabilitiesResponse> listener) {
-        if (featureService.clusterHasFeature(clusterService.state(), RestNodesCapabilitiesAction.CAPABILITIES_ACTION) == false) {
-            // not everything in the cluster supports capabilities.
-            // Therefore we don't support whatever it is we're being asked for
-            listener.onResponse(new NodesCapabilitiesResponse(clusterService.getClusterName(), List.of(), List.of()) {
-                @Override
-                public Optional<Boolean> isSupported() {
-                    return Optional.of(false);
-                }
-            });
-        } else {
-            super.doExecute(task, request, listener);
-        }
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java
index e194101738489..ae4d6cb92c08f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java
@@ -15,9 +15,11 @@
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.elasticsearch.cluster.routing.allocation.NodeAllocationStats;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.discovery.DiscoveryStats;
 import org.elasticsearch.http.HttpStats;
@@ -38,10 +40,13 @@
 import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Objects;
 
+import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.chunk;
+
 /**
  * Node statistics (dynamic, changes depending on when created).
  */
@@ -342,7 +347,7 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
-        return ChunkedToXContent.builder(outerParams).append((builder, params) -> {
+        return Iterators.concat(chunk((builder, params) -> {
             builder.field("name", getNode().getName());
             builder.field("transport_address", getNode().getAddress().toString());
             builder.field("host", getNode().getHostName());
@@ -353,7 +358,6 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerP
                 builder.value(role.roleName());
             }
             builder.endArray();
-
             if (getNode().getAttributes().isEmpty() == false) {
                 builder.startObject("attributes");
                 for (Map.Entry<String, String> attrEntry : getNode().getAttributes().entrySet()) {
@@ -361,30 +365,38 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerP
                 }
                 builder.endObject();
             }
+
             return builder;
-        })
-
-            .appendIfPresent(getIndices())
-            .append((builder, p) -> builder.value(ifPresent(getOs()), p).value(ifPresent(getProcess()), p).value(ifPresent(getJvm()), p))
-
-            .appendIfPresent(getThreadPool())
-            .appendIfPresent(getFs())
-            .appendIfPresent(getTransport())
-            .appendIfPresent(getHttp())
-            .appendIfPresent(getBreaker())
-            .appendIfPresent(getScriptStats())
-            .appendIfPresent(getDiscoveryStats())
-            .appendIfPresent(getIngestStats())
-            .appendIfPresent(getAdaptiveSelectionStats())
-            .appendIfPresent(getScriptCacheStats())
-            .append(
+        }),
+            ifPresent(getIndices()).toXContentChunked(outerParams),
+            chunk((builder, p) -> builder.value(ifPresent(getOs()), p).value(ifPresent(getProcess()), p).value(ifPresent(getJvm()), p)),
+            ifPresent(getThreadPool()).toXContentChunked(outerParams),
+            singleChunkIfPresent(getFs()),
+            ifPresent(getTransport()).toXContentChunked(outerParams),
+            ifPresent(getHttp()).toXContentChunked(outerParams),
+            singleChunkIfPresent(getBreaker()),
+            ifPresent(getScriptStats()).toXContentChunked(outerParams),
+            singleChunkIfPresent(getDiscoveryStats()),
+            ifPresent(getIngestStats()).toXContentChunked(outerParams),
+            singleChunkIfPresent(getAdaptiveSelectionStats()),
+            singleChunkIfPresent(getScriptCacheStats()),
+            chunk(
                 (builder, p) -> builder.value(ifPresent(getIndexingPressureStats()), p)
                     .value(ifPresent(getRepositoriesStats()), p)
                     .value(ifPresent(getNodeAllocationStats()), p)
-            );
+            )
+        );
+    }
+
+    private static ChunkedToXContent ifPresent(@Nullable ChunkedToXContent chunkedToXContent) {
+        return Objects.requireNonNullElse(chunkedToXContent, ChunkedToXContent.EMPTY);
     }
 
     private static ToXContent ifPresent(@Nullable ToXContent toXContent) {
         return Objects.requireNonNullElse(toXContent, ToXContent.EMPTY);
     }
+
+    private static Iterator<ToXContent> singleChunkIfPresent(ToXContent toXContent) {
+        return toXContent == null ? Collections.emptyIterator() : ChunkedToXContentHelper.chunk(toXContent);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsResponse.java
index 1f0e0daf77d7b..25d34957b6958 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsResponse.java
@@ -14,9 +14,10 @@
 import org.elasticsearch.action.support.nodes.BaseNodesXContentResponse;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
@@ -41,12 +42,15 @@ protected void writeNodesTo(StreamOutput out, List<NodeStats> nodes) throws IOEx
 
     @Override
     protected Iterator<? extends ToXContent> xContentChunks(ToXContent.Params outerParams) {
-        return ChunkedToXContent.builder(outerParams)
-            .object(
-                "nodes",
-                getNodes().iterator(),
-                (b, ns) -> b.object(ns.getNode().getId(), ob -> ob.field("timestamp", ns.getTimestamp()).append(ns))
-            );
+        return Iterators.concat(
+            ChunkedToXContentHelper.startObject("nodes"),
+            Iterators.flatMap(getNodes().iterator(), nodeStats -> Iterators.concat(Iterators.single((builder, params) -> {
+                builder.startObject(nodeStats.getNode().getId());
+                builder.field("timestamp", nodeStats.getTimestamp());
+                return builder;
+            }), nodeStats.toXContentChunked(outerParams), ChunkedToXContentHelper.endObject())),
+            ChunkedToXContentHelper.endObject()
+        );
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
index 45ee00a98c2e2..f04db5eec667b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
@@ -13,11 +13,12 @@
 import org.elasticsearch.action.support.master.IsAcknowledgedSupplier;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.allocation.RoutingExplanations;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.UpdateForV10;
@@ -25,6 +26,7 @@
 import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.Objects;
 
@@ -92,14 +94,17 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerP
         if (emitState(outerParams)) {
             deprecationLogger.critical(DeprecationCategory.API, "reroute_cluster_state", STATE_FIELD_DEPRECATION_MESSAGE);
         }
-        return ChunkedToXContent.builder(outerParams).object(b -> {
-            b.field(ACKNOWLEDGED_KEY, isAcknowledged());
-            if (emitState(outerParams)) {
-                b.xContentObject("state", state);
-            }
-            if (outerParams.paramAsBoolean("explain", false)) {
-                b.append(explanations);
-            }
-        });
+        return Iterators.concat(
+            Iterators.single((builder, params) -> builder.startObject().field(ACKNOWLEDGED_KEY, isAcknowledged())),
+            emitState(outerParams)
+                ? ChunkedToXContentHelper.object("state", state.toXContentChunked(outerParams))
+                : Collections.emptyIterator(),
+            Iterators.single((builder, params) -> {
+                if (params.paramAsBoolean("explain", false)) {
+                    explanations.toXContent(builder, params);
+                }
+                return builder.endObject();
+            })
+        );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java
index 7e3c38c735091..ca02d19749ae7 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java
@@ -13,13 +13,18 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
-import org.elasticsearch.action.support.master.MasterNodeReadRequest;
+import org.elasticsearch.action.support.local.LocalClusterStateRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV10;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
+import java.util.Map;
 import java.util.Objects;
 
 public class ClusterGetSettingsAction extends ActionType<ClusterGetSettingsAction.Response> {
@@ -34,25 +39,29 @@ public ClusterGetSettingsAction() {
     /**
      * Request to retrieve the cluster settings
      */
-    public static class Request extends MasterNodeReadRequest<Request> {
+    public static class Request extends LocalClusterStateRequest {
         public Request(TimeValue masterNodeTimeout) {
             super(masterNodeTimeout);
         }
 
+        /**
+         * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to read these requests until
+         * we no longer need to support calling this action remotely.
+         */
+        @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA)
         public Request(StreamInput in) throws IOException {
             super(in);
             assert in.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0);
         }
 
         @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            assert out.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0);
-            super.writeTo(out);
+        public ActionRequestValidationException validate() {
+            return null;
         }
 
         @Override
-        public ActionRequestValidationException validate() {
-            return null;
+        public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+            return new CancellableTask(id, type, action, "", parentTaskId, headers);
         }
     }
 
@@ -79,20 +88,17 @@ public int hashCode() {
             return Objects.hash(persistentSettings, transientSettings, settings);
         }
 
-        public Response(StreamInput in) throws IOException {
-            super(in);
-            assert in.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0);
-            persistentSettings = Settings.readSettingsFromStream(in);
-            transientSettings = Settings.readSettingsFromStream(in);
-            settings = Settings.readSettingsFromStream(in);
-        }
-
         public Response(Settings persistentSettings, Settings transientSettings, Settings settings) {
             this.persistentSettings = Objects.requireNonNullElse(persistentSettings, Settings.EMPTY);
             this.transientSettings = Objects.requireNonNullElse(transientSettings, Settings.EMPTY);
             this.settings = Objects.requireNonNullElse(settings, Settings.EMPTY);
         }
 
+        /**
+         * NB prior to 9.0 get-component was a TransportMasterNodeReadAction so for BwC we must remain able to write these responses until
+         * we no longer need to support calling this action remotely.
+         */
+        @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA)
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             assert out.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java
index dce6a38001392..71b976e012aad 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java
@@ -11,57 +11,66 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
+import org.elasticsearch.action.support.ChannelActionListener;
+import org.elasticsearch.action.support.local.TransportLocalClusterStateAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-public class TransportClusterGetSettingsAction extends TransportMasterNodeReadAction<
+public class TransportClusterGetSettingsAction extends TransportLocalClusterStateAction<
     ClusterGetSettingsAction.Request,
     ClusterGetSettingsAction.Response> {
 
     private final SettingsFilter settingsFilter;
 
+    /**
+     * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until
+     * we no longer need to support calling this action remotely.
+     */
+    @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA)
+    @SuppressWarnings("this-escape")
     @Inject
     public TransportClusterGetSettingsAction(
         TransportService transportService,
         ClusterService clusterService,
-        ThreadPool threadPool,
         SettingsFilter settingsFilter,
-        ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver
+        ActionFilters actionFilters
     ) {
         super(
             ClusterGetSettingsAction.NAME,
-            false,
-            transportService,
-            clusterService,
-            threadPool,
             actionFilters,
-            ClusterGetSettingsAction.Request::new,
-            indexNameExpressionResolver,
-            ClusterGetSettingsAction.Response::new,
+            transportService.getTaskManager(),
+            clusterService,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
-
         this.settingsFilter = settingsFilter;
+
+        transportService.registerRequestHandler(
+            actionName,
+            executor,
+            false,
+            true,
+            ClusterGetSettingsAction.Request::new,
+            (request, channel, task) -> executeDirect(task, request, new ChannelActionListener<>(channel))
+        );
     }
 
     @Override
-    protected void masterOperation(
+    protected void localClusterStateOperation(
         Task task,
         ClusterGetSettingsAction.Request request,
         ClusterState state,
         ActionListener<ClusterGetSettingsAction.Response> listener
     ) throws Exception {
+        ((CancellableTask) task).ensureNotCancelled();
         Metadata metadata = state.metadata();
         listener.onResponse(
             new ClusterGetSettingsAction.Response(
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
index b6ced06623306..bfdf41e58f6d3 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
@@ -61,7 +61,7 @@ public class CreateSnapshotRequest extends MasterNodeRequest<CreateSnapshotReque
 
     public static final TransportVersion SETTINGS_IN_REQUEST_VERSION = TransportVersions.V_8_0_0;
 
-    public static int MAXIMUM_METADATA_BYTES = 1024; // chosen arbitrarily
+    public static final int MAXIMUM_METADATA_BYTES = 1024; // chosen arbitrarily
 
     private String snapshot;
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotResponse.java
index 012a57e046b8d..46d6d95f2a152 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotResponse.java
@@ -22,7 +22,7 @@
 import java.util.Optional;
 
 public class GetShardSnapshotResponse extends ActionResponse {
-    public static GetShardSnapshotResponse EMPTY = new GetShardSnapshotResponse(null, Collections.emptyMap());
+    public static final GetShardSnapshotResponse EMPTY = new GetShardSnapshotResponse(null, Collections.emptyMap());
 
     private final ShardSnapshotInfo latestShardSnapshot;
     private final Map<String, RepositoryException> repositoryFailures;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
index b7a513b3cb08e..24c427c32d69a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
-import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -44,9 +43,7 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
     private String snapshot;
     private String repository;
     private String[] indices = Strings.EMPTY_ARRAY;
-    private IndicesOptions indicesOptions = DataStream.isFailureStoreFeatureFlagEnabled()
-        ? IndicesOptions.strictExpandOpenIncludeFailureStore()
-        : IndicesOptions.strictExpandOpen();
+    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
     private String[] featureStates = Strings.EMPTY_ARRAY;
     private String renamePattern;
     private String renameReplacement;
@@ -54,7 +51,7 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
     private boolean includeGlobalState = false;
     private boolean partial = false;
     private boolean includeAliases = true;
-    public static TransportVersion VERSION_SUPPORTING_QUIET_PARAMETER = TransportVersions.V_8_4_0;
+    public static final TransportVersion VERSION_SUPPORTING_QUIET_PARAMETER = TransportVersions.V_8_4_0;
     private boolean quiet = false;
     private Settings indexSettings = Settings.EMPTY;
     private String[] ignoreIndexSettings = Strings.EMPTY_ARRAY;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java
index 3bbaa80ec200e..8500302e4f755 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java
@@ -41,7 +41,6 @@
  * <br>
  */
 public final class CCSTelemetrySnapshot implements Writeable, ToXContentFragment {
-    public static final String CCS_TELEMETRY_FIELD_NAME = "_search";
     private long totalCount;
     private long successCount;
     private final Map<String, Long> failureReasons;
@@ -66,6 +65,9 @@ public final class CCSTelemetrySnapshot implements Writeable, ToXContentFragment
 
     private final Map<String, Long> clientCounts;
     private final Map<String, PerClusterCCSTelemetry> byRemoteCluster;
+    // Whether we should use per-MRT (minimize roundtrips) metrics.
+    // ES|QL does not have "minimize_roundtrips" option, so we don't collect those metrics for ES|QL usage.
+    private boolean useMRT = true;
 
     /**
     * Creates a new stats instance with the provided info.
@@ -191,6 +193,11 @@ public Map<String, PerClusterCCSTelemetry> getByRemoteCluster() {
         return Collections.unmodifiableMap(byRemoteCluster);
     }
 
+    public CCSTelemetrySnapshot setUseMRT(boolean useMRT) {
+        this.useMRT = useMRT;
+        return this;
+    }
+
     public static class PerClusterCCSTelemetry implements Writeable, ToXContentFragment {
         private long count;
         private long skippedCount;
@@ -270,6 +277,11 @@ public boolean equals(Object o) {
         public int hashCode() {
             return Objects.hash(count, skippedCount, took);
         }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this, true, true);
+        }
     }
 
     /**
@@ -291,8 +303,10 @@ public void add(CCSTelemetrySnapshot stats) {
         stats.featureCounts.forEach((k, v) -> featureCounts.merge(k, v, Long::sum));
         stats.clientCounts.forEach((k, v) -> clientCounts.merge(k, v, Long::sum));
         took.add(stats.took);
-        tookMrtTrue.add(stats.tookMrtTrue);
-        tookMrtFalse.add(stats.tookMrtFalse);
+        if (useMRT) {
+            tookMrtTrue.add(stats.tookMrtTrue);
+            tookMrtFalse.add(stats.tookMrtFalse);
+        }
         remotesPerSearchMax = Math.max(remotesPerSearchMax, stats.remotesPerSearchMax);
         if (totalCount > 0 && oldCount > 0) {
             // Weighted average
@@ -328,30 +342,28 @@ private static void publishLatency(XContentBuilder builder, String name, LongMet
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(CCS_TELEMETRY_FIELD_NAME);
-        {
-            builder.field("total", totalCount);
-            builder.field("success", successCount);
-            builder.field("skipped", skippedRemotes);
-            publishLatency(builder, "took", took);
+        builder.field("total", totalCount);
+        builder.field("success", successCount);
+        builder.field("skipped", skippedRemotes);
+        publishLatency(builder, "took", took);
+        if (useMRT) {
             publishLatency(builder, "took_mrt_true", tookMrtTrue);
             publishLatency(builder, "took_mrt_false", tookMrtFalse);
-            builder.field("remotes_per_search_max", remotesPerSearchMax);
-            builder.field("remotes_per_search_avg", remotesPerSearchAvg);
-            builder.field("failure_reasons", failureReasons);
-            builder.field("features", featureCounts);
-            builder.field("clients", clientCounts);
-            builder.startObject("clusters");
-            {
-                for (var entry : byRemoteCluster.entrySet()) {
-                    String remoteName = entry.getKey();
-                    if (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(remoteName)) {
-                        remoteName = SearchResponse.LOCAL_CLUSTER_NAME_REPRESENTATION;
-                    }
-                    builder.field(remoteName, entry.getValue());
+        }
+        builder.field("remotes_per_search_max", remotesPerSearchMax);
+        builder.field("remotes_per_search_avg", remotesPerSearchAvg);
+        builder.field("failure_reasons", failureReasons);
+        builder.field("features", featureCounts);
+        builder.field("clients", clientCounts);
+        builder.startObject("clusters");
+        {
+            for (var entry : byRemoteCluster.entrySet()) {
+                String remoteName = entry.getKey();
+                if (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(remoteName)) {
+                    remoteName = SearchResponse.LOCAL_CLUSTER_NAME_REPRESENTATION;
                 }
+                builder.field(remoteName, entry.getValue());
             }
-            builder.endObject();
         }
         builder.endObject();
         return builder;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java
index 9e58d6d8febef..29a7dcb5d07d8 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java
@@ -10,6 +10,7 @@
 package org.elasticsearch.action.admin.cluster.stats;
 
 import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ShardOperationFailedException;
@@ -20,6 +21,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.query.SearchTimeoutException;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskCancelledException;
 
 import java.util.Arrays;
@@ -84,6 +86,15 @@ public Builder setClient(String client) {
             return this;
         }
 
+        public Builder setClientFromTask(Task task) {
+            String client = task.getHeader(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER);
+            if (client != null) {
+                return setClient(client);
+            } else {
+                return this;
+            }
+        }
+
         public Builder skippedRemote(String remote) {
             this.skippedRemotes.add(remote);
             return this;
@@ -133,6 +144,10 @@ public static Result getFailureType(Exception e) {
             if (ExceptionsHelper.unwrapCorruption(e) != null) {
                 return Result.CORRUPTION;
             }
+            ElasticsearchStatusException se = (ElasticsearchStatusException) ExceptionsHelper.unwrap(e, ElasticsearchStatusException.class);
+            if (se != null && se.getDetailedMessage().contains("license")) {
+                return Result.LICENSE;
+            }
             // This is kind of last resort check - if we still don't know the reason but all shard failures are remote,
             // we assume it's remote's fault somehow.
             if (e instanceof SearchPhaseExecutionException spe) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java
index 6c8178282d3c3..3f04eceed7eb5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java
@@ -47,6 +47,7 @@ public enum Result {
         TIMEOUT("timeout"),
         CORRUPTION("corruption"),
         SECURITY("security"),
+        LICENSE("license"),
         // May be helpful if there's a lot of other reasons, and it may be hard to calculate the unknowns for some clients.
         UNKNOWN("other");
 
@@ -106,8 +107,14 @@ public String getName() {
 
     private final Map<String, LongAdder> clientCounts;
     private final Map<String, PerClusterCCSTelemetry> byRemoteCluster;
+    // Should we calculate separate metrics per MRT?
+    private final boolean useMRT;
 
     public CCSUsageTelemetry() {
+        this(true);
+    }
+
+    public CCSUsageTelemetry(boolean useMRT) {
         this.byRemoteCluster = new ConcurrentHashMap<>();
         totalCount = new LongAdder();
         successCount = new LongAdder();
@@ -119,6 +126,7 @@ public CCSUsageTelemetry() {
         skippedRemotes = new LongAdder();
         featureCounts = new ConcurrentHashMap<>();
         clientCounts = new ConcurrentHashMap<>();
+        this.useMRT = useMRT;
     }
 
     public void updateUsage(CCSUsage ccsUsage) {
@@ -134,10 +142,12 @@ private void doUpdate(CCSUsage ccsUsage) {
         if (isSuccess(ccsUsage)) {
             successCount.increment();
             took.record(searchTook);
-            if (isMRT(ccsUsage)) {
-                tookMrtTrue.record(searchTook);
-            } else {
-                tookMrtFalse.record(searchTook);
+            if (useMRT) {
+                if (isMRT(ccsUsage)) {
+                    tookMrtTrue.record(searchTook);
+                } else {
+                    tookMrtFalse.record(searchTook);
+                }
             }
             ccsUsage.getPerClusterUsage().forEach((r, u) -> byRemoteCluster.computeIfAbsent(r, PerClusterCCSTelemetry::new).update(u));
         } else {
@@ -243,6 +253,6 @@ public CCSTelemetrySnapshot getCCSTelemetrySnapshot() {
             Collections.unmodifiableMap(Maps.transformValues(featureCounts, LongAdder::longValue)),
             Collections.unmodifiableMap(Maps.transformValues(clientCounts, LongAdder::longValue)),
             Collections.unmodifiableMap(Maps.transformValues(byRemoteCluster, PerClusterCCSTelemetry::getSnapshot))
-        );
+        ).setUseMRT(useMRT);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java
index abeb73e5d8c3e..48b4e967742cd 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java
@@ -31,7 +31,8 @@ public class ClusterStatsNodeResponse extends BaseNodeResponse {
     private final ClusterHealthStatus clusterStatus;
     private final SearchUsageStats searchUsageStats;
     private final RepositoryUsageStats repositoryUsageStats;
-    private final CCSTelemetrySnapshot ccsMetrics;
+    private final CCSTelemetrySnapshot searchCcsMetrics;
+    private final CCSTelemetrySnapshot esqlCcsMetrics;
 
     public ClusterStatsNodeResponse(StreamInput in) throws IOException {
         super(in);
@@ -46,10 +47,15 @@ public ClusterStatsNodeResponse(StreamInput in) throws IOException {
         }
         if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) {
             repositoryUsageStats = RepositoryUsageStats.readFrom(in);
-            ccsMetrics = new CCSTelemetrySnapshot(in);
+            searchCcsMetrics = new CCSTelemetrySnapshot(in);
         } else {
             repositoryUsageStats = RepositoryUsageStats.EMPTY;
-            ccsMetrics = new CCSTelemetrySnapshot();
+            searchCcsMetrics = new CCSTelemetrySnapshot();
+        }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_TELEMETRY_STATS)) {
+            esqlCcsMetrics = new CCSTelemetrySnapshot(in);
+        } else {
+            esqlCcsMetrics = new CCSTelemetrySnapshot();
         }
     }
 
@@ -61,7 +67,8 @@ public ClusterStatsNodeResponse(
         ShardStats[] shardsStats,
         SearchUsageStats searchUsageStats,
         RepositoryUsageStats repositoryUsageStats,
-        CCSTelemetrySnapshot ccsTelemetrySnapshot
+        CCSTelemetrySnapshot ccsTelemetrySnapshot,
+        CCSTelemetrySnapshot esqlTelemetrySnapshot
     ) {
         super(node);
         this.nodeInfo = nodeInfo;
@@ -70,7 +77,8 @@ public ClusterStatsNodeResponse(
         this.clusterStatus = clusterStatus;
         this.searchUsageStats = Objects.requireNonNull(searchUsageStats);
         this.repositoryUsageStats = Objects.requireNonNull(repositoryUsageStats);
-        this.ccsMetrics = ccsTelemetrySnapshot;
+        this.searchCcsMetrics = ccsTelemetrySnapshot;
+        this.esqlCcsMetrics = esqlTelemetrySnapshot;
     }
 
     public NodeInfo nodeInfo() {
@@ -101,8 +109,12 @@ public RepositoryUsageStats repositoryUsageStats() {
         return repositoryUsageStats;
     }
 
-    public CCSTelemetrySnapshot getCcsMetrics() {
-        return ccsMetrics;
+    public CCSTelemetrySnapshot getSearchCcsMetrics() {
+        return searchCcsMetrics;
+    }
+
+    public CCSTelemetrySnapshot getEsqlCcsMetrics() {
+        return esqlCcsMetrics;
     }
 
     @Override
@@ -117,8 +129,11 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) {
             repositoryUsageStats.writeTo(out);
-            ccsMetrics.writeTo(out);
+            searchCcsMetrics.writeTo(out);
         } // else just drop these stats, ok for bwc
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_TELEMETRY_STATS)) {
+            esqlCcsMetrics.writeTo(out);
+        }
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
index 5f7c45c5807a5..ed8ca2f94a78b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
@@ -36,10 +36,14 @@ public class ClusterStatsResponse extends BaseNodesResponse<ClusterStatsNodeResp
     final ClusterSnapshotStats clusterSnapshotStats;
     final RepositoryUsageStats repositoryUsageStats;
     final CCSTelemetrySnapshot ccsMetrics;
+    final CCSTelemetrySnapshot esqlMetrics;
     final long timestamp;
     final String clusterUUID;
     private final Map<String, RemoteClusterStats> remoteClustersStats;
 
+    public static final String CCS_TELEMETRY_FIELD_NAME = "_search";
+    public static final String ESQL_TELEMETRY_FIELD_NAME = "_esql";
+
     public ClusterStatsResponse(
         long timestamp,
         String clusterUUID,
@@ -58,6 +62,7 @@ public ClusterStatsResponse(
         nodesStats = new ClusterStatsNodes(nodes);
         indicesStats = new ClusterStatsIndices(nodes, mappingStats, analysisStats, versionStats);
         ccsMetrics = new CCSTelemetrySnapshot();
+        esqlMetrics = new CCSTelemetrySnapshot().setUseMRT(false);
         ClusterHealthStatus status = null;
         for (ClusterStatsNodeResponse response : nodes) {
             // only the master node populates the status
@@ -66,7 +71,10 @@ public ClusterStatsResponse(
                 break;
             }
         }
-        nodes.forEach(node -> ccsMetrics.add(node.getCcsMetrics()));
+        nodes.forEach(node -> {
+            ccsMetrics.add(node.getSearchCcsMetrics());
+            esqlMetrics.add(node.getEsqlCcsMetrics());
+        });
         this.status = status;
         this.clusterSnapshotStats = clusterSnapshotStats;
 
@@ -147,9 +155,18 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (remoteClustersStats != null) {
             builder.field("clusters", remoteClustersStats);
         }
+        builder.startObject(CCS_TELEMETRY_FIELD_NAME);
         ccsMetrics.toXContent(builder, params);
         builder.endObject();
 
+        if (esqlMetrics.getTotalCount() > 0) {
+            builder.startObject(ESQL_TELEMETRY_FIELD_NAME);
+            esqlMetrics.toXContent(builder, params);
+            builder.endObject();
+        }
+
+        builder.endObject();
+
         return builder;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java
index 1bc2e1d13c864..29a124b3d0b20 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java
@@ -21,7 +21,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.features.NodeFeature;
-import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -71,7 +71,7 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) {
             }
             AnalysisStats.countMapping(mappingCounts, indexMetadata);
 
-            var sourceMode = SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings());
+            var sourceMode = IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings());
             sourceModeUsageCount.merge(sourceMode.toString().toLowerCase(Locale.ENGLISH), 1, Integer::sum);
         }
         final AtomicLong totalFieldCount = new AtomicLong();
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index 2c20daa5d7afb..6f69def7aa4e0 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -103,6 +103,7 @@ public class TransportClusterStatsAction extends TransportNodesAction<
     private final RepositoriesService repositoriesService;
     private final SearchUsageHolder searchUsageHolder;
     private final CCSUsageTelemetry ccsUsageHolder;
+    private final CCSUsageTelemetry esqlUsageHolder;
 
     private final Executor clusterStateStatsExecutor;
     private final MetadataStatsCache<MappingStats> mappingStatsCache;
@@ -135,6 +136,7 @@ public TransportClusterStatsAction(
         this.repositoriesService = repositoriesService;
         this.searchUsageHolder = usageService.getSearchUsageHolder();
         this.ccsUsageHolder = usageService.getCcsUsageHolder();
+        this.esqlUsageHolder = usageService.getEsqlUsageHolder();
         this.clusterStateStatsExecutor = threadPool.executor(ThreadPool.Names.MANAGEMENT);
         this.mappingStatsCache = new MetadataStatsCache<>(threadPool.getThreadContext(), MappingStats::of);
         this.analysisStatsCache = new MetadataStatsCache<>(threadPool.getThreadContext(), AnalysisStats::of);
@@ -293,6 +295,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
 
         final RepositoryUsageStats repositoryUsageStats = repositoriesService.getUsageStats();
         final CCSTelemetrySnapshot ccsTelemetry = ccsUsageHolder.getCCSTelemetrySnapshot();
+        final CCSTelemetrySnapshot esqlTelemetry = esqlUsageHolder.getCCSTelemetrySnapshot();
 
         return new ClusterStatsNodeResponse(
             nodeInfo.getNode(),
@@ -302,7 +305,8 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
             shardsStats.toArray(new ShardStats[shardsStats.size()]),
             searchUsageStats,
             repositoryUsageStats,
-            ccsTelemetry
+            ccsTelemetry,
+            esqlTelemetry
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
index d66cab1d2d717..aa861148f022c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
@@ -74,7 +74,7 @@ public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesReq
                 .allowAliasToMultipleIndices(true)
                 .allowClosedIndices(true)
                 .ignoreThrottled(false)
-                .allowFailureIndices(true)
+                .allowSelectors(false)
                 .build()
         )
         .build();
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java
index af95787a5fcca..0950e16229a2b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.action.AliasesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.local.LocalClusterStateRequest;
-import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.tasks.CancellableTask;
@@ -23,23 +22,13 @@
 
 public class GetAliasesRequest extends LocalClusterStateRequest implements AliasesRequest {
 
-    public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandHidden();
+    public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandHiddenNoSelectors();
 
     private String[] aliases;
     private String[] originalAliases;
     private String[] indices = Strings.EMPTY_ARRAY;
     private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS;
 
-    @Deprecated
-    public GetAliasesRequest(String... aliases) {
-        this(MasterNodeRequest.TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, aliases);
-    }
-
-    @Deprecated
-    public GetAliasesRequest() {
-        this(Strings.EMPTY_ARRAY);
-    }
-
     public GetAliasesRequest(TimeValue masterTimeout, String... aliases) {
         super(masterTimeout);
         this.aliases = aliases;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequestBuilder.java
index 80fb7d794c32f..dc6374fed21fd 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequestBuilder.java
@@ -12,10 +12,11 @@
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.ElasticsearchClient;
+import org.elasticsearch.core.TimeValue;
 
 public class GetAliasesRequestBuilder extends ActionRequestBuilder<GetAliasesRequest, GetAliasesResponse> {
-    public GetAliasesRequestBuilder(ElasticsearchClient client, String... aliases) {
-        super(client, GetAliasesAction.INSTANCE, new GetAliasesRequest(aliases));
+    public GetAliasesRequestBuilder(ElasticsearchClient client, TimeValue masterTimeout, String... aliases) {
+        super(client, GetAliasesAction.INSTANCE, new GetAliasesRequest(masterTimeout, aliases));
     }
 
     public GetAliasesRequestBuilder setAliases(String... aliases) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
index c233ed57b748e..cc96954c8a8e4 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
@@ -480,13 +480,19 @@ public CreateIndexRequest requireDataStream(boolean requireDataStream) {
         return this;
     }
 
+    /**
+     * Returns whether the failure store should be initialized. N.B. If true, failure store index creation will be performed regardless of
+     * whether the template indicates that the failure store is enabled.
+     */
     public boolean isInitializeFailureStore() {
         return initializeFailureStore;
     }
 
     /**
      * Set whether this CreateIndexRequest should initialize the failure store on data stream creation. This can be necessary when, for
-     * example, a failure occurs while trying to ingest a document into a data stream that has to be auto-created.
+     * example, a failure occurs while trying to ingest a document into a data stream that has to be auto-created. N.B. If true, failure
+     * store index creation will be performed regardless of whether the template indicates that the failure store is enabled. It is the
+     * caller's responsibility to ensure that this is correct.
      */
     public CreateIndexRequest initializeFailureStore(boolean initializeFailureStore) {
         this.initializeFailureStore = initializeFailureStore;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java
index 96fae42b05505..48a2ec171b780 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java
@@ -43,7 +43,7 @@ public class DeleteIndexRequest extends AcknowledgedRequest<DeleteIndexRequest>
                 .allowAliasToMultipleIndices(false)
                 .allowClosedIndices(true)
                 .ignoreThrottled(false)
-                .allowFailureIndices(true)
+                .allowSelectors(false)
                 .build()
         )
         .build();
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java
index 801dbbdee0858..be7aaeec8f69e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.info.ClusterInfoRequest;
-import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.ArrayUtils;
@@ -95,13 +94,7 @@ public static Feature[] fromRequest(RestRequest request) {
     private transient boolean includeDefaults = false;
 
     public GetIndexRequest() {
-        super(
-            DataStream.isFailureStoreFeatureFlagEnabled()
-                ? IndicesOptions.builder(IndicesOptions.strictExpandOpen())
-                    .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
-                    .build()
-                : IndicesOptions.strictExpandOpen()
-        );
+        super(IndicesOptions.strictExpandOpen());
     }
 
     public GetIndexRequest(StreamInput in) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java
index dd4114c947174..84789d8a2acfb 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.info.ClusterInfoRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
@@ -21,7 +22,9 @@
 
 public class GetMappingsRequest extends ClusterInfoRequest<GetMappingsRequest> {
 
-    public GetMappingsRequest() {}
+    public GetMappingsRequest(TimeValue masterTimeout) {
+        super(masterTimeout);
+    }
 
     public GetMappingsRequest(StreamInput in) throws IOException {
         super(in);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java
index 3f54138581398..a12ba4f60c26a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java
@@ -11,13 +11,14 @@
 
 import org.elasticsearch.action.support.master.info.ClusterInfoRequestBuilder;
 import org.elasticsearch.client.internal.ElasticsearchClient;
+import org.elasticsearch.core.TimeValue;
 
 public class GetMappingsRequestBuilder extends ClusterInfoRequestBuilder<
     GetMappingsRequest,
     GetMappingsResponse,
     GetMappingsRequestBuilder> {
 
-    public GetMappingsRequestBuilder(ElasticsearchClient client, String... indices) {
-        super(client, GetMappingsAction.INSTANCE, new GetMappingsRequest().indices(indices));
+    public GetMappingsRequestBuilder(ElasticsearchClient client, TimeValue masterTimeout, String... indices) {
+        super(client, GetMappingsAction.INSTANCE, new GetMappingsRequest(masterTimeout).indices(indices));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
index 7b782c6da5a84..05cc0d2cf05d8 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
@@ -82,7 +82,7 @@ public class PutMappingRequest extends AcknowledgedRequest<PutMappingRequest> im
                 .allowClosedIndices(true)
                 .allowAliasToMultipleIndices(true)
                 .ignoreThrottled(false)
-                .allowFailureIndices(false)
+                .allowSelectors(false)
         )
         .build();
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
index 749470e181deb..24f8735b6bd7f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
@@ -20,6 +20,8 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataMappingService;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -40,6 +42,7 @@
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.SortedMap;
 
 /**
  * Put mapping action.
@@ -106,7 +109,14 @@ protected void masterOperation(
                 return;
             }
 
-            final String message = checkForSystemIndexViolations(systemIndices, concreteIndices, request);
+            String message = checkForFailureStoreViolations(clusterService.state(), concreteIndices, request);
+            if (message != null) {
+                logger.warn(message);
+                listener.onFailure(new IllegalStateException(message));
+                return;
+            }
+
+            message = checkForSystemIndexViolations(systemIndices, concreteIndices, request);
             if (message != null) {
                 logger.warn(message);
                 listener.onFailure(new IllegalStateException(message));
@@ -172,6 +182,33 @@ static void performMappingUpdate(
         metadataMappingService.putMapping(updateRequest, wrappedListener);
     }
 
+    static String checkForFailureStoreViolations(ClusterState clusterState, Index[] concreteIndices, PutMappingRequest request) {
+        // Requests that a cluster generates itself are permitted to make changes to mappings
+        // so that rolling upgrade scenarios still work. We check this via the request's origin.
+        if (Strings.isNullOrEmpty(request.origin()) == false) {
+            return null;
+        }
+
+        List<String> violations = new ArrayList<>();
+        SortedMap<String, IndexAbstraction> indicesLookup = clusterState.metadata().getIndicesLookup();
+        for (Index index : concreteIndices) {
+            IndexAbstraction indexAbstraction = indicesLookup.get(index.getName());
+            if (indexAbstraction != null) {
+                DataStream maybeDataStream = indexAbstraction.getParentDataStream();
+                if (maybeDataStream != null && maybeDataStream.isFailureStoreIndex(index.getName())) {
+                    violations.add(index.getName());
+                }
+            }
+        }
+
+        if (violations.isEmpty() == false) {
+            return "Cannot update mappings in "
+                + violations
+                + ": mappings for indices contained in data stream failure stores cannot be updated";
+        }
+        return null;
+    }
+
     static String checkForSystemIndexViolations(SystemIndices systemIndices, Index[] concreteIndices, PutMappingRequest request) {
         // Requests that a cluster generates itself are permitted to have a difference in mappings
         // so that rolling upgrade scenarios still work. We check this via the request's origin.
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java
index 9c5b6097b11bd..ebc9b0fea1be4 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java
@@ -9,12 +9,14 @@
 
 package org.elasticsearch.action.admin.indices.resolve;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.tasks.CancellableTask;
@@ -30,6 +32,7 @@
 public class ResolveClusterActionRequest extends ActionRequest implements IndicesRequest.Replaceable {
 
     public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpen();
+    public static final String TRANSPORT_VERSION_ERROR_MESSAGE_PREFIX = "ResolveClusterAction requires at least version";
 
     private String[] names;
     /*
@@ -65,12 +68,7 @@ public ResolveClusterActionRequest(String[] names, IndicesOptions indicesOptions
     public ResolveClusterActionRequest(StreamInput in) throws IOException {
         super(in);
         if (in.getTransportVersion().before(TransportVersions.V_8_13_0)) {
-            throw new UnsupportedOperationException(
-                "ResolveClusterAction requires at least version "
-                    + TransportVersions.V_8_13_0.toReleaseVersion()
-                    + " but was "
-                    + in.getTransportVersion().toReleaseVersion()
-            );
+            throw new UnsupportedOperationException(createVersionErrorMessage(in.getTransportVersion()));
         }
         this.names = in.readStringArray();
         this.indicesOptions = IndicesOptions.readIndicesOptions(in);
@@ -81,17 +79,21 @@ public ResolveClusterActionRequest(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         if (out.getTransportVersion().before(TransportVersions.V_8_13_0)) {
-            throw new UnsupportedOperationException(
-                "ResolveClusterAction requires at least version "
-                    + TransportVersions.V_8_13_0.toReleaseVersion()
-                    + " but was "
-                    + out.getTransportVersion().toReleaseVersion()
-            );
+            throw new UnsupportedOperationException(createVersionErrorMessage(out.getTransportVersion()));
         }
         out.writeStringArray(names);
         indicesOptions.writeIndicesOptions(out);
     }
 
+    private String createVersionErrorMessage(TransportVersion versionFound) {
+        return Strings.format(
+            "%s %s but was %s",
+            TRANSPORT_VERSION_ERROR_MESSAGE_PREFIX,
+            TransportVersions.V_8_13_0.toReleaseVersion(),
+            versionFound.toReleaseVersion()
+        );
+    }
+
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java
index f5c100b7884bb..4aa022aff1c80 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java
@@ -59,6 +59,7 @@
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.action.search.TransportSearchHelper.checkCCSVersionCompatibility;
 
@@ -598,12 +599,13 @@ private static void mergeResults(
 
         private static void enrichIndexAbstraction(
             ClusterState clusterState,
-            ResolvedExpression indexAbstraction,
+            ResolvedExpression resolvedExpression,
             List<ResolvedIndex> indices,
             List<ResolvedAlias> aliases,
             List<ResolvedDataStream> dataStreams
         ) {
-            IndexAbstraction ia = clusterState.metadata().getIndicesLookup().get(indexAbstraction.resource());
+            SortedMap<String, IndexAbstraction> indicesLookup = clusterState.metadata().getIndicesLookup();
+            IndexAbstraction ia = indicesLookup.get(resolvedExpression.resource());
             if (ia != null) {
                 switch (ia.getType()) {
                     case CONCRETE_INDEX -> {
@@ -632,13 +634,24 @@ private static void enrichIndexAbstraction(
                         );
                     }
                     case ALIAS -> {
-                        String[] indexNames = ia.getIndices().stream().map(Index::getName).toArray(String[]::new);
+                        String[] indexNames = getAliasIndexStream(resolvedExpression, ia, indicesLookup).map(Index::getName)
+                            .toArray(String[]::new);
                         Arrays.sort(indexNames);
                         aliases.add(new ResolvedAlias(ia.getName(), indexNames));
                     }
                     case DATA_STREAM -> {
                         DataStream dataStream = (DataStream) ia;
-                        String[] backingIndices = dataStream.getIndices().stream().map(Index::getName).toArray(String[]::new);
+                        Stream<Index> dataStreamIndices = resolvedExpression.selector() == null
+                            ? dataStream.getIndices().stream()
+                            : switch (resolvedExpression.selector()) {
+                                case DATA -> dataStream.getBackingIndices().getIndices().stream();
+                                case FAILURES -> dataStream.getFailureIndices().getIndices().stream();
+                                case ALL_APPLICABLE -> Stream.concat(
+                                    dataStream.getBackingIndices().getIndices().stream(),
+                                    dataStream.getFailureIndices().getIndices().stream()
+                                );
+                            };
+                        String[] backingIndices = dataStreamIndices.map(Index::getName).toArray(String[]::new);
                         dataStreams.add(new ResolvedDataStream(dataStream.getName(), backingIndices, DataStream.TIMESTAMP_FIELD_NAME));
                     }
                     default -> throw new IllegalStateException("unknown index abstraction type: " + ia.getType());
@@ -646,6 +659,52 @@ private static void enrichIndexAbstraction(
             }
         }
 
+        private static Stream<Index> getAliasIndexStream(
+            ResolvedExpression resolvedExpression,
+            IndexAbstraction ia,
+            SortedMap<String, IndexAbstraction> indicesLookup
+        ) {
+            Stream<Index> aliasIndices;
+            if (resolvedExpression.selector() == null) {
+                aliasIndices = ia.getIndices().stream();
+            } else {
+                aliasIndices = switch (resolvedExpression.selector()) {
+                    case DATA -> ia.getIndices().stream();
+                    case FAILURES -> {
+                        assert ia.isDataStreamRelated() : "Illegal selector [failures] used on non data stream alias";
+                        yield ia.getIndices()
+                            .stream()
+                            .map(Index::getName)
+                            .map(indicesLookup::get)
+                            .map(IndexAbstraction::getParentDataStream)
+                            .filter(Objects::nonNull)
+                            .distinct()
+                            .map(DataStream::getFailureIndices)
+                            .flatMap(failureIndices -> failureIndices.getIndices().stream());
+                    }
+                    case ALL_APPLICABLE -> {
+                        if (ia.isDataStreamRelated()) {
+                            yield Stream.concat(
+                                ia.getIndices().stream(),
+                                ia.getIndices()
+                                    .stream()
+                                    .map(Index::getName)
+                                    .map(indicesLookup::get)
+                                    .map(IndexAbstraction::getParentDataStream)
+                                    .filter(Objects::nonNull)
+                                    .distinct()
+                                    .map(DataStream::getFailureIndices)
+                                    .flatMap(failureIndices -> failureIndices.getIndices().stream())
+                            );
+                        } else {
+                            yield ia.getIndices().stream();
+                        }
+                    }
+                };
+            }
+            return aliasIndices;
+        }
+
         enum Attribute {
             OPEN,
             CLOSED,
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java
index c30a2a44274a7..50dbaf33d2e4f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java
@@ -51,7 +51,6 @@
 public class TransportResolveClusterAction extends HandledTransportAction<ResolveClusterActionRequest, ResolveClusterActionResponse> {
 
     private static final Logger logger = LogManager.getLogger(TransportResolveClusterAction.class);
-    private static final String TRANSPORT_VERSION_ERROR_MESSAGE = "ResolveClusterAction requires at least Transport Version";
 
     public static final String NAME = "indices:admin/resolve/cluster";
     public static final ActionType<ResolveClusterActionResponse> TYPE = new ActionType<>(NAME);
@@ -141,7 +140,7 @@ protected void doExecuteForked(Task task, ResolveClusterActionRequest request, A
                 RemoteClusterClient remoteClusterClient = remoteClusterService.getRemoteClusterClient(
                     clusterAlias,
                     searchCoordinationExecutor,
-                    RemoteClusterService.DisconnectedStrategy.RECONNECT_IF_DISCONNECTED
+                    RemoteClusterService.DisconnectedStrategy.FAIL_IF_DISCONNECTED
                 );
                 var remoteRequest = new ResolveClusterActionRequest(originalIndices.indices(), request.indicesOptions());
                 // allow cancellation requests to propagate to remote clusters
@@ -175,7 +174,13 @@ public void onFailure(Exception failure) {
                             failure,
                             ElasticsearchSecurityException.class
                         ) instanceof ElasticsearchSecurityException ese) {
-                            clusterInfoMap.put(clusterAlias, new ResolveClusterInfo(true, skipUnavailable, ese.getMessage()));
+                            /*
+                             * some ElasticsearchSecurityExceptions come from the local cluster security interceptor after you've
+                             * issued the client.execute call but before any call went to the remote cluster, so with an
+                             * ElasticsearchSecurityException you can't tell whether the remote cluster is available or not, so mark
+                             * it as connected=false
+                             */
+                            clusterInfoMap.put(clusterAlias, new ResolveClusterInfo(false, skipUnavailable, ese.getMessage()));
                         } else if (ExceptionsHelper.unwrap(failure, IndexNotFoundException.class) instanceof IndexNotFoundException infe) {
                             clusterInfoMap.put(clusterAlias, new ResolveClusterInfo(true, skipUnavailable, infe.getMessage()));
                         } else {
@@ -184,7 +189,7 @@ public void onFailure(Exception failure) {
                             // this error at the Transport layer BEFORE it sends the request to the remote cluster, since there
                             // are version guards on the Writeables for this Action, namely ResolveClusterActionRequest.writeTo
                             if (cause instanceof UnsupportedOperationException
-                                && cause.getMessage().contains(TRANSPORT_VERSION_ERROR_MESSAGE)) {
+                                && cause.getMessage().contains(ResolveClusterActionRequest.TRANSPORT_VERSION_ERROR_MESSAGE_PREFIX)) {
                                 // Since this cluster does not have _resolve/cluster, we call the _resolve/index
                                 // endpoint to fill in the matching_indices field of the response for that cluster
                                 ResolveIndexAction.Request resolveIndexRequest = new ResolveIndexAction.Request(
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java
index a677897d79633..c845d1a3854c9 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java
@@ -21,6 +21,8 @@
 import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SelectorResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
@@ -31,7 +33,6 @@
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Iterators;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
@@ -53,8 +54,6 @@ public final class LazyRolloverAction extends ActionType<RolloverResponse> {
 
     private static final Logger logger = LogManager.getLogger(LazyRolloverAction.class);
 
-    public static final NodeFeature DATA_STREAM_LAZY_ROLLOVER = new NodeFeature("data_stream.rollover.lazy");
-
     public static final LazyRolloverAction INSTANCE = new LazyRolloverAction();
     public static final String NAME = "indices:admin/data_stream/lazy_rollover";
 
@@ -119,32 +118,38 @@ protected void masterOperation(
                 : "The auto rollover action does not expect any other parameters in the request apart from the data stream name";
 
             Metadata metadata = clusterState.metadata();
-            DataStream dataStream = metadata.dataStreams().get(rolloverRequest.getRolloverTarget());
+            ResolvedExpression resolvedRolloverTarget = SelectorResolver.parseExpression(
+                rolloverRequest.getRolloverTarget(),
+                rolloverRequest.indicesOptions()
+            );
+            boolean isFailureStoreRollover = resolvedRolloverTarget.selector() != null
+                && resolvedRolloverTarget.selector().shouldIncludeFailures();
+
+            DataStream dataStream = metadata.dataStreams().get(resolvedRolloverTarget.resource());
             // Skip submitting the task if we detect that the lazy rollover has been already executed.
-            if (isLazyRolloverNeeded(dataStream, rolloverRequest.targetsFailureStore()) == false) {
-                DataStream.DataStreamIndices targetIndices = dataStream.getDataStreamIndices(rolloverRequest.targetsFailureStore());
+            if (isLazyRolloverNeeded(dataStream, isFailureStoreRollover) == false) {
+                DataStream.DataStreamIndices targetIndices = dataStream.getDataStreamIndices(isFailureStoreRollover);
                 listener.onResponse(noopLazyRolloverResponse(targetIndices));
                 return;
             }
             // We evaluate the names of the source index as well as what our newly created index would be.
             final MetadataRolloverService.NameResolution trialRolloverNames = MetadataRolloverService.resolveRolloverNames(
                 clusterState,
-                rolloverRequest.getRolloverTarget(),
+                resolvedRolloverTarget.resource(),
                 rolloverRequest.getNewIndexName(),
                 rolloverRequest.getCreateIndexRequest(),
-                rolloverRequest.targetsFailureStore()
+                isFailureStoreRollover
             );
             final String trialSourceIndexName = trialRolloverNames.sourceName();
             final String trialRolloverIndexName = trialRolloverNames.rolloverName();
             MetadataCreateIndexService.validateIndexName(trialRolloverIndexName, clusterState.metadata(), clusterState.routingTable());
 
-            assert metadata.dataStreams().containsKey(rolloverRequest.getRolloverTarget()) : "Auto-rollover applies only to data streams";
+            assert metadata.dataStreams().containsKey(resolvedRolloverTarget.resource()) : "Auto-rollover applies only to data streams";
 
             String source = "lazy_rollover source [" + trialSourceIndexName + "] to target [" + trialRolloverIndexName + "]";
             // We create a new rollover request to ensure that it doesn't contain any other parameters apart from the data stream name
             // This will provide a more resilient user experience
-            var newRolloverRequest = new RolloverRequest(rolloverRequest.getRolloverTarget(), null);
-            newRolloverRequest.setIndicesOptions(rolloverRequest.indicesOptions());
+            var newRolloverRequest = new RolloverRequest(resolvedRolloverTarget.combined(), null);
             LazyRolloverTask rolloverTask = new LazyRolloverTask(newRolloverRequest, listener);
             lazyRolloverTaskQueue.submitTask(source, rolloverTask, rolloverRequest.masterNodeTimeout());
         }
@@ -223,12 +228,19 @@ public ClusterState executeTask(
             AllocationActionMultiListener<RolloverResponse> allocationActionMultiListener
         ) throws Exception {
 
+            ResolvedExpression resolvedRolloverTarget = SelectorResolver.parseExpression(
+                rolloverRequest.getRolloverTarget(),
+                rolloverRequest.indicesOptions()
+            );
+            boolean isFailureStoreRollover = resolvedRolloverTarget.selector() != null
+                && resolvedRolloverTarget.selector().shouldIncludeFailures();
+
             // If the data stream has been rolled over since it was marked for lazy rollover, this operation is a noop
-            final DataStream dataStream = currentState.metadata().dataStreams().get(rolloverRequest.getRolloverTarget());
+            final DataStream dataStream = currentState.metadata().dataStreams().get(resolvedRolloverTarget.resource());
             assert dataStream != null;
 
-            if (isLazyRolloverNeeded(dataStream, rolloverRequest.targetsFailureStore()) == false) {
-                final DataStream.DataStreamIndices targetIndices = dataStream.getDataStreamIndices(rolloverRequest.targetsFailureStore());
+            if (isLazyRolloverNeeded(dataStream, isFailureStoreRollover) == false) {
+                final DataStream.DataStreamIndices targetIndices = dataStream.getDataStreamIndices(isFailureStoreRollover);
                 var noopResponse = noopLazyRolloverResponse(targetIndices);
                 notifyAllListeners(rolloverTaskContexts, context -> context.getTask().listener.onResponse(noopResponse));
                 return currentState;
@@ -237,7 +249,7 @@ public ClusterState executeTask(
             // Perform the actual rollover
             final var rolloverResult = rolloverService.rolloverClusterState(
                 currentState,
-                rolloverRequest.getRolloverTarget(),
+                resolvedRolloverTarget.resource(),
                 rolloverRequest.getNewIndexName(),
                 rolloverRequest.getCreateIndexRequest(),
                 List.of(),
@@ -246,7 +258,7 @@ public ClusterState executeTask(
                 false,
                 null,
                 null,
-                rolloverRequest.targetsFailureStore()
+                isFailureStoreRollover
             );
             results.add(rolloverResult);
             logger.trace("lazy rollover result [{}]", rolloverResult);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
index 552ce727d4249..608d32d50a856 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -16,7 +16,8 @@
 import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
-import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SelectorResolver;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.mapper.MapperService;
@@ -81,7 +82,7 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
     private RolloverConditions conditions = new RolloverConditions();
     // the index name "_na_" is never read back, what matters are settings, mappings and aliases
     private CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_");
-    private IndicesOptions indicesOptions = IndicesOptions.strictSingleIndexNoExpandForbidClosed();
+    private IndicesOptions indicesOptions = IndicesOptions.strictSingleIndexNoExpandForbidClosedAllowSelectors();
 
     public RolloverRequest(StreamInput in) throws IOException {
         super(in);
@@ -125,12 +126,15 @@ public ActionRequestValidationException validate() {
             );
         }
 
-        var selector = indicesOptions.selectorOptions().defaultSelector();
-        if (selector == IndexComponentSelector.ALL_APPLICABLE) {
-            validationException = addValidationError(
-                "rollover cannot be applied to both regular and failure indices at the same time",
-                validationException
-            );
+        if (rolloverTarget != null) {
+            ResolvedExpression resolvedExpression = SelectorResolver.parseExpression(rolloverTarget, indicesOptions);
+            IndexComponentSelector selector = resolvedExpression.selector();
+            if (IndexComponentSelector.ALL_APPLICABLE.equals(selector)) {
+                validationException = addValidationError(
+                    "rollover cannot be applied to both regular and failure indices at the same time",
+                    validationException
+                );
+            }
         }
 
         return validationException;
@@ -162,13 +166,6 @@ public IndicesOptions indicesOptions() {
         return indicesOptions;
     }
 
-    /**
-     * @return true of the rollover request targets the failure store, false otherwise.
-     */
-    public boolean targetsFailureStore() {
-        return DataStream.isFailureStoreFeatureFlagEnabled() && indicesOptions.includeFailureIndices();
-    }
-
     public void setIndicesOptions(IndicesOptions indicesOptions) {
         this.indicesOptions = indicesOptions;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index c5c874f9bcddf..4f0aa9c5bade4 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -36,6 +36,8 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexMetadataStats;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SelectorResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
@@ -149,8 +151,7 @@ protected ClusterBlockException checkBlock(RolloverRequest request, ClusterState
                 .matchOpen(request.indicesOptions().expandWildcardsOpen())
                 .matchClosed(request.indicesOptions().expandWildcardsClosed())
                 .build(),
-            IndicesOptions.GatekeeperOptions.DEFAULT,
-            request.indicesOptions().selectorOptions()
+            IndicesOptions.GatekeeperOptions.DEFAULT
         );
 
         return state.blocks()
@@ -170,11 +171,18 @@ protected void masterOperation(
 
         assert task instanceof CancellableTask;
         Metadata metadata = clusterState.metadata();
+
+        // Parse the rollover request's target since the expression it may contain a selector on it
+        ResolvedExpression resolvedRolloverTarget = SelectorResolver.parseExpression(
+            rolloverRequest.getRolloverTarget(),
+            rolloverRequest.indicesOptions()
+        );
+        boolean targetFailureStore = resolvedRolloverTarget.selector() != null && resolvedRolloverTarget.selector().shouldIncludeFailures();
+
         // We evaluate the names of the index for which we should evaluate conditions, as well as what our newly created index *would* be.
-        boolean targetFailureStore = rolloverRequest.targetsFailureStore();
         final MetadataRolloverService.NameResolution trialRolloverNames = MetadataRolloverService.resolveRolloverNames(
             clusterState,
-            rolloverRequest.getRolloverTarget(),
+            resolvedRolloverTarget.resource(),
             rolloverRequest.getNewIndexName(),
             rolloverRequest.getCreateIndexRequest(),
             targetFailureStore
@@ -183,7 +191,7 @@ protected void masterOperation(
         final String trialRolloverIndexName = trialRolloverNames.rolloverName();
         MetadataCreateIndexService.validateIndexName(trialRolloverIndexName, metadata, clusterState.routingTable());
 
-        boolean isDataStream = metadata.dataStreams().containsKey(rolloverRequest.getRolloverTarget());
+        boolean isDataStream = metadata.dataStreams().containsKey(resolvedRolloverTarget.resource());
         if (rolloverRequest.isLazy()) {
             if (isDataStream == false || rolloverRequest.getConditions().hasConditions()) {
                 String message;
@@ -201,7 +209,7 @@ protected void masterOperation(
             }
             if (rolloverRequest.isDryRun() == false) {
                 metadataDataStreamsService.setRolloverOnWrite(
-                    rolloverRequest.getRolloverTarget(),
+                    resolvedRolloverTarget.resource(),
                     true,
                     targetFailureStore,
                     rolloverRequest.ackTimeout(),
@@ -225,7 +233,7 @@ protected void masterOperation(
 
         final IndexAbstraction rolloverTargetAbstraction = clusterState.metadata()
             .getIndicesLookup()
-            .get(rolloverRequest.getRolloverTarget());
+            .get(resolvedRolloverTarget.resource());
         if (rolloverTargetAbstraction.getType() == IndexAbstraction.Type.ALIAS && rolloverTargetAbstraction.isDataStreamRelated()) {
             listener.onFailure(
                 new IllegalStateException("Aliases to data streams cannot be rolled over. Please rollover the data stream itself.")
@@ -246,10 +254,10 @@ protected void masterOperation(
         final var statsIndicesOptions = new IndicesOptions(
             IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS,
             IndicesOptions.WildcardOptions.builder().matchClosed(true).allowEmptyExpressions(false).build(),
-            IndicesOptions.GatekeeperOptions.DEFAULT,
-            rolloverRequest.indicesOptions().selectorOptions()
+            IndicesOptions.GatekeeperOptions.DEFAULT
         );
-        IndicesStatsRequest statsRequest = new IndicesStatsRequest().indices(rolloverRequest.getRolloverTarget())
+        // Make sure to recombine any selectors on the stats request
+        IndicesStatsRequest statsRequest = new IndicesStatsRequest().indices(resolvedRolloverTarget.combined())
             .clear()
             .indicesOptions(statsIndicesOptions)
             .docs(true)
@@ -266,9 +274,7 @@ protected void masterOperation(
             listener.delegateFailureAndWrap((delegate, statsResponse) -> {
 
                 AutoShardingResult rolloverAutoSharding = null;
-                final IndexAbstraction indexAbstraction = clusterState.metadata()
-                    .getIndicesLookup()
-                    .get(rolloverRequest.getRolloverTarget());
+                final IndexAbstraction indexAbstraction = clusterState.metadata().getIndicesLookup().get(resolvedRolloverTarget.resource());
                 if (indexAbstraction.getType().equals(IndexAbstraction.Type.DATA_STREAM)) {
                     DataStream dataStream = (DataStream) indexAbstraction;
                     final Optional<IndexStats> indexStats = Optional.ofNullable(statsResponse)
@@ -492,14 +498,20 @@ public ClusterState executeTask(
         ) throws Exception {
             final var rolloverTask = rolloverTaskContext.getTask();
             final var rolloverRequest = rolloverTask.rolloverRequest();
+            ResolvedExpression resolvedRolloverTarget = SelectorResolver.parseExpression(
+                rolloverRequest.getRolloverTarget(),
+                rolloverRequest.indicesOptions()
+            );
+            boolean targetFailureStore = resolvedRolloverTarget.selector() != null
+                && resolvedRolloverTarget.selector().shouldIncludeFailures();
 
             // Regenerate the rollover names, as a rollover could have happened in between the pre-check and the cluster state update
             final var rolloverNames = MetadataRolloverService.resolveRolloverNames(
                 currentState,
-                rolloverRequest.getRolloverTarget(),
+                resolvedRolloverTarget.resource(),
                 rolloverRequest.getNewIndexName(),
                 rolloverRequest.getCreateIndexRequest(),
-                rolloverRequest.targetsFailureStore()
+                targetFailureStore
             );
 
             // Re-evaluate the conditions, now with our final source index name
@@ -532,7 +544,7 @@ public ClusterState executeTask(
 
                 final IndexAbstraction rolloverTargetAbstraction = currentState.metadata()
                     .getIndicesLookup()
-                    .get(rolloverRequest.getRolloverTarget());
+                    .get(resolvedRolloverTarget.resource());
 
                 final IndexMetadataStats sourceIndexStats = rolloverTargetAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM
                     ? IndexMetadataStats.fromStatsResponse(rolloverSourceIndex, rolloverTask.statsResponse())
@@ -541,7 +553,7 @@ public ClusterState executeTask(
                 // Perform the actual rollover
                 final var rolloverResult = rolloverService.rolloverClusterState(
                     currentState,
-                    rolloverRequest.getRolloverTarget(),
+                    resolvedRolloverTarget.resource(),
                     rolloverRequest.getNewIndexName(),
                     rolloverRequest.getCreateIndexRequest(),
                     metConditions,
@@ -550,7 +562,7 @@ public ClusterState executeTask(
                     false,
                     sourceIndexStats,
                     rolloverTask.autoShardingResult(),
-                    rolloverRequest.targetsFailureStore()
+                    targetFailureStore
                 );
                 results.add(rolloverResult);
                 logger.trace("rollover result [{}]", rolloverResult);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java
index 45d784d301bf1..d34e71f715a5d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java
@@ -78,9 +78,7 @@ protected Iterator<ToXContent> customXContentChunks(ToXContent.Params params) {
                 getIndices().values().iterator(),
                 indexSegments -> Iterators.concat(
 
-                    ChunkedToXContentHelper.singleChunk(
-                        (builder, p) -> builder.startObject(indexSegments.getIndex()).startObject(Fields.SHARDS)
-                    ),
+                    ChunkedToXContentHelper.chunk((builder, p) -> builder.startObject(indexSegments.getIndex()).startObject(Fields.SHARDS)),
                     Iterators.flatMap(
                         indexSegments.iterator(),
                         indexSegment -> Iterators.concat(
@@ -90,7 +88,7 @@ protected Iterator<ToXContent> customXContentChunks(ToXContent.Params params) {
                                 indexSegment.iterator(),
                                 shardSegments -> Iterators.concat(
 
-                                    ChunkedToXContentHelper.singleChunk((builder, p) -> {
+                                    ChunkedToXContentHelper.chunk((builder, p) -> {
                                         builder.startObject();
 
                                         builder.startObject(Fields.ROUTING);
@@ -112,7 +110,7 @@ protected Iterator<ToXContent> customXContentChunks(ToXContent.Params params) {
                                         shardSegments.iterator(),
                                         segment -> Iterators.concat(
 
-                                            ChunkedToXContentHelper.singleChunk((builder, p) -> {
+                                            ChunkedToXContentHelper.chunk((builder, p) -> {
                                                 builder.startObject(segment.getName());
                                                 builder.field(Fields.GENERATION, segment.getGeneration());
                                                 builder.field(Fields.NUM_DOCS, segment.getNumDocs());
@@ -132,7 +130,7 @@ protected Iterator<ToXContent> customXContentChunks(ToXContent.Params params) {
                                                 return builder;
                                             }),
                                             getSegmentSortChunks(segment.getSegmentSort()),
-                                            ChunkedToXContentHelper.singleChunk((builder, p) -> {
+                                            ChunkedToXContentHelper.chunk((builder, p) -> {
                                                 if (segment.attributes != null && segment.attributes.isEmpty() == false) {
                                                     builder.field("attributes", segment.attributes);
                                                 }
@@ -141,13 +139,13 @@ protected Iterator<ToXContent> customXContentChunks(ToXContent.Params params) {
                                             })
                                         )
                                     ),
-                                    ChunkedToXContentHelper.singleChunk((builder, p) -> builder.endObject().endObject())
+                                    ChunkedToXContentHelper.chunk((builder, p) -> builder.endObject().endObject())
                                 )
                             ),
                             ChunkedToXContentHelper.endArray()
                         )
                     ),
-                    ChunkedToXContentHelper.singleChunk((builder, p) -> builder.endObject().endObject())
+                    ChunkedToXContentHelper.chunk((builder, p) -> builder.endObject().endObject())
                 )
             ),
             ChunkedToXContentHelper.endObject()
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java
index 50b46aa5e284f..dfb22a4f0cbae 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java
@@ -50,9 +50,9 @@ protected Iterator<ToXContent> customXContentChunks(ToXContent.Params params) {
         return Iterators.flatMap(
             stats.entrySet().stream().sorted(Map.Entry.comparingByKey()).iterator(),
             entry -> Iterators.concat(
-                ChunkedToXContentHelper.singleChunk((builder, p) -> builder.startObject(entry.getKey()).startArray("shards")),
+                ChunkedToXContentHelper.chunk((builder, p) -> builder.startObject(entry.getKey()).startArray("shards")),
                 entry.getValue().iterator(),
-                ChunkedToXContentHelper.singleChunk((builder, p) -> builder.endArray().endObject())
+                ChunkedToXContentHelper.chunk((builder, p) -> builder.endArray().endObject())
             )
         );
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java
index 5bdecd10075e6..6106e620521f7 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.features.NodeFeature;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -22,9 +21,6 @@
 
 public class IndexStats implements Iterable<IndexShardStats> {
 
-    // feature was effectively reverted but we still need to keep this constant around
-    public static final NodeFeature REVERTED_TIER_CREATION_DATE = new NodeFeature("stats.tier_creation_date");
-
     private final String index;
 
     private final String uuid;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsFeatures.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsFeatures.java
deleted file mode 100644
index 558343db1023a..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsFeatures.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.action.admin.indices.stats;
-
-import org.elasticsearch.features.FeatureSpecification;
-import org.elasticsearch.features.NodeFeature;
-
-import java.util.Set;
-
-public class IndicesStatsFeatures implements FeatureSpecification {
-
-    @Override
-    public Set<NodeFeature> getFeatures() {
-        return Set.of(IndexStats.REVERTED_TIER_CREATION_DATE);
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java
index 91e0e7cbc1dff..d6c9a3e2e544b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java
@@ -204,7 +204,7 @@ protected Iterator<ToXContent> customXContentChunks(ToXContent.Params params) {
         if (level == ClusterStatsLevel.INDICES || level == ClusterStatsLevel.SHARDS) {
             return Iterators.concat(
 
-                ChunkedToXContentHelper.singleChunk((builder, p) -> {
+                ChunkedToXContentHelper.chunk((builder, p) -> {
                     commonStats(builder, p);
                     return builder.startObject(Fields.INDICES);
                 }),
@@ -212,7 +212,7 @@ protected Iterator<ToXContent> customXContentChunks(ToXContent.Params params) {
                     getIndices().values().iterator(),
                     indexStats -> Iterators.concat(
 
-                        ChunkedToXContentHelper.singleChunk((builder, p) -> {
+                        ChunkedToXContentHelper.chunk((builder, p) -> {
                             builder.startObject(indexStats.getIndex());
                             builder.field("uuid", indexStats.getUuid());
                             if (indexStats.getHealth() != null) {
@@ -257,7 +257,7 @@ protected Iterator<ToXContent> customXContentChunks(ToXContent.Params params) {
                 ChunkedToXContentHelper.endObject()
             );
         } else {
-            return ChunkedToXContentHelper.singleChunk((builder, p) -> {
+            return ChunkedToXContentHelper.chunk((builder, p) -> {
                 commonStats(builder, p);
                 return builder;
             });
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java
index f729455edcc24..4f61b89aeaf5e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java
@@ -14,12 +14,17 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration;
-import org.elasticsearch.action.support.master.MasterNodeReadRequest;
+import org.elasticsearch.action.support.local.LocalClusterStateRequest;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV10;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -43,22 +48,23 @@ private GetComponentTemplateAction() {
     /**
      * Request that to retrieve one or more component templates
      */
-    public static class Request extends MasterNodeReadRequest<Request> {
+    public static class Request extends LocalClusterStateRequest {
 
         @Nullable
         private String name;
         private boolean includeDefaults;
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
-        }
-
-        public Request(String name) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterTimeout, String name) {
+            super(masterTimeout);
             this.name = name;
             this.includeDefaults = false;
         }
 
+        /**
+         * NB prior to 9.0 get-component was a TransportMasterNodeReadAction so for BwC we must remain able to read these requests until
+         * we no longer need to support calling this action remotely.
+         */
+        @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
         public Request(StreamInput in) throws IOException {
             super(in);
             name = in.readOptionalString();
@@ -70,17 +76,13 @@ public Request(StreamInput in) throws IOException {
         }
 
         @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeOptionalString(name);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) {
-                out.writeBoolean(includeDefaults);
-            }
+        public ActionRequestValidationException validate() {
+            return null;
         }
 
         @Override
-        public ActionRequestValidationException validate() {
-            return null;
+        public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+            return new CancellableTask(id, type, action, "", parentTaskId, headers);
         }
 
         /**
@@ -123,19 +125,6 @@ public static class Response extends ActionResponse implements ToXContentObject
         @Nullable
         private final RolloverConfiguration rolloverConfiguration;
 
-        public Response(StreamInput in) throws IOException {
-            super(in);
-            componentTemplates = in.readMap(ComponentTemplate::new);
-            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) {
-                rolloverConfiguration = in.readOptionalWriteable(RolloverConfiguration::new);
-            } else {
-                rolloverConfiguration = null;
-            }
-            if (in.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) {
-                in.readOptionalWriteable(DataStreamGlobalRetention::read);
-            }
-        }
-
         /**
          * Please use {@link GetComponentTemplateAction.Response#Response(Map)}
          */
@@ -183,6 +172,11 @@ public DataStreamGlobalRetention getGlobalRetention() {
             return null;
         }
 
+        /**
+         * NB prior to 9.0 get-component was a TransportMasterNodeReadAction so for BwC we must remain able to write these responses until
+         * we no longer need to support calling this action remotely.
+         */
+        @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeMap(componentTemplates, StreamOutput::writeWriteable);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java
index 67f87476ea6a5..5d322bba17c5e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java
@@ -14,12 +14,17 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration;
-import org.elasticsearch.action.support.master.MasterNodeReadRequest;
+import org.elasticsearch.action.support.local.LocalClusterStateRequest;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV10;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -40,17 +45,18 @@ private GetComposableIndexTemplateAction() {
     /**
      * Request that to retrieve one or more index templates
      */
-    public static class Request extends MasterNodeReadRequest<Request> {
+    public static class Request extends LocalClusterStateRequest {
 
         @Nullable
         private final String name;
         private boolean includeDefaults;
 
         /**
+         * @param masterTimeout Timeout for waiting for new cluster state in case it is blocked.
          * @param name A template name or pattern, or {@code null} to retrieve all templates.
          */
-        public Request(@Nullable String name) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterTimeout, @Nullable String name) {
+            super(masterTimeout);
             if (name != null && name.contains(",")) {
                 throw new IllegalArgumentException("template name may not contain ','");
             }
@@ -58,6 +64,11 @@ public Request(@Nullable String name) {
             this.includeDefaults = false;
         }
 
+        /**
+         * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to read these requests until
+         * we no longer need to support calling this action remotely.
+         */
+        @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
         public Request(StreamInput in) throws IOException {
             super(in);
             name = in.readOptionalString();
@@ -68,15 +79,6 @@ public Request(StreamInput in) throws IOException {
             }
         }
 
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeOptionalString(name);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) {
-                out.writeBoolean(includeDefaults);
-            }
-        }
-
         public void includeDefaults(boolean includeDefaults) {
             this.includeDefaults = includeDefaults;
         }
@@ -90,6 +92,11 @@ public ActionRequestValidationException validate() {
             return null;
         }
 
+        @Override
+        public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+            return new CancellableTask(id, type, action, "", parentTaskId, headers);
+        }
+
         /**
          * The name of the index templates.
          */
@@ -124,19 +131,6 @@ public static class Response extends ActionResponse implements ToXContentObject
         @Nullable
         private final RolloverConfiguration rolloverConfiguration;
 
-        public Response(StreamInput in) throws IOException {
-            super(in);
-            indexTemplates = in.readMap(ComposableIndexTemplate::new);
-            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) {
-                rolloverConfiguration = in.readOptionalWriteable(RolloverConfiguration::new);
-            } else {
-                rolloverConfiguration = null;
-            }
-            if (in.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) {
-                in.readOptionalWriteable(DataStreamGlobalRetention::read);
-            }
-        }
-
         /**
          * Please use {@link GetComposableIndexTemplateAction.Response#Response(Map)}
          */
@@ -184,6 +178,11 @@ public RolloverConfiguration getRolloverConfiguration() {
             return rolloverConfiguration;
         }
 
+        /**
+         * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to write these responses until
+         * we no longer need to support calling this action remotely.
+         */
+        @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeMap(indexTemplates, StreamOutput::writeWriteable);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java
index e3f765eb8ae88..1c911b4a0a97c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java
@@ -9,38 +9,42 @@
 package org.elasticsearch.action.admin.indices.template.get;
 
 import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.support.master.MasterNodeReadRequest;
+import org.elasticsearch.action.support.local.LocalClusterStateRequest;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV10;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
+import java.util.Map;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
 /**
  * Request that allows to retrieve index templates
  */
-public class GetIndexTemplatesRequest extends MasterNodeReadRequest<GetIndexTemplatesRequest> {
+public class GetIndexTemplatesRequest extends LocalClusterStateRequest {
 
     private final String[] names;
 
-    public GetIndexTemplatesRequest(String... names) {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+    public GetIndexTemplatesRequest(TimeValue masterTimeout, String... names) {
+        super(masterTimeout);
         this.names = names;
     }
 
+    /**
+     * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to read these requests until
+     * we no longer need to support calling this action remotely.
+     */
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
     public GetIndexTemplatesRequest(StreamInput in) throws IOException {
         super(in);
         names = in.readStringArray();
     }
 
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeStringArray(names);
-    }
-
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
@@ -56,6 +60,11 @@ public ActionRequestValidationException validate() {
         return validationException;
     }
 
+    @Override
+    public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+        return new CancellableTask(id, type, action, "", parentTaskId, headers);
+    }
+
     /**
      * The names of the index templates.
      */
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequestBuilder.java
index 97aabbd4631b7..ed53154cc5c93 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequestBuilder.java
@@ -8,15 +8,13 @@
  */
 package org.elasticsearch.action.admin.indices.template.get;
 
-import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
+import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.internal.ElasticsearchClient;
+import org.elasticsearch.core.TimeValue;
 
-public class GetIndexTemplatesRequestBuilder extends MasterNodeReadOperationRequestBuilder<
-    GetIndexTemplatesRequest,
-    GetIndexTemplatesResponse,
-    GetIndexTemplatesRequestBuilder> {
+public class GetIndexTemplatesRequestBuilder extends ActionRequestBuilder<GetIndexTemplatesRequest, GetIndexTemplatesResponse> {
 
-    public GetIndexTemplatesRequestBuilder(ElasticsearchClient client, String... names) {
-        super(client, GetIndexTemplatesAction.INSTANCE, new GetIndexTemplatesRequest(names));
+    public GetIndexTemplatesRequestBuilder(ElasticsearchClient client, TimeValue masterTimeout, String... names) {
+        super(client, GetIndexTemplatesAction.INSTANCE, new GetIndexTemplatesRequest(masterTimeout, names));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
index 2d854d2c6fa45..69ae3f1db92ae 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
@@ -10,8 +10,8 @@
 
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -26,11 +26,6 @@ public class GetIndexTemplatesResponse extends ActionResponse implements ToXCont
 
     private final List<IndexTemplateMetadata> indexTemplates;
 
-    public GetIndexTemplatesResponse(StreamInput in) throws IOException {
-        super(in);
-        indexTemplates = in.readCollectionAsList(IndexTemplateMetadata::readFrom);
-    }
-
     public GetIndexTemplatesResponse(List<IndexTemplateMetadata> indexTemplates) {
         this.indexTemplates = indexTemplates;
     }
@@ -39,6 +34,11 @@ public List<IndexTemplateMetadata> getIndexTemplates() {
         return indexTemplates;
     }
 
+    /**
+     * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to write these responses until
+     * we no longer need to support calling this action remotely.
+     */
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeCollection(indexTemplates);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java
index d04ccd39be04b..40e1a988dc2f2 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java
@@ -12,51 +12,61 @@
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
+import org.elasticsearch.action.support.ChannelActionListener;
+import org.elasticsearch.action.support.local.TransportLocalClusterStateAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.util.HashMap;
 import java.util.Map;
 
-public class TransportGetComponentTemplateAction extends TransportMasterNodeReadAction<
+public class TransportGetComponentTemplateAction extends TransportLocalClusterStateAction<
     GetComponentTemplateAction.Request,
     GetComponentTemplateAction.Response> {
 
     private final ClusterSettings clusterSettings;
 
+    /**
+     * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until
+     * we no longer need to support calling this action remotely.
+     */
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
+    @SuppressWarnings("this-escape")
     @Inject
     public TransportGetComponentTemplateAction(
         TransportService transportService,
         ClusterService clusterService,
-        ThreadPool threadPool,
-        ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver
+        ActionFilters actionFilters
     ) {
         super(
             GetComponentTemplateAction.NAME,
-            transportService,
-            clusterService,
-            threadPool,
             actionFilters,
-            GetComponentTemplateAction.Request::new,
-            indexNameExpressionResolver,
-            GetComponentTemplateAction.Response::new,
+            transportService.getTaskManager(),
+            clusterService,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
         clusterSettings = clusterService.getClusterSettings();
+
+        transportService.registerRequestHandler(
+            actionName,
+            executor,
+            false,
+            true,
+            GetComponentTemplateAction.Request::new,
+            (request, channel, task) -> executeDirect(task, request, new ChannelActionListener<>(channel))
+        );
     }
 
     @Override
@@ -65,12 +75,13 @@ protected ClusterBlockException checkBlock(GetComponentTemplateAction.Request re
     }
 
     @Override
-    protected void masterOperation(
+    protected void localClusterStateOperation(
         Task task,
         GetComponentTemplateAction.Request request,
         ClusterState state,
         ActionListener<GetComponentTemplateAction.Response> listener
     ) {
+        final var cancellableTask = (CancellableTask) task;
         Map<String, ComponentTemplate> allTemplates = state.metadata().componentTemplates();
         Map<String, ComponentTemplate> results;
 
@@ -93,6 +104,7 @@ protected void masterOperation(
 
             }
         }
+        cancellableTask.ensureNotCancelled();
         if (request.includeDefaults()) {
             listener.onResponse(
                 new GetComponentTemplateAction.Response(
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java
index 608e7529d34bf..3d5800de2cb3a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java
@@ -12,51 +12,61 @@
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
+import org.elasticsearch.action.support.ChannelActionListener;
+import org.elasticsearch.action.support.local.TransportLocalClusterStateAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.util.HashMap;
 import java.util.Map;
 
-public class TransportGetComposableIndexTemplateAction extends TransportMasterNodeReadAction<
+public class TransportGetComposableIndexTemplateAction extends TransportLocalClusterStateAction<
     GetComposableIndexTemplateAction.Request,
     GetComposableIndexTemplateAction.Response> {
 
     private final ClusterSettings clusterSettings;
 
+    /**
+     * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until
+     * we no longer need to support calling this action remotely.
+     */
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
+    @SuppressWarnings("this-escape")
     @Inject
     public TransportGetComposableIndexTemplateAction(
         TransportService transportService,
         ClusterService clusterService,
-        ThreadPool threadPool,
-        ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver
+        ActionFilters actionFilters
     ) {
         super(
             GetComposableIndexTemplateAction.NAME,
-            transportService,
-            clusterService,
-            threadPool,
             actionFilters,
-            GetComposableIndexTemplateAction.Request::new,
-            indexNameExpressionResolver,
-            GetComposableIndexTemplateAction.Response::new,
+            transportService.getTaskManager(),
+            clusterService,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
         clusterSettings = clusterService.getClusterSettings();
+
+        transportService.registerRequestHandler(
+            actionName,
+            executor,
+            false,
+            true,
+            GetComposableIndexTemplateAction.Request::new,
+            (request, channel, task) -> executeDirect(task, request, new ChannelActionListener<>(channel))
+        );
     }
 
     @Override
@@ -65,12 +75,13 @@ protected ClusterBlockException checkBlock(GetComposableIndexTemplateAction.Requ
     }
 
     @Override
-    protected void masterOperation(
+    protected void localClusterStateOperation(
         Task task,
         GetComposableIndexTemplateAction.Request request,
         ClusterState state,
         ActionListener<GetComposableIndexTemplateAction.Response> listener
     ) {
+        final var cancellableTask = (CancellableTask) task;
         Map<String, ComposableIndexTemplate> allTemplates = state.metadata().templatesV2();
         Map<String, ComposableIndexTemplate> results;
         // If we did not ask for a specific name, then we return all templates
@@ -91,6 +102,7 @@ protected void masterOperation(
                 throw new ResourceNotFoundException("index template matching [" + request.name() + "] not found");
             }
         }
+        cancellableTask.ensureNotCancelled();
         if (request.includeDefaults()) {
             listener.onResponse(
                 new GetComposableIndexTemplateAction.Response(
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java
index 99763d4d76dc6..c53ce4a6841ab 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java
@@ -10,45 +10,53 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
+import org.elasticsearch.action.support.ChannelActionListener;
+import org.elasticsearch.action.support.local.TransportLocalClusterStateAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
-public class TransportGetIndexTemplatesAction extends TransportMasterNodeReadAction<GetIndexTemplatesRequest, GetIndexTemplatesResponse> {
+public class TransportGetIndexTemplatesAction extends TransportLocalClusterStateAction<
+    GetIndexTemplatesRequest,
+    GetIndexTemplatesResponse> {
 
+    /**
+     * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until
+     * we no longer need to support calling this action remotely.
+     */
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
+    @SuppressWarnings("this-escape")
     @Inject
-    public TransportGetIndexTemplatesAction(
-        TransportService transportService,
-        ClusterService clusterService,
-        ThreadPool threadPool,
-        ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver
-    ) {
+    public TransportGetIndexTemplatesAction(TransportService transportService, ClusterService clusterService, ActionFilters actionFilters) {
         super(
             GetIndexTemplatesAction.NAME,
-            transportService,
-            clusterService,
-            threadPool,
             actionFilters,
-            GetIndexTemplatesRequest::new,
-            indexNameExpressionResolver,
-            GetIndexTemplatesResponse::new,
+            transportService.getTaskManager(),
+            clusterService,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
+
+        transportService.registerRequestHandler(
+            actionName,
+            executor,
+            false,
+            true,
+            GetIndexTemplatesRequest::new,
+            (request, channel, task) -> executeDirect(task, request, new ChannelActionListener<>(channel))
+        );
     }
 
     @Override
@@ -57,12 +65,13 @@ protected ClusterBlockException checkBlock(GetIndexTemplatesRequest request, Clu
     }
 
     @Override
-    protected void masterOperation(
+    protected void localClusterStateOperation(
         Task task,
         GetIndexTemplatesRequest request,
         ClusterState state,
         ActionListener<GetIndexTemplatesResponse> listener
     ) {
+        final var cancellableTask = (CancellableTask) task;
         List<IndexTemplateMetadata> results;
 
         // If we did not ask for a specific name, then we return all templates
@@ -84,6 +93,7 @@ protected void masterOperation(
             }
         }
 
+        cancellableTask.ensureNotCancelled();
         listener.onResponse(new GetIndexTemplatesResponse(results));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java
index ce29d65ececf9..003be58d19554 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java
@@ -12,16 +12,20 @@
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
-import org.elasticsearch.action.support.master.MasterNodeReadRequest;
+import org.elasticsearch.action.support.local.LocalClusterStateRequest;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
+import java.util.Map;
 import java.util.Objects;
 
-public class SimulateIndexTemplateRequest extends MasterNodeReadRequest<SimulateIndexTemplateRequest> {
+public class SimulateIndexTemplateRequest extends LocalClusterStateRequest {
 
     private String indexName;
 
@@ -30,14 +34,18 @@ public class SimulateIndexTemplateRequest extends MasterNodeReadRequest<Simulate
 
     private boolean includeDefaults = false;
 
-    public SimulateIndexTemplateRequest(String indexName) {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+    public SimulateIndexTemplateRequest(TimeValue masterTimeout, String indexName) {
+        super(masterTimeout);
         if (Strings.isNullOrEmpty(indexName)) {
             throw new IllegalArgumentException("index name cannot be null or empty");
         }
         this.indexName = indexName;
     }
 
+    /**
+     * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to read these requests until
+     * we no longer need to support calling this action remotely.
+     */
     public SimulateIndexTemplateRequest(StreamInput in) throws IOException {
         super(in);
         indexName = in.readString();
@@ -47,16 +55,6 @@ public SimulateIndexTemplateRequest(StreamInput in) throws IOException {
         }
     }
 
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeString(indexName);
-        out.writeOptionalWriteable(indexTemplateRequest);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) {
-            out.writeBoolean(includeDefaults);
-        }
-    }
-
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
@@ -66,6 +64,11 @@ public ActionRequestValidationException validate() {
         return validationException;
     }
 
+    @Override
+    public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+        return new CancellableTask(id, type, action, "", parentTaskId, headers);
+    }
+
     public String getIndexName() {
         return indexName;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java
index a521dac60e96a..1a04b6e4d7633 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java
@@ -12,13 +12,11 @@
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
 import org.elasticsearch.cluster.metadata.ResettableValue;
 import org.elasticsearch.cluster.metadata.Template;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -67,27 +65,11 @@ public RolloverConfiguration getRolloverConfiguration() {
         return rolloverConfiguration;
     }
 
-    public SimulateIndexTemplateResponse(StreamInput in) throws IOException {
-        super(in);
-        resolvedTemplate = in.readOptionalWriteable(Template::new);
-        if (in.readBoolean()) {
-            int overlappingTemplatesCount = in.readInt();
-            overlappingTemplates = Maps.newMapWithExpectedSize(overlappingTemplatesCount);
-            for (int i = 0; i < overlappingTemplatesCount; i++) {
-                String templateName = in.readString();
-                overlappingTemplates.put(templateName, in.readStringCollectionAsList());
-            }
-        } else {
-            this.overlappingTemplates = null;
-        }
-        rolloverConfiguration = in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)
-            ? in.readOptionalWriteable(RolloverConfiguration::new)
-            : null;
-        if (in.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) {
-            in.readOptionalWriteable(DataStreamGlobalRetention::read);
-        }
-    }
-
+    /**
+     * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to write these responses until
+     * we no longer need to support calling this action remotely.
+     */
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(resolvedTemplate);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java
index 75cc72416a854..15015b910767e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java
@@ -14,12 +14,16 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
-import org.elasticsearch.action.support.master.MasterNodeReadRequest;
+import org.elasticsearch.action.support.local.LocalClusterStateRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
+import java.util.Map;
 import java.util.Objects;
 
 /**
@@ -35,7 +39,7 @@ private SimulateTemplateAction() {
         super(NAME);
     }
 
-    public static class Request extends MasterNodeReadRequest<Request> {
+    public static class Request extends LocalClusterStateRequest {
 
         @Nullable
         private String templateName;
@@ -44,26 +48,15 @@ public static class Request extends MasterNodeReadRequest<Request> {
         private TransportPutComposableIndexTemplateAction.Request indexTemplateRequest;
         private boolean includeDefaults = false;
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
-        }
-
-        public Request(String templateName) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
-            if (templateName == null) {
-                throw new IllegalArgumentException("template name cannot be null");
-            }
+        public Request(TimeValue masterTimeout, String templateName) {
+            super(masterTimeout);
             this.templateName = templateName;
         }
 
-        public Request(TransportPutComposableIndexTemplateAction.Request indexTemplateRequest) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
-            if (indexTemplateRequest == null) {
-                throw new IllegalArgumentException("index template body must be present");
-            }
-            this.indexTemplateRequest = indexTemplateRequest;
-        }
-
+        /**
+         * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to read these requests until
+         * we no longer need to support calling this action remotely.
+         */
         public Request(StreamInput in) throws IOException {
             super(in);
             templateName = in.readOptionalString();
@@ -73,16 +66,6 @@ public Request(StreamInput in) throws IOException {
             }
         }
 
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeOptionalString(templateName);
-            out.writeOptionalWriteable(indexTemplateRequest);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) {
-                out.writeBoolean(includeDefaults);
-            }
-        }
-
         @Override
         public ActionRequestValidationException validate() {
             ActionRequestValidationException validationException = null;
@@ -98,6 +81,11 @@ public ActionRequestValidationException validate() {
             return validationException;
         }
 
+        @Override
+        public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+            return new CancellableTask(id, type, action, "", parentTaskId, headers);
+        }
+
         @Nullable
         public String getTemplateName() {
             return templateName;
@@ -112,11 +100,6 @@ public TransportPutComposableIndexTemplateAction.Request getIndexTemplateRequest
             return indexTemplateRequest;
         }
 
-        public Request templateName(String templateName) {
-            this.templateName = templateName;
-            return this;
-        }
-
         public Request indexTemplateRequest(TransportPutComposableIndexTemplateAction.Request indexTemplateRequest) {
             this.indexTemplateRequest = indexTemplateRequest;
             return this;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java
index 5f98852148ed4..74936128caa25 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java
@@ -11,7 +11,8 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
+import org.elasticsearch.action.support.ChannelActionListener;
+import org.elasticsearch.action.support.local.TransportLocalClusterStateAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
@@ -20,7 +21,6 @@
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
@@ -31,6 +31,7 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexSettingProviders;
@@ -42,7 +43,6 @@
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
@@ -65,7 +65,7 @@
 import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveLifecycle;
 import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveSettings;
 
-public class TransportSimulateIndexTemplateAction extends TransportMasterNodeReadAction<
+public class TransportSimulateIndexTemplateAction extends TransportLocalClusterStateAction<
     SimulateIndexTemplateRequest,
     SimulateIndexTemplateResponse> {
 
@@ -77,14 +77,18 @@ public class TransportSimulateIndexTemplateAction extends TransportMasterNodeRea
     private final ClusterSettings clusterSettings;
     private final boolean isDslOnlyMode;
 
+    /**
+     * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until
+     * we no longer need to support calling this action remotely.
+     */
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
+    @SuppressWarnings("this-escape")
     @Inject
     public TransportSimulateIndexTemplateAction(
         TransportService transportService,
         ClusterService clusterService,
-        ThreadPool threadPool,
         MetadataIndexTemplateService indexTemplateService,
         ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver,
         NamedXContentRegistry xContentRegistry,
         IndicesService indicesService,
         SystemIndices systemIndices,
@@ -92,13 +96,9 @@ public TransportSimulateIndexTemplateAction(
     ) {
         super(
             SimulateIndexTemplateAction.NAME,
-            transportService,
-            clusterService,
-            threadPool,
             actionFilters,
-            SimulateIndexTemplateRequest::new,
-            indexNameExpressionResolver,
-            SimulateIndexTemplateResponse::new,
+            transportService.getTaskManager(),
+            clusterService,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
         this.indexTemplateService = indexTemplateService;
@@ -108,10 +108,19 @@ public TransportSimulateIndexTemplateAction(
         this.indexSettingProviders = indexSettingProviders.getIndexSettingProviders();
         this.clusterSettings = clusterService.getClusterSettings();
         this.isDslOnlyMode = isDataStreamsLifecycleOnlyMode(clusterService.getSettings());
+
+        transportService.registerRequestHandler(
+            actionName,
+            executor,
+            false,
+            true,
+            SimulateIndexTemplateRequest::new,
+            (request, channel, task) -> executeDirect(task, request, new ChannelActionListener<>(channel))
+        );
     }
 
     @Override
-    protected void masterOperation(
+    protected void localClusterStateOperation(
         Task task,
         SimulateIndexTemplateRequest request,
         ClusterState state,
@@ -214,8 +223,7 @@ public static ClusterState resolveTemporaryState(
             .build();
 
         final IndexMetadata indexMetadata = IndexMetadata.builder(indexName)
-            // handle mixed-cluster states by passing in minTransportVersion to reset event.ingested range to UNKNOWN if an older version
-            .eventIngestedRange(getEventIngestedRange(indexName, simulatedState), simulatedState.getMinTransportVersion())
+            .eventIngestedRange(getEventIngestedRange(indexName, simulatedState))
             .settings(dummySettings)
             .build();
         return ClusterState.builder(simulatedState)
@@ -304,8 +312,7 @@ public static Template resolveTemplate(
         dummySettings.put(templateSettings);
 
         final IndexMetadata indexMetadata = IndexMetadata.builder(indexName)
-            // handle mixed-cluster states by passing in minTransportVersion to reset event.ingested range to UNKNOWN if an older version
-            .eventIngestedRange(getEventIngestedRange(indexName, simulatedState), simulatedState.getMinTransportVersion())
+            .eventIngestedRange(getEventIngestedRange(indexName, simulatedState))
             .settings(dummySettings)
             .build();
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java
index 30bbad0b57df0..692f027b23f9e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java
@@ -11,26 +11,26 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
+import org.elasticsearch.action.support.ChannelActionListener;
+import org.elasticsearch.action.support.local.TransportLocalClusterStateAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexSettingProviders;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
@@ -48,7 +48,7 @@
  * Handles simulating an index template either by name (looking it up in the
  * cluster state), or by a provided template configuration
  */
-public class TransportSimulateTemplateAction extends TransportMasterNodeReadAction<
+public class TransportSimulateTemplateAction extends TransportLocalClusterStateAction<
     SimulateTemplateAction.Request,
     SimulateIndexTemplateResponse> {
 
@@ -60,14 +60,18 @@ public class TransportSimulateTemplateAction extends TransportMasterNodeReadActi
     private final ClusterSettings clusterSettings;
     private final boolean isDslOnlyMode;
 
+    /**
+     * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until
+     * we no longer need to support calling this action remotely.
+     */
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
+    @SuppressWarnings("this-escape")
     @Inject
     public TransportSimulateTemplateAction(
         TransportService transportService,
         ClusterService clusterService,
-        ThreadPool threadPool,
         MetadataIndexTemplateService indexTemplateService,
         ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver,
         NamedXContentRegistry xContentRegistry,
         IndicesService indicesService,
         SystemIndices systemIndices,
@@ -75,13 +79,9 @@ public TransportSimulateTemplateAction(
     ) {
         super(
             SimulateTemplateAction.NAME,
-            transportService,
-            clusterService,
-            threadPool,
             actionFilters,
-            SimulateTemplateAction.Request::new,
-            indexNameExpressionResolver,
-            SimulateIndexTemplateResponse::new,
+            transportService.getTaskManager(),
+            clusterService,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
         this.indexTemplateService = indexTemplateService;
@@ -91,10 +91,19 @@ public TransportSimulateTemplateAction(
         this.indexSettingProviders = indexSettingProviders.getIndexSettingProviders();
         this.clusterSettings = clusterService.getClusterSettings();
         this.isDslOnlyMode = isDataStreamsLifecycleOnlyMode(clusterService.getSettings());
+
+        transportService.registerRequestHandler(
+            actionName,
+            executor,
+            false,
+            true,
+            SimulateTemplateAction.Request::new,
+            (request, channel, task) -> executeDirect(task, request, new ChannelActionListener<>(channel))
+        );
     }
 
     @Override
-    protected void masterOperation(
+    protected void localClusterStateOperation(
         Task task,
         SimulateTemplateAction.Request request,
         ClusterState state,
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java
index 62a9b88cb6a57..5851549977eab 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java
@@ -14,22 +14,10 @@
 
 import java.util.Set;
 
-import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS;
-import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS;
-import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_ADDITION;
-import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION;
-import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION_TEMPLATES;
-import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING;
+import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_IGNORED_FIELDS;
 
 public class BulkFeatures implements FeatureSpecification {
     public Set<NodeFeature> getFeatures() {
-        return Set.of(
-            SIMULATE_MAPPING_VALIDATION,
-            SIMULATE_MAPPING_VALIDATION_TEMPLATES,
-            SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS,
-            SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS,
-            SIMULATE_MAPPING_ADDITION,
-            SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING
-        );
+        return Set.of(SIMULATE_IGNORED_FIELDS);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java
index 4df228240add5..dd473869fb2d9 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.client.internal.node.NodeClient;
@@ -33,6 +33,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -98,6 +99,7 @@ final class BulkOperation extends ActionRunnable<BulkResponse> {
     private final Set<Integer> failedRolloverRequests = ConcurrentCollections.newConcurrentSet();
     private final Map<ShardId, Exception> shortCircuitShardFailures = ConcurrentCollections.newConcurrentMap();
     private final FailureStoreMetrics failureStoreMetrics;
+    private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings;
 
     BulkOperation(
         Task task,
@@ -111,7 +113,8 @@ final class BulkOperation extends ActionRunnable<BulkResponse> {
         LongSupplier relativeTimeProvider,
         long startTimeNanos,
         ActionListener<BulkResponse> listener,
-        FailureStoreMetrics failureStoreMetrics
+        FailureStoreMetrics failureStoreMetrics,
+        DataStreamFailureStoreSettings dataStreamFailureStoreSettings
     ) {
         this(
             task,
@@ -127,7 +130,8 @@ final class BulkOperation extends ActionRunnable<BulkResponse> {
             listener,
             new ClusterStateObserver(clusterService, bulkRequest.timeout(), logger, threadPool.getThreadContext()),
             new FailureStoreDocumentConverter(),
-            failureStoreMetrics
+            failureStoreMetrics,
+            dataStreamFailureStoreSettings
         );
     }
 
@@ -145,7 +149,8 @@ final class BulkOperation extends ActionRunnable<BulkResponse> {
         ActionListener<BulkResponse> listener,
         ClusterStateObserver observer,
         FailureStoreDocumentConverter failureStoreDocumentConverter,
-        FailureStoreMetrics failureStoreMetrics
+        FailureStoreMetrics failureStoreMetrics,
+        DataStreamFailureStoreSettings dataStreamFailureStoreSettings
     ) {
         super(listener);
         this.task = task;
@@ -164,6 +169,7 @@ final class BulkOperation extends ActionRunnable<BulkResponse> {
         this.rolloverClient = new OriginSettingClient(client, LAZY_ROLLOVER_ORIGIN);
         this.shortCircuitShardFailures.putAll(bulkRequest.incrementalState().shardLevelFailures());
         this.failureStoreMetrics = failureStoreMetrics;
+        this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings;
     }
 
     @Override
@@ -210,11 +216,9 @@ private void rollOverFailureStores(Runnable runnable) {
         }
         try (RefCountingRunnable refs = new RefCountingRunnable(runnable)) {
             for (String dataStream : failureStoresToBeRolledOver) {
-                RolloverRequest rolloverRequest = new RolloverRequest(dataStream, null);
-                rolloverRequest.setIndicesOptions(
-                    IndicesOptions.builder(rolloverRequest.indicesOptions())
-                        .selectorOptions(IndicesOptions.SelectorOptions.FAILURES)
-                        .build()
+                RolloverRequest rolloverRequest = new RolloverRequest(
+                    IndexNameExpressionResolver.combineSelector(dataStream, IndexComponentSelector.FAILURES),
+                    null
                 );
                 // We are executing a lazy rollover because it is an action specialised for this situation, when we want an
                 // unconditional and performant rollover.
@@ -544,7 +548,7 @@ private IndexDocFailureStoreStatus processFailure(BulkItemRequest bulkItemReques
             // Do not redirect documents to a failure store that were already headed to one.
             var isFailureStoreRequest = isFailureStoreRequest(docWriteRequest);
             if (isFailureStoreRequest == false
-                && failureStoreCandidate.isFailureStoreEnabled()
+                && failureStoreCandidate.isFailureStoreEffectivelyEnabled(dataStreamFailureStoreSettings)
                 && error instanceof VersionConflictEngineException == false
                 && error instanceof EsRejectedExecutionException == false) {
                 // Prepare the data stream failure store if necessary
@@ -577,7 +581,7 @@ private IndexDocFailureStoreStatus processFailure(BulkItemRequest bulkItemReques
                 if (isFailureStoreRequest) {
                     return IndexDocFailureStoreStatus.FAILED;
                 }
-                if (failureStoreCandidate.isFailureStoreEnabled() == false) {
+                if (failureStoreCandidate.isFailureStoreEffectivelyEnabled(dataStreamFailureStoreSettings) == false) {
                     return IndexDocFailureStoreStatus.NOT_ENABLED;
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java
index 12d3aa67ca9bb..28e345a527085 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ToXContent;
@@ -158,13 +157,14 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).object(ob -> ob.append((b, p) -> {
-            b.field(ERRORS, hasFailures());
-            b.field(TOOK, tookInMillis);
+        return Iterators.concat(Iterators.single((builder, p) -> {
+            builder.startObject();
+            builder.field(ERRORS, hasFailures());
+            builder.field(TOOK, tookInMillis);
             if (ingestTookInMillis != BulkResponse.NO_INGEST_TOOK) {
-                b.field(INGEST_TOOK, ingestTookInMillis);
+                builder.field(INGEST_TOOK, ingestTookInMillis);
             }
-            return b;
-        }).array(ITEMS, Iterators.forArray(responses)));
+            return builder.startArray(ITEMS);
+        }), Iterators.forArray(responses), Iterators.<ToXContent>single((builder, p) -> builder.endArray().endObject()));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java
index e83bca4b661c9..24534826f8e3e 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java
@@ -343,9 +343,11 @@ public boolean isForceExecution() {
      * @param indexName The index name to check.
      * @param metadata Cluster state metadata.
      * @param epochMillis A timestamp to use when resolving date math in the index name.
-     * @return true if this is not a simulation, and the given index name corresponds to a data stream with a failure store
-     * or if it matches a template that has a data stream failure store enabled. Returns false if the index name corresponds to a
-     * data stream, but it doesn't have the failure store enabled. Returns null when it doesn't correspond to a data stream.
+     * @return true if this is not a simulation, and the given index name corresponds to a data stream with a failure store, or if it
+     *     matches a template that has a data stream failure store enabled, or if it matches a data stream template with no failure store
+     *     option specified and the name matches the cluster setting to enable the failure store. Returns false if the index name
+     *     corresponds to a data stream, but it doesn't have the failure store enabled by one of those conditions. Returns null when it
+     *     doesn't correspond to a data stream.
      */
     protected abstract Boolean resolveFailureStore(String indexName, Metadata metadata, long epochMillis);
 
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
index e2c73349b93ec..523381321ada7 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.DocWriteRequest;
-import org.elasticsearch.action.DocWriteRequest.OpType;
 import org.elasticsearch.action.admin.indices.create.AutoCreateAction;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
@@ -26,7 +25,7 @@
 import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.action.support.WriteResponse;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
@@ -35,6 +34,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
 import org.elasticsearch.cluster.metadata.DataStreamOptions;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -44,7 +44,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.index.IndexingPressure;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.indices.SystemIndices;
@@ -80,11 +79,11 @@ public class TransportBulkAction extends TransportAbstractBulkAction {
     private static final Logger logger = LogManager.getLogger(TransportBulkAction.class);
     public static final String LAZY_ROLLOVER_ORIGIN = "lazy_rollover";
 
-    private final FeatureService featureService;
     private final NodeClient client;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
     private final OriginSettingClient rolloverClient;
     private final FailureStoreMetrics failureStoreMetrics;
+    private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings;
 
     @Inject
     public TransportBulkAction(
@@ -92,27 +91,27 @@ public TransportBulkAction(
         TransportService transportService,
         ClusterService clusterService,
         IngestService ingestService,
-        FeatureService featureService,
         NodeClient client,
         ActionFilters actionFilters,
         IndexNameExpressionResolver indexNameExpressionResolver,
         IndexingPressure indexingPressure,
         SystemIndices systemIndices,
-        FailureStoreMetrics failureStoreMetrics
+        FailureStoreMetrics failureStoreMetrics,
+        DataStreamFailureStoreSettings dataStreamFailureStoreSettings
     ) {
         this(
             threadPool,
             transportService,
             clusterService,
             ingestService,
-            featureService,
             client,
             actionFilters,
             indexNameExpressionResolver,
             indexingPressure,
             systemIndices,
             threadPool::relativeTimeInNanos,
-            failureStoreMetrics
+            failureStoreMetrics,
+            dataStreamFailureStoreSettings
         );
     }
 
@@ -121,14 +120,14 @@ public TransportBulkAction(
         TransportService transportService,
         ClusterService clusterService,
         IngestService ingestService,
-        FeatureService featureService,
         NodeClient client,
         ActionFilters actionFilters,
         IndexNameExpressionResolver indexNameExpressionResolver,
         IndexingPressure indexingPressure,
         SystemIndices systemIndices,
         LongSupplier relativeTimeProvider,
-        FailureStoreMetrics failureStoreMetrics
+        FailureStoreMetrics failureStoreMetrics,
+        DataStreamFailureStoreSettings dataStreamFailureStoreSettings
     ) {
         this(
             TYPE,
@@ -137,14 +136,14 @@ public TransportBulkAction(
             transportService,
             clusterService,
             ingestService,
-            featureService,
             client,
             actionFilters,
             indexNameExpressionResolver,
             indexingPressure,
             systemIndices,
             relativeTimeProvider,
-            failureStoreMetrics
+            failureStoreMetrics,
+            dataStreamFailureStoreSettings
         );
     }
 
@@ -155,14 +154,14 @@ public TransportBulkAction(
         TransportService transportService,
         ClusterService clusterService,
         IngestService ingestService,
-        FeatureService featureService,
         NodeClient client,
         ActionFilters actionFilters,
         IndexNameExpressionResolver indexNameExpressionResolver,
         IndexingPressure indexingPressure,
         SystemIndices systemIndices,
         LongSupplier relativeTimeProvider,
-        FailureStoreMetrics failureStoreMetrics
+        FailureStoreMetrics failureStoreMetrics,
+        DataStreamFailureStoreSettings dataStreamFailureStoreSettings
     ) {
         super(
             bulkAction,
@@ -176,8 +175,8 @@ public TransportBulkAction(
             systemIndices,
             relativeTimeProvider
         );
+        this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings;
         Objects.requireNonNull(relativeTimeProvider);
-        this.featureService = featureService;
         this.client = client;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
         this.rolloverClient = new OriginSettingClient(client, LAZY_ROLLOVER_ORIGIN);
@@ -276,13 +275,12 @@ private void populateMissingTargets(
         // A map for memorizing which indices exist.
         Map<String, Boolean> indexExistence = new HashMap<>();
         Function<String, Boolean> indexExistenceComputation = (index) -> indexNameExpressionResolver.hasIndexAbstraction(index, state);
-        boolean lazyRolloverFeature = featureService.clusterHasFeature(state, LazyRolloverAction.DATA_STREAM_LAZY_ROLLOVER);
         boolean lazyRolloverFailureStoreFeature = DataStream.isFailureStoreFeatureFlagEnabled();
         Set<String> indicesThatRequireAlias = new HashSet<>();
 
         for (DocWriteRequest<?> request : bulkRequest.requests) {
             // Delete requests should not attempt to create the index (if the index does not exist), unless an external versioning is used.
-            if (request.opType() == OpType.DELETE
+            if (request.opType() == DocWriteRequest.OpType.DELETE
                 && request.versionType() != VersionType.EXTERNAL
                 && request.versionType() != VersionType.EXTERNAL_GTE) {
                 continue;
@@ -321,18 +319,15 @@ private void populateMissingTargets(
                 }
             }
             // Determine which data streams and failure stores need to be rolled over.
-            if (lazyRolloverFeature) {
-                DataStream dataStream = state.metadata().dataStreams().get(request.index());
-                if (dataStream != null) {
-                    if (writeToFailureStore == false && dataStream.getBackingIndices().isRolloverOnWrite()) {
-                        dataStreamsToBeRolledOver.add(request.index());
-                    } else if (lazyRolloverFailureStoreFeature
-                        && writeToFailureStore
-                        && dataStream.getFailureIndices().isRolloverOnWrite()) {
-                            failureStoresToBeRolledOver.add(request.index());
-                        }
+            DataStream dataStream = state.metadata().dataStreams().get(request.index());
+            if (dataStream != null) {
+                if (writeToFailureStore == false && dataStream.getBackingIndices().isRolloverOnWrite()) {
+                    dataStreamsToBeRolledOver.add(request.index());
+                } else if (lazyRolloverFailureStoreFeature && writeToFailureStore && dataStream.getFailureIndices().isRolloverOnWrite()) {
+                    failureStoresToBeRolledOver.add(request.index());
                 }
             }
+
         }
     }
 
@@ -418,11 +413,7 @@ private void rollOverDataStreams(
             RolloverRequest rolloverRequest = new RolloverRequest(dataStream, null);
             rolloverRequest.masterNodeTimeout(bulkRequest.timeout);
             if (targetFailureStore) {
-                rolloverRequest.setIndicesOptions(
-                    IndicesOptions.builder(rolloverRequest.indicesOptions())
-                        .selectorOptions(IndicesOptions.SelectorOptions.FAILURES)
-                        .build()
-                );
+                rolloverRequest.setRolloverTarget(IndexNameExpressionResolver.combineSelector(dataStream, IndexComponentSelector.FAILURES));
             }
             // We are executing a lazy rollover because it is an action specialised for this situation, when we want an
             // unconditional and performant rollover.
@@ -431,9 +422,8 @@ private void rollOverDataStreams(
                 @Override
                 public void onResponse(RolloverResponse result) {
                     logger.debug(
-                        "Data stream{} {} has {} over, the latest index is {}",
-                        rolloverRequest.targetsFailureStore() ? " failure store" : "",
-                        dataStream,
+                        "Data stream [{}] has {} over, the latest index is {}",
+                        rolloverRequest.getRolloverTarget(),
                         result.isRolledOver() ? "been successfully rolled" : "skipped rolling",
                         result.getNewIndex()
                     );
@@ -492,7 +482,7 @@ private void failRequestsWhenPrerequisiteActionFailed(
 
     static void prohibitAppendWritesInBackingIndices(DocWriteRequest<?> writeRequest, IndexAbstraction indexAbstraction) {
         DocWriteRequest.OpType opType = writeRequest.opType();
-        if ((opType == OpType.CREATE || opType == OpType.INDEX) == false) {
+        if ((opType == DocWriteRequest.OpType.CREATE || opType == DocWriteRequest.OpType.INDEX) == false) {
             // op type not create or index, then bail early
             return;
         }
@@ -588,7 +578,8 @@ void executeBulk(
             relativeTimeNanosProvider,
             startTimeNanos,
             listener,
-            failureStoreMetrics
+            failureStoreMetrics,
+            dataStreamFailureStoreSettings
         ).run();
     }
 
@@ -596,7 +587,7 @@ void executeBulk(
      * See {@link #resolveFailureStore(String, Metadata, long)}
      */
     // Visibility for testing
-    static Boolean resolveFailureInternal(String indexName, Metadata metadata, long epochMillis) {
+    Boolean resolveFailureInternal(String indexName, Metadata metadata, long epochMillis) {
         if (DataStream.isFailureStoreFeatureFlagEnabled() == false) {
             return null;
         }
@@ -604,7 +595,7 @@ static Boolean resolveFailureInternal(String indexName, Metadata metadata, long
         if (resolution != null) {
             return resolution;
         }
-        return resolveFailureStoreFromTemplate(indexName, metadata);
+        return resolveFailureStoreFromTemplate(indexName, metadata, epochMillis);
     }
 
     @Override
@@ -619,7 +610,7 @@ protected Boolean resolveFailureStore(String indexName, Metadata metadata, long
      * @param epochMillis A timestamp to use when resolving date math in the index name.
      * @return true if the given index name corresponds to an existing data stream with a failure store enabled.
      */
-    private static Boolean resolveFailureStoreFromMetadata(String indexName, Metadata metadata, long epochMillis) {
+    private Boolean resolveFailureStoreFromMetadata(String indexName, Metadata metadata, long epochMillis) {
         if (indexName == null) {
             return null;
         }
@@ -636,7 +627,7 @@ private static Boolean resolveFailureStoreFromMetadata(String indexName, Metadat
         DataStream targetDataStream = DataStream.resolveDataStream(indexAbstraction, metadata);
 
         // We will store the failure if the write target belongs to a data stream with a failure store.
-        return targetDataStream != null && targetDataStream.isFailureStoreEnabled();
+        return targetDataStream != null && targetDataStream.isFailureStoreEffectivelyEnabled(dataStreamFailureStoreSettings);
     }
 
     /**
@@ -644,18 +635,20 @@ private static Boolean resolveFailureStoreFromMetadata(String indexName, Metadat
      * a data stream feature, the method returns true/false only if it is a data stream template, otherwise null.
      * @param indexName The index name to check.
      * @param metadata Cluster state metadata.
-     * @return true the associated index template has failure store enabled, false if the failure store is disabled or it's not specified,
-     * and null if the template is not a data stream template.
-     * Visible for testing
+     * @param epochMillis A timestamp to use when resolving date math in the index name.
+     * @return true the associated index template has failure store enabled, false if the failure store is disabled, true or false according
+     *     to the cluster setting if there is a data stream template with no failure store option specified, and null if no template is
+     *     found or if the template is not a data stream template.
      */
     @Nullable
-    static Boolean resolveFailureStoreFromTemplate(String indexName, Metadata metadata) {
+    private Boolean resolveFailureStoreFromTemplate(String indexName, Metadata metadata, long epochMillis) {
         if (indexName == null) {
             return null;
         }
 
         // Check to see if the index name matches any templates such that an index would have been attributed
         // We don't check v1 templates at all because failure stores can only exist on data streams via a v2 template
+        // N.B. This currently does date math resolution itself and does *not* use epochMillis (it gets the system time again)
         String template = MetadataIndexTemplateService.findV2Template(metadata, indexName, false);
         if (template != null) {
             // Check if this is a data stream template or if it is just a normal index.
@@ -666,7 +659,12 @@ static Boolean resolveFailureStoreFromTemplate(String indexName, Metadata metada
                     composableIndexTemplate,
                     metadata.componentTemplates()
                 ).mapAndGet(DataStreamOptions.Template::toDataStreamOptions);
-                return dataStreamOptions != null && dataStreamOptions.isFailureStoreEnabled();
+                return DataStream.isFailureStoreEffectivelyEnabled(
+                    dataStreamOptions,
+                    dataStreamFailureStoreSettings,
+                    IndexNameExpressionResolver.resolveDateMathExpression(indexName, epochMillis),
+                    systemIndices
+                );
             }
         }
 
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
index 74143cc5c059b..89cee714a9ff2 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
@@ -48,9 +48,11 @@
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.index.mapper.MapperException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MappingLookup;
+import org.elasticsearch.index.mapper.RoutingFieldMapper;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.IndexShard;
@@ -326,7 +328,8 @@ static boolean executeBulkItemRequest(
         if (opType == DocWriteRequest.OpType.UPDATE) {
             final UpdateRequest updateRequest = (UpdateRequest) context.getCurrent();
             try {
-                updateResult = updateHelper.prepare(updateRequest, context.getPrimary(), nowInMillisSupplier);
+                var gFields = getStoredFieldsSpec(context.getPrimary());
+                updateResult = updateHelper.prepare(updateRequest, context.getPrimary(), nowInMillisSupplier, gFields);
             } catch (Exception failure) {
                 // we may fail translating a update to index or delete operation
                 // we use index result to communicate failure while translating update request
@@ -401,6 +404,16 @@ static boolean executeBulkItemRequest(
         return true;
     }
 
+    private static String[] getStoredFieldsSpec(IndexShard indexShard) {
+        if (InferenceMetadataFieldsMapper.isEnabled(indexShard.mapperService().mappingLookup())) {
+            if (indexShard.mapperService().mappingLookup().inferenceFields().size() > 0) {
+                // Retrieves the inference metadata field containing the inference results for all semantic fields defined in the mapping.
+                return new String[] { RoutingFieldMapper.NAME, InferenceMetadataFieldsMapper.NAME };
+            }
+        }
+        return new String[] { RoutingFieldMapper.NAME };
+    }
+
     private static boolean handleMappingUpdateRequired(
         BulkPrimaryExecutionContext context,
         MappingUpdatePerformer mappingUpdater,
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java
index 1353fa78595ef..18c420d99f525 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java
@@ -9,6 +9,8 @@
 
 package org.elasticsearch.action.bulk;
 
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.admin.indices.template.post.TransportSimulateIndexTemplateAction;
@@ -33,6 +35,7 @@
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexSettingProviders;
@@ -40,6 +43,8 @@
 import org.elasticsearch.index.IndexingPressure;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.mapper.IgnoredFieldMapper;
+import org.elasticsearch.index.mapper.LuceneDocument;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.seqno.SequenceNumbers;
@@ -60,6 +65,7 @@
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -77,14 +83,8 @@
  * shards are not actually modified).
  */
 public class TransportSimulateBulkAction extends TransportAbstractBulkAction {
-    public static final NodeFeature SIMULATE_MAPPING_VALIDATION = new NodeFeature("simulate.mapping.validation");
-    public static final NodeFeature SIMULATE_MAPPING_VALIDATION_TEMPLATES = new NodeFeature("simulate.mapping.validation.templates");
-    public static final NodeFeature SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS = new NodeFeature(
-        "simulate.component.template.substitutions"
-    );
-    public static final NodeFeature SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS = new NodeFeature("simulate.index.template.substitutions");
-    public static final NodeFeature SIMULATE_MAPPING_ADDITION = new NodeFeature("simulate.mapping.addition");
-    public static final NodeFeature SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING = new NodeFeature("simulate.support.non.template.mapping");
+    public static final NodeFeature SIMULATE_IGNORED_FIELDS = new NodeFeature("simulate.ignored.fields");
+
     private final IndicesService indicesService;
     private final NamedXContentRegistry xContentRegistry;
     private final Set<IndexSettingProvider> indexSettingProviders;
@@ -137,12 +137,13 @@ protected void doInternalExecute(
             DocWriteRequest<?> docRequest = bulkRequest.requests.get(i);
             assert docRequest instanceof IndexRequest : "TransportSimulateBulkAction should only ever be called with IndexRequests";
             IndexRequest request = (IndexRequest) docRequest;
-            Exception mappingValidationException = validateMappings(
+            Tuple<Collection<String>, Exception> validationResult = validateMappings(
                 componentTemplateSubstitutions,
                 indexTemplateSubstitutions,
                 mappingAddition,
                 request
             );
+            Exception mappingValidationException = validationResult.v2();
             responses.set(
                 i,
                 BulkItemResponse.success(
@@ -155,6 +156,7 @@ protected void doInternalExecute(
                         request.source(),
                         request.getContentType(),
                         request.getExecutedPipelines(),
+                        validationResult.v1(),
                         mappingValidationException
                     )
                 )
@@ -168,11 +170,12 @@ protected void doInternalExecute(
     /**
      * This creates a temporary index with the mappings of the index in the request, and then attempts to index the source from the request
      * into it. If there is a mapping exception, that exception is returned. On success the returned exception is null.
-     * @parem componentTemplateSubstitutions The component template definitions to use in place of existing ones for validation
+     * @param componentTemplateSubstitutions The component template definitions to use in place of existing ones for validation
      * @param request The IndexRequest whose source will be validated against the mapping (if it exists) of its index
-     * @return a mapping exception if the source does not match the mappings, otherwise null
+     * @return a Tuple containing: (1) in v1 the names of any fields that would be ignored upon indexing and (2) in v2 the mapping
+     * exception if the source does not match the mappings, otherwise null
      */
-    private Exception validateMappings(
+    private Tuple<Collection<String>, Exception> validateMappings(
         Map<String, ComponentTemplate> componentTemplateSubstitutions,
         Map<String, ComposableIndexTemplate> indexTemplateSubstitutions,
         Map<String, Object> mappingAddition,
@@ -189,6 +192,7 @@ private Exception validateMappings(
 
         ClusterState state = clusterService.state();
         Exception mappingValidationException = null;
+        Collection<String> ignoredFields = List.of();
         IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index());
         try {
             if (indexAbstraction != null
@@ -275,7 +279,7 @@ private Exception validateMappings(
                     );
                     CompressedXContent mappings = template.mappings();
                     CompressedXContent mergedMappings = mergeMappings(mappings, mappingAddition);
-                    validateUpdatedMappings(mappings, mergedMappings, request, sourceToParse);
+                    ignoredFields = validateUpdatedMappings(mappings, mergedMappings, request, sourceToParse);
                 } else {
                     List<IndexTemplateMetadata> matchingTemplates = findV1Templates(simulatedState.metadata(), request.index(), false);
                     if (matchingTemplates.isEmpty() == false) {
@@ -289,7 +293,7 @@ private Exception validateMappings(
                             xContentRegistry
                         );
                         final CompressedXContent combinedMappings = mergeMappings(new CompressedXContent(mappingsMap), mappingAddition);
-                        validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
+                        ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
                     } else if (indexAbstraction != null && mappingAddition.isEmpty() == false) {
                         /*
                          * The index matched no templates of any kind, including the substitutions. But it might have a mapping. So we
@@ -298,7 +302,7 @@ private Exception validateMappings(
                         MappingMetadata mappingFromIndex = clusterService.state().metadata().index(indexAbstraction.getName()).mapping();
                         CompressedXContent currentIndexCompressedXContent = mappingFromIndex == null ? null : mappingFromIndex.source();
                         CompressedXContent combinedMappings = mergeMappings(currentIndexCompressedXContent, mappingAddition);
-                        validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
+                        ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
                     } else {
                         /*
                          * The index matched no templates and had no mapping of its own. If there were component template substitutions
@@ -306,27 +310,28 @@ private Exception validateMappings(
                          * and validate.
                          */
                         final CompressedXContent combinedMappings = mergeMappings(null, mappingAddition);
-                        validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
+                        ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
                     }
                 }
             }
         } catch (Exception e) {
             mappingValidationException = e;
         }
-        return mappingValidationException;
+        return Tuple.tuple(ignoredFields, mappingValidationException);
     }
 
     /*
-     * Validates that when updatedMappings are applied
+     * Validates that when updatedMappings are applied. If any fields would be ignored while indexing, then those field names are returned.
+     * Otherwise the returned Collection is empty.
      */
-    private void validateUpdatedMappings(
+    private Collection<String> validateUpdatedMappings(
         @Nullable CompressedXContent originalMappings,
         @Nullable CompressedXContent updatedMappings,
         IndexRequest request,
         SourceToParse sourceToParse
     ) throws IOException {
         if (updatedMappings == null) {
-            return; // no validation to do
+            return List.of(); // no validation to do
         }
         Settings dummySettings = Settings.builder()
             .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
@@ -343,7 +348,7 @@ private void validateUpdatedMappings(
             .settings(dummySettings)
             .putMapping(new MappingMetadata(updatedMappings))
             .build();
-        indicesService.withTempIndexService(originalIndexMetadata, indexService -> {
+        Engine.Index result = indicesService.withTempIndexService(originalIndexMetadata, indexService -> {
             indexService.mapperService().merge(updatedIndexMetadata, MapperService.MergeReason.MAPPING_UPDATE);
             return IndexShard.prepareIndex(
                 indexService.mapperService(),
@@ -360,6 +365,24 @@ private void validateUpdatedMappings(
                 0
             );
         });
+        final Collection<String> ignoredFields;
+        if (result == null) {
+            ignoredFields = List.of();
+        } else {
+            List<LuceneDocument> luceneDocuments = result.parsedDoc().docs();
+            assert luceneDocuments == null || luceneDocuments.size() == 1 : "Expected a single lucene document from index attempt";
+            if (luceneDocuments != null && luceneDocuments.size() == 1) {
+                ignoredFields = luceneDocuments.getFirst()
+                    .getFields()
+                    .stream()
+                    .filter(field -> field.name().equals(IgnoredFieldMapper.NAME) && field instanceof StringField)
+                    .map(IndexableField::stringValue)
+                    .toList();
+            } else {
+                ignoredFields = List.of();
+            }
+        }
+        return ignoredFields;
     }
 
     private static CompressedXContent mergeMappings(@Nullable CompressedXContent originalMapping, Map<String, Object> mappingAddition)
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java
index a0a05138406c5..62caba8f7ed96 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java
@@ -9,16 +9,18 @@
 
 package org.elasticsearch.action.datastreams;
 
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression;
 import org.elasticsearch.index.Index;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.SortedMap;
-import java.util.stream.Stream;
 
 public class DataStreamsActionUtil {
 
@@ -47,25 +49,79 @@ public static IndicesOptions updateIndicesOptions(IndicesOptions indicesOptions)
         return indicesOptions;
     }
 
-    public static Stream<String> resolveConcreteIndexNames(
+    public static List<String> resolveConcreteIndexNames(
         IndexNameExpressionResolver indexNameExpressionResolver,
         ClusterState clusterState,
         String[] names,
         IndicesOptions indicesOptions
     ) {
-        List<String> abstractionNames = getDataStreamNames(indexNameExpressionResolver, clusterState, names, indicesOptions);
+        List<ResolvedExpression> abstractionNames = indexNameExpressionResolver.dataStreams(
+            clusterState,
+            updateIndicesOptions(indicesOptions),
+            names
+        );
         SortedMap<String, IndexAbstraction> indicesLookup = clusterState.getMetadata().getIndicesLookup();
 
-        return abstractionNames.stream().flatMap(abstractionName -> {
+        List<String> results = new ArrayList<>(abstractionNames.size());
+        for (ResolvedExpression abstractionName : abstractionNames) {
+            IndexAbstraction indexAbstraction = indicesLookup.get(abstractionName.resource());
+            assert indexAbstraction != null;
+            if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) {
+                selectDataStreamIndicesNames(
+                    (DataStream) indexAbstraction,
+                    IndexComponentSelector.FAILURES.equals(abstractionName.selector()),
+                    results
+                );
+            }
+        }
+        return results;
+    }
+
+    /**
+     * Resolves a list of expressions into data stream names and then collects the concrete indices
+     * that are applicable for those data streams based on the selector provided in the arguments.
+     * @param indexNameExpressionResolver resolver object
+     * @param clusterState state to query
+     * @param names data stream expressions
+     * @param selector which component indices of the data stream should be returned
+     * @param indicesOptions options for expression resolution
+     * @return A stream of concrete index names that belong to the components specified
+     *         on the data streams returned from the expressions given
+     */
+    public static List<String> resolveConcreteIndexNamesWithSelector(
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        ClusterState clusterState,
+        String[] names,
+        IndexComponentSelector selector,
+        IndicesOptions indicesOptions
+    ) {
+        assert indicesOptions.allowSelectors() == false : "If selectors are enabled, use resolveConcreteIndexNames instead";
+        List<String> abstractionNames = indexNameExpressionResolver.dataStreamNames(
+            clusterState,
+            updateIndicesOptions(indicesOptions),
+            names
+        );
+        SortedMap<String, IndexAbstraction> indicesLookup = clusterState.getMetadata().getIndicesLookup();
+
+        List<String> results = new ArrayList<>(abstractionNames.size());
+        for (String abstractionName : abstractionNames) {
             IndexAbstraction indexAbstraction = indicesLookup.get(abstractionName);
             assert indexAbstraction != null;
             if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) {
-                DataStream dataStream = (DataStream) indexAbstraction;
-                List<Index> indices = dataStream.getIndices();
-                return indices.stream().map(Index::getName);
-            } else {
-                return Stream.empty();
+                if (selector.shouldIncludeData()) {
+                    selectDataStreamIndicesNames((DataStream) indexAbstraction, false, results);
+                }
+                if (selector.shouldIncludeFailures()) {
+                    selectDataStreamIndicesNames((DataStream) indexAbstraction, true, results);
+                }
             }
-        });
+        }
+        return results;
+    }
+
+    private static void selectDataStreamIndicesNames(DataStream indexAbstraction, boolean failureStore, List<String> accumulator) {
+        for (Index index : indexAbstraction.getDataStreamIndices(failureStore).getIndices()) {
+            accumulator.add(index.getName());
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java
index 9266bae439b73..82afeec752378 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java
@@ -38,8 +38,6 @@ public DataStreamsStatsAction() {
 
     public static class Request extends BroadcastRequest<Request> {
         public Request() {
-            // this doesn't really matter since data stream name resolution isn't affected by IndicesOptions and
-            // a data stream's backing indices are retrieved from its metadata
             super(
                 null,
                 IndicesOptions.builder()
@@ -58,10 +56,9 @@ public Request() {
                             .allowAliasToMultipleIndices(true)
                             .allowClosedIndices(true)
                             .ignoreThrottled(false)
-                            .allowFailureIndices(true)
+                            .allowSelectors(false)
                             .build()
                     )
-                    .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
                     .build()
             );
         }
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java
index 4f647d4f02884..640c88918ffc0 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java
@@ -61,7 +61,7 @@ public static class Request extends MasterNodeRequest<Request> implements Indice
                     .allowAliasToMultipleIndices(false)
                     .allowClosedIndices(true)
                     .ignoreThrottled(false)
-                    .allowFailureIndices(true)
+                    .allowSelectors(false)
                     .build()
             )
             .build();
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java
index 93c40ad18cc8a..c55957787aee7 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java
@@ -72,10 +72,11 @@ public static class Request extends MasterNodeReadRequest<Request> implements In
                     .allowAliasToMultipleIndices(false)
                     .allowClosedIndices(true)
                     .ignoreThrottled(false)
-                    .allowFailureIndices(true)
+                    .allowSelectors(false)
                     .build()
             )
             .build();
+
         private boolean includeDefaults = false;
         private boolean verbose = false;
 
@@ -234,6 +235,7 @@ public static class DataStreamInfo implements SimpleDiffable<DataStreamInfo>, To
 
             private final DataStream dataStream;
             private final ClusterHealthStatus dataStreamStatus;
+            private final boolean failureStoreEffectivelyEnabled; // Must be serialized independently of dataStream as depends on settings
             @Nullable
             private final String indexTemplate;
             @Nullable
@@ -247,6 +249,7 @@ public static class DataStreamInfo implements SimpleDiffable<DataStreamInfo>, To
 
             public DataStreamInfo(
                 DataStream dataStream,
+                boolean failureStoreEffectivelyEnabled,
                 ClusterHealthStatus dataStreamStatus,
                 @Nullable String indexTemplate,
                 @Nullable String ilmPolicyName,
@@ -256,6 +259,7 @@ public DataStreamInfo(
                 @Nullable Long maximumTimestamp
             ) {
                 this.dataStream = dataStream;
+                this.failureStoreEffectivelyEnabled = failureStoreEffectivelyEnabled;
                 this.dataStreamStatus = dataStreamStatus;
                 this.indexTemplate = indexTemplate;
                 this.ilmPolicyName = ilmPolicyName;
@@ -267,22 +271,32 @@ public DataStreamInfo(
 
             @SuppressWarnings("unchecked")
             DataStreamInfo(StreamInput in) throws IOException {
-                this(
-                    DataStream.read(in),
-                    ClusterHealthStatus.readFrom(in),
-                    in.readOptionalString(),
-                    in.readOptionalString(),
-                    in.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0) ? in.readOptionalWriteable(TimeSeries::new) : null,
-                    in.getTransportVersion().onOrAfter(V_8_11_X) ? in.readMap(Index::new, IndexProperties::new) : Map.of(),
-                    in.getTransportVersion().onOrAfter(V_8_11_X) ? in.readBoolean() : true,
-                    in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readOptionalVLong() : null
-                );
+                this.dataStream = DataStream.read(in);
+                this.failureStoreEffectivelyEnabled = in.getTransportVersion()
+                    .onOrAfter(TransportVersions.FAILURE_STORE_ENABLED_BY_CLUSTER_SETTING)
+                        ? in.readBoolean()
+                        : dataStream.isFailureStoreExplicitlyEnabled(); // Revert to the behaviour before this field was added
+                this.dataStreamStatus = ClusterHealthStatus.readFrom(in);
+                this.indexTemplate = in.readOptionalString();
+                this.ilmPolicyName = in.readOptionalString();
+                this.timeSeries = in.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0)
+                    ? in.readOptionalWriteable(TimeSeries::new)
+                    : null;
+                this.indexSettingsValues = in.getTransportVersion().onOrAfter(V_8_11_X)
+                    ? in.readMap(Index::new, IndexProperties::new)
+                    : Map.of();
+                this.templatePreferIlmValue = in.getTransportVersion().onOrAfter(V_8_11_X) ? in.readBoolean() : true;
+                this.maximumTimestamp = in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readOptionalVLong() : null;
             }
 
             public DataStream getDataStream() {
                 return dataStream;
             }
 
+            public boolean isFailureStoreEffectivelyEnabled() {
+                return failureStoreEffectivelyEnabled;
+            }
+
             public ClusterHealthStatus getDataStreamStatus() {
                 return dataStreamStatus;
             }
@@ -318,6 +332,9 @@ public Long getMaximumTimestamp() {
             @Override
             public void writeTo(StreamOutput out) throws IOException {
                 dataStream.writeTo(out);
+                if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_ENABLED_BY_CLUSTER_SETTING)) {
+                    out.writeBoolean(failureStoreEffectivelyEnabled);
+                }
                 dataStreamStatus.writeTo(out);
                 out.writeOptionalString(indexTemplate);
                 out.writeOptionalString(ilmPolicyName);
@@ -398,7 +415,7 @@ public XContentBuilder toXContent(
                 }
                 if (DataStream.isFailureStoreFeatureFlagEnabled()) {
                     builder.startObject(DataStream.FAILURE_STORE_FIELD.getPreferredName());
-                    builder.field(FAILURE_STORE_ENABLED.getPreferredName(), dataStream.isFailureStoreEnabled());
+                    builder.field(FAILURE_STORE_ENABLED.getPreferredName(), failureStoreEffectivelyEnabled);
                     builder.field(
                         DataStream.ROLLOVER_ON_WRITE_FIELD.getPreferredName(),
                         dataStream.getFailureIndices().isRolloverOnWrite()
@@ -477,6 +494,7 @@ public boolean equals(Object o) {
                 DataStreamInfo that = (DataStreamInfo) o;
                 return templatePreferIlmValue == that.templatePreferIlmValue
                     && Objects.equals(dataStream, that.dataStream)
+                    && failureStoreEffectivelyEnabled == that.failureStoreEffectivelyEnabled
                     && dataStreamStatus == that.dataStreamStatus
                     && Objects.equals(indexTemplate, that.indexTemplate)
                     && Objects.equals(ilmPolicyName, that.ilmPolicyName)
@@ -490,6 +508,7 @@ public int hashCode() {
                 return Objects.hash(
                     dataStream,
                     dataStreamStatus,
+                    failureStoreEffectivelyEnabled,
                     indexTemplate,
                     ilmPolicyName,
                     timeSeries,
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingService.java b/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingService.java
index 580cf92d15e2c..2eed45e5afa6d 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingService.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingService.java
@@ -23,8 +23,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.features.FeatureService;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.Index;
 
 import java.util.List;
@@ -43,8 +41,6 @@ public class DataStreamAutoShardingService {
     private static final Logger logger = LogManager.getLogger(DataStreamAutoShardingService.class);
     public static final String DATA_STREAMS_AUTO_SHARDING_ENABLED = "data_streams.auto_sharding.enabled";
 
-    public static final NodeFeature DATA_STREAM_AUTO_SHARDING_FEATURE = new NodeFeature("data_stream.auto_sharding");
-
     public static final Setting<List<String>> DATA_STREAMS_AUTO_SHARDING_EXCLUDES_SETTING = Setting.listSetting(
         "data_streams.auto_sharding.excludes",
         List.of(),
@@ -101,7 +97,6 @@ public class DataStreamAutoShardingService {
     );
     private final ClusterService clusterService;
     private final boolean isAutoShardingEnabled;
-    private final FeatureService featureService;
     private final LongSupplier nowSupplier;
     private volatile TimeValue increaseShardsCooldown;
     private volatile TimeValue reduceShardsCooldown;
@@ -109,12 +104,7 @@ public class DataStreamAutoShardingService {
     private volatile int maxWriteThreads;
     private volatile List<String> dataStreamExcludePatterns;
 
-    public DataStreamAutoShardingService(
-        Settings settings,
-        ClusterService clusterService,
-        FeatureService featureService,
-        LongSupplier nowSupplier
-    ) {
+    public DataStreamAutoShardingService(Settings settings, ClusterService clusterService, LongSupplier nowSupplier) {
         this.clusterService = clusterService;
         this.isAutoShardingEnabled = settings.getAsBoolean(DATA_STREAMS_AUTO_SHARDING_ENABLED, false);
         this.increaseShardsCooldown = DATA_STREAMS_AUTO_SHARDING_INCREASE_SHARDS_COOLDOWN.get(settings);
@@ -122,7 +112,6 @@ public DataStreamAutoShardingService(
         this.minWriteThreads = CLUSTER_AUTO_SHARDING_MIN_WRITE_THREADS.get(settings);
         this.maxWriteThreads = CLUSTER_AUTO_SHARDING_MAX_WRITE_THREADS.get(settings);
         this.dataStreamExcludePatterns = DATA_STREAMS_AUTO_SHARDING_EXCLUDES_SETTING.get(settings);
-        this.featureService = featureService;
         this.nowSupplier = nowSupplier;
     }
 
@@ -168,15 +157,6 @@ public AutoShardingResult calculate(ClusterState state, DataStream dataStream, @
             return NOT_APPLICABLE_RESULT;
         }
 
-        if (featureService.clusterHasFeature(state, DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE) == false) {
-            logger.debug(
-                "Data stream auto sharding service cannot compute the optimal number of shards for data stream [{}] because the cluster "
-                    + "doesn't have the auto sharding feature",
-                dataStream.getName()
-            );
-            return NOT_APPLICABLE_RESULT;
-        }
-
         if (dataStreamExcludePatterns.stream().anyMatch(pattern -> Regex.simpleMatch(pattern, dataStream.getName()))) {
             logger.debug(
                 "Data stream [{}] is excluded from auto sharding via the [{}] setting",
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java
index a43d29501a7ee..401bd7a27c6fa 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java
@@ -63,7 +63,7 @@ public static class Request extends MasterNodeReadRequest<Request> implements In
                     .allowAliasToMultipleIndices(false)
                     .allowClosedIndices(true)
                     .ignoreThrottled(false)
-                    .allowFailureIndices(true)
+                    .allowSelectors(false)
                     .build()
             )
             .build();
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java
index b054d12890366..c2b7de8d5df8b 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java
@@ -94,7 +94,7 @@ public static Request parseRequest(XContentParser parser, Factory factory) {
                     .allowAliasToMultipleIndices(false)
                     .allowClosedIndices(true)
                     .ignoreThrottled(false)
-                    .allowFailureIndices(false)
+                    .allowSelectors(false)
                     .build()
             )
             .build();
diff --git a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleAction.java b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleAction.java
index 62771230636c1..cce01aca7685a 100644
--- a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleAction.java
+++ b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleAction.java
@@ -82,7 +82,7 @@ public String[] indices() {
 
         @Override
         public IndicesOptions indicesOptions() {
-            return IndicesOptions.STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED;
+            return IndicesOptions.strictSingleIndexNoExpandForbidClosed();
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/RequestDispatcher.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/RequestDispatcher.java
index 802f5d196569a..fce925d868532 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/RequestDispatcher.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/RequestDispatcher.java
@@ -95,7 +95,7 @@ final class RequestDispatcher {
         for (String index : indices) {
             final GroupShardsIterator<ShardIterator> shardIts;
             try {
-                shardIts = clusterService.operationRouting().searchShards(clusterState, new String[] { index }, null, null, null, null);
+                shardIts = clusterService.operationRouting().searchShards(clusterState, new String[] { index }, null, null);
             } catch (Exception e) {
                 onIndexFailure.accept(index, e);
                 continue;
diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
index 22537f1f51216..e6469ab75299e 100644
--- a/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.VersionType;
@@ -52,7 +51,6 @@ public class MultiGetRequest extends ActionRequest
         CompositeIndicesRequest,
         RealtimeRequest,
         ToXContentObject {
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(MultiGetRequest.class);
 
     private static final ParseField DOCS = new ParseField("docs");
     private static final ParseField INDEX = new ParseField("_index");
diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
index a2c7c8664e81a..29b926598ac32 100644
--- a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
+++ b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.PlainShardIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
-import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.TimeValue;
@@ -109,7 +108,7 @@ protected ShardIterator shards(ClusterState state, InternalRequest request) {
         if (iterator == null) {
             return null;
         }
-        return new PlainShardIterator(iterator.shardId(), iterator.getShardRoutings().stream().filter(ShardRouting::isSearchable).toList());
+        return PlainShardIterator.allSearchableShards(iterator);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
index 01dc705d7146b..e5087a790a292 100644
--- a/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
+++ b/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.routing.OperationRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -81,7 +82,7 @@ protected void doExecute(Task task, final MultiGetRequest request, final ActionL
                     lastResolvedIndex = Tuple.tuple(item.index(), concreteSingleIndex);
                 }
                 item.routing(clusterState.metadata().resolveIndexRouting(item.routing(), item.index()));
-                shardId = clusterService.operationRouting().shardId(clusterState, concreteSingleIndex, item.id(), item.routing());
+                shardId = OperationRouting.shardId(clusterState, concreteSingleIndex, item.id(), item.routing());
             } catch (RoutingMissingException e) {
                 responses.set(i, newItemFailure(e.getIndex().getName(), e.getId(), e));
                 continue;
diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
index 0fa770df8e4ef..d9a04acc0466e 100644
--- a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
+++ b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.PlainShardIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
-import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.TimeValue;
@@ -113,7 +112,7 @@ protected ShardIterator shards(ClusterState state, InternalRequest request) {
         if (iterator == null) {
             return null;
         }
-        return new PlainShardIterator(iterator.shardId(), iterator.getShardRoutings().stream().filter(ShardRouting::isSearchable).toList());
+        return PlainShardIterator.allSearchableShards(iterator);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
index d5b8b657bd14e..4343451256920 100644
--- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
@@ -51,6 +51,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.OptionalInt;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
@@ -78,7 +79,6 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
     private static final TransportVersion PIPELINES_HAVE_RUN_FIELD_ADDED = TransportVersions.V_8_10_X;
 
     private static final Supplier<String> ID_GENERATOR = UUIDs::base64UUID;
-    private static final Supplier<String> K_SORTED_TIME_BASED_ID_GENERATOR = UUIDs::base64TimeBasedKOrderedUUID;
 
     /**
      * Max length of the source document to include into string()
@@ -705,9 +705,18 @@ public void autoGenerateId() {
     }
 
     public void autoGenerateTimeBasedId() {
+        autoGenerateTimeBasedId(OptionalInt.empty());
+    }
+
+    /**
+     *  Set the {@code #id()} to an automatically generated one, optimized for storage (compression) efficiency.
+     *  If a routing hash is passed, it is included in the generated id starting at 9 bytes before the end.
+     * @param hash optional routing hash value, used to route requests by id to the right shard.
+     */
+    public void autoGenerateTimeBasedId(OptionalInt hash) {
         assertBeforeGeneratingId();
         autoGenerateTimestamp();
-        id(K_SORTED_TIME_BASED_ID_GENERATOR.get());
+        id(UUIDs.base64TimeBasedKOrderedUUIDWithHash(hash));
     }
 
     private void autoGenerateTimestamp() {
@@ -899,12 +908,18 @@ public IndexRequest setRequireAlias(boolean requireAlias) {
     }
 
     /**
-     * Transient flag denoting that the local request should be routed to a failure store. Not persisted across the wire.
+     * Returns a transient flag denoting that the local request should be routed to a failure store. Not persisted across the wire. N.B. If
+     * true, the failure store will be used regardless of whether the metadata indicates that the failure store is enabled.
      */
     public boolean isWriteToFailureStore() {
         return writeToFailureStore;
     }
 
+    /**
+     * Sets a transient flag denoting that the local request should be routed to a failure store. Not persisted across the wire. N.B. If
+     * true, the failure store will be used regardless of whether the metadata indicates that the failure store is enabled. It is the
+     * caller's responsibility to ensure that this is correct.
+     */
     public IndexRequest setWriteToFailureStore(boolean writeToFailureStore) {
         this.writeToFailureStore = writeToFailureStore;
         return this;
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java
index 9d883cb075ede..307996a4c72cb 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.List;
 
 /**
@@ -34,6 +35,7 @@
 public class SimulateIndexResponse extends IndexResponse {
     private final BytesReference source;
     private final XContentType sourceXContentType;
+    private final Collection<String> ignoredFields;
     private final Exception exception;
 
     @SuppressWarnings("this-escape")
@@ -47,6 +49,11 @@ public SimulateIndexResponse(StreamInput in) throws IOException {
         } else {
             this.exception = null;
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_IGNORED_FIELDS)) {
+            this.ignoredFields = in.readStringCollectionAsList();
+        } else {
+            this.ignoredFields = List.of();
+        }
     }
 
     @SuppressWarnings("this-escape")
@@ -57,6 +64,7 @@ public SimulateIndexResponse(
         BytesReference source,
         XContentType sourceXContentType,
         List<String> pipelines,
+        Collection<String> ignoredFields,
         @Nullable Exception exception
     ) {
         // We don't actually care about most of the IndexResponse fields:
@@ -73,6 +81,7 @@ public SimulateIndexResponse(
         this.source = source;
         this.sourceXContentType = sourceXContentType;
         setShardInfo(ShardInfo.EMPTY);
+        this.ignoredFields = ignoredFields;
         this.exception = exception;
     }
 
@@ -84,6 +93,16 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
         builder.field("_source", XContentHelper.convertToMap(source, false, sourceXContentType).v2());
         assert executedPipelines != null : "executedPipelines is null when it shouldn't be - we always list pipelines in simulate mode";
         builder.array("executed_pipelines", executedPipelines.toArray());
+        if (ignoredFields.isEmpty() == false) {
+            builder.startArray("ignored_fields");
+            for (String ignoredField : ignoredFields) {
+                builder.startObject();
+                builder.field("field", ignoredField);
+                builder.endObject();
+            }
+            ;
+            builder.endArray();
+        }
         if (exception != null) {
             builder.startObject("error");
             ElasticsearchException.generateThrowableXContent(builder, params, exception);
@@ -105,6 +124,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
             out.writeException(exception);
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_IGNORED_FIELDS)) {
+            out.writeStringCollection(ignoredFields);
+        }
     }
 
     public Exception getException() {
diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index 47abfe266c524..73e6a0306247d 100644
--- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -231,7 +231,7 @@ public final void start() {
     }
 
     @Override
-    public final void run() {
+    protected final void run() {
         for (final SearchShardIterator iterator : toSkipShardsIts) {
             assert iterator.skip();
             skipShard(iterator);
@@ -286,7 +286,7 @@ private static boolean assertExecuteOnStartThread() {
         return true;
     }
 
-    protected void performPhaseOnShard(final int shardIndex, final SearchShardIterator shardIt, final SearchShardTarget shard) {
+    private void performPhaseOnShard(final int shardIndex, final SearchShardIterator shardIt, final SearchShardTarget shard) {
         if (throttleConcurrentRequests) {
             var pendingExecutions = pendingExecutionsPerNode.computeIfAbsent(
                 shard.getNodeId(),
@@ -349,7 +349,7 @@ protected abstract void executePhaseOnShard(
      * of the next phase. If there are no successful operations in the context when this method is executed the search is aborted and
      * a response is returned to the user indicating that all shards have failed.
      */
-    protected void executeNextPhase(SearchPhase currentPhase, Supplier<SearchPhase> nextPhaseSupplier) {
+    protected void executeNextPhase(String currentPhase, Supplier<SearchPhase> nextPhaseSupplier) {
         /* This is the main search phase transition where we move to the next phase. If all shards
          * failed or if there was a failure and partial results are not allowed, then we immediately
          * fail. Otherwise we continue to the next phase.
@@ -360,7 +360,7 @@ protected void executeNextPhase(SearchPhase currentPhase, Supplier<SearchPhase>
             Throwable cause = shardSearchFailures.length == 0
                 ? null
                 : ElasticsearchException.guessRootCauses(shardSearchFailures[0].getCause())[0];
-            logger.debug(() -> "All shards failed for phase: [" + currentPhase.getName() + "]", cause);
+            logger.debug(() -> "All shards failed for phase: [" + currentPhase + "]", cause);
             onPhaseFailure(currentPhase, "all shards failed", cause);
         } else {
             Boolean allowPartialResults = request.allowPartialSearchResults();
@@ -373,7 +373,7 @@ protected void executeNextPhase(SearchPhase currentPhase, Supplier<SearchPhase>
                         int numShardFailures = shardSearchFailures.length;
                         shardSearchFailures = ExceptionsHelper.groupBy(shardSearchFailures);
                         Throwable cause = ElasticsearchException.guessRootCauses(shardSearchFailures[0].getCause())[0];
-                        logger.debug(() -> format("%s shards failed for phase: [%s]", numShardFailures, currentPhase.getName()), cause);
+                        logger.debug(() -> format("%s shards failed for phase: [%s]", numShardFailures, currentPhase), cause);
                     }
                     onPhaseFailure(currentPhase, "Partial shards failure", null);
                 } else {
@@ -386,7 +386,7 @@ protected void executeNextPhase(SearchPhase currentPhase, Supplier<SearchPhase>
                             successfulOps.get(),
                             skippedOps.get(),
                             getNumShards(),
-                            currentPhase.getName()
+                            currentPhase
                         );
                     }
                     onPhaseFailure(currentPhase, "Partial shards failure (" + discrepancy + " shards unavailable)", null);
@@ -400,7 +400,7 @@ protected void executeNextPhase(SearchPhase currentPhase, Supplier<SearchPhase>
                     .collect(Collectors.joining(","));
                 logger.trace(
                     "[{}] Moving to next phase: [{}], based on results from: {} (cluster state version: {})",
-                    currentPhase.getName(),
+                    currentPhase,
                     nextPhase.getName(),
                     resultsFrom,
                     clusterStateVersion
@@ -413,11 +413,11 @@ protected void executeNextPhase(SearchPhase currentPhase, Supplier<SearchPhase>
     private void executePhase(SearchPhase phase) {
         try {
             phase.run();
-        } catch (Exception e) {
+        } catch (RuntimeException e) {
             if (logger.isDebugEnabled()) {
                 logger.debug(() -> format("Failed to execute [%s] while moving to [%s] phase", request, phase.getName()), e);
             }
-            onPhaseFailure(phase, "", e);
+            onPhaseFailure(phase.getName(), "", e);
         }
     }
 
@@ -521,7 +521,6 @@ void onShardFailure(final int shardIndex, SearchShardTarget shardTarget, Excepti
                 successfulOps.decrementAndGet(); // if this shard was successful before (initial phase) we have to adjust the counter
             }
         }
-        results.consumeShardFailure(shardIndex);
     }
 
     private static boolean isTaskCancelledException(Exception e) {
@@ -694,8 +693,8 @@ public void sendSearchResponse(SearchResponseSections internalSearchResponse, At
      * @param msg an optional message
      * @param cause the cause of the phase failure
      */
-    public void onPhaseFailure(SearchPhase phase, String msg, Throwable cause) {
-        raisePhaseFailure(new SearchPhaseExecutionException(phase.getName(), msg, cause, buildShardFailures()));
+    public void onPhaseFailure(String phase, String msg, Throwable cause) {
+        raisePhaseFailure(new SearchPhaseExecutionException(phase, msg, cause, buildShardFailures()));
     }
 
     /**
@@ -740,7 +739,7 @@ void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connecti
      * @see #onShardResult(SearchPhaseResult, SearchShardIterator)
      */
     private void onPhaseDone() {  // as a tribute to @kimchy aka. finishHim()
-        executeNextPhase(this, this::getNextPhase);
+        executeNextPhase(getName(), this::getNextPhase);
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeRequest.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeRequest.java
index ba1afaf4678fb..7890a0f9f9738 100644
--- a/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeRequest.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.builder.SubSearchSourceBuilder;
@@ -49,7 +48,8 @@ public class CanMatchNodeRequest extends TransportRequest implements IndicesRequ
     private final SearchType searchType;
     private final Boolean requestCache;
     private final boolean allowPartialSearchResults;
-    private final Scroll scroll;
+    @Nullable
+    private final TimeValue scroll;
     private final int numberOfShards;
     private final long nowInMillis;
     @Nullable
@@ -195,7 +195,7 @@ public CanMatchNodeRequest(StreamInput in) throws IOException {
                 );
             }
         }
-        scroll = in.readOptionalWriteable(Scroll::new);
+        scroll = in.readOptionalTimeValue();
         requestCache = in.readOptionalBoolean();
         allowPartialSearchResults = in.readBoolean();
         numberOfShards = in.readVInt();
@@ -216,7 +216,7 @@ public void writeTo(StreamOutput out) throws IOException {
             // types not supported so send an empty array to previous versions
             out.writeStringArray(Strings.EMPTY_ARRAY);
         }
-        out.writeOptionalWriteable(scroll);
+        out.writeOptionalTimeValue(scroll);
         out.writeOptionalBoolean(requestCache);
         out.writeBoolean(allowPartialSearchResults);
         out.writeVInt(numberOfShards);
diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
index eaf62d1e57e66..d45a8a6f01cd1 100644
--- a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
@@ -42,7 +42,6 @@
 import java.util.concurrent.atomic.AtomicReferenceArray;
 import java.util.function.BiFunction;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.core.Types.forciblyCast;
@@ -74,7 +73,9 @@ final class CanMatchPreFilterSearchPhase {
     private final Executor executor;
     private final boolean requireAtLeastOneMatch;
 
-    private final CanMatchSearchPhaseResults results;
+    private final FixedBitSet possibleMatches;
+    private final MinAndMax<?>[] minAndMaxes;
+    private int numPossibleMatches;
     private final CoordinatorRewriteContextProvider coordinatorRewriteContextProvider;
 
     CanMatchPreFilterSearchPhase(
@@ -105,12 +106,13 @@ final class CanMatchPreFilterSearchPhase {
         this.requireAtLeastOneMatch = requireAtLeastOneMatch;
         this.coordinatorRewriteContextProvider = coordinatorRewriteContextProvider;
         this.executor = executor;
-        results = new CanMatchSearchPhaseResults(shardsIts.size());
-
+        final int size = shardsIts.size();
+        possibleMatches = new FixedBitSet(size);
+        minAndMaxes = new MinAndMax<?>[size];
         // we compute the shard index based on the natural order of the shards
         // that participate in the search request. This means that this number is
         // consistent between two requests that target the same shards.
-        final SearchShardIterator[] naturalOrder = new SearchShardIterator[shardsIts.size()];
+        final SearchShardIterator[] naturalOrder = new SearchShardIterator[size];
         int i = 0;
         for (SearchShardIterator shardsIt : shardsIts) {
             naturalOrder[i++] = shardsIt;
@@ -138,7 +140,7 @@ private void runCoordinatorRewritePhase() {
                 request,
                 searchShardIterator.getOriginalIndices().indicesOptions(),
                 Collections.emptyList(),
-                getNumShards(),
+                shardsIts.size(),
                 timeProvider.absoluteStartMillis(),
                 searchShardIterator.getClusterAlias()
             );
@@ -177,7 +179,27 @@ private void runCoordinatorRewritePhase() {
     private void consumeResult(boolean canMatch, ShardSearchRequest request) {
         CanMatchShardResponse result = new CanMatchShardResponse(canMatch, null);
         result.setShardIndex(request.shardRequestIndex());
-        results.consumeResult(result, () -> {});
+        consumeResult(result, () -> {});
+    }
+
+    private void consumeResult(CanMatchShardResponse result, Runnable next) {
+        try {
+            final boolean canMatch = result.canMatch();
+            final MinAndMax<?> minAndMax = result.estimatedMinAndMax();
+            if (canMatch || minAndMax != null) {
+                consumeResult(result.getShardIndex(), canMatch, minAndMax);
+            }
+        } finally {
+            next.run();
+        }
+    }
+
+    private synchronized void consumeResult(int shardIndex, boolean canMatch, MinAndMax<?> minAndMax) {
+        if (canMatch) {
+            possibleMatches.set(shardIndex);
+            numPossibleMatches++;
+        }
+        minAndMaxes[shardIndex] = minAndMax;
     }
 
     private void checkNoMissingShards(GroupShardsIterator<SearchShardIterator> shards) {
@@ -235,32 +257,38 @@ protected void doRun() {
                     continue;
                 }
 
+                var sendingTarget = entry.getKey();
                 try {
-                    searchTransportService.sendCanMatch(getConnection(entry.getKey()), canMatchNodeRequest, task, new ActionListener<>() {
-                        @Override
-                        public void onResponse(CanMatchNodeResponse canMatchNodeResponse) {
-                            assert canMatchNodeResponse.getResponses().size() == canMatchNodeRequest.getShardLevelRequests().size();
-                            for (int i = 0; i < canMatchNodeResponse.getResponses().size(); i++) {
-                                CanMatchNodeResponse.ResponseOrFailure response = canMatchNodeResponse.getResponses().get(i);
-                                if (response.getResponse() != null) {
-                                    CanMatchShardResponse shardResponse = response.getResponse();
-                                    shardResponse.setShardIndex(shardLevelRequests.get(i).getShardRequestIndex());
-                                    onOperation(shardResponse.getShardIndex(), shardResponse);
-                                } else {
-                                    Exception failure = response.getException();
-                                    assert failure != null;
-                                    onOperationFailed(shardLevelRequests.get(i).getShardRequestIndex(), failure);
+                    searchTransportService.sendCanMatch(
+                        nodeIdToConnection.apply(sendingTarget.clusterAlias, sendingTarget.nodeId),
+                        canMatchNodeRequest,
+                        task,
+                        new ActionListener<>() {
+                            @Override
+                            public void onResponse(CanMatchNodeResponse canMatchNodeResponse) {
+                                assert canMatchNodeResponse.getResponses().size() == canMatchNodeRequest.getShardLevelRequests().size();
+                                for (int i = 0; i < canMatchNodeResponse.getResponses().size(); i++) {
+                                    CanMatchNodeResponse.ResponseOrFailure response = canMatchNodeResponse.getResponses().get(i);
+                                    if (response.getResponse() != null) {
+                                        CanMatchShardResponse shardResponse = response.getResponse();
+                                        shardResponse.setShardIndex(shardLevelRequests.get(i).getShardRequestIndex());
+                                        onOperation(shardResponse.getShardIndex(), shardResponse);
+                                    } else {
+                                        Exception failure = response.getException();
+                                        assert failure != null;
+                                        onOperationFailed(shardLevelRequests.get(i).getShardRequestIndex(), failure);
+                                    }
                                 }
                             }
-                        }
 
-                        @Override
-                        public void onFailure(Exception e) {
-                            for (CanMatchNodeRequest.Shard shard : shardLevelRequests) {
-                                onOperationFailed(shard.getShardRequestIndex(), e);
+                            @Override
+                            public void onFailure(Exception e) {
+                                for (CanMatchNodeRequest.Shard shard : shardLevelRequests) {
+                                    onOperationFailed(shard.getShardRequestIndex(), e);
+                                }
                             }
                         }
-                    });
+                    );
                 } catch (Exception e) {
                     for (CanMatchNodeRequest.Shard shard : shardLevelRequests) {
                         onOperationFailed(shard.getShardRequestIndex(), e);
@@ -271,7 +299,7 @@ public void onFailure(Exception e) {
 
         private void onOperation(int idx, CanMatchShardResponse response) {
             failedResponses.set(idx, null);
-            results.consumeResult(response, () -> {
+            consumeResult(response, () -> {
                 if (countDown.countDown()) {
                     finishRound();
                 }
@@ -280,7 +308,8 @@ private void onOperation(int idx, CanMatchShardResponse response) {
 
         private void onOperationFailed(int idx, Exception e) {
             failedResponses.set(idx, e);
-            results.consumeShardFailure(idx);
+            // we have to carry over shard failures in order to account for them in the response.
+            consumeResult(idx, true, null);
             if (countDown.countDown()) {
                 finishRound();
             }
@@ -334,14 +363,14 @@ private CanMatchNodeRequest createCanMatchRequest(Map.Entry<SendingTarget, List<
             request,
             first.getOriginalIndices().indicesOptions(),
             shardLevelRequests,
-            getNumShards(),
+            shardsIts.size(),
             timeProvider.absoluteStartMillis(),
             first.getClusterAlias()
         );
     }
 
     private void finishPhase() {
-        listener.onResponse(getIterator(results, shardsIts));
+        listener.onResponse(getIterator(shardsIts));
     }
 
     private static final float DEFAULT_INDEX_BOOST = 1.0f;
@@ -364,7 +393,7 @@ public CanMatchNodeRequest.Shard buildShardLevelRequest(SearchShardIterator shar
     }
 
     public void start() {
-        if (getNumShards() == 0) {
+        if (shardsIts.size() == 0) {
             finishPhase();
             return;
         }
@@ -386,85 +415,13 @@ protected void doRun() {
         });
     }
 
-    public void onPhaseFailure(String msg, Exception cause) {
+    private void onPhaseFailure(String msg, Exception cause) {
         listener.onFailure(new SearchPhaseExecutionException("can_match", msg, cause, ShardSearchFailure.EMPTY_ARRAY));
     }
 
-    public Transport.Connection getConnection(SendingTarget sendingTarget) {
-        return nodeIdToConnection.apply(sendingTarget.clusterAlias, sendingTarget.nodeId);
-    }
-
-    private int getNumShards() {
-        return shardsIts.size();
-    }
-
-    private static final class CanMatchSearchPhaseResults extends SearchPhaseResults<CanMatchShardResponse> {
-        private final FixedBitSet possibleMatches;
-        private final MinAndMax<?>[] minAndMaxes;
-        private int numPossibleMatches;
-
-        CanMatchSearchPhaseResults(int size) {
-            super(size);
-            possibleMatches = new FixedBitSet(size);
-            minAndMaxes = new MinAndMax<?>[size];
-        }
-
-        @Override
-        void consumeResult(CanMatchShardResponse result, Runnable next) {
-            try {
-                final boolean canMatch = result.canMatch();
-                final MinAndMax<?> minAndMax = result.estimatedMinAndMax();
-                if (canMatch || minAndMax != null) {
-                    consumeResult(result.getShardIndex(), canMatch, minAndMax);
-                }
-            } finally {
-                next.run();
-            }
-        }
-
-        @Override
-        boolean hasResult(int shardIndex) {
-            return false; // unneeded
-        }
-
-        @Override
-        void consumeShardFailure(int shardIndex) {
-            // we have to carry over shard failures in order to account for them in the response.
-            consumeResult(shardIndex, true, null);
-        }
-
-        private synchronized void consumeResult(int shardIndex, boolean canMatch, MinAndMax<?> minAndMax) {
-            if (canMatch) {
-                possibleMatches.set(shardIndex);
-                numPossibleMatches++;
-            }
-            minAndMaxes[shardIndex] = minAndMax;
-        }
-
-        synchronized int getNumPossibleMatches() {
-            return numPossibleMatches;
-        }
-
-        synchronized FixedBitSet getPossibleMatches() {
-            return possibleMatches;
-        }
-
-        @Override
-        Stream<CanMatchShardResponse> getSuccessfulResults() {
-            return Stream.empty();
-        }
-
-        @Override
-        public void close() {}
-    }
-
-    private GroupShardsIterator<SearchShardIterator> getIterator(
-        CanMatchSearchPhaseResults results,
-        GroupShardsIterator<SearchShardIterator> shardsIts
-    ) {
-        FixedBitSet possibleMatches = results.getPossibleMatches();
+    private synchronized GroupShardsIterator<SearchShardIterator> getIterator(GroupShardsIterator<SearchShardIterator> shardsIts) {
         // TODO: pick the local shard when possible
-        if (requireAtLeastOneMatch && results.getNumPossibleMatches() == 0) {
+        if (requireAtLeastOneMatch && numPossibleMatches == 0) {
             // this is a special case where we have no hit but we need to get at least one search response in order
             // to produce a valid search result with all the aggs etc.
             // Since it's possible that some of the shards that we're skipping are
@@ -491,11 +448,11 @@ private GroupShardsIterator<SearchShardIterator> getIterator(
                 iter.skip(true);
             }
         }
-        if (shouldSortShards(results.minAndMaxes) == false) {
+        if (shouldSortShards(minAndMaxes) == false) {
             return shardsIts;
         }
         FieldSortBuilder fieldSort = FieldSortBuilder.getPrimaryFieldSortOrNull(request.source());
-        return new GroupShardsIterator<>(sortShards(shardsIts, results.minAndMaxes, fieldSort.order()));
+        return new GroupShardsIterator<>(sortShards(shardsIts, minAndMaxes, fieldSort.order()));
     }
 
     private static List<SearchShardIterator> sortShards(
diff --git a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
index cc8c4becea9a9..faeb552530e47 100644
--- a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
@@ -39,6 +39,9 @@
  * @see CountedCollector#onFailure(int, SearchShardTarget, Exception)
  */
 final class DfsQueryPhase extends SearchPhase {
+
+    public static final String NAME = "dfs_query";
+
     private final SearchPhaseResults<SearchPhaseResult> queryResult;
     private final List<DfsSearchResult> searchResults;
     private final AggregatedDfs dfs;
@@ -56,7 +59,7 @@ final class DfsQueryPhase extends SearchPhase {
         Function<SearchPhaseResults<SearchPhaseResult>, SearchPhase> nextPhaseFactory,
         AbstractSearchAsyncAction<?> context
     ) {
-        super("dfs_query");
+        super(NAME);
         this.progressListener = context.getTask().getProgressListener();
         this.queryResult = queryResult;
         this.searchResults = searchResults;
@@ -68,13 +71,13 @@ final class DfsQueryPhase extends SearchPhase {
     }
 
     @Override
-    public void run() {
+    protected void run() {
         // TODO we can potentially also consume the actual per shard results from the initial phase here in the aggregateDfs
         // to free up memory early
         final CountedCollector<SearchPhaseResult> counter = new CountedCollector<>(
             queryResult,
             searchResults.size(),
-            () -> context.executeNextPhase(this, () -> nextPhaseFactory.apply(queryResult)),
+            () -> context.executeNextPhase(NAME, () -> nextPhaseFactory.apply(queryResult)),
             context
         );
 
@@ -106,7 +109,7 @@ protected void innerOnResponse(QuerySearchResult response) {
                             response.setSearchProfileDfsPhaseResult(dfsResult.searchProfileDfsPhaseResult());
                             counter.onResult(response);
                         } catch (Exception e) {
-                            context.onPhaseFailure(DfsQueryPhase.this, "", e);
+                            context.onPhaseFailure(NAME, "", e);
                         }
                     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
index e8d94c32bdcc7..b0b3f15265920 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
@@ -31,12 +31,15 @@
  * forwards to the next phase immediately.
  */
 final class ExpandSearchPhase extends SearchPhase {
+
+    static final String NAME = "expand";
+
     private final AbstractSearchAsyncAction<?> context;
     private final SearchHits searchHits;
     private final Supplier<SearchPhase> nextPhase;
 
     ExpandSearchPhase(AbstractSearchAsyncAction<?> context, SearchHits searchHits, Supplier<SearchPhase> nextPhase) {
-        super("expand");
+        super(NAME);
         this.context = context;
         this.searchHits = searchHits;
         this.nextPhase = nextPhase;
@@ -51,7 +54,7 @@ private boolean isCollapseRequest() {
     }
 
     @Override
-    public void run() {
+    protected void run() {
         if (isCollapseRequest() == false || searchHits.getHits().length == 0) {
             onPhaseDone();
         } else {
@@ -123,7 +126,7 @@ private void doRun() {
     }
 
     private void phaseFailure(Exception ex) {
-        context.onPhaseFailure(this, "failed to expand hits", ex);
+        context.onPhaseFailure(NAME, "failed to expand hits", ex);
     }
 
     private static SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder options, CollapseBuilder innerCollapseBuilder) {
@@ -168,6 +171,6 @@ private static SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilde
     }
 
     private void onPhaseDone() {
-        context.executeNextPhase(this, nextPhase);
+        context.executeNextPhase(NAME, nextPhase);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchLookupFieldsPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchLookupFieldsPhase.java
index d8671bcadf86d..2e98d50196490 100644
--- a/server/src/main/java/org/elasticsearch/action/search/FetchLookupFieldsPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/FetchLookupFieldsPhase.java
@@ -33,6 +33,9 @@
  * @see org.elasticsearch.index.mapper.LookupRuntimeFieldType
  */
 final class FetchLookupFieldsPhase extends SearchPhase {
+
+    static final String NAME = "fetch_lookup_fields";
+
     private final AbstractSearchAsyncAction<?> context;
     private final SearchResponseSections searchResponse;
     private final AtomicArray<SearchPhaseResult> queryResults;
@@ -42,7 +45,7 @@ final class FetchLookupFieldsPhase extends SearchPhase {
         SearchResponseSections searchResponse,
         AtomicArray<SearchPhaseResult> queryResults
     ) {
-        super("fetch_lookup_fields");
+        super(NAME);
         this.context = context;
         this.searchResponse = searchResponse;
         this.queryResults = queryResults;
@@ -74,7 +77,7 @@ private static List<Cluster> groupLookupFieldsByClusterAlias(SearchHits searchHi
     }
 
     @Override
-    public void run() {
+    protected void run() {
         final List<Cluster> clusters = groupLookupFieldsByClusterAlias(searchResponse.hits);
         if (clusters.isEmpty()) {
             context.sendSearchResponse(searchResponse, queryResults);
@@ -129,7 +132,7 @@ public void onResponse(MultiSearchResponse items) {
                     }
                 }
                 if (failure != null) {
-                    context.onPhaseFailure(FetchLookupFieldsPhase.this, "failed to fetch lookup fields", failure);
+                    context.onPhaseFailure(NAME, "failed to fetch lookup fields", failure);
                 } else {
                     context.sendSearchResponse(searchResponse, queryResults);
                 }
@@ -137,7 +140,7 @@ public void onResponse(MultiSearchResponse items) {
 
             @Override
             public void onFailure(Exception e) {
-                context.onPhaseFailure(FetchLookupFieldsPhase.this, "failed to fetch lookup fields", e);
+                context.onPhaseFailure(NAME, "failed to fetch lookup fields", e);
             }
         });
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
index 0fbface3793a8..119cfcab76105 100644
--- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
@@ -34,6 +34,9 @@
  * Then it reaches out to all relevant shards to fetch the topN hits.
  */
 final class FetchSearchPhase extends SearchPhase {
+
+    static final String NAME = "fetch";
+
     private final AtomicArray<SearchPhaseResult> searchPhaseShardResults;
     private final BiFunction<SearchResponseSections, AtomicArray<SearchPhaseResult>, SearchPhase> nextPhaseFactory;
     private final AbstractSearchAsyncAction<?> context;
@@ -70,7 +73,7 @@ final class FetchSearchPhase extends SearchPhase {
         @Nullable SearchPhaseController.ReducedQueryPhase reducedQueryPhase,
         BiFunction<SearchResponseSections, AtomicArray<SearchPhaseResult>, SearchPhase> nextPhaseFactory
     ) {
-        super("fetch");
+        super(NAME);
         if (context.getNumShards() != resultConsumer.getNumShards()) {
             throw new IllegalStateException(
                 "number of shards must match the length of the query results but doesn't:"
@@ -90,7 +93,7 @@ final class FetchSearchPhase extends SearchPhase {
     }
 
     @Override
-    public void run() {
+    protected void run() {
         context.execute(new AbstractRunnable() {
 
             @Override
@@ -100,7 +103,7 @@ protected void doRun() throws Exception {
 
             @Override
             public void onFailure(Exception e) {
-                context.onPhaseFailure(FetchSearchPhase.this, "", e);
+                context.onPhaseFailure(NAME, "", e);
             }
         });
     }
@@ -222,7 +225,7 @@ public void innerOnResponse(FetchSearchResult result) {
                     progressListener.notifyFetchResult(shardIndex);
                     counter.onResult(result);
                 } catch (Exception e) {
-                    context.onPhaseFailure(FetchSearchPhase.this, "", e);
+                    context.onPhaseFailure(NAME, "", e);
                 }
             }
 
@@ -269,9 +272,9 @@ private void moveToNextPhase(
         AtomicArray<? extends SearchPhaseResult> fetchResultsArr,
         SearchPhaseController.ReducedQueryPhase reducedQueryPhase
     ) {
-        context.executeNextPhase(this, () -> {
+        context.executeNextPhase(NAME, () -> {
             var resp = SearchPhaseController.merge(context.getRequest().scroll() != null, reducedQueryPhase, fetchResultsArr);
-            context.addReleasable(resp::decRef);
+            context.addReleasable(resp);
             return nextPhaseFactory.apply(resp, searchPhaseShardResults);
         });
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java
index 199228c9f992c..e9302883457e1 100644
--- a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java
@@ -37,6 +37,8 @@
  */
 public class RankFeaturePhase extends SearchPhase {
 
+    static final String NAME = "rank-feature";
+
     private static final Logger logger = LogManager.getLogger(RankFeaturePhase.class);
     private final AbstractSearchAsyncAction<?> context;
     final SearchPhaseResults<SearchPhaseResult> queryPhaseResults;
@@ -51,7 +53,7 @@ public class RankFeaturePhase extends SearchPhase {
         AbstractSearchAsyncAction<?> context,
         RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext
     ) {
-        super("rank-feature");
+        super(NAME);
         assert rankFeaturePhaseRankCoordinatorContext != null;
         this.rankFeaturePhaseRankCoordinatorContext = rankFeaturePhaseRankCoordinatorContext;
         if (context.getNumShards() != queryPhaseResults.getNumShards()) {
@@ -71,7 +73,7 @@ public class RankFeaturePhase extends SearchPhase {
     }
 
     @Override
-    public void run() {
+    protected void run() {
         context.execute(new AbstractRunnable() {
             @Override
             protected void doRun() throws Exception {
@@ -84,7 +86,7 @@ protected void doRun() throws Exception {
 
             @Override
             public void onFailure(Exception e) {
-                context.onPhaseFailure(RankFeaturePhase.this, "", e);
+                context.onPhaseFailure(NAME, "", e);
             }
         });
     }
@@ -139,7 +141,7 @@ protected void innerOnResponse(RankFeatureResult response) {
                     progressListener.notifyRankFeatureResult(shardIndex);
                     rankRequestCounter.onResult(response);
                 } catch (Exception e) {
-                    context.onPhaseFailure(RankFeaturePhase.this, "", e);
+                    context.onPhaseFailure(NAME, "", e);
                 }
             }
 
@@ -194,7 +196,7 @@ public void onResponse(RankFeatureDoc[] docsWithUpdatedScores) {
 
                 @Override
                 public void onFailure(Exception e) {
-                    context.onPhaseFailure(RankFeaturePhase.this, "Computing updated ranks for results failed", e);
+                    context.onPhaseFailure(NAME, "Computing updated ranks for results failed", e);
                 }
             }
         );
@@ -239,6 +241,6 @@ private float maxScore(ScoreDoc[] scoreDocs) {
     }
 
     void moveToNextPhase(SearchPhaseResults<SearchPhaseResult> phaseResults, SearchPhaseController.ReducedQueryPhase reducedQueryPhase) {
-        context.executeNextPhase(this, () -> new FetchSearchPhase(phaseResults, aggregatedDfs, context, reducedQueryPhase));
+        context.executeNextPhase(NAME, () -> new FetchSearchPhase(phaseResults, aggregatedDfs, context, reducedQueryPhase));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java
index 7d849a72abf9d..702369dc38390 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java
@@ -9,25 +9,25 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
-import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.transport.Transport;
 
-import java.io.IOException;
 import java.util.Objects;
 import java.util.function.Function;
 
 /**
  * Base class for all individual search phases like collecting distributed frequencies, fetching documents, querying shards.
  */
-abstract class SearchPhase implements CheckedRunnable<IOException> {
+abstract class SearchPhase {
     private final String name;
 
     protected SearchPhase(String name) {
         this.name = Objects.requireNonNull(name, "name must not be null");
     }
 
+    protected abstract void run();
+
     /**
      * Returns the phases name.
      */
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
index b118c2560925e..69e7fba4dd0d5 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
@@ -551,9 +551,8 @@ static ReducedQueryPhase reducedQueryPhase(
         assert numReducePhases >= 0 : "num reduce phases must be >= 0 but was: " + numReducePhases;
         numReducePhases++; // increment for this phase
         if (queryResults.isEmpty()) { // early terminate we have nothing to reduce
-            final TotalHits totalHits = topDocsStats.getTotalHits();
             return new ReducedQueryPhase(
-                totalHits,
+                topDocsStats.getTotalHits(),
                 topDocsStats.fetchHits,
                 topDocsStats.getMaxScore(),
                 false,
@@ -570,8 +569,7 @@ static ReducedQueryPhase reducedQueryPhase(
                 true
             );
         }
-        int total = queryResults.size();
-        final Collection<SearchPhaseResult> nonNullResults = new ArrayList<>();
+        final List<QuerySearchResult> nonNullResults = new ArrayList<>();
         boolean hasSuggest = false;
         boolean hasProfileResults = false;
         for (SearchPhaseResult queryResult : queryResults) {
@@ -581,12 +579,11 @@ static ReducedQueryPhase reducedQueryPhase(
             }
             hasSuggest |= res.suggest() != null;
             hasProfileResults |= res.hasProfileResults();
-            nonNullResults.add(queryResult);
+            nonNullResults.add(res);
         }
-        queryResults = nonNullResults;
-        validateMergeSortValueFormats(queryResults);
-        if (queryResults.isEmpty()) {
-            var ex = new IllegalStateException("must have at least one non-empty search result, got 0 out of " + total);
+        validateMergeSortValueFormats(nonNullResults);
+        if (nonNullResults.isEmpty()) {
+            var ex = new IllegalStateException("must have at least one non-empty search result, got 0 out of " + queryResults.size());
             assert false : ex;
             throw ex;
         }
@@ -594,13 +591,12 @@ static ReducedQueryPhase reducedQueryPhase(
         // count the total (we use the query result provider here, since we might not get any hits (we scrolled past them))
         final Map<String, List<Suggestion<?>>> groupedSuggestions = hasSuggest ? new HashMap<>() : Collections.emptyMap();
         final Map<String, SearchProfileQueryPhaseResult> profileShardResults = hasProfileResults
-            ? Maps.newMapWithExpectedSize(queryResults.size())
+            ? Maps.newMapWithExpectedSize(nonNullResults.size())
             : Collections.emptyMap();
         int from = 0;
         int size = 0;
         DocValueFormat[] sortValueFormats = null;
-        for (SearchPhaseResult entry : queryResults) {
-            QuerySearchResult result = entry.queryResult();
+        for (QuerySearchResult result : nonNullResults) {
             from = result.from();
             // sorted queries can set the size to 0 if they have enough competitive hits.
             size = Math.max(result.size(), size);
@@ -611,8 +607,7 @@ static ReducedQueryPhase reducedQueryPhase(
             if (hasSuggest) {
                 assert result.suggest() != null;
                 for (Suggestion<? extends Suggestion.Entry<? extends Suggestion.Entry.Option>> suggestion : result.suggest()) {
-                    List<Suggestion<?>> suggestionList = groupedSuggestions.computeIfAbsent(suggestion.getName(), s -> new ArrayList<>());
-                    suggestionList.add(suggestion);
+                    groupedSuggestions.computeIfAbsent(suggestion.getName(), s -> new ArrayList<>()).add(suggestion);
                     if (suggestion instanceof CompletionSuggestion completionSuggestion) {
                         completionSuggestion.setShardIndex(result.getShardIndex());
                     }
@@ -620,53 +615,48 @@ static ReducedQueryPhase reducedQueryPhase(
             }
             assert bufferedTopDocs.isEmpty() || result.hasConsumedTopDocs() : "firstResult has no aggs but we got non null buffered aggs?";
             if (hasProfileResults) {
-                String key = result.getSearchShardTarget().toString();
-                profileShardResults.put(key, result.consumeProfileResult());
+                profileShardResults.put(result.getSearchShardTarget().toString(), result.consumeProfileResult());
             }
         }
-        final Suggest reducedSuggest;
-        final List<CompletionSuggestion> reducedCompletionSuggestions;
-        if (groupedSuggestions.isEmpty()) {
-            reducedSuggest = null;
-            reducedCompletionSuggestions = Collections.emptyList();
-        } else {
-            reducedSuggest = new Suggest(Suggest.reduce(groupedSuggestions));
-            reducedCompletionSuggestions = reducedSuggest.filter(CompletionSuggestion.class);
-        }
-        final InternalAggregations aggregations = bufferedAggs == null
-            ? null
-            : InternalAggregations.topLevelReduceDelayable(
-                bufferedAggs,
-                performFinalReduce ? aggReduceContextBuilder.forFinalReduction() : aggReduceContextBuilder.forPartialReduction()
-            );
-        final SearchProfileResultsBuilder profileBuilder = profileShardResults.isEmpty()
-            ? null
-            : new SearchProfileResultsBuilder(profileShardResults);
+        final Suggest reducedSuggest = groupedSuggestions.isEmpty() ? null : new Suggest(Suggest.reduce(groupedSuggestions));
         final SortedTopDocs sortedTopDocs;
         if (queryPhaseRankCoordinatorContext == null) {
-            sortedTopDocs = sortDocs(isScrollRequest, bufferedTopDocs, from, size, reducedCompletionSuggestions);
+            sortedTopDocs = sortDocs(
+                isScrollRequest,
+                bufferedTopDocs,
+                from,
+                size,
+                reducedSuggest == null ? Collections.emptyList() : reducedSuggest.filter(CompletionSuggestion.class)
+            );
         } else {
-            ScoreDoc[] rankedDocs = queryPhaseRankCoordinatorContext.rankQueryPhaseResults(
-                queryResults.stream().map(SearchPhaseResult::queryResult).toList(),
-                topDocsStats
+            sortedTopDocs = new SortedTopDocs(
+                queryPhaseRankCoordinatorContext.rankQueryPhaseResults(nonNullResults, topDocsStats),
+                false,
+                null,
+                null,
+                null,
+                0
             );
-            sortedTopDocs = new SortedTopDocs(rankedDocs, false, null, null, null, 0);
             size = sortedTopDocs.scoreDocs.length;
             // we need to reset from here as pagination and result trimming has already taken place
             // within the `QueryPhaseRankCoordinatorContext#rankQueryPhaseResults` and we don't want
             // to apply it again in the `getHits` method.
             from = 0;
         }
-        final TotalHits totalHits = topDocsStats.getTotalHits();
         return new ReducedQueryPhase(
-            totalHits,
+            topDocsStats.getTotalHits(),
             topDocsStats.fetchHits,
             topDocsStats.getMaxScore(),
             topDocsStats.timedOut,
             topDocsStats.terminatedEarly,
             reducedSuggest,
-            aggregations,
-            profileBuilder,
+            bufferedAggs == null
+                ? null
+                : InternalAggregations.topLevelReduceDelayable(
+                    bufferedAggs,
+                    performFinalReduce ? aggReduceContextBuilder.forFinalReduction() : aggReduceContextBuilder.forPartialReduction()
+                ),
+            profileShardResults.isEmpty() ? null : new SearchProfileResultsBuilder(profileShardResults),
             sortedTopDocs,
             sortValueFormats,
             queryPhaseRankCoordinatorContext,
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
index 2d798f2da3a4f..fc79e85f9326d 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
@@ -73,15 +73,21 @@ public RestStatus status() {
             // on coordinator node. so get the status from cause instead of returning SERVICE_UNAVAILABLE blindly
             return getCause() == null ? RestStatus.SERVICE_UNAVAILABLE : ExceptionsHelper.status(getCause());
         }
-        RestStatus status = shardFailures[0].status();
-        if (shardFailures.length > 1) {
-            for (int i = 1; i < shardFailures.length; i++) {
-                if (shardFailures[i].status().getStatus() >= RestStatus.INTERNAL_SERVER_ERROR.getStatus()) {
-                    status = shardFailures[i].status();
-                }
+        RestStatus status = null;
+        for (ShardSearchFailure shardFailure : shardFailures) {
+            RestStatus shardStatus = shardFailure.status();
+            int statusCode = shardStatus.getStatus();
+
+            // Return if it's an error that can be retried.
+            // These currently take precedence over other status code(s).
+            if (statusCode >= 502 && statusCode <= 504) {
+                return shardStatus;
+            } else if (statusCode >= 500) {
+                status = shardStatus;
             }
         }
-        return status;
+
+        return status == null ? shardFailures[0].status() : status;
     }
 
     public ShardSearchFailure[] shardFailures() {
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseResults.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseResults.java
index 73bb0d545a2e0..54f9f5549f30c 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseResults.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseResults.java
@@ -49,8 +49,6 @@ final int getNumShards() {
      */
     abstract boolean hasResult(int shardIndex);
 
-    void consumeShardFailure(int shardIndex) {}
-
     AtomicArray<Result> getAtomicArray() {
         throw new UnsupportedOperationException();
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 2e1d58e042f09..8b77ec7fb5463 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.Rewriteable;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.PointInTimeBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.internal.SearchContext;
@@ -82,7 +81,7 @@ public class SearchRequest extends ActionRequest implements IndicesRequest.Repla
 
     private Boolean allowPartialSearchResults;
 
-    private Scroll scroll;
+    private TimeValue scrollKeepAlive;
 
     private int batchedReduceSize = DEFAULT_BATCHED_REDUCE_SIZE;
 
@@ -206,7 +205,7 @@ private SearchRequest(
         this.preFilterShardSize = searchRequest.preFilterShardSize;
         this.requestCache = searchRequest.requestCache;
         this.routing = searchRequest.routing;
-        this.scroll = searchRequest.scroll;
+        this.scrollKeepAlive = searchRequest.scrollKeepAlive;
         this.searchType = searchRequest.searchType;
         this.source = searchRequest.source;
         this.localClusterAlias = localClusterAlias;
@@ -229,7 +228,7 @@ public SearchRequest(StreamInput in) throws IOException {
         indices = in.readStringArray();
         routing = in.readOptionalString();
         preference = in.readOptionalString();
-        scroll = in.readOptionalWriteable(Scroll::new);
+        scrollKeepAlive = in.readOptionalTimeValue();
         source = in.readOptionalWriteable(SearchSourceBuilder::new);
         if (in.getTransportVersion().before(TransportVersions.V_8_0_0)) {
             // types no longer relevant so ignore
@@ -276,7 +275,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeStringArray(indices);
         out.writeOptionalString(routing);
         out.writeOptionalString(preference);
-        out.writeOptionalWriteable(scroll);
+        out.writeOptionalTimeValue(scrollKeepAlive);
         out.writeOptionalWriteable(source);
         if (out.getTransportVersion().before(TransportVersions.V_8_0_0)) {
             // types not supported so send an empty array to previous versions
@@ -525,23 +524,16 @@ public String[] indices() {
     /**
      * If set, will enable scrolling of the search request.
      */
-    public Scroll scroll() {
-        return scroll;
-    }
-
-    /**
-     * If set, will enable scrolling of the search request.
-     */
-    public SearchRequest scroll(Scroll scroll) {
-        this.scroll = scroll;
-        return this;
+    public TimeValue scroll() {
+        return scrollKeepAlive;
     }
 
     /**
      * If set, will enable scrolling of the search request for the specified timeout.
      */
     public SearchRequest scroll(TimeValue keepAlive) {
-        return scroll(new Scroll(keepAlive));
+        this.scrollKeepAlive = keepAlive;
+        return this;
     }
 
     /**
@@ -681,7 +673,7 @@ public boolean hasKnnSearch() {
     }
 
     public int resolveTrackTotalHitsUpTo() {
-        return resolveTrackTotalHitsUpTo(scroll, source);
+        return resolveTrackTotalHitsUpTo(scrollKeepAlive, source);
     }
 
     /**
@@ -731,7 +723,7 @@ public SearchRequest rewrite(QueryRewriteContext ctx) throws IOException {
         return hasChanged ? new SearchRequest(this).source(source) : this;
     }
 
-    public static int resolveTrackTotalHitsUpTo(Scroll scroll, SearchSourceBuilder source) {
+    public static int resolveTrackTotalHitsUpTo(TimeValue scroll, SearchSourceBuilder source) {
         if (scroll != null) {
             // no matter what the value of track_total_hits is
             return SearchContext.TRACK_TOTAL_HITS_ACCURATE;
@@ -752,8 +744,8 @@ public final String buildDescription() {
         Strings.arrayToDelimitedString(indices, ",", sb);
         sb.append("]");
         sb.append(", search_type[").append(searchType).append("]");
-        if (scroll != null) {
-            sb.append(", scroll[").append(scroll.keepAlive()).append("]");
+        if (scrollKeepAlive != null) {
+            sb.append(", scroll[").append(scrollKeepAlive).append("]");
         }
         if (source != null) {
             sb.append(", source[").append(source.toString(FORMAT_PARAMS)).append("]");
@@ -784,7 +776,7 @@ public boolean equals(Object o) {
             && Objects.equals(preference, that.preference)
             && Objects.equals(source, that.source)
             && Objects.equals(requestCache, that.requestCache)
-            && Objects.equals(scroll, that.scroll)
+            && Objects.equals(scrollKeepAlive, that.scrollKeepAlive)
             && Objects.equals(batchedReduceSize, that.batchedReduceSize)
             && Objects.equals(maxConcurrentShardRequests, that.maxConcurrentShardRequests)
             && Objects.equals(preFilterShardSize, that.preFilterShardSize)
@@ -805,7 +797,7 @@ public int hashCode() {
             preference,
             source,
             requestCache,
-            scroll,
+            scrollKeepAlive,
             indicesOptions,
             batchedReduceSize,
             maxConcurrentShardRequests,
@@ -836,7 +828,7 @@ public String toString() {
             + ", requestCache="
             + requestCache
             + ", scroll="
-            + scroll
+            + scrollKeepAlive
             + ", maxConcurrentShardRequests="
             + maxConcurrentShardRequests
             + ", batchedReduceSize="
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index 2927c394da3d4..d309ef3a7498a 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.builder.PointInTimeBuilder;
@@ -62,14 +61,6 @@ public SearchRequestBuilder setSearchType(SearchType searchType) {
         return this;
     }
 
-    /**
-     * If set, will enable scrolling of the search request.
-     */
-    public SearchRequestBuilder setScroll(Scroll scroll) {
-        request.scroll(scroll);
-        return this;
-    }
-
     /**
      * If set, will enable scrolling of the search request for the specified timeout.
      */
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
index 041b3ae73c1ee..787dc14f6cd96 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
@@ -14,12 +14,13 @@
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.RefCounted;
@@ -342,8 +343,7 @@ public ShardSearchFailure[] getShardFailures() {
     }
 
     /**
-     * If scrolling was enabled ({@link SearchRequest#scroll(org.elasticsearch.search.Scroll)}, the
-     * scroll id that can be used to continue scrolling.
+     * If scrolling was enabled ({@link SearchRequest#scroll(TimeValue)}, the scroll id that can be used to continue scrolling.
      */
     public String getScrollId() {
         return scrollId;
@@ -382,17 +382,24 @@ public Clusters getClusters() {
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         assert hasReferences();
-        return ChunkedToXContent.builder(params).xContentObject(innerToXContentChunked(params));
+        return Iterators.concat(
+            ChunkedToXContentHelper.startObject(),
+            this.innerToXContentChunked(params),
+            ChunkedToXContentHelper.endObject()
+        );
     }
 
     public Iterator<? extends ToXContent> innerToXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params)
-            .append(SearchResponse.this::headerToXContent)
-            .append(clusters)
-            .append(hits)
-            .appendIfPresent(aggregations)
-            .appendIfPresent(suggest)
-            .appendIfPresent(profileResults);
+        return Iterators.concat(
+            ChunkedToXContentHelper.chunk(SearchResponse.this::headerToXContent),
+            Iterators.single(clusters),
+            Iterators.concat(
+                hits.toXContentChunked(params),
+                aggregations == null ? Collections.emptyIterator() : ChunkedToXContentHelper.chunk(aggregations),
+                suggest == null ? Collections.emptyIterator() : ChunkedToXContentHelper.chunk(suggest),
+                profileResults == null ? Collections.emptyIterator() : ChunkedToXContentHelper.chunk(profileResults)
+            )
+        );
     }
 
     public XContentBuilder headerToXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java
index 8c9a42a61e33e..9d85348b80d62 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java
@@ -9,14 +9,12 @@
 
 package org.elasticsearch.action.search;
 
-import org.elasticsearch.core.RefCounted;
-import org.elasticsearch.core.SimpleRefCounted;
+import org.elasticsearch.core.Releasable;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.profile.SearchProfileResults;
 import org.elasticsearch.search.profile.SearchProfileShardResult;
 import org.elasticsearch.search.suggest.Suggest;
-import org.elasticsearch.transport.LeakTracker;
 
 import java.util.Collections;
 import java.util.Map;
@@ -25,7 +23,7 @@
  * Holds some sections that a search response is composed of (hits, aggs, suggestions etc.) during some steps of the search response
  * building.
  */
-public class SearchResponseSections implements RefCounted {
+public class SearchResponseSections implements Releasable {
 
     public static final SearchResponseSections EMPTY_WITH_TOTAL_HITS = new SearchResponseSections(
         SearchHits.EMPTY_WITH_TOTAL_HITS,
@@ -53,8 +51,6 @@ public class SearchResponseSections implements RefCounted {
     protected final Boolean terminatedEarly;
     protected final int numReducePhases;
 
-    private final RefCounted refCounted;
-
     public SearchResponseSections(
         SearchHits hits,
         InternalAggregations aggregations,
@@ -72,7 +68,6 @@ public SearchResponseSections(
         this.timedOut = timedOut;
         this.terminatedEarly = terminatedEarly;
         this.numReducePhases = numReducePhases;
-        refCounted = hits.getHits().length > 0 ? LeakTracker.wrap(new SimpleRefCounted()) : ALWAYS_REFERENCED;
     }
 
     public final SearchHits hits() {
@@ -97,26 +92,7 @@ public final Map<String, SearchProfileShardResult> profile() {
     }
 
     @Override
-    public void incRef() {
-        refCounted.incRef();
-    }
-
-    @Override
-    public boolean tryIncRef() {
-        return refCounted.tryIncRef();
-    }
-
-    @Override
-    public boolean decRef() {
-        if (refCounted.decRef()) {
-            hits.decRef();
-            return true;
-        }
-        return false;
-    }
-
-    @Override
-    public boolean hasReferences() {
-        return refCounted.hasReferences();
+    public void close() {
+        hits.decRef();
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
index 60e96a8cce8ab..caba10d5cee02 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
@@ -40,7 +40,7 @@
  * fan out to nodes and execute the query part of the scroll request. Subclasses can for instance
  * run separate fetch phases etc.
  */
-abstract class SearchScrollAsyncAction<T extends SearchPhaseResult> implements Runnable {
+abstract class SearchScrollAsyncAction<T extends SearchPhaseResult> {
     protected final Logger logger;
     protected final ActionListener<SearchResponse> listener;
     protected final ParsedScrollId scrollId;
@@ -229,7 +229,7 @@ protected SearchPhase sendResponsePhase(
     ) {
         return new SearchPhase("fetch") {
             @Override
-            public void run() {
+            protected void run() {
                 sendResponse(queryPhase, fetchResults);
             }
         };
@@ -246,8 +246,7 @@ protected final void sendResponse(
             if (request.scroll() != null) {
                 scrollId = request.scrollId();
             }
-            var sections = SearchPhaseController.merge(true, queryPhase, fetchResults);
-            try {
+            try (var sections = SearchPhaseController.merge(true, queryPhase, fetchResults)) {
                 ActionListener.respondAndRelease(
                     listener,
                     new SearchResponse(
@@ -262,8 +261,6 @@ protected final void sendResponse(
                         null
                     )
                 );
-            } finally {
-                sections.decRef();
             }
         } catch (Exception e) {
             listener.onFailure(new ReduceSearchPhaseException("fetch", "inner finish failed", e, buildShardFailures()));
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
index 8c33e3ca7da4b..fd3d748a556be 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
@@ -65,7 +65,7 @@ protected void executeInitialPhase(
     protected SearchPhase moveToNextPhase(BiFunction<String, String, DiscoveryNode> clusterNodeLookup) {
         return new SearchPhase("fetch") {
             @Override
-            public void run() {
+            protected void run() {
                 final SearchPhaseController.ReducedQueryPhase reducedQueryPhase = SearchPhaseController.reducedScrollQueryPhase(
                     queryResults.asList()
                 );
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java
index 71b88b03a5463..1d34288665ae4 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -30,7 +29,7 @@
 public class SearchScrollRequest extends ActionRequest implements ToXContentObject {
 
     private String scrollId;
-    private Scroll scroll;
+    private TimeValue scroll;
 
     public SearchScrollRequest() {}
 
@@ -41,14 +40,14 @@ public SearchScrollRequest(String scrollId) {
     public SearchScrollRequest(StreamInput in) throws IOException {
         super(in);
         scrollId = in.readString();
-        scroll = in.readOptionalWriteable(Scroll::new);
+        scroll = in.readOptionalTimeValue();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeString(scrollId);
-        out.writeOptionalWriteable(scroll);
+        out.writeOptionalTimeValue(scroll);
     }
 
     @Override
@@ -79,23 +78,16 @@ public ParsedScrollId parseScrollId() {
     /**
      * If set, will enable scrolling of the search request.
      */
-    public Scroll scroll() {
+    public TimeValue scroll() {
         return scroll;
     }
 
-    /**
-     * If set, will enable scrolling of the search request.
-     */
-    public SearchScrollRequest scroll(Scroll scroll) {
-        this.scroll = scroll;
-        return this;
-    }
-
     /**
      * If set, will enable scrolling of the search request for the specified timeout.
      */
     public SearchScrollRequest scroll(TimeValue keepAlive) {
-        return scroll(new Scroll(keepAlive));
+        this.scroll = keepAlive;
+        return this;
     }
 
     @Override
@@ -135,7 +127,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.field("scroll_id", scrollId);
         if (scroll != null) {
-            builder.field("scroll", scroll.keepAlive().getStringRep());
+            builder.field("scroll", scroll.getStringRep());
         }
         builder.endObject();
         return builder;
@@ -157,7 +149,7 @@ public void fromXContent(XContentParser parser) throws IOException {
                 } else if ("scroll_id".equals(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
                     scrollId(parser.text());
                 } else if ("scroll".equals(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
-                    scroll(new Scroll(TimeValue.parseTimeValue(parser.text(), null, "scroll")));
+                    scroll(TimeValue.parseTimeValue(parser.text(), null, "scroll"));
                 } else {
                     throw new IllegalArgumentException(
                         "Unknown parameter [" + currentFieldName + "] in request body or parameter is of the wrong type[" + token + "] "
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java
index 24dac98166ce0..57a6e44f4c2b0 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.internal.ElasticsearchClient;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.search.Scroll;
 
 /**
  * A search scroll action request builder.
@@ -35,14 +34,6 @@ public SearchScrollRequestBuilder setScrollId(String scrollId) {
         return this;
     }
 
-    /**
-     * If set, will enable scrolling of the search request.
-     */
-    public SearchScrollRequestBuilder setScroll(Scroll scroll) {
-        request.scroll(scroll);
-        return this;
-    }
-
     /**
      * If set, will enable scrolling of the search request for the specified timeout.
      */
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
index cfc2e1bcdaf2b..2041754bc2bcc 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -456,7 +456,8 @@ public static void registerRequestHandler(TransportService transportService, Sea
             (request, channel, task) -> searchService.executeQueryPhase(
                 request,
                 (SearchShardTask) task,
-                new ChannelActionListener<>(channel)
+                new ChannelActionListener<>(channel),
+                channel.getVersion()
             )
         );
         TransportActionProxy.registerProxyAction(transportService, QUERY_ID_ACTION_NAME, true, QuerySearchResult::new);
@@ -468,7 +469,8 @@ public static void registerRequestHandler(TransportService transportService, Sea
             (request, channel, task) -> searchService.executeQueryPhase(
                 request,
                 (SearchShardTask) task,
-                new ChannelActionListener<>(channel)
+                new ChannelActionListener<>(channel),
+                channel.getVersion()
             )
         );
         TransportActionProxy.registerProxyAction(transportService, QUERY_SCROLL_ACTION_NAME, true, ScrollQuerySearchResult::new);
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java
index 36ca0fba94372..6c95a3c8fd436 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java
@@ -270,7 +270,7 @@ protected void executePhaseOnShard(
                 protected SearchPhase getNextPhase() {
                     return new SearchPhase(getName()) {
                         @Override
-                        public void run() {
+                        protected void run() {
                             sendSearchResponse(SearchResponseSections.EMPTY_WITH_TOTAL_HITS, results.getAtomicArray());
                         }
                     };
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index ae27406bf396d..70a7f4c8cad0c 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -388,10 +388,7 @@ void executeRequest(
                     if (original.pointInTimeBuilder() != null) {
                         tl.setFeature(CCSUsageTelemetry.PIT_FEATURE);
                     }
-                    String client = task.getHeader(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER);
-                    if (client != null) {
-                        tl.setClient(client);
-                    }
+                    tl.setClient(task);
                     // Check if any of the index patterns are wildcard patterns
                     var localIndices = resolvedIndices.getLocalIndices();
                     if (localIndices != null && Arrays.stream(localIndices.indices()).anyMatch(Regex::isSimpleMatchPattern)) {
@@ -508,6 +505,7 @@ void executeRequest(
                 }
             }
         });
+
         final SearchSourceBuilder source = original.source();
         if (shouldOpenPIT(source)) {
             // disabling shard reordering for request
@@ -1883,7 +1881,7 @@ private interface TelemetryListener {
 
         void setFeature(String feature);
 
-        void setClient(String client);
+        void setClient(Task task);
     }
 
     private class SearchResponseActionListener extends DelegatingActionListener<SearchResponse, SearchResponse>
@@ -1917,8 +1915,8 @@ public void setFeature(String feature) {
         }
 
         @Override
-        public void setClient(String client) {
-            usageBuilder.setClient(client);
+        public void setClient(Task task) {
+            usageBuilder.setClientFromTask(task);
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java
index cffba76988f7d..b232cd16ba65e 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java
@@ -91,7 +91,7 @@ public void onFailure(Exception e) {
         };
         try {
             ParsedScrollId scrollId = parseScrollId(request.scrollId());
-            Runnable action = switch (scrollId.getType()) {
+            var action = switch (scrollId.getType()) {
                 case QUERY_THEN_FETCH_TYPE -> new SearchScrollQueryThenFetchAsyncAction(
                     logger,
                     clusterService,
diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
index ebbd47336e3da..4525259451481 100644
--- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
+++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.core.Nullable;
@@ -47,37 +46,13 @@
  * @param gatekeeperOptions, applies to all the resolved indices and defines if throttled will be included and if certain type of
  *                        aliases or indices are allowed, or they will throw an error. It acts as a gatekeeper when an action
  *                        does not support certain options.
- * @param selectorOptions, applies to all resolved expressions, and it specifies the index component that should be included, if there
- *                         is no index component defined on the expression level.
  */
 public record IndicesOptions(
     ConcreteTargetOptions concreteTargetOptions,
     WildcardOptions wildcardOptions,
-    GatekeeperOptions gatekeeperOptions,
-    SelectorOptions selectorOptions
+    GatekeeperOptions gatekeeperOptions
 ) implements ToXContentFragment {
 
-    /**
-     * @deprecated this query param will be replaced by the selector `::` on the expression level
-     */
-    @Deprecated
-    public static final String FAILURE_STORE_QUERY_PARAM = "failure_store";
-    /**
-     * @deprecated this value will be replaced by the selector `::*` on the expression level
-     */
-    @Deprecated
-    public static final String INCLUDE_ALL = "include";
-    /**
-     * @deprecated this value will be replaced by the selector `::data` on the expression level
-     */
-    @Deprecated
-    public static final String INCLUDE_ONLY_REGULAR_INDICES = "exclude";
-    /**
-     * @deprecated this value will be replaced by the selector `::failures` on the expression level
-     */
-    @Deprecated
-    public static final String INCLUDE_ONLY_FAILURE_INDICES = "only";
-
     public static IndicesOptions.Builder builder() {
         return new Builder();
     }
@@ -324,14 +299,14 @@ public static Builder builder(WildcardOptions wildcardOptions) {
      * - The ignoreThrottled flag, which is a deprecated flag that will filter out frozen indices.
      * @param allowAliasToMultipleIndices, allow aliases to multiple indices, true by default.
      * @param allowClosedIndices, allow closed indices, true by default.
-     * @param allowFailureIndices, allow failure indices in the response, true by default
+     * @param allowSelectors, allow selectors within index expressions, true by default.
      * @param ignoreThrottled, filters out throttled (aka frozen indices), defaults to true. This is deprecated and the only one
      *                         that only filters and never throws an error.
      */
     public record GatekeeperOptions(
         boolean allowAliasToMultipleIndices,
         boolean allowClosedIndices,
-        boolean allowFailureIndices,
+        boolean allowSelectors,
         @Deprecated boolean ignoreThrottled
     ) implements ToXContentFragment {
 
@@ -355,7 +330,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         public static class Builder {
             private boolean allowAliasToMultipleIndices;
             private boolean allowClosedIndices;
-            private boolean allowFailureIndices;
+            private boolean allowSelectors;
             private boolean ignoreThrottled;
 
             public Builder() {
@@ -365,7 +340,7 @@ public Builder() {
             Builder(GatekeeperOptions options) {
                 allowAliasToMultipleIndices = options.allowAliasToMultipleIndices;
                 allowClosedIndices = options.allowClosedIndices;
-                allowFailureIndices = options.allowFailureIndices;
+                allowSelectors = options.allowSelectors;
                 ignoreThrottled = options.ignoreThrottled;
             }
 
@@ -388,11 +363,12 @@ public Builder allowClosedIndices(boolean allowClosedIndices) {
             }
 
             /**
-             * Failure indices are accepted when true, otherwise the resolution will throw an error.
+             * Selectors are allowed within index expressions when true, otherwise the resolution will treat their presence as a syntax
+             * error when resolving index expressions.
              * Defaults to true.
              */
-            public Builder allowFailureIndices(boolean allowFailureIndices) {
-                this.allowFailureIndices = allowFailureIndices;
+            public Builder allowSelectors(boolean allowSelectors) {
+                this.allowSelectors = allowSelectors;
                 return this;
             }
 
@@ -405,7 +381,7 @@ public Builder ignoreThrottled(boolean ignoreThrottled) {
             }
 
             public GatekeeperOptions build() {
-                return new GatekeeperOptions(allowAliasToMultipleIndices, allowClosedIndices, allowFailureIndices, ignoreThrottled);
+                return new GatekeeperOptions(allowAliasToMultipleIndices, allowClosedIndices, allowSelectors, ignoreThrottled);
             }
         }
 
@@ -418,50 +394,6 @@ public static Builder builder(GatekeeperOptions gatekeeperOptions) {
         }
     }
 
-    /**
-     * Defines which selectors should be used by default for an index operation in the event that no selectors are provided.
-     */
-    public record SelectorOptions(IndexComponentSelector defaultSelector) implements Writeable {
-
-        public static final SelectorOptions ALL_APPLICABLE = new SelectorOptions(IndexComponentSelector.ALL_APPLICABLE);
-        public static final SelectorOptions DATA = new SelectorOptions(IndexComponentSelector.DATA);
-        public static final SelectorOptions FAILURES = new SelectorOptions(IndexComponentSelector.FAILURES);
-        /**
-         * Default instance. Uses <pre>::data</pre> as the default selector if none are present in an index expression.
-         */
-        public static final SelectorOptions DEFAULT = DATA;
-
-        public static SelectorOptions read(StreamInput in) throws IOException {
-            if (in.getTransportVersion().before(TransportVersions.INTRODUCE_ALL_APPLICABLE_SELECTOR)) {
-                EnumSet<IndexComponentSelector> set = in.readEnumSet(IndexComponentSelector.class);
-                if (set.isEmpty() || set.size() == 2) {
-                    assert set.contains(IndexComponentSelector.DATA) && set.contains(IndexComponentSelector.FAILURES)
-                        : "The enum set only supported ::data and ::failures";
-                    return SelectorOptions.ALL_APPLICABLE;
-                } else if (set.contains(IndexComponentSelector.DATA)) {
-                    return SelectorOptions.DATA;
-                } else {
-                    return SelectorOptions.FAILURES;
-                }
-            } else {
-                return new SelectorOptions(IndexComponentSelector.read(in));
-            }
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            if (out.getTransportVersion().before(TransportVersions.INTRODUCE_ALL_APPLICABLE_SELECTOR)) {
-                switch (defaultSelector) {
-                    case ALL_APPLICABLE -> out.writeEnumSet(EnumSet.of(IndexComponentSelector.DATA, IndexComponentSelector.FAILURES));
-                    case DATA -> out.writeEnumSet(EnumSet.of(IndexComponentSelector.DATA));
-                    case FAILURES -> out.writeEnumSet(EnumSet.of(IndexComponentSelector.FAILURES));
-                }
-            } else {
-                defaultSelector.writeTo(out);
-            }
-        }
-    }
-
     /**
      * This class is maintained for backwards compatibility and performance purposes. We use it for serialisation along with {@link Option}.
      */
@@ -497,7 +429,8 @@ private enum Option {
         ERROR_WHEN_CLOSED_INDICES,
         IGNORE_THROTTLED,
 
-        ALLOW_FAILURE_INDICES // Added in 8.14
+        ALLOW_FAILURE_INDICES, // Added in 8.14, Removed in 8.18
+        ALLOW_SELECTORS // Added in 8.18
     }
 
     private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(IndicesOptions.class);
@@ -510,8 +443,7 @@ private enum Option {
     public static final IndicesOptions DEFAULT = new IndicesOptions(
         ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS,
         WildcardOptions.DEFAULT,
-        GatekeeperOptions.DEFAULT,
-        SelectorOptions.DEFAULT
+        GatekeeperOptions.DEFAULT
     );
 
     public static final IndicesOptions STRICT_EXPAND_OPEN = IndicesOptions.builder()
@@ -528,10 +460,9 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowAliasToMultipleIndices(true)
                 .allowClosedIndices(true)
-                .allowFailureIndices(true)
+                .allowSelectors(true)
                 .ignoreThrottled(false)
         )
-        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions STRICT_EXPAND_OPEN_FAILURE_STORE = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -547,10 +478,9 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowAliasToMultipleIndices(true)
                 .allowClosedIndices(true)
-                .allowFailureIndices(true)
+                .allowSelectors(true)
                 .ignoreThrottled(false)
         )
-        .selectorOptions(SelectorOptions.ALL_APPLICABLE)
         .build();
     public static final IndicesOptions LENIENT_EXPAND_OPEN = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
@@ -566,10 +496,9 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowAliasToMultipleIndices(true)
                 .allowClosedIndices(true)
-                .allowFailureIndices(true)
+                .allowSelectors(true)
                 .ignoreThrottled(false)
         )
-        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions LENIENT_EXPAND_OPEN_NO_SELECTORS = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
@@ -585,7 +514,7 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowAliasToMultipleIndices(true)
                 .allowClosedIndices(true)
-                .allowFailureIndices(false)
+                .allowSelectors(false)
                 .ignoreThrottled(false)
         )
         .build();
@@ -603,10 +532,9 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowAliasToMultipleIndices(true)
                 .allowClosedIndices(true)
-                .allowFailureIndices(true)
+                .allowSelectors(true)
                 .ignoreThrottled(false)
         )
-        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
@@ -622,10 +550,9 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowAliasToMultipleIndices(true)
                 .allowClosedIndices(true)
-                .allowFailureIndices(true)
+                .allowSelectors(true)
                 .ignoreThrottled(false)
         )
-        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED_HIDDEN = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
@@ -636,10 +563,9 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowAliasToMultipleIndices(true)
                 .allowClosedIndices(true)
-                .allowFailureIndices(true)
+                .allowSelectors(true)
                 .ignoreThrottled(false)
         )
-        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
@@ -650,7 +576,7 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowAliasToMultipleIndices(true)
                 .allowClosedIndices(true)
-                .allowFailureIndices(false)
+                .allowSelectors(false)
                 .ignoreThrottled(false)
         )
         .build();
@@ -668,10 +594,9 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowAliasToMultipleIndices(true)
                 .allowClosedIndices(true)
-                .allowFailureIndices(true)
+                .allowSelectors(true)
                 .ignoreThrottled(false)
         )
-        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -682,10 +607,9 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowAliasToMultipleIndices(true)
                 .allowClosedIndices(true)
-                .allowFailureIndices(true)
+                .allowSelectors(true)
                 .ignoreThrottled(false)
         )
-        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTORS = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -696,7 +620,7 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowAliasToMultipleIndices(true)
                 .allowClosedIndices(true)
-                .allowFailureIndices(false)
+                .allowSelectors(false)
                 .ignoreThrottled(false)
         )
         .build();
@@ -714,10 +638,9 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowAliasToMultipleIndices(true)
                 .allowClosedIndices(true)
-                .allowFailureIndices(true)
+                .allowSelectors(true)
                 .ignoreThrottled(false)
         )
-        .selectorOptions(SelectorOptions.ALL_APPLICABLE)
         .build();
     public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN_FAILURE_STORE = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -728,10 +651,9 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowAliasToMultipleIndices(true)
                 .allowClosedIndices(true)
-                .allowFailureIndices(true)
+                .allowSelectors(true)
                 .ignoreThrottled(false)
         )
-        .selectorOptions(SelectorOptions.ALL_APPLICABLE)
         .build();
     public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_FAILURE_STORE = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -747,10 +669,9 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowAliasToMultipleIndices(true)
                 .allowClosedIndices(true)
-                .allowFailureIndices(true)
+                .allowSelectors(true)
                 .ignoreThrottled(false)
         )
-        .selectorOptions(SelectorOptions.ALL_APPLICABLE)
         .build();
     public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -766,10 +687,9 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowClosedIndices(false)
                 .allowAliasToMultipleIndices(true)
-                .allowFailureIndices(true)
+                .allowSelectors(true)
                 .ignoreThrottled(false)
         )
-        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -785,10 +705,9 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowClosedIndices(false)
                 .allowAliasToMultipleIndices(true)
-                .allowFailureIndices(true)
+                .allowSelectors(true)
                 .ignoreThrottled(false)
         )
-        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -804,10 +723,9 @@ private enum Option {
             GatekeeperOptions.builder()
                 .ignoreThrottled(true)
                 .allowClosedIndices(false)
-                .allowFailureIndices(true)
+                .allowSelectors(true)
                 .allowAliasToMultipleIndices(true)
         )
-        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -823,10 +741,27 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowAliasToMultipleIndices(false)
                 .allowClosedIndices(false)
-                .allowFailureIndices(true)
+                .allowSelectors(false)
+                .ignoreThrottled(false)
+        )
+        .build();
+    public static final IndicesOptions STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED_ALLOW_SELECTORS = IndicesOptions.builder()
+        .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+        .wildcardOptions(
+            WildcardOptions.builder()
+                .matchOpen(false)
+                .matchClosed(false)
+                .includeHidden(false)
+                .allowEmptyExpressions(true)
+                .resolveAliases(true)
+        )
+        .gatekeeperOptions(
+            GatekeeperOptions.builder()
+                .allowAliasToMultipleIndices(false)
+                .allowClosedIndices(false)
+                .allowSelectors(true)
                 .ignoreThrottled(false)
         )
-        .selectorOptions(SelectorOptions.DATA)
         .build();
     public static final IndicesOptions STRICT_NO_EXPAND_FORBID_CLOSED = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -842,10 +777,9 @@ private enum Option {
             GatekeeperOptions.builder()
                 .allowClosedIndices(false)
                 .allowAliasToMultipleIndices(true)
-                .allowFailureIndices(true)
+                .allowSelectors(true)
                 .ignoreThrottled(false)
         )
-        .selectorOptions(SelectorOptions.DATA)
         .build();
 
     /**
@@ -903,10 +837,10 @@ public boolean forbidClosedIndices() {
     }
 
     /**
-     * @return Whether execution on failure indices is allowed.
+     * @return Whether selectors (::) are allowed in the index expression.
      */
-    public boolean allowFailureIndices() {
-        return gatekeeperOptions.allowFailureIndices();
+    public boolean allowSelectors() {
+        return DataStream.isFailureStoreFeatureFlagEnabled() && gatekeeperOptions.allowSelectors();
     }
 
     /**
@@ -930,20 +864,6 @@ public boolean ignoreThrottled() {
         return gatekeeperOptions().ignoreThrottled();
     }
 
-    /**
-     * @return whether regular indices (stand-alone or backing indices) will be included in the response
-     */
-    public boolean includeRegularIndices() {
-        return selectorOptions().defaultSelector().shouldIncludeData();
-    }
-
-    /**
-     * @return whether failure indices (only supported by certain data streams) will be included in the response
-     */
-    public boolean includeFailureIndices() {
-        return selectorOptions().defaultSelector().shouldIncludeFailures();
-    }
-
     public void writeIndicesOptions(StreamOutput out) throws IOException {
         EnumSet<Option> backwardsCompatibleOptions = EnumSet.noneOf(Option.class);
         if (allowNoIndices()) {
@@ -964,9 +884,13 @@ public void writeIndicesOptions(StreamOutput out) throws IOException {
         if (ignoreUnavailable()) {
             backwardsCompatibleOptions.add(Option.ALLOW_UNAVAILABLE_CONCRETE_TARGETS);
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
-            if (allowFailureIndices()) {
+        // Until the feature flag is removed we access the field directly from the gatekeeper options.
+        if (gatekeeperOptions().allowSelectors()) {
+            if (out.getTransportVersion()
+                .between(TransportVersions.V_8_14_0, TransportVersions.REPLACE_FAILURE_STORE_OPTIONS_WITH_SELECTOR_SYNTAX)) {
                 backwardsCompatibleOptions.add(Option.ALLOW_FAILURE_INDICES);
+            } else if (out.getTransportVersion().onOrAfter(TransportVersions.REPLACE_FAILURE_STORE_OPTIONS_WITH_SELECTOR_SYNTAX)) {
+                backwardsCompatibleOptions.add(Option.ALLOW_SELECTORS);
             }
         }
         out.writeEnumSet(backwardsCompatibleOptions);
@@ -983,11 +907,17 @@ public void writeIndicesOptions(StreamOutput out) throws IOException {
         }
         out.writeEnumSet(states);
         if (out.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) {
-            out.writeBoolean(includeRegularIndices());
-            out.writeBoolean(includeFailureIndices());
+            out.writeBoolean(true);
+            out.writeBoolean(false);
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) {
-            selectorOptions.writeTo(out);
+        if (out.getTransportVersion()
+            .between(TransportVersions.V_8_16_0, TransportVersions.REPLACE_FAILURE_STORE_OPTIONS_WITH_SELECTOR_SYNTAX)) {
+            if (out.getTransportVersion().before(TransportVersions.INTRODUCE_ALL_APPLICABLE_SELECTOR)) {
+                out.writeVInt(1); // Enum set sized 1
+                out.writeVInt(0); // ordinal 0 (::data selector)
+            } else {
+                out.writeByte((byte) 0); // ordinal 0 (::data selector)
+            }
         }
     }
 
@@ -998,39 +928,44 @@ public static IndicesOptions readIndicesOptions(StreamInput in) throws IOExcepti
             options.contains(Option.ALLOW_EMPTY_WILDCARD_EXPRESSIONS),
             options.contains(Option.EXCLUDE_ALIASES)
         );
-        boolean allowFailureIndices = true;
-        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
-            allowFailureIndices = options.contains(Option.ALLOW_FAILURE_INDICES);
+        boolean allowSelectors = true;
+        if (in.getTransportVersion()
+            .between(TransportVersions.V_8_14_0, TransportVersions.REPLACE_FAILURE_STORE_OPTIONS_WITH_SELECTOR_SYNTAX)) {
+            // We've effectively replaced the allow failure indices setting with allow selectors. If it is configured on an older version
+            // then use its value for allow selectors.
+            allowSelectors = options.contains(Option.ALLOW_FAILURE_INDICES);
+        } else if (in.getTransportVersion().onOrAfter(TransportVersions.REPLACE_FAILURE_STORE_OPTIONS_WITH_SELECTOR_SYNTAX)) {
+            allowSelectors = options.contains(Option.ALLOW_SELECTORS);
         }
         GatekeeperOptions gatekeeperOptions = GatekeeperOptions.builder()
             .allowClosedIndices(options.contains(Option.ERROR_WHEN_CLOSED_INDICES) == false)
             .allowAliasToMultipleIndices(options.contains(Option.ERROR_WHEN_ALIASES_TO_MULTIPLE_INDICES) == false)
-            .allowFailureIndices(allowFailureIndices)
+            .allowSelectors(allowSelectors)
             .ignoreThrottled(options.contains(Option.IGNORE_THROTTLED))
             .build();
-        SelectorOptions selectorOptions = SelectorOptions.DEFAULT;
         if (in.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) {
             // Reading from an older node, which will be sending two booleans that we must read out and ignore.
-            var includeData = in.readBoolean();
-            var includeFailures = in.readBoolean();
-            if (includeData && includeFailures) {
-                selectorOptions = SelectorOptions.ALL_APPLICABLE;
-            } else if (includeData) {
-                selectorOptions = SelectorOptions.DATA;
+            in.readBoolean();
+            in.readBoolean();
+        }
+        if (in.getTransportVersion()
+            .between(TransportVersions.V_8_16_0, TransportVersions.REPLACE_FAILURE_STORE_OPTIONS_WITH_SELECTOR_SYNTAX)) {
+            // Reading from an older node, which will be sending either an enum set or a single byte that needs to be read out and ignored.
+            if (in.getTransportVersion().before(TransportVersions.INTRODUCE_ALL_APPLICABLE_SELECTOR)) {
+                int size = in.readVInt();
+                for (int i = 0; i < size; i++) {
+                    in.readVInt();
+                }
             } else {
-                selectorOptions = SelectorOptions.FAILURES;
+                in.readByte();
             }
         }
-        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) {
-            selectorOptions = SelectorOptions.read(in);
-        }
         return new IndicesOptions(
             options.contains(Option.ALLOW_UNAVAILABLE_CONCRETE_TARGETS)
                 ? ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS
                 : ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS,
             wildcardOptions,
-            gatekeeperOptions,
-            selectorOptions
+            gatekeeperOptions
         );
     }
 
@@ -1038,7 +973,6 @@ public static class Builder {
         private ConcreteTargetOptions concreteTargetOptions;
         private WildcardOptions wildcardOptions;
         private GatekeeperOptions gatekeeperOptions;
-        private SelectorOptions selectorOptions;
 
         Builder() {
             this(DEFAULT);
@@ -1048,7 +982,6 @@ public static class Builder {
             concreteTargetOptions = indicesOptions.concreteTargetOptions;
             wildcardOptions = indicesOptions.wildcardOptions;
             gatekeeperOptions = indicesOptions.gatekeeperOptions;
-            selectorOptions = indicesOptions.selectorOptions;
         }
 
         public Builder concreteTargetOptions(ConcreteTargetOptions concreteTargetOptions) {
@@ -1076,13 +1009,8 @@ public Builder gatekeeperOptions(GatekeeperOptions.Builder generalOptions) {
             return this;
         }
 
-        public Builder selectorOptions(SelectorOptions selectorOptions) {
-            this.selectorOptions = selectorOptions;
-            return this;
-        }
-
         public IndicesOptions build() {
-            return new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions, selectorOptions);
+            return new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions);
         }
     }
 
@@ -1181,12 +1109,10 @@ public static IndicesOptions fromOptions(
             .allowClosedIndices(forbidClosedIndices == false)
             .ignoreThrottled(ignoreThrottled)
             .build();
-        final SelectorOptions selectorOptions = SelectorOptions.DEFAULT;
         return new IndicesOptions(
             ignoreUnavailable ? ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS : ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS,
             wildcards,
-            gatekeeperOptions,
-            selectorOptions
+            gatekeeperOptions
         );
     }
 
@@ -1200,7 +1126,6 @@ public static IndicesOptions fromRequest(RestRequest request, IndicesOptions def
             request.param(ConcreteTargetOptions.IGNORE_UNAVAILABLE),
             request.param(WildcardOptions.ALLOW_NO_INDICES),
             request.param(GatekeeperOptions.IGNORE_THROTTLED),
-            DataStream.isFailureStoreFeatureFlagEnabled() ? request.param(FAILURE_STORE_QUERY_PARAM) : INCLUDE_ONLY_REGULAR_INDICES,
             defaultSettings
         );
     }
@@ -1216,7 +1141,6 @@ public static IndicesOptions fromMap(Map<String, Object> map, IndicesOptions def
                 map.containsKey(GatekeeperOptions.IGNORE_THROTTLED)
                     ? map.get(GatekeeperOptions.IGNORE_THROTTLED)
                     : map.get("ignoreThrottled"),
-                map.containsKey(FAILURE_STORE_QUERY_PARAM) ? map.get(FAILURE_STORE_QUERY_PARAM) : map.get("failureStore"),
                 defaultSettings
             );
         }
@@ -1243,9 +1167,7 @@ public static boolean isIndicesOptions(String name) {
             || GatekeeperOptions.IGNORE_THROTTLED.equals(name)
             || "ignoreThrottled".equals(name)
             || WildcardOptions.ALLOW_NO_INDICES.equals(name)
-            || "allowNoIndices".equals(name)
-            || (DataStream.isFailureStoreFeatureFlagEnabled() && FAILURE_STORE_QUERY_PARAM.equals(name))
-            || (DataStream.isFailureStoreFeatureFlagEnabled() && "failureStore".equals(name));
+            || "allowNoIndices".equals(name);
     }
 
     public static IndicesOptions fromParameters(
@@ -1276,52 +1198,20 @@ public static IndicesOptions fromParameters(
 
         WildcardOptions wildcards = WildcardOptions.parseParameters(wildcardsString, allowNoIndicesString, defaultSettings.wildcardOptions);
         GatekeeperOptions gatekeeperOptions = GatekeeperOptions.parseParameter(ignoreThrottled, defaultSettings.gatekeeperOptions);
-        SelectorOptions selectorOptions = DataStream.isFailureStoreFeatureFlagEnabled()
-            ? parseFailureStoreParameters(failureStoreString, defaultSettings.selectorOptions)
-            : SelectorOptions.DEFAULT;
 
         // note that allowAliasesToMultipleIndices is not exposed, always true (only for internal use)
         return IndicesOptions.builder()
             .concreteTargetOptions(ConcreteTargetOptions.fromParameter(ignoreUnavailableString, defaultSettings.concreteTargetOptions))
             .wildcardOptions(wildcards)
             .gatekeeperOptions(gatekeeperOptions)
-            .selectorOptions(selectorOptions)
             .build();
     }
 
-    /**
-     * @deprecated This method parses the query parameter failure_store. This is a deprecated param, and it will be replaced
-     * the selector suffix, for example `my-data-stream::data` or `my-data-stream::failures`
-     */
-    @Deprecated
-    private static SelectorOptions parseFailureStoreParameters(Object failureStoreValue, SelectorOptions defaultOptions) {
-        if (failureStoreValue == null) {
-            return defaultOptions;
-        }
-        return switch (failureStoreValue.toString()) {
-            case INCLUDE_ALL -> SelectorOptions.ALL_APPLICABLE;
-            case INCLUDE_ONLY_REGULAR_INDICES -> SelectorOptions.DATA;
-            case INCLUDE_ONLY_FAILURE_INDICES -> SelectorOptions.FAILURES;
-            default -> throw new IllegalArgumentException("No valid " + FAILURE_STORE_QUERY_PARAM + " value [" + failureStoreValue + "]");
-        };
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
         concreteTargetOptions.toXContent(builder, params);
         wildcardOptions.toXContent(builder, params);
         gatekeeperOptions.toXContent(builder, params);
-        if (DataStream.isFailureStoreFeatureFlagEnabled()) {
-            String displayValue;
-            if (SelectorOptions.ALL_APPLICABLE.equals(selectorOptions())) {
-                displayValue = INCLUDE_ALL;
-            } else if (SelectorOptions.DATA.equals(selectorOptions())) {
-                displayValue = INCLUDE_ONLY_REGULAR_INDICES;
-            } else {
-                displayValue = INCLUDE_ONLY_FAILURE_INDICES;
-            }
-            builder.field(FAILURE_STORE_QUERY_PARAM, displayValue);
-        }
         return builder;
     }
 
@@ -1329,7 +1219,6 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
     private static final ParseField IGNORE_UNAVAILABLE_FIELD = new ParseField(ConcreteTargetOptions.IGNORE_UNAVAILABLE);
     private static final ParseField IGNORE_THROTTLED_FIELD = new ParseField(GatekeeperOptions.IGNORE_THROTTLED).withAllDeprecated();
     private static final ParseField ALLOW_NO_INDICES_FIELD = new ParseField(WildcardOptions.ALLOW_NO_INDICES);
-    private static final ParseField FAILURE_STORE_FIELD = new ParseField(FAILURE_STORE_QUERY_PARAM);
 
     public static IndicesOptions fromXContent(XContentParser parser) throws IOException {
         return fromXContent(parser, null);
@@ -1340,7 +1229,6 @@ public static IndicesOptions fromXContent(XContentParser parser, @Nullable Indic
         WildcardOptions.Builder wildcards = defaults == null ? null : WildcardOptions.builder(defaults.wildcardOptions());
         GatekeeperOptions.Builder generalOptions = GatekeeperOptions.builder()
             .ignoreThrottled(defaults != null && defaults.gatekeeperOptions().ignoreThrottled());
-        SelectorOptions selectorOptions = defaults == null ? SelectorOptions.DEFAULT : defaults.selectorOptions();
         Boolean allowNoIndices = defaults == null ? null : defaults.allowNoIndices();
         Boolean ignoreUnavailable = defaults == null ? null : defaults.ignoreUnavailable();
         Token token = parser.currentToken() == Token.START_OBJECT ? parser.currentToken() : parser.nextToken();
@@ -1390,14 +1278,11 @@ public static IndicesOptions fromXContent(XContentParser parser, @Nullable Indic
                     allowNoIndices = parser.booleanValue();
                 } else if (IGNORE_THROTTLED_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     generalOptions.ignoreThrottled(parser.booleanValue());
-                } else if (DataStream.isFailureStoreFeatureFlagEnabled()
-                    && FAILURE_STORE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                        selectorOptions = parseFailureStoreParameters(parser.text(), selectorOptions);
-                    } else {
-                        throw new ElasticsearchParseException(
-                            "could not read indices options. Unexpected index option [" + currentFieldName + "]"
-                        );
-                    }
+                } else {
+                    throw new ElasticsearchParseException(
+                        "could not read indices options. Unexpected index option [" + currentFieldName + "]"
+                    );
+                }
             } else {
                 throw new ElasticsearchParseException("could not read indices options. Unexpected object field [" + currentFieldName + "]");
             }
@@ -1423,7 +1308,6 @@ public static IndicesOptions fromXContent(XContentParser parser, @Nullable Indic
             .concreteTargetOptions(new ConcreteTargetOptions(ignoreUnavailable))
             .wildcardOptions(wildcards)
             .gatekeeperOptions(generalOptions)
-            .selectorOptions(selectorOptions)
             .build();
     }
 
@@ -1435,14 +1319,6 @@ public static IndicesOptions strictExpandOpen() {
         return STRICT_EXPAND_OPEN;
     }
 
-    /**
-     * @return indices options that requires every specified index to exist, expands wildcards only to open indices and
-     * allows that no indices are resolved from wildcard expressions (not returning an error).
-     */
-    public static IndicesOptions strictExpandOpenIncludeFailureStore() {
-        return STRICT_EXPAND_OPEN_FAILURE_STORE;
-    }
-
     /**
      * @return indices options that requires every specified index to exist, expands wildcards only to open indices,
      * allows that no indices are resolved from wildcard expressions (not returning an error) and forbids the
@@ -1477,6 +1353,15 @@ public static IndicesOptions strictExpandHidden() {
         return STRICT_EXPAND_OPEN_CLOSED_HIDDEN;
     }
 
+    /**
+     * @return indices option that requires every specified index to exist, expands wildcards to both open and closed indices, includes
+     * hidden indices, allows that no indices are resolved from wildcard expressions (not returning an error), and disallows selectors
+     * in the expression (no :: separators).
+     */
+    public static IndicesOptions strictExpandHiddenNoSelectors() {
+        return STRICT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTORS;
+    }
+
     /**
      * @return indices option that expands wildcards to both open and closed indices, includes failure store
      * (with data stream) and allows that indices can be missing and no indices are resolved from wildcard expressions
@@ -1510,6 +1395,14 @@ public static IndicesOptions strictSingleIndexNoExpandForbidClosed() {
         return STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED;
     }
 
+    /**
+     * @return indices option that requires each specified index or alias to exist, doesn't expand wildcards and
+     * throws error if any of the aliases resolves to multiple indices
+     */
+    public static IndicesOptions strictSingleIndexNoExpandForbidClosedAllowSelectors() {
+        return STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED_ALLOW_SELECTORS;
+    }
+
     /**
      * @return indices options that ignores unavailable indices, expands wildcards only to open indices and
      * allows that no indices are resolved from wildcard expressions (not returning an error).
@@ -1518,6 +1411,15 @@ public static IndicesOptions lenientExpandOpen() {
         return LENIENT_EXPAND_OPEN;
     }
 
+    /**
+     * @return indices options that ignores unavailable indices, expands wildcards only to open indices,
+     * allows that no indices are resolved from wildcard expressions (not returning an error), and disallows
+     * selectors in the expression (no :: separators).
+     */
+    public static IndicesOptions lenientExpandOpenNoSelectors() {
+        return LENIENT_EXPAND_OPEN_NO_SELECTORS;
+    }
+
     /**
      * @return indices options that ignores unavailable indices, expands wildcards to open and hidden indices, and
      * allows that no indices are resolved from wildcard expressions (not returning an error).
@@ -1563,14 +1465,8 @@ public String toString() {
             + ignoreAliases()
             + ", ignore_throttled="
             + ignoreThrottled()
-            + (DataStream.isFailureStoreFeatureFlagEnabled()
-                ? ", include_regular_indices="
-                    + includeRegularIndices()
-                    + ", include_failure_indices="
-                    + includeFailureIndices()
-                    + ", allow_failure_indices="
-                    + allowFailureIndices()
-                : "")
+            // Until the feature flag is removed we access the field directly from the gatekeeper options.
+            + (DataStream.isFailureStoreFeatureFlagEnabled() ? ", allow_selectors=" + gatekeeperOptions().allowSelectors() : "")
             + ']';
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/ChunkedBroadcastResponse.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/ChunkedBroadcastResponse.java
index 4b65af23c622a..18172bf4e8f31 100644
--- a/server/src/main/java/org/elasticsearch/action/support/broadcast/ChunkedBroadcastResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/ChunkedBroadcastResponse.java
@@ -9,8 +9,9 @@
 package org.elasticsearch.action.support.broadcast;
 
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.xcontent.ToXContent;
@@ -35,8 +36,11 @@ public ChunkedBroadcastResponse(
 
     @Override
     public final Iterator<ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params)
-            .object(ob -> ob.append((b, p) -> RestActions.buildBroadcastShardsHeader(b, p, this)).append(this::customXContentChunks));
+        return Iterators.concat(Iterators.single((b, p) -> {
+            b.startObject();
+            RestActions.buildBroadcastShardsHeader(b, p, this);
+            return b;
+        }), customXContentChunks(params), ChunkedToXContentHelper.endObject());
     }
 
     protected abstract Iterator<ToXContent> customXContentChunks(ToXContent.Params params);
diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
index 5c6171dc22536..4c691334b6fa8 100644
--- a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
@@ -612,7 +612,7 @@ public void writeTo(StreamOutput out) throws IOException {
      * which there is no shard-level return value.
      */
     public static final class EmptyResult implements Writeable {
-        public static EmptyResult INSTANCE = new EmptyResult();
+        public static final EmptyResult INSTANCE = new EmptyResult();
 
         private EmptyResult() {}
 
diff --git a/server/src/main/java/org/elasticsearch/action/support/local/LocalClusterStateRequest.java b/server/src/main/java/org/elasticsearch/action/support/local/LocalClusterStateRequest.java
index dfbcb21c2a959..ec7c72c0ab428 100644
--- a/server/src/main/java/org/elasticsearch/action/support/local/LocalClusterStateRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/local/LocalClusterStateRequest.java
@@ -9,10 +9,13 @@
 
 package org.elasticsearch.action.support.local;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV10;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -32,6 +35,20 @@ protected LocalClusterStateRequest(TimeValue masterTimeout) {
         this.masterTimeout = Objects.requireNonNull(masterTimeout);
     }
 
+    /**
+     * This constructor exists solely for BwC purposes. It should exclusively be used by requests that used to extend
+     * {@link org.elasticsearch.action.support.master.MasterNodeReadRequest} and still need to be able to serialize incoming request.
+     */
+    @UpdateForV10(owner = UpdateForV10.Owner.DISTRIBUTED_COORDINATION)
+    protected LocalClusterStateRequest(StreamInput in) throws IOException {
+        super(in);
+        masterTimeout = in.readTimeValue();
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
+            in.readVLong();
+        }
+        in.readBoolean();
+    }
+
     @Override
     public final void writeTo(StreamOutput out) throws IOException {
         TransportAction.localOnly();
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java
index 943b03588d4ba..634a103e9754a 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 
@@ -27,11 +28,22 @@ public abstract class ClusterInfoRequest<Request extends ClusterInfoRequest<Requ
 
     private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
 
+    public ClusterInfoRequest(TimeValue masterTimeout) {
+        super(masterTimeout);
+    }
+
+    public ClusterInfoRequest(TimeValue masterTimeout, IndicesOptions indicesOptions) {
+        super(masterTimeout);
+        this.indicesOptions = indicesOptions;
+    }
+
+    @Deprecated(forRemoval = true)
     public ClusterInfoRequest() {
         super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
     }
 
     // So subclasses can override the default indices options, if needed
+    @Deprecated(forRemoval = true)
     protected ClusterInfoRequest(IndicesOptions indicesOptions) {
         super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
         this.indicesOptions = indicesOptions;
diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponse.java b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponse.java
index b9fb2807a05c6..2a27606569062 100644
--- a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponse.java
@@ -11,7 +11,8 @@
 
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.xcontent.ToXContent;
@@ -29,12 +30,11 @@ protected BaseNodesXContentResponse(ClusterName clusterName, List<TNodeResponse>
 
     @Override
     public final Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params)
-            .object(
-                ob -> ob.append((b, p) -> RestActions.buildNodesHeader(b, p, this))
-                    .field("cluster_name", getClusterName().value())
-                    .append(xContentChunks(params))
-            );
+        return Iterators.concat(Iterators.single((b, p) -> {
+            b.startObject();
+            RestActions.buildNodesHeader(b, p, this);
+            return b.field("cluster_name", getClusterName().value());
+        }), xContentChunks(params), ChunkedToXContentHelper.endObject());
     }
 
     protected abstract Iterator<? extends ToXContent> xContentChunks(ToXContent.Params outerParams);
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java
index c8b53e0185606..b3230fae9834a 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.routing.OperationRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -72,8 +73,12 @@ protected void doExecute(Task task, final MultiTermVectorsRequest request, final
                     clusterState.metadata().resolveIndexRouting(termVectorsRequest.routing(), termVectorsRequest.index())
                 );
                 String concreteSingleIndex = indexNameExpressionResolver.concreteSingleIndex(clusterState, termVectorsRequest).getName();
-                shardId = clusterService.operationRouting()
-                    .shardId(clusterState, concreteSingleIndex, termVectorsRequest.id(), termVectorsRequest.routing());
+                shardId = OperationRouting.shardId(
+                    clusterState,
+                    concreteSingleIndex,
+                    termVectorsRequest.id(),
+                    termVectorsRequest.routing()
+                );
             } catch (RoutingMissingException e) {
                 responses.set(
                     i,
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
index 7567577c78452..02479a9f8d143 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
@@ -62,16 +62,27 @@ public TransportTermVectorsAction(
 
     @Override
     protected ShardIterator shards(ClusterState state, InternalRequest request) {
+        final var operationRouting = clusterService.operationRouting();
         if (request.request().doc() != null && request.request().routing() == null) {
             // artificial document without routing specified, ignore its "id" and use either random shard or according to preference
-            GroupShardsIterator<ShardIterator> groupShardsIter = clusterService.operationRouting()
-                .searchShards(state, new String[] { request.concreteIndex() }, null, request.request().preference());
+            GroupShardsIterator<ShardIterator> groupShardsIter = operationRouting.searchShards(
+                state,
+                new String[] { request.concreteIndex() },
+                null,
+                request.request().preference()
+            );
             return groupShardsIter.iterator().next();
         }
 
-        ShardIterator shards = clusterService.operationRouting()
-            .getShards(state, request.concreteIndex(), request.request().id(), request.request().routing(), request.request().preference());
-        return clusterService.operationRouting().useOnlyPromotableShardsForStateless(shards);
+        return operationRouting.useOnlyPromotableShardsForStateless(
+            operationRouting.getShards(
+                state,
+                request.concreteIndex(),
+                request.request().id(),
+                request.request().routing(),
+                request.request().preference()
+            )
+        );
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
index 0749512635f83..ee84bcd15824d 100644
--- a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.mapper.InferenceFieldMapper;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.shard.IndexShard;
@@ -374,7 +375,7 @@ private static UpdateHelper.Result deleteInferenceResults(
         IndexMetadata indexMetadata,
         MappingLookup mappingLookup
     ) {
-        if (result.getResponseResult() != DocWriteResponse.Result.UPDATED) {
+        if (result.getResponseResult() != DocWriteResponse.Result.UPDATED || InferenceMetadataFieldsMapper.isEnabled(mappingLookup)) {
             return result;
         }
 
@@ -403,7 +404,7 @@ private static UpdateHelper.Result deleteInferenceResults(
             String inferenceFieldName = entry.getKey();
             Mapper mapper = mappingLookup.getMapper(inferenceFieldName);
 
-            if (mapper instanceof InferenceFieldMapper inferenceFieldMapper) {
+            if (mapper instanceof InferenceFieldMapper) {
                 String[] sourceFields = entry.getValue().getSourceFields();
                 for (String sourceField : sourceFields) {
                     if (sourceField.equals(inferenceFieldName) == false
@@ -412,7 +413,7 @@ private static UpdateHelper.Result deleteInferenceResults(
                         // This has two important side effects:
                         // - The inference field value will remain parsable by its mapper
                         // - The inference results will be removed, forcing them to be re-generated downstream
-                        updatedSource.put(inferenceFieldName, inferenceFieldMapper.getOriginalValue(updatedSource));
+                        updatedSource.put(inferenceFieldName, getOriginalValueLegacy(inferenceFieldName, updatedSource));
                         updatedSourceModified = true;
                         break;
                     }
@@ -435,4 +436,24 @@ private static UpdateHelper.Result deleteInferenceResults(
 
         return returnedResult;
     }
+
+    /**
+     * Get the field's original value (i.e. the value the user specified) from the provided source.
+     *
+     * @param sourceAsMap The source as a map
+     * @return The field's original value, or {@code null} if none was provided
+     */
+    private static Object getOriginalValueLegacy(String fullPath, Map<String, Object> sourceAsMap) {
+        // TODO: Fix bug here when semantic text field is in an object
+        Object fieldValue = sourceAsMap.get(fullPath);
+        if (fieldValue == null) {
+            return null;
+        } else if (fieldValue instanceof Map<?, ?> == false) {
+            // Don't try to further validate the non-map value, that will be handled when the source is fully parsed
+            return fieldValue;
+        }
+
+        Map<String, Object> fieldValueMap = XContentMapValues.nodeMapValue(fieldValue, "Field [" + fullPath + "]");
+        return XContentMapValues.extractValue("text", fieldValueMap);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
index a645c156b63c7..39600c7eca661 100644
--- a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
+++ b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
@@ -60,7 +60,15 @@ public UpdateHelper(ScriptService scriptService) {
      * Prepares an update request by converting it into an index or delete request or an update response (no action).
      */
     public Result prepare(UpdateRequest request, IndexShard indexShard, LongSupplier nowInMillis) throws IOException {
-        final GetResult getResult = indexShard.getService().getForUpdate(request.id(), request.ifSeqNo(), request.ifPrimaryTerm());
+        // TODO: Don't hard-code gFields
+        return prepare(request, indexShard, nowInMillis, new String[] { RoutingFieldMapper.NAME });
+    }
+
+    /**
+     * Prepares an update request by converting it into an index or delete request or an update response (no action).
+     */
+    public Result prepare(UpdateRequest request, IndexShard indexShard, LongSupplier nowInMillis, String[] gFields) throws IOException {
+        final GetResult getResult = indexShard.getService().getForUpdate(request.id(), request.ifSeqNo(), request.ifPrimaryTerm(), gFields);
         return prepare(indexShard, request, getResult, nowInMillis);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
index 56d185645e149..4c7fb96c5b1d5 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -33,6 +33,7 @@ class Bootstrap {
 
     // arguments from the CLI process
     private final ServerArgs args;
+    private final boolean useEntitlements;
 
     // controller for spawning component subprocesses
     private final Spawner spawner = new Spawner();
@@ -46,10 +47,11 @@ class Bootstrap {
     // loads information about plugins required for entitlements in phase 2, used by plugins service in phase 3
     private final SetOnce<PluginsLoader> pluginsLoader = new SetOnce<>();
 
-    Bootstrap(PrintStream out, PrintStream err, ServerArgs args) {
+    Bootstrap(PrintStream out, PrintStream err, ServerArgs args, boolean useEntitlements) {
         this.out = out;
         this.err = err;
         this.args = args;
+        this.useEntitlements = useEntitlements;
     }
 
     ServerArgs args() {
@@ -60,6 +62,10 @@ Spawner spawner() {
         return spawner;
     }
 
+    public boolean useEntitlements() {
+        return useEntitlements;
+    }
+
     void setSecureSettings(SecureSettings secureSettings) {
         this.secureSettings.set(secureSettings);
     }
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
index 7b2f0c2c894be..b5b616fff0182 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
@@ -212,7 +212,6 @@ static List<BootstrapCheck> checks() {
         checks.add(new OnErrorCheck());
         checks.add(new OnOutOfMemoryErrorCheck());
         checks.add(new EarlyAccessCheck());
-        checks.add(new AllPermissionCheck());
         checks.add(new DiscoveryConfiguredCheck());
         checks.add(new ByteOrderCheck());
         return Collections.unmodifiableList(checks);
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
index 9be23c91db072..7c6dee1e420b2 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.util.concurrent.RunOnce;
 import org.elasticsearch.core.AbstractRefCounted;
+import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap;
@@ -54,6 +55,7 @@
 import java.nio.file.Path;
 import java.security.Permission;
 import java.security.Security;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.CountDownLatch;
@@ -108,6 +110,9 @@ private static Bootstrap initPhase1() {
         final PrintStream out = getStdout();
         final PrintStream err = getStderr();
         final ServerArgs args;
+        final boolean entitlementsExplicitlyEnabled = Booleans.parseBoolean(System.getProperty("es.entitlements.enabled", "false"));
+        // java 24+ only supports entitlements, but it may be enabled on earlier versions explicitly
+        final boolean useEntitlements = RuntimeVersionFeature.isSecurityManagerAvailable() == false || entitlementsExplicitlyEnabled;
         try {
             initSecurityProperties();
 
@@ -116,7 +121,7 @@ private static Bootstrap initPhase1() {
              * the presence of a security manager or lack thereof act as if there is a security manager present (e.g., DNS cache policy).
              * This forces such policies to take effect immediately.
              */
-            if (RuntimeVersionFeature.isSecurityManagerAvailable()) {
+            if (useEntitlements == false && RuntimeVersionFeature.isSecurityManagerAvailable()) {
                 org.elasticsearch.bootstrap.Security.setSecurityManager(new SecurityManager() {
                     @Override
                     public void checkPermission(Permission perm) {
@@ -149,7 +154,7 @@ public void checkPermission(Permission perm) {
             return null; // unreachable, to satisfy compiler
         }
 
-        return new Bootstrap(out, err, args);
+        return new Bootstrap(out, err, args, useEntitlements);
     }
 
     /**
@@ -213,9 +218,8 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException {
         // load the plugin Java modules and layers now for use in entitlements
         var pluginsLoader = PluginsLoader.createPluginsLoader(nodeEnv.modulesFile(), nodeEnv.pluginsFile());
         bootstrap.setPluginsLoader(pluginsLoader);
-        var pluginsResolver = PluginsResolver.create(pluginsLoader);
 
-        if (Boolean.parseBoolean(System.getProperty("es.entitlements.enabled"))) {
+        if (bootstrap.useEntitlements()) {
             LogManager.getLogger(Elasticsearch.class).info("Bootstrapping Entitlements");
 
             List<EntitlementBootstrap.PluginData> pluginData = Stream.concat(
@@ -227,6 +231,8 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException {
                     .map(bundle -> new EntitlementBootstrap.PluginData(bundle.getDir(), bundle.pluginDescriptor().isModular(), true))
             ).toList();
 
+            var pluginsResolver = PluginsResolver.create(pluginsLoader);
+
             EntitlementBootstrap.bootstrap(pluginData, pluginsResolver::resolveClassToPluginName);
         } else if (RuntimeVersionFeature.isSecurityManagerAvailable()) {
             // install SM after natives, shutdown hooks, etc.
@@ -279,7 +285,11 @@ protected void validateNodeBeforeAcceptingRequests(
                 final BoundTransportAddress boundTransportAddress,
                 List<BootstrapCheck> checks
             ) throws NodeValidationException {
-                BootstrapChecks.check(context, boundTransportAddress, checks);
+                var additionalChecks = new ArrayList<>(checks);
+                if (bootstrap.useEntitlements() == false) {
+                    additionalChecks.add(new BootstrapChecks.AllPermissionCheck());
+                }
+                BootstrapChecks.check(context, boundTransportAddress, additionalChecks);
             }
         };
         INSTANCE = new Elasticsearch(bootstrap.spawner(), node);
diff --git a/server/src/main/java/org/elasticsearch/client/internal/IndicesAdminClient.java b/server/src/main/java/org/elasticsearch/client/internal/IndicesAdminClient.java
index 7a5e027a850fe..481b59c72729c 100644
--- a/server/src/main/java/org/elasticsearch/client/internal/IndicesAdminClient.java
+++ b/server/src/main/java/org/elasticsearch/client/internal/IndicesAdminClient.java
@@ -115,6 +115,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.threadpool.ThreadPool;
 
 /**
@@ -266,8 +267,8 @@ public ActionFuture<GetMappingsResponse> getMappings(GetMappingsRequest request)
         return execute(GetMappingsAction.INSTANCE, request);
     }
 
-    public GetMappingsRequestBuilder prepareGetMappings(String... indices) {
-        return new GetMappingsRequestBuilder(this, indices);
+    public GetMappingsRequestBuilder prepareGetMappings(TimeValue masterTimeout, String... indices) {
+        return new GetMappingsRequestBuilder(this, masterTimeout, indices);
     }
 
     public void getFieldMappings(GetFieldMappingsRequest request, ActionListener<GetFieldMappingsResponse> listener) {
@@ -336,8 +337,8 @@ public void getAliases(GetAliasesRequest request, ActionListener<GetAliasesRespo
         execute(GetAliasesAction.INSTANCE, request, listener);
     }
 
-    public GetAliasesRequestBuilder prepareGetAliases(String... aliases) {
-        return new GetAliasesRequestBuilder(this, aliases);
+    public GetAliasesRequestBuilder prepareGetAliases(TimeValue masterTimeout, String... aliases) {
+        return new GetAliasesRequestBuilder(this, masterTimeout, aliases);
     }
 
     public ActionFuture<GetIndexResponse> getIndex(GetIndexRequest request) {
@@ -420,8 +421,8 @@ public void getTemplates(final GetIndexTemplatesRequest request, final ActionLis
         execute(GetIndexTemplatesAction.INSTANCE, request, listener);
     }
 
-    public GetIndexTemplatesRequestBuilder prepareGetTemplates(String... names) {
-        return new GetIndexTemplatesRequestBuilder(this, names);
+    public GetIndexTemplatesRequestBuilder prepareGetTemplates(TimeValue masterTimeout, String... names) {
+        return new GetIndexTemplatesRequestBuilder(this, masterTimeout, names);
     }
 
     public ActionFuture<ValidateQueryResponse> validateQuery(final ValidateQueryRequest request) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterFeatures.java b/server/src/main/java/org/elasticsearch/cluster/ClusterFeatures.java
index ad285cbd391cd..495306a334c2c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterFeatures.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterFeatures.java
@@ -9,11 +9,13 @@
 
 package org.elasticsearch.cluster;
 
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
-import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.xcontent.ToXContent;
 
@@ -79,28 +81,61 @@ public Map<String, Set<String>> nodeFeatures() {
         return nodeFeatures;
     }
 
-    /**
-     * The features in all nodes in the cluster.
-     * <p>
-     * NOTE: This should not be used directly.
-     * Please use {@link org.elasticsearch.features.FeatureService#clusterHasFeature} instead.
-     */
-    public Set<String> allNodeFeatures() {
+    private Set<String> allNodeFeatures() {
         if (allNodeFeatures == null) {
             allNodeFeatures = Set.copyOf(calculateAllNodeFeatures(nodeFeatures.values()));
         }
         return allNodeFeatures;
     }
 
+    /**
+     * Returns {@code true} if {@code node} can have assumed features.
+     * @see org.elasticsearch.env.BuildVersion#canRemoveAssumedFeatures
+     */
+    public static boolean featuresCanBeAssumedForNode(DiscoveryNode node) {
+        return node.getBuildVersion().canRemoveAssumedFeatures();
+    }
+
+    /**
+     * Returns {@code true} if one or more nodes in {@code nodes} can have assumed features.
+     * @see org.elasticsearch.env.BuildVersion#canRemoveAssumedFeatures
+     */
+    public static boolean featuresCanBeAssumedForNodes(DiscoveryNodes nodes) {
+        return nodes.getAllNodes().stream().anyMatch(n -> n.getBuildVersion().canRemoveAssumedFeatures());
+    }
+
     /**
      * {@code true} if {@code feature} is present on all nodes in the cluster.
      * <p>
      * NOTE: This should not be used directly.
      * Please use {@link org.elasticsearch.features.FeatureService#clusterHasFeature} instead.
      */
-    @SuppressForbidden(reason = "directly reading cluster features")
-    public boolean clusterHasFeature(NodeFeature feature) {
-        return allNodeFeatures().contains(feature.id());
+    public boolean clusterHasFeature(DiscoveryNodes nodes, NodeFeature feature) {
+        assert nodes.getNodes().keySet().equals(nodeFeatures.keySet())
+            : "Cluster features nodes " + nodeFeatures.keySet() + " is different to discovery nodes " + nodes.getNodes().keySet();
+
+        // basic case
+        boolean allNodesHaveFeature = allNodeFeatures().contains(feature.id());
+        if (allNodesHaveFeature) {
+            return true;
+        }
+
+        // if the feature is assumed, check the versions more closely
+        // it's actually ok if the feature is assumed, and all nodes missing the feature can assume it
+        // TODO: do we need some kind of transient cache of this calculation?
+        if (feature.assumedAfterNextCompatibilityBoundary()) {
+            for (var nf : nodeFeatures.entrySet()) {
+                if (nf.getValue().contains(feature.id()) == false
+                    && featuresCanBeAssumedForNode(nodes.getNodes().get(nf.getKey())) == false) {
+                    return false;
+                }
+            }
+
+            // all nodes missing the feature can assume it - so that's alright then
+            return true;
+        }
+
+        return false;
     }
 
     /**
@@ -244,12 +279,15 @@ public ClusterFeatures apply(ClusterFeatures part) {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params)
-            .array(nodeFeatures.entrySet().stream().sorted(Map.Entry.comparingByKey()).iterator(), e -> (builder, p) -> {
+        return Iterators.concat(
+            ChunkedToXContentHelper.startArray(),
+            nodeFeatures.entrySet().stream().sorted(Map.Entry.comparingByKey()).<ToXContent>map(e -> (builder, p) -> {
                 String[] features = e.getValue().toArray(String[]::new);
                 Arrays.sort(features);
                 return builder.startObject().field("node_id", e.getKey()).array("features", features).endObject();
-            });
+            }).iterator(),
+            ChunkedToXContentHelper.endArray()
+        );
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java
index 6f37f48894c28..230677a6c86c2 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -34,6 +35,9 @@
 
 import static org.elasticsearch.cluster.routing.ShardRouting.newUnassigned;
 import static org.elasticsearch.cluster.routing.UnassignedInfo.Reason.REINITIALIZED;
+import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.chunk;
+import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.endArray;
+import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.startObject;
 
 /**
  * ClusterInfo is an object representing a map of nodes to {@link DiskUsage}
@@ -133,7 +137,7 @@ private static ShardRouting createFakeShardRoutingFromNodeAndShard(NodeAndShard
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).object("nodes", leastAvailableSpaceUsage.entrySet().iterator(), c -> (builder, p) -> {
+        return Iterators.concat(startObject("nodes"), Iterators.map(leastAvailableSpaceUsage.entrySet().iterator(), c -> (builder, p) -> {
             builder.startObject(c.getKey());
             { // node
                 builder.field("node_name", c.getValue().nodeName());
@@ -152,31 +156,48 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
                 builder.endObject(); // end "most_available"
             }
             return builder.endObject(); // end $nodename
-        })
-            .object(
-                "shard_sizes",
+        }),
+            chunk(
+                (builder, p) -> builder.endObject() // end "nodes"
+                    .startObject("shard_sizes")
+            ),
+
+            Iterators.map(
                 shardSizes.entrySet().iterator(),
                 c -> (builder, p) -> builder.humanReadableField(c.getKey() + "_bytes", c.getKey(), ByteSizeValue.ofBytes(c.getValue()))
-            )
-            .object(
-                "shard_data_set_sizes",
+            ),
+            chunk(
+                (builder, p) -> builder.endObject() // end "shard_sizes"
+                    .startObject("shard_data_set_sizes")
+            ),
+            Iterators.map(
                 shardDataSetSizes.entrySet().iterator(),
                 c -> (builder, p) -> builder.humanReadableField(
                     c.getKey() + "_bytes",
                     c.getKey().toString(),
                     ByteSizeValue.ofBytes(c.getValue())
                 )
-            )
-            .object("shard_paths", dataPath.entrySet().iterator(), (xb, c) -> xb.field(c.getKey().toString(), c.getValue()))
-            .array("reserved_sizes", reservedSpace.entrySet().iterator(), c -> (builder, p) -> {
+            ),
+            chunk(
+                (builder, p) -> builder.endObject() // end "shard_data_set_sizes"
+                    .startObject("shard_paths")
+            ),
+            Iterators.map(dataPath.entrySet().iterator(), c -> (builder, p) -> builder.field(c.getKey().toString(), c.getValue())),
+            chunk(
+                (builder, p) -> builder.endObject() // end "shard_paths"
+                    .startArray("reserved_sizes")
+            ),
+            Iterators.map(reservedSpace.entrySet().iterator(), c -> (builder, p) -> {
                 builder.startObject();
                 {
                     builder.field("node_id", c.getKey().nodeId);
                     builder.field("path", c.getKey().path);
-                    c.getValue().toXContent(builder, p);
+                    c.getValue().toXContent(builder, params);
                 }
                 return builder.endObject(); // NodeAndPath
-            });
+            }),
+            endArray() // end "reserved_sizes"
+        );
     }
 
     /**
@@ -285,7 +306,7 @@ public String toString() {
 
     // exposed for tests, computed here rather than exposing all the collections separately
     int getChunkCount() {
-        return leastAvailableSpaceUsage.size() + shardSizes.size() + shardDataSetSizes.size() + dataPath.size() + reservedSpace.size() + 10;
+        return leastAvailableSpaceUsage.size() + shardSizes.size() + shardDataSetSizes.size() + dataPath.size() + reservedSpace.size() + 6;
     }
 
     public record NodeAndShard(String nodeId, ShardId shardId) implements Writeable {
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index c2da33f8f4135..efa20a41518ca 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -33,7 +33,7 @@
 import org.elasticsearch.cluster.routing.allocation.AllocationService.RerouteStrategy;
 import org.elasticsearch.cluster.routing.allocation.AllocationStatsService;
 import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator;
-import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsProvider;
+import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator;
 import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceShardsAllocator;
@@ -139,7 +139,10 @@ public ClusterModule(
         this.clusterPlugins = clusterPlugins;
         this.deciderList = createAllocationDeciders(settings, clusterService.getClusterSettings(), clusterPlugins);
         this.allocationDeciders = new AllocationDeciders(deciderList);
-        var nodeAllocationStatsProvider = new NodeAllocationStatsProvider(writeLoadForecaster, clusterService.getClusterSettings());
+        var nodeAllocationStatsAndWeightsCalculator = new NodeAllocationStatsAndWeightsCalculator(
+            writeLoadForecaster,
+            clusterService.getClusterSettings()
+        );
         this.shardsAllocator = createShardsAllocator(
             settings,
             clusterService.getClusterSettings(),
@@ -149,7 +152,7 @@ public ClusterModule(
             this::reconcile,
             writeLoadForecaster,
             telemetryProvider,
-            nodeAllocationStatsProvider
+            nodeAllocationStatsAndWeightsCalculator
         );
         this.clusterService = clusterService;
         this.indexNameExpressionResolver = new IndexNameExpressionResolver(threadPool.getThreadContext(), systemIndices);
@@ -167,7 +170,7 @@ public ClusterModule(
             clusterService,
             clusterInfoService,
             shardsAllocator,
-            nodeAllocationStatsProvider
+            nodeAllocationStatsAndWeightsCalculator
         );
         this.telemetryProvider = telemetryProvider;
     }
@@ -409,7 +412,7 @@ private static ShardsAllocator createShardsAllocator(
         DesiredBalanceReconcilerAction reconciler,
         WriteLoadForecaster writeLoadForecaster,
         TelemetryProvider telemetryProvider,
-        NodeAllocationStatsProvider nodeAllocationStatsProvider
+        NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator
     ) {
         Map<String, Supplier<ShardsAllocator>> allocators = new HashMap<>();
         allocators.put(BALANCED_ALLOCATOR, () -> new BalancedShardsAllocator(clusterSettings, writeLoadForecaster));
@@ -422,7 +425,7 @@ private static ShardsAllocator createShardsAllocator(
                 clusterService,
                 reconciler,
                 telemetryProvider,
-                nodeAllocationStatsProvider
+                nodeAllocationStatsAndWeightsCalculator
             )
         );
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterSnapshotStats.java b/server/src/main/java/org/elasticsearch/cluster/ClusterSnapshotStats.java
index ac96a2d55bc71..b859b0d13a586 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterSnapshotStats.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterSnapshotStats.java
@@ -36,7 +36,7 @@ public record ClusterSnapshotStats(
     List<PerRepositoryStats> statsByRepository
 ) implements ToXContentObject, Writeable {
 
-    public static ClusterSnapshotStats EMPTY = new ClusterSnapshotStats(0, 0, 0, 0, List.of());
+    public static final ClusterSnapshotStats EMPTY = new ClusterSnapshotStats(0, 0, 0, 0, List.of());
 
     public static ClusterSnapshotStats of(ClusterState clusterState, long currentTimeMillis) {
         return of(
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
index 1364f0c78f3be..62f2947d06a41 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
@@ -627,7 +627,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerP
         return Iterators.concat(
 
             // header chunk
-            Iterators.single(((builder, params) -> {
+            Iterators.single((builder, params) -> {
                 // always provide the cluster_uuid as part of the top-level response (also part of the metadata response)
                 builder.field("cluster_uuid", metadata().clusterUUID());
 
@@ -643,7 +643,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerP
                 }
 
                 return builder;
-            })),
+            }),
 
             // blocks
             chunkedSection(metrics.contains(Metric.BLOCKS), (builder, params) -> {
@@ -758,8 +758,10 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerP
 
             // customs
             metrics.contains(Metric.CUSTOMS)
-                ? ChunkedToXContent.builder(outerParams)
-                    .forEach(customs.entrySet().iterator(), (b, e) -> b.xContentObject(e.getKey(), e.getValue()))
+                ? Iterators.flatMap(
+                    customs.entrySet().iterator(),
+                    e -> ChunkedToXContentHelper.object(e.getKey(), e.getValue().toXContentChunked(outerParams))
+                )
                 : Collections.emptyIterator()
         );
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
index bd438d66549aa..ed6ca57d67b25 100644
--- a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
+++ b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
@@ -754,7 +754,7 @@ public ClusterState execute(BatchExecutionContext<StartedShardUpdateTask> batchE
                         metadataBuilder.put(
                             IndexMetadata.builder(metadataBuilder.getSafe(updatedTimeRangesEntry.getKey()))
                                 .timestampRange(timeRanges.timestampRange())
-                                .eventIngestedRange(timeRanges.eventIngestedRange(), maybeUpdatedState.getMinTransportVersion())
+                                .eventIngestedRange(timeRanges.eventIngestedRange())
                         );
                     }
                     maybeUpdatedState = ClusterState.builder(maybeUpdatedState).metadata(metadataBuilder).build();
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java
index 5235293a54d95..916a192d53871 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.features.FeatureService;
+import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
@@ -39,6 +40,7 @@
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -47,6 +49,7 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.cluster.metadata.IndexMetadataVerifier.isReadOnlySupportedVersion;
 import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
 
 public class NodeJoinExecutor implements ClusterStateTaskExecutor<JoinTask> {
@@ -137,8 +140,8 @@ public ClusterState execute(BatchExecutionContext<JoinTask> batchExecutionContex
 
         DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(newState.nodes());
         Map<String, CompatibilityVersions> compatibilityVersionsMap = new HashMap<>(newState.compatibilityVersions());
-        Map<String, Set<String>> nodeFeatures = new HashMap<>(newState.nodeFeatures());
-        Set<String> allNodesFeatures = ClusterFeatures.calculateAllNodeFeatures(nodeFeatures.values());
+        Map<String, Set<String>> nodeFeatures = new HashMap<>(newState.nodeFeatures()); // as present in cluster state
+        Set<String> effectiveClusterFeatures = calculateEffectiveClusterFeatures(newState.nodes(), nodeFeatures);
 
         assert nodesBuilder.isLocalNodeElectedMaster();
 
@@ -174,14 +177,22 @@ public ClusterState execute(BatchExecutionContext<JoinTask> batchExecutionContex
                         }
                         blockForbiddenVersions(compatibilityVersions.transportVersion());
                         ensureNodesCompatibility(node.getVersion(), minClusterNodeVersion, maxClusterNodeVersion);
-                        enforceNodeFeatureBarrier(node.getId(), allNodesFeatures, features);
+                        Set<String> newNodeEffectiveFeatures = enforceNodeFeatureBarrier(node, effectiveClusterFeatures, features);
                         // we do this validation quite late to prevent race conditions between nodes joining and importing dangling indices
                         // we have to reject nodes that don't support all indices we have in this cluster
-                        ensureIndexCompatibility(node.getMinIndexVersion(), node.getMaxIndexVersion(), initialState.getMetadata());
+                        ensureIndexCompatibility(
+                            node.getMinIndexVersion(),
+                            node.getMinReadOnlyIndexVersion(),
+                            node.getMaxIndexVersion(),
+                            initialState.getMetadata()
+                        );
+
                         nodesBuilder.add(node);
                         compatibilityVersionsMap.put(node.getId(), compatibilityVersions);
+                        // store the actual node features here, not including assumed features, as this is persisted in cluster state
                         nodeFeatures.put(node.getId(), features);
-                        allNodesFeatures.retainAll(features);
+                        effectiveClusterFeatures.retainAll(newNodeEffectiveFeatures);
+
                         nodesChanged = true;
                         minClusterNodeVersion = Version.min(minClusterNodeVersion, node.getVersion());
                         maxClusterNodeVersion = Version.max(maxClusterNodeVersion, node.getVersion());
@@ -355,14 +366,49 @@ private static void blockForbiddenVersions(TransportVersion joiningTransportVers
         }
     }
 
+    /**
+     * Calculate the cluster's effective features. This includes all features that are assumed on any nodes in the cluster,
+     * that are also present across the whole cluster as a result.
+     */
+    private Set<String> calculateEffectiveClusterFeatures(DiscoveryNodes nodes, Map<String, Set<String>> nodeFeatures) {
+        if (featureService.featuresCanBeAssumedForNodes(nodes)) {
+            Set<String> assumedFeatures = featureService.getNodeFeatures()
+                .values()
+                .stream()
+                .filter(NodeFeature::assumedAfterNextCompatibilityBoundary)
+                .map(NodeFeature::id)
+                .collect(Collectors.toSet());
+
+            // add all assumed features to the featureset of all nodes of the next major version
+            nodeFeatures = new HashMap<>(nodeFeatures);
+            for (var node : nodes.getNodes().entrySet()) {
+                if (featureService.featuresCanBeAssumedForNode(node.getValue())) {
+                    assert nodeFeatures.containsKey(node.getKey()) : "Node " + node.getKey() + " does not have any features";
+                    nodeFeatures.computeIfPresent(node.getKey(), (k, v) -> {
+                        var newFeatures = new HashSet<>(v);
+                        return newFeatures.addAll(assumedFeatures) ? newFeatures : v;
+                    });
+                }
+            }
+        }
+
+        return ClusterFeatures.calculateAllNodeFeatures(nodeFeatures.values());
+    }
+
     /**
      * Ensures that all indices are compatible with the given index version. This will ensure that all indices in the given metadata
      * will not be created with a newer version of elasticsearch as well as that all indices are newer or equal to the minimum index
      * compatibility version.
      * @see IndexVersions#MINIMUM_COMPATIBLE
+     * @see IndexVersions#MINIMUM_READONLY_COMPATIBLE
      * @throws IllegalStateException if any index is incompatible with the given version
      */
-    public static void ensureIndexCompatibility(IndexVersion minSupportedVersion, IndexVersion maxSupportedVersion, Metadata metadata) {
+    public static void ensureIndexCompatibility(
+        IndexVersion minSupportedVersion,
+        IndexVersion minReadOnlySupportedVersion,
+        IndexVersion maxSupportedVersion,
+        Metadata metadata
+    ) {
         // we ensure that all indices in the cluster we join are compatible with us no matter if they are
         // closed or not we can't read mappings of these indices so we need to reject the join...
         for (IndexMetadata idxMetadata : metadata) {
@@ -377,14 +423,17 @@ public static void ensureIndexCompatibility(IndexVersion minSupportedVersion, In
                 );
             }
             if (idxMetadata.getCompatibilityVersion().before(minSupportedVersion)) {
-                throw new IllegalStateException(
-                    "index "
-                        + idxMetadata.getIndex()
-                        + " version not supported: "
-                        + idxMetadata.getCompatibilityVersion().toReleaseVersion()
-                        + " minimum compatible index version is: "
-                        + minSupportedVersion.toReleaseVersion()
-                );
+                boolean isReadOnlySupported = isReadOnlySupportedVersion(idxMetadata, minSupportedVersion, minReadOnlySupportedVersion);
+                if (isReadOnlySupported == false) {
+                    throw new IllegalStateException(
+                        "index "
+                            + idxMetadata.getIndex()
+                            + " version not supported: "
+                            + idxMetadata.getCompatibilityVersion().toReleaseVersion()
+                            + " minimum compatible index version is: "
+                            + minSupportedVersion.toReleaseVersion()
+                    );
+                }
             }
         }
     }
@@ -461,13 +510,44 @@ public static void ensureVersionBarrier(Version joiningNodeVersion, Version minC
         }
     }
 
-    private void enforceNodeFeatureBarrier(String nodeId, Set<String> existingNodesFeatures, Set<String> newNodeFeatures) {
+    /**
+     * Enforces the feature join barrier - a joining node should have all features already present in all existing nodes in the cluster
+     *
+     * @return The set of features that this node has (including assumed features)
+     */
+    private Set<String> enforceNodeFeatureBarrier(DiscoveryNode node, Set<String> effectiveClusterFeatures, Set<String> newNodeFeatures) {
         // prevent join if it does not have one or more features that all other nodes have
-        Set<String> missingFeatures = new HashSet<>(existingNodesFeatures);
+        Set<String> missingFeatures = new HashSet<>(effectiveClusterFeatures);
         missingFeatures.removeAll(newNodeFeatures);
 
-        if (missingFeatures.isEmpty() == false) {
-            throw new IllegalStateException("Node " + nodeId + " is missing required features " + missingFeatures);
+        if (missingFeatures.isEmpty()) {
+            // nothing missing - all ok
+            return newNodeFeatures;
+        }
+
+        if (featureService.featuresCanBeAssumedForNode(node)) {
+            // it might still be ok for this node to join if this node can have assumed features,
+            // and all the missing features are assumed
+            // we can get the NodeFeature object direct from this node's registered features
+            // as all existing nodes in the cluster have the features present in existingNodesFeatures, including this one
+            newNodeFeatures = new HashSet<>(newNodeFeatures);
+            for (Iterator<String> it = missingFeatures.iterator(); it.hasNext();) {
+                String feature = it.next();
+                NodeFeature nf = featureService.getNodeFeatures().get(feature);
+                if (nf.assumedAfterNextCompatibilityBoundary()) {
+                    // its ok for this feature to be missing from this node
+                    it.remove();
+                    // and it should be assumed to still be in the cluster
+                    newNodeFeatures.add(feature);
+                }
+                // even if we don't remove it, still continue, so the exception message below is accurate
+            }
+        }
+
+        if (missingFeatures.isEmpty()) {
+            return newNodeFeatures;
+        } else {
+            throw new IllegalStateException("Node " + node.getId() + " is missing required features " + missingFeatures);
         }
     }
 
@@ -477,7 +557,12 @@ public static Collection<BiConsumer<DiscoveryNode, ClusterState>> addBuiltInJoin
         final Collection<BiConsumer<DiscoveryNode, ClusterState>> validators = new ArrayList<>();
         validators.add((node, state) -> {
             ensureNodesCompatibility(node.getVersion(), state.getNodes());
-            ensureIndexCompatibility(node.getMinIndexVersion(), node.getMaxIndexVersion(), state.getMetadata());
+            ensureIndexCompatibility(
+                node.getMinIndexVersion(),
+                node.getMinReadOnlyIndexVersion(),
+                node.getMaxIndexVersion(),
+                state.getMetadata()
+            );
         });
         validators.addAll(onJoinValidators);
         return Collections.unmodifiableCollection(validators);
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
index 2aedfcef272ed..ed1ad1fc9ebbb 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
@@ -102,8 +102,8 @@ public static ComponentTemplateMetadata fromXContent(XContentParser parser) thro
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).xContentObjectFields(COMPONENT_TEMPLATE.getPreferredName(), componentTemplates);
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.xContentObjectFields(COMPONENT_TEMPLATE.getPreferredName(), componentTemplates);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java
index 998217e93c426..45433c4cba107 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java
@@ -427,7 +427,7 @@ public DataStreamTemplate(boolean hidden, boolean allowCustomRouting) {
         /**
          * A mapping snippet for a backing index with `_data_stream_timestamp` meta field mapper properly configured.
          */
-        public static CompressedXContent DATA_STREAM_MAPPING_SNIPPET;
+        public static final CompressedXContent DATA_STREAM_MAPPING_SNIPPET;
 
         static {
             try {
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java
index f34375d9c8a6d..95f08963b62b4 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
@@ -103,8 +103,8 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).xContentObjectFields(INDEX_TEMPLATE.getPreferredName(), indexTemplates);
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.xContentObjectFields(INDEX_TEMPLATE.getPreferredName(), indexTemplates);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
index 7745ec9cc75b2..286ecb3113337 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -49,7 +50,6 @@
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
@@ -82,7 +82,7 @@ public static boolean isFailureStoreFeatureFlagEnabled() {
     public static final DateFormatter DATE_FORMATTER = DateFormatter.forPattern("uuuu.MM.dd");
     public static final String TIMESTAMP_FIELD_NAME = "@timestamp";
     // Timeseries indices' leaf readers should be sorted by desc order of their timestamp field, as it allows search time optimizations
-    public static Comparator<LeafReader> TIMESERIES_LEAF_READERS_SORTER = Comparator.comparingLong((LeafReader r) -> {
+    public static final Comparator<LeafReader> TIMESERIES_LEAF_READERS_SORTER = Comparator.comparingLong((LeafReader r) -> {
         try {
             PointValues points = r.getPointValues(TIMESTAMP_FIELD_NAME);
             if (points != null) {
@@ -118,6 +118,7 @@ public static boolean isFailureStoreFeatureFlagEnabled() {
     private final DataStreamIndices backingIndices;
     private final DataStreamIndices failureIndices;
 
+    // visible for testing
     public DataStream(
         String name,
         List<Index> indices,
@@ -151,7 +152,6 @@ public DataStream(
         );
     }
 
-    // visible for testing
     DataStream(
         String name,
         long generation,
@@ -300,7 +300,15 @@ public boolean rolloverOnWrite() {
      * @return true if it's a system index or has a dot-prefixed name.
      */
     public boolean isInternal() {
-        return isSystem() || name.charAt(0) == '.';
+        return isSystem() || isDotPrefixName(name);
+    }
+
+    private static boolean isInternalName(String name, SystemIndices systemIndices) {
+        return isDotPrefixName(name) || systemIndices.isSystemDataStream(name);
+    }
+
+    private static boolean isDotPrefixName(String name) {
+        return name.charAt(0) == '.';
     }
 
     /**
@@ -419,12 +427,55 @@ public boolean isAllowCustomRouting() {
     }
 
     /**
-     * Determines if this data stream has its failure store enabled or not. Currently, the failure store
-     * is enabled only when a user has explicitly requested it.
-     * @return true, if the user has explicitly enabled the failure store.
+     * Determines whether this data stream has its failure store enabled explicitly in its metadata.
      */
-    public boolean isFailureStoreEnabled() {
-        return dataStreamOptions.isFailureStoreEnabled();
+    public boolean isFailureStoreExplicitlyEnabled() {
+        return dataStreamOptions.failureStore() != null && Boolean.TRUE.equals(dataStreamOptions.failureStore().enabled());
+    }
+
+    /**
+     * Returns whether this data stream has its failure store enabled, either explicitly in its metadata or implicitly via settings.
+     *
+     * <p>If the failure store is either explicitly enabled or explicitly disabled in its options metadata, that value is returned. If not,
+     * it checks whether its name matches one of the patterns in the settings, and that the data stream is not internal (i.e. neither a
+     * dot-prefixed nor a system data stream).
+     *
+     * @param dataStreamFailureStoreSettings The settings to use to determine whether the failure store should be implicitly enabled
+     */
+    public boolean isFailureStoreEffectivelyEnabled(DataStreamFailureStoreSettings dataStreamFailureStoreSettings) {
+        return isFailureStoreEffectivelyEnabled(dataStreamOptions, dataStreamFailureStoreSettings, name, isInternal());
+    }
+
+    /**
+     * Returns whether a data stream has its failure store enabled, either explicitly in its metadata or implicitly via settings, based
+     * on the given parameters. The logic is equivalent to that in
+     * {@link #isFailureStoreEffectivelyEnabled(DataStreamFailureStoreSettings)}.
+     *
+     * @param options The {@link DataStreamOptions} for the data stream (which may be null)
+     * @param dataStreamFailureStoreSettings The settings to use to determine whether the failure store should be implicitly enabled
+     * @param name The name of the data stream
+     * @param systemIndices The {@link SystemIndices} instance to use to determine whether this is a system data stream
+     */
+    public static boolean isFailureStoreEffectivelyEnabled(
+        @Nullable DataStreamOptions options,
+        DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
+        String name,
+        SystemIndices systemIndices
+    ) {
+        return isFailureStoreEffectivelyEnabled(options, dataStreamFailureStoreSettings, name, isInternalName(name, systemIndices));
+    }
+
+    private static boolean isFailureStoreEffectivelyEnabled(
+        DataStreamOptions options,
+        DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
+        String name,
+        boolean isInternal
+    ) {
+        if (options != null && options.failureStore() != null && options.failureStore().enabled() != null) {
+            return options.failureStore().enabled();
+        } else {
+            return (isInternal == false) && dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName(name);
+        }
     }
 
     @Nullable
@@ -794,27 +845,57 @@ public DataStream promoteDataStream() {
 
     /**
      * Reconciles this data stream with a list of indices available in a snapshot. Allows snapshots to store accurate data
-     * stream definitions that do not reference backing indices not contained in the snapshot.
+     * stream definitions that do not reference backing indices and failure indices not contained in the snapshot.
      *
      * @param indicesInSnapshot List of indices in the snapshot
+     * @param snapshotMetadataBuilder a metadata builder with the current view of the snapshot metadata
      * @return Reconciled {@link DataStream} instance or {@code null} if no reconciled version of this data stream could be built from the
      *         given indices
      */
     @Nullable
-    public DataStream snapshot(Collection<String> indicesInSnapshot) {
+    public DataStream snapshot(Set<String> indicesInSnapshot, Metadata.Builder snapshotMetadataBuilder) {
+        boolean backingIndicesChanged = false;
+        boolean failureIndicesChanged = false;
+
         // do not include indices not available in the snapshot
-        List<Index> reconciledIndices = new ArrayList<>(this.backingIndices.indices);
-        if (reconciledIndices.removeIf(x -> indicesInSnapshot.contains(x.getName()) == false) == false) {
+        List<Index> reconciledBackingIndices = this.backingIndices.indices;
+        if (isAnyIndexMissing(this.backingIndices.getIndices(), snapshotMetadataBuilder, indicesInSnapshot)) {
+            reconciledBackingIndices = new ArrayList<>(this.backingIndices.indices);
+            backingIndicesChanged = reconciledBackingIndices.removeIf(x -> indicesInSnapshot.contains(x.getName()) == false);
+            if (reconciledBackingIndices.isEmpty()) {
+                return null;
+            }
+        }
+
+        List<Index> reconciledFailureIndices = this.failureIndices.indices;
+        if (DataStream.isFailureStoreFeatureFlagEnabled()
+            && isAnyIndexMissing(failureIndices.indices, snapshotMetadataBuilder, indicesInSnapshot)) {
+            reconciledFailureIndices = new ArrayList<>(this.failureIndices.indices);
+            failureIndicesChanged = reconciledFailureIndices.removeIf(x -> indicesInSnapshot.contains(x.getName()) == false);
+        }
+
+        if (backingIndicesChanged == false && failureIndicesChanged == false) {
             return this;
         }
 
-        if (reconciledIndices.size() == 0) {
-            return null;
+        Builder builder = copy();
+        if (backingIndicesChanged) {
+            builder.setBackingIndices(backingIndices.copy().setIndices(reconciledBackingIndices).build());
+        }
+        if (failureIndicesChanged) {
+            builder.setFailureIndices(failureIndices.copy().setIndices(reconciledFailureIndices).build());
         }
+        return builder.setMetadata(metadata == null ? null : new HashMap<>(metadata)).build();
+    }
 
-        return copy().setBackingIndices(backingIndices.copy().setIndices(reconciledIndices).build())
-            .setMetadata(metadata == null ? null : new HashMap<>(metadata))
-            .build();
+    private static boolean isAnyIndexMissing(List<Index> indices, Metadata.Builder builder, Set<String> indicesInSnapshot) {
+        for (Index index : indices) {
+            final String indexName = index.getName();
+            if (builder.get(indexName) == null || indicesInSnapshot.contains(indexName) == false) {
+                return true;
+            }
+        }
+        return false;
     }
 
     /**
@@ -1077,7 +1158,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         if (out.getTransportVersion()
             .between(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION, DataStream.ADD_DATA_STREAM_OPTIONS_VERSION)) {
-            out.writeBoolean(isFailureStoreEnabled());
+            out.writeBoolean(isFailureStoreExplicitlyEnabled());
         }
         if (out.getTransportVersion().onOrAfter(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION)) {
             out.writeCollection(failureIndices.indices);
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreSettings.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreSettings.java
new file mode 100644
index 0000000000000..c5076d01eabb0
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreSettings.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Setting;
+
+import java.util.List;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.core.Predicates.never;
+
+/**
+ * Holder for the data stream global settings relating to the data stream failure store. This defines, validates, and monitors the settings.
+ */
+public class DataStreamFailureStoreSettings {
+
+    private static final Logger logger = LogManager.getLogger(DataStreamFailureStoreSettings.class);
+
+    public static final Setting<List<String>> DATA_STREAM_FAILURE_STORED_ENABLED_SETTING = Setting.stringListSetting(
+        "data_streams.failure_store.enabled",
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    private Predicate<String> failureStoreEnabledByName;
+
+    private DataStreamFailureStoreSettings() {
+        failureStoreEnabledByName = never();
+    }
+
+    /**
+     * Creates an instance and initialises the cluster settings listeners.
+     *
+     * @param clusterSettings The cluster settings to initialize the instance from and to watch for updates to
+     */
+    public static DataStreamFailureStoreSettings create(ClusterSettings clusterSettings) {
+        DataStreamFailureStoreSettings dataStreamFailureStoreSettings = new DataStreamFailureStoreSettings();
+        if (DataStream.isFailureStoreFeatureFlagEnabled()) {
+            clusterSettings.initializeAndWatch(
+                DATA_STREAM_FAILURE_STORED_ENABLED_SETTING,
+                dataStreamFailureStoreSettings::setEnabledByNamePatterns
+            );
+        }
+        return dataStreamFailureStoreSettings;
+    }
+
+    /**
+     * Returns whether the settings indicate that the failure store should be enabled by the cluster settings for the given name.
+     *
+     * @param name The data stream name
+     */
+    public boolean failureStoreEnabledForDataStreamName(String name) {
+        assert DataStream.isFailureStoreFeatureFlagEnabled() : "Testing whether failure store is enabled should be behind by feature flag";
+        return failureStoreEnabledByName.test(name);
+    }
+
+    private void setEnabledByNamePatterns(List<String> patterns) {
+        failureStoreEnabledByName = Regex.simpleMatcher(patterns.toArray(String[]::new));
+        logger.info("Updated data stream name patterns for enabling failure store to [{}]", patterns);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java
index af807e87a848e..17267525d4bdf 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.features.NodeFeature;
 
 import java.io.IOException;
 
@@ -23,7 +22,6 @@
  */
 public record DataStreamGlobalRetention(@Nullable TimeValue defaultRetention, @Nullable TimeValue maxRetention) implements Writeable {
 
-    public static final NodeFeature GLOBAL_RETENTION = new NodeFeature("data_stream.lifecycle.global_retention");
     public static final TimeValue MIN_RETENTION_VALUE = TimeValue.timeValueSeconds(10);
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java
index e49b991c21887..67117df97441b 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java
@@ -17,9 +17,10 @@
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -231,10 +232,13 @@ public static DataStreamMetadata fromXContent(XContentParser parser) throws IOEx
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params)
-            .xContentObjectFields(DATA_STREAM.getPreferredName(), dataStreams)
-            .xContentObject(DATA_STREAM_ALIASES.getPreferredName(), dataStreamAliases.values().iterator());
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.concat(
+            ChunkedToXContentHelper.xContentObjectFields(DATA_STREAM.getPreferredName(), dataStreams),
+            ChunkedToXContentHelper.startObject(DATA_STREAM_ALIASES.getPreferredName()),
+            dataStreamAliases.values().iterator(),
+            ChunkedToXContentHelper.endObject()
+        );
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java
index 51e13c05e6892..423b698442581 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java
@@ -70,16 +70,6 @@ public boolean isEmpty() {
         return failureStore == null;
     }
 
-    /**
-     * Determines if this data stream has its failure store enabled or not. Currently, the failure store
-     * is enabled only when a user has explicitly requested it.
-     *
-     * @return true, if the user has explicitly enabled the failure store.
-     */
-    public boolean isFailureStoreEnabled() {
-        return failureStore != null && Boolean.TRUE.equals(failureStore.enabled());
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(failureStore);
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java
index de3343c1944c1..8366083b1907e 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.common.unit.Processors;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.UpdateForV9;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -46,9 +45,6 @@
 
 public final class DesiredNode implements Writeable, ToXContentObject, Comparable<DesiredNode> {
 
-    public static final NodeFeature RANGE_FLOAT_PROCESSORS_SUPPORTED = new NodeFeature("desired_node.range_float_processors");
-    public static final NodeFeature DESIRED_NODE_VERSION_DEPRECATED = new NodeFeature("desired_node.version_deprecated");
-
     public static final TransportVersion RANGE_FLOAT_PROCESSORS_SUPPORT_TRANSPORT_VERSION = TransportVersions.V_8_3_0;
 
     private static final ParseField SETTINGS_FIELD = new ParseField("settings");
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java
index aa0b1c045be5c..015c90ebe450e 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java
@@ -9,8 +9,11 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel;
 
@@ -49,6 +52,18 @@ public List<String> resolveIndexAbstractions(
                 indexAbstraction = index;
             }
 
+            // Always check to see if there's a selector on the index expression
+            Tuple<String, String> expressionAndSelector = IndexNameExpressionResolver.splitSelectorExpression(indexAbstraction);
+            String selectorString = expressionAndSelector.v2();
+            if (indicesOptions.allowSelectors() == false && selectorString != null) {
+                throw new IllegalArgumentException(
+                    "Index component selectors are not supported in this context but found selector in expression ["
+                        + indexAbstraction
+                        + "]"
+                );
+            }
+            indexAbstraction = expressionAndSelector.v1();
+
             // we always need to check for date math expressions
             indexAbstraction = IndexNameExpressionResolver.resolveDateMathExpression(indexAbstraction);
 
@@ -59,13 +74,15 @@ public List<String> resolveIndexAbstractions(
                     if (Regex.simpleMatch(indexAbstraction, authorizedIndex)
                         && isIndexVisible(
                             indexAbstraction,
+                            selectorString,
                             authorizedIndex,
                             indicesOptions,
                             metadata,
                             indexNameExpressionResolver,
                             includeDataStreams
                         )) {
-                        resolvedIndices.add(authorizedIndex);
+                        // Resolve any ::* suffixes on the expression. We need to resolve them all to their final valid selectors
+                        resolveSelectorsAndCombine(authorizedIndex, selectorString, indicesOptions, resolvedIndices, metadata);
                     }
                 }
                 if (resolvedIndices.isEmpty()) {
@@ -81,21 +98,63 @@ && isIndexVisible(
                     }
                 }
             } else {
+                // Resolve any ::* suffixes on the expression. We need to resolve them all to their final valid selectors
+                Set<String> resolvedIndices = new HashSet<>();
+                resolveSelectorsAndCombine(indexAbstraction, selectorString, indicesOptions, resolvedIndices, metadata);
                 if (minus) {
-                    finalIndices.remove(indexAbstraction);
+                    finalIndices.removeAll(resolvedIndices);
                 } else if (indicesOptions.ignoreUnavailable() == false || isAuthorized.test(indexAbstraction)) {
                     // Unauthorized names are considered unavailable, so if `ignoreUnavailable` is `true` they should be silently
                     // discarded from the `finalIndices` list. Other "ways of unavailable" must be handled by the action
                     // handler, see: https://github.com/elastic/elasticsearch/issues/90215
-                    finalIndices.add(indexAbstraction);
+                    finalIndices.addAll(resolvedIndices);
                 }
             }
         }
         return finalIndices;
     }
 
+    private static void resolveSelectorsAndCombine(
+        String indexAbstraction,
+        String selectorString,
+        IndicesOptions indicesOptions,
+        Set<String> collect,
+        Metadata metadata
+    ) {
+        if (indicesOptions.allowSelectors()) {
+            IndexAbstraction abstraction = metadata.getIndicesLookup().get(indexAbstraction);
+            // We can't determine which selectors are valid for a nonexistent abstraction, so simply propagate them as if they supported
+            // all of them so we don't drop anything.
+            boolean acceptsAllSelectors = abstraction == null || abstraction.isDataStreamRelated();
+
+            // Supply default if needed
+            if (selectorString == null) {
+                selectorString = IndexComponentSelector.DATA.getKey();
+            }
+
+            if (Regex.isMatchAllPattern(selectorString)) {
+                // Always accept data
+                collect.add(IndexNameExpressionResolver.combineSelectorExpression(indexAbstraction, IndexComponentSelector.DATA.getKey()));
+                // Only put failures on the expression if the abstraction supports it.
+                if (acceptsAllSelectors) {
+                    collect.add(
+                        IndexNameExpressionResolver.combineSelectorExpression(indexAbstraction, IndexComponentSelector.FAILURES.getKey())
+                    );
+                }
+            } else {
+                // A non-wildcard selector is always passed along as-is, it's validity for this kind of abstraction is tested later
+                collect.add(IndexNameExpressionResolver.combineSelectorExpression(indexAbstraction, selectorString));
+            }
+        } else {
+            assert selectorString == null
+                : "A selector string [" + selectorString + "] is present but selectors are disabled in this context";
+            collect.add(indexAbstraction);
+        }
+    }
+
     public static boolean isIndexVisible(
         String expression,
+        @Nullable String selectorString,
         String index,
         IndicesOptions indicesOptions,
         Metadata metadata,
@@ -111,7 +170,15 @@ public static boolean isIndexVisible(
         if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS) {
             // it's an alias, ignore expandWildcardsOpen and expandWildcardsClosed.
             // complicated to support those options with aliases pointing to multiple indices...
-            return isVisible && indicesOptions.ignoreAliases() == false;
+            isVisible = isVisible && indicesOptions.ignoreAliases() == false;
+            if (isVisible && selectorString != null) {
+                // Check if a selector was present, and if it is, check if this alias is applicable to it
+                IndexComponentSelector selector = IndexComponentSelector.getByKey(selectorString);
+                if (IndexComponentSelector.FAILURES.equals(selector)) {
+                    isVisible = indexAbstraction.isDataStreamRelated();
+                }
+            }
+            return isVisible;
         }
         if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) {
             if (includeDataStreams == false) {
@@ -142,6 +209,13 @@ public static boolean isIndexVisible(
                 return isSystemIndexVisible(resolver, indexAbstraction);
             }
         }
+        if (selectorString != null && Regex.isMatchAllPattern(selectorString) == false) {
+            // Check if a selector was present, and if it is, check if this index is applicable to it
+            IndexComponentSelector selector = IndexComponentSelector.getByKey(selectorString);
+            if (IndexComponentSelector.FAILURES.equals(selector)) {
+                return false;
+            }
+        }
 
         IndexMetadata indexMetadata = metadata.index(indexAbstraction.getIndices().get(0));
         if (indexMetadata.getState() == IndexMetadata.State.CLOSE && indicesOptions.expandWildcardsClosed()) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
index 62867b4260bfd..320be8acb0af9 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
@@ -19,7 +19,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.xcontent.ContextParser;
 import org.elasticsearch.xcontent.ObjectParser;
@@ -128,8 +128,8 @@ public boolean containsIndex(final Index index) {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).array(TOMBSTONES_FIELD.getPreferredName(), tombstones.iterator());
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.array(TOMBSTONES_FIELD.getPreferredName(), tombstones.iterator());
     }
 
     public static IndexGraveyard fromXContent(final XContentParser parser) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
index 896e1295e7cce..04f9448a936be 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
@@ -41,7 +41,6 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.gateway.MetadataStateFormat;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexMode;
@@ -942,22 +941,12 @@ public IndexMetadata withIncrementedPrimaryTerm(int shardId) {
     /**
      * @param timestampRange new @timestamp range
      * @param eventIngestedRange new 'event.ingested' range
-     * @param minClusterTransportVersion minimum transport version used between nodes of this cluster
      * @return copy of this instance with updated timestamp range
      */
-    public IndexMetadata withTimestampRanges(
-        IndexLongFieldRange timestampRange,
-        IndexLongFieldRange eventIngestedRange,
-        TransportVersion minClusterTransportVersion
-    ) {
+    public IndexMetadata withTimestampRanges(IndexLongFieldRange timestampRange, IndexLongFieldRange eventIngestedRange) {
         if (timestampRange.equals(this.timestampRange) && eventIngestedRange.equals(this.eventIngestedRange)) {
             return this;
         }
-        @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_FOUNDATIONS) // remove this check when 8.15 is no longer communicable
-        IndexLongFieldRange allowedEventIngestedRange = eventIngestedRange;
-        if (minClusterTransportVersion.before(TransportVersions.V_8_15_0)) {
-            allowedEventIngestedRange = IndexLongFieldRange.UNKNOWN;
-        }
         return new IndexMetadata(
             this.index,
             this.version,
@@ -988,7 +977,7 @@ public IndexMetadata withTimestampRanges(
             this.isSystem,
             this.isHidden,
             timestampRange,
-            allowedEventIngestedRange,
+            eventIngestedRange,
             this.priority,
             this.creationDate,
             this.ignoreDiskWatermarks,
@@ -1706,12 +1695,7 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeOptionalDouble(indexWriteLoadForecast);
                 out.writeOptionalLong(shardSizeInBytesForecast);
             }
-            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
-                eventIngestedRange.writeTo(out);
-            } else {
-                assert eventIngestedRange == IndexLongFieldRange.UNKNOWN
-                    : "eventIngestedRange should be UNKNOWN until all nodes are on the new version but is " + eventIngestedRange;
-            }
+            eventIngestedRange.writeTo(out);
         }
 
         @Override
@@ -1814,11 +1798,7 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function<String,
             builder.indexWriteLoadForecast(in.readOptionalDouble());
             builder.shardSizeInBytesForecast(in.readOptionalLong());
         }
-        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
-            builder.eventIngestedRange(IndexLongFieldRange.readFrom(in), in.getTransportVersion());
-        } else {
-            builder.eventIngestedRange(IndexLongFieldRange.UNKNOWN, in.getTransportVersion());
-        }
+        builder.eventIngestedRange(IndexLongFieldRange.readFrom(in));
         return builder.build(true);
     }
 
@@ -2179,28 +2159,12 @@ public Builder timestampRange(IndexLongFieldRange timestampRange) {
             return this;
         }
 
-        // only for use within this class file where minClusterTransportVersion is not known (e.g., IndexMetadataDiff.apply)
-        Builder eventIngestedRange(IndexLongFieldRange eventIngestedRange) {
+        public Builder eventIngestedRange(IndexLongFieldRange eventIngestedRange) {
             assert eventIngestedRange != null : "eventIngestedRange cannot be null";
             this.eventIngestedRange = eventIngestedRange;
             return this;
         }
 
-        public Builder eventIngestedRange(IndexLongFieldRange eventIngestedRange, TransportVersion minClusterTransportVersion) {
-            assert eventIngestedRange != null : "eventIngestedRange cannot be null";
-            assert minClusterTransportVersion != null || eventIngestedRange == IndexLongFieldRange.UNKNOWN
-                : "eventIngestedRange must be UNKNOWN when minClusterTransportVersion is null, but minClusterTransportVersion: "
-                    + minClusterTransportVersion
-                    + "; eventIngestedRange = "
-                    + eventIngestedRange;
-            if (minClusterTransportVersion != null && minClusterTransportVersion.before(TransportVersions.V_8_15_0)) {
-                this.eventIngestedRange = IndexLongFieldRange.UNKNOWN;
-            } else {
-                this.eventIngestedRange = eventIngestedRange;
-            }
-            return this;
-        }
-
         public Builder stats(IndexMetadataStats stats) {
             this.stats = stats;
             return this;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java
index 0b8095c24519f..be2563c4732b7 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java
@@ -88,8 +88,12 @@ public IndexMetadataVerifier(
      * If the index does not need upgrade it returns the index metadata unchanged, otherwise it returns a modified index metadata. If index
      * cannot be updated the method throws an exception.
      */
-    public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) {
-        checkSupportedVersion(indexMetadata, minimumIndexCompatibilityVersion);
+    public IndexMetadata verifyIndexMetadata(
+        IndexMetadata indexMetadata,
+        IndexVersion minimumIndexCompatibilityVersion,
+        IndexVersion minimumReadOnlyIndexCompatibilityVersion
+    ) {
+        checkSupportedVersion(indexMetadata, minimumIndexCompatibilityVersion, minimumReadOnlyIndexCompatibilityVersion);
 
         // First convert any shared_cache searchable snapshot indices to only use _tier_preference: data_frozen
         IndexMetadata newMetadata = convertSharedCacheTierPreference(indexMetadata);
@@ -105,26 +109,81 @@ public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, IndexVersi
     }
 
     /**
-     * Check that the index version is compatible. Elasticsearch does not support indices created before the
-     * previous major version.
+     * Check that the index version is compatible. Elasticsearch supports reading and writing indices created in the current version ("N")
+     + as well as the previous major version ("N-1"). Elasticsearch only supports reading indices created down to the penultimate version
+     + ("N-2") and does not support reading nor writing any version below that.
      */
-    private static void checkSupportedVersion(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) {
-        boolean isSupportedVersion = indexMetadata.getCompatibilityVersion().onOrAfter(minimumIndexCompatibilityVersion);
-        if (isSupportedVersion == false) {
-            throw new IllegalStateException(
-                "The index "
-                    + indexMetadata.getIndex()
-                    + " has current compatibility version ["
-                    + indexMetadata.getCompatibilityVersion().toReleaseVersion()
-                    + "] but the minimum compatible version is ["
-                    + minimumIndexCompatibilityVersion.toReleaseVersion()
-                    + "]. It should be re-indexed in Elasticsearch "
-                    + (Version.CURRENT.major - 1)
-                    + ".x before upgrading to "
-                    + Build.current().version()
-                    + "."
-            );
+    private static void checkSupportedVersion(
+        IndexMetadata indexMetadata,
+        IndexVersion minimumIndexCompatibilityVersion,
+        IndexVersion minimumReadOnlyIndexCompatibilityVersion
+    ) {
+        if (isFullySupportedVersion(indexMetadata, minimumIndexCompatibilityVersion)) {
+            return;
+        }
+        if (isReadOnlySupportedVersion(indexMetadata, minimumIndexCompatibilityVersion, minimumReadOnlyIndexCompatibilityVersion)) {
+            return;
+        }
+        throw new IllegalStateException(
+            "The index "
+                + indexMetadata.getIndex()
+                + " has current compatibility version ["
+                + indexMetadata.getCompatibilityVersion().toReleaseVersion()
+                + "] but the minimum compatible version is ["
+                + minimumIndexCompatibilityVersion.toReleaseVersion()
+                + "]. It should be re-indexed in Elasticsearch "
+                + (Version.CURRENT.major - 1)
+                + ".x before upgrading to "
+                + Build.current().version()
+                + "."
+        );
+    }
+
+    private static boolean isFullySupportedVersion(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) {
+        return indexMetadata.getCompatibilityVersion().onOrAfter(minimumIndexCompatibilityVersion);
+    }
+
+    /**
+     * Returns {@code true} if the index version is compatible in read-only mode. As of today, only searchable snapshots and archive indices
+     * in version N-2 with a write block are read-only compatible. This method throws an {@link IllegalStateException} if the index is
+     * either a searchable snapshot or an archive index with a read-only compatible version but is missing the write block.
+     *
+     * @param indexMetadata                         the index metadata
+     * @param minimumIndexCompatibilityVersion      the min. index compatible version for reading and writing indices (used in assertion)
+     * @param minReadOnlyIndexCompatibilityVersion  the min. index compatible version for only reading indices
+     *
+     * @return {@code true} if the index version is compatible in read-only mode, {@code false} otherwise.
+     * @throws IllegalStateException if the index is read-only compatible but has no write block in place.
+     */
+    public static boolean isReadOnlySupportedVersion(
+        IndexMetadata indexMetadata,
+        IndexVersion minimumIndexCompatibilityVersion,
+        IndexVersion minReadOnlyIndexCompatibilityVersion
+    ) {
+        boolean isReadOnlySupportedVersion = indexMetadata.getCompatibilityVersion().onOrAfter(minReadOnlyIndexCompatibilityVersion);
+        assert isFullySupportedVersion(indexMetadata, minimumIndexCompatibilityVersion) == false;
+
+        if (isReadOnlySupportedVersion
+            && (indexMetadata.isSearchableSnapshot() || indexMetadata.getCreationVersion().isLegacyIndexVersion())) {
+            boolean isReadOnly = IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(indexMetadata.getSettings());
+            if (isReadOnly == false) {
+                throw new IllegalStateException(
+                    "The index "
+                        + indexMetadata.getIndex()
+                        + " created in version ["
+                        + indexMetadata.getCreationVersion()
+                        + "] with current compatibility version ["
+                        + indexMetadata.getCompatibilityVersion().toReleaseVersion()
+                        + "] must be marked as read-only using the setting ["
+                        + IndexMetadata.SETTING_BLOCKS_WRITE
+                        + "] set to [true] before upgrading to "
+                        + Build.current().version()
+                        + '.'
+                );
+            }
+            return true;
         }
+        return false;
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index 2ce91b66fa789..8b88f7609fa8c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexAbstraction.Type;
@@ -28,11 +29,11 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Predicates;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
-import org.elasticsearch.indices.FailureIndexNotSupportedException;
 import org.elasticsearch.indices.IndexClosedException;
 import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.indices.SystemIndices;
@@ -54,6 +55,8 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.SortedMap;
+import java.util.function.BiFunction;
+import java.util.function.Function;
 import java.util.function.LongSupplier;
 import java.util.function.Predicate;
 
@@ -82,11 +85,21 @@ public IndexNameExpressionResolver(ThreadContext threadContext, SystemIndices sy
     }
 
     /**
-     * This represents a resolved expression in the form of the name of a resource in the cluster.
-     * Soon it will facilitate an index component selector, which will define which part of the resource the expression is targeting.
+     * This represents a resolved expression in the form of the name of a resource in the cluster and a potential selector
+     * which defines which part of the resource the expression is targeting.
      * @param resource the name of a resource that an expression refers to.
+     * @param selector optionally indicates which part of a resource to target during an operation. A null value indicates selectors are
+     *                 unsupported in an API and the default data component of the expression should be used where applicable.
      */
-    public record ResolvedExpression(String resource) {}
+    public record ResolvedExpression(String resource, @Nullable IndexComponentSelector selector) {
+        public ResolvedExpression(String indexAbstraction) {
+            this(indexAbstraction, null);
+        }
+
+        public String combined() {
+            return combineSelector(resource, selector);
+        }
+    }
 
     /**
      * Same as {@link #concreteIndexNames(ClusterState, IndicesOptions, String...)}, but the index expressions and options
@@ -194,6 +207,10 @@ public String[] concreteIndexNames(ClusterState state, IndicesOptions options, I
     }
 
     public List<String> dataStreamNames(ClusterState state, IndicesOptions options, String... indexExpressions) {
+        return dataStreams(state, options, indexExpressions).stream().map(ResolvedExpression::resource).distinct().toList();
+    }
+
+    public List<ResolvedExpression> dataStreams(ClusterState state, IndicesOptions options, String... indexExpressions) {
         Context context = new Context(
             state,
             options,
@@ -206,13 +223,10 @@ public List<String> dataStreamNames(ClusterState state, IndicesOptions options,
             getNetNewSystemIndexPredicate()
         );
         final Collection<ResolvedExpression> expressions = resolveExpressionsToResources(context, indexExpressions);
-        return expressions.stream()
-            .map(ResolvedExpression::resource)
-            .map(x -> state.metadata().getIndicesLookup().get(x))
-            .filter(Objects::nonNull)
-            .filter(ia -> ia.getType() == Type.DATA_STREAM)
-            .map(IndexAbstraction::getName)
-            .toList();
+        return expressions.stream().filter(expression -> {
+            IndexAbstraction ia = state.metadata().getIndicesLookup().get(expression.resource());
+            return ia != null && Type.DATA_STREAM == ia.getType();
+        }).toList();
     }
 
     /**
@@ -271,14 +285,25 @@ protected static Collection<ResolvedExpression> resolveExpressionsToResources(Co
         // If we do not expand wildcards, then empty or _all expression result in an empty list
         boolean expandWildcards = context.getOptions().expandWildcardExpressions();
         if (expandWildcards == false) {
-            if (expressions == null || expressions.length == 0 || expressions.length == 1 && Metadata.ALL.equals(expressions[0])) {
+            if (expressions == null
+                || expressions.length == 0
+                || expressions.length == 1
+                    && SelectorResolver.selectorsValidatedAndMatchesPredicate(expressions[0], context, Metadata.ALL::equals)) {
                 return List.of();
             }
         } else {
+            Predicate<String> isMatchAll = (((Predicate<String>) Metadata.ALL::equals)).or(Regex::isMatchAllPattern);
             if (expressions == null
                 || expressions.length == 0
-                || expressions.length == 1 && (Metadata.ALL.equals(expressions[0]) || Regex.isMatchAllPattern(expressions[0]))) {
-                return WildcardExpressionResolver.resolveAll(context);
+                || expressions.length == 1
+                    && (SelectorResolver.selectorsValidatedAndMatchesPredicate(expressions[0], context, isMatchAll))) {
+                IndexComponentSelector selector;
+                if (expressions != null && expressions.length == 1) {
+                    selector = SelectorResolver.parseMatchAllToSelector(context, expressions[0]);
+                } else {
+                    selector = IndexComponentSelector.DATA;
+                }
+                return WildcardExpressionResolver.resolveAll(context, selector);
             } else if (isNoneExpression(expressions)) {
                 return List.of();
             }
@@ -296,6 +321,11 @@ protected static Collection<ResolvedExpression> resolveExpressionsToResources(Co
             boolean isExclusion = wildcardSeen && originalExpression.startsWith("-");
             String baseExpression = isExclusion ? originalExpression.substring(1) : originalExpression;
 
+            // Parse a potential selector from the expression
+            ResolvedExpression partiallyResolvedExpression = SelectorResolver.parseExpression(baseExpression, context.getOptions());
+            baseExpression = partiallyResolvedExpression.resource();
+            IndexComponentSelector selector = partiallyResolvedExpression.selector();
+
             // Resolve date math
             baseExpression = DateMathExpressionResolver.resolveExpression(baseExpression, context::getStartTime);
 
@@ -303,14 +333,18 @@ protected static Collection<ResolvedExpression> resolveExpressionsToResources(Co
             validateResourceExpression(context, baseExpression, expressions);
 
             // Check if it's wildcard
-            boolean isWildcard = expandWildcards && WildcardExpressionResolver.isWildcard(originalExpression);
+            boolean isWildcard = expandWildcards && WildcardExpressionResolver.isWildcard(baseExpression);
             wildcardSeen |= isWildcard;
 
             if (isWildcard) {
-                Set<ResolvedExpression> matchingResources = WildcardExpressionResolver.matchWildcardToResources(context, baseExpression);
+                Set<ResolvedExpression> matchingResources = WildcardExpressionResolver.matchWildcardToResources(
+                    context,
+                    baseExpression,
+                    selector
+                );
 
                 if (context.getOptions().allowNoIndices() == false && matchingResources.isEmpty()) {
-                    throw notFoundException(baseExpression);
+                    throw notFoundException(combineSelector(baseExpression, partiallyResolvedExpression.selector()));
                 }
 
                 if (isExclusion) {
@@ -320,9 +354,21 @@ protected static Collection<ResolvedExpression> resolveExpressionsToResources(Co
                 }
             } else {
                 if (isExclusion) {
-                    resources.remove(new ResolvedExpression(baseExpression));
-                } else if (ensureAliasOrIndexExists(context, baseExpression)) {
-                    resources.add(new ResolvedExpression(baseExpression));
+                    if (IndexComponentSelector.ALL_APPLICABLE.equals(selector)) {
+                        resources.remove(new ResolvedExpression(baseExpression, IndexComponentSelector.DATA));
+                        resources.remove(new ResolvedExpression(baseExpression, IndexComponentSelector.FAILURES));
+                    } else {
+                        resources.remove(new ResolvedExpression(baseExpression, selector));
+                    }
+                } else if (ensureAliasOrIndexExists(context, baseExpression, selector)) {
+                    if (IndexComponentSelector.ALL_APPLICABLE.equals(selector)) {
+                        resources.add(new ResolvedExpression(baseExpression, IndexComponentSelector.DATA));
+                        if (context.getState().getMetadata().getIndicesLookup().get(baseExpression).isDataStreamRelated()) {
+                            resources.add(new ResolvedExpression(baseExpression, IndexComponentSelector.FAILURES));
+                        }
+                    } else {
+                        resources.add(new ResolvedExpression(baseExpression, selector));
+                    }
                 }
             }
         }
@@ -458,17 +504,17 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
                 }
                 if (indexAbstraction.isDataStreamRelated()) {
                     DataStream dataStream = indicesLookup.get(indexAbstraction.getWriteIndex().getName()).getParentDataStream();
-                    resolveWriteIndexForDataStreams(context, dataStream, concreteIndicesResult);
+                    resolveWriteIndexForDataStreams(context, dataStream, concreteIndicesResult, expression.selector());
                 } else {
                     if (addIndex(writeIndex, null, context)) {
                         concreteIndicesResult.add(writeIndex);
                     }
                 }
             } else if (indexAbstraction.getType() == Type.DATA_STREAM && context.isResolveToWriteIndex()) {
-                resolveWriteIndexForDataStreams(context, (DataStream) indexAbstraction, concreteIndicesResult);
+                resolveWriteIndexForDataStreams(context, (DataStream) indexAbstraction, concreteIndicesResult, expression.selector());
             } else {
-                if (resolvesToMoreThanOneIndex(indexAbstraction, context)
-                    && context.getOptions().allowAliasesToMultipleIndices() == false) {
+                if (context.getOptions().allowAliasesToMultipleIndices() == false
+                    && resolvesToMoreThanOneIndex(indexAbstraction, context, expression)) {
                     String[] indexNames = new String[indexAbstraction.getIndices().size()];
                     int i = 0;
                     for (Index indexName : indexAbstraction.getIndices()) {
@@ -485,20 +531,12 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
                 }
 
                 if (indexAbstraction.getType() == Type.DATA_STREAM) {
-                    resolveIndicesForDataStream(context, (DataStream) indexAbstraction, concreteIndicesResult);
+                    resolveIndicesForDataStream(context, (DataStream) indexAbstraction, concreteIndicesResult, expression.selector());
                 } else if (indexAbstraction.getType() == Type.ALIAS
                     && indexAbstraction.isDataStreamRelated()
-                    && DataStream.isFailureStoreFeatureFlagEnabled()
-                    && context.getOptions().includeFailureIndices()) {
-                        // Collect the data streams involved
-                        Set<DataStream> aliasDataStreams = new HashSet<>();
-                        List<Index> indices = indexAbstraction.getIndices();
-                        for (int i = 0, n = indices.size(); i < n; i++) {
-                            Index index = indices.get(i);
-                            aliasDataStreams.add(indicesLookup.get(index.getName()).getParentDataStream());
-                        }
-                        for (DataStream dataStream : aliasDataStreams) {
-                            resolveIndicesForDataStream(context, dataStream, concreteIndicesResult);
+                    && shouldIncludeFailureIndices(context.getOptions(), expression.selector())) {
+                        for (DataStream dataStream : getAliasDataStreams(indexAbstraction, indicesLookup)) {
+                            resolveIndicesForDataStream(context, dataStream, concreteIndicesResult, expression.selector());
                         }
                     } else {
                         List<Index> indices = indexAbstraction.getIndices();
@@ -520,8 +558,29 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
         return resultArray;
     }
 
-    private static void resolveIndicesForDataStream(Context context, DataStream dataStream, Set<Index> concreteIndicesResult) {
-        if (shouldIncludeRegularIndices(context.getOptions())) {
+    private static Set<DataStream> getAliasDataStreams(IndexAbstraction indexAbstraction, Map<String, IndexAbstraction> indicesLookup) {
+        // Collect the data streams involved with the alias
+        assert indexAbstraction.getType().equals(Type.ALIAS) && indexAbstraction.isDataStreamRelated()
+            : "Non data stream alias [" + indexAbstraction.getName() + "]";
+        Set<DataStream> aliasDataStreams = new HashSet<>();
+        List<Index> indices = indexAbstraction.getIndices();
+        for (int i = 0, n = indices.size(); i < n; i++) {
+            Index index = indices.get(i);
+            DataStream parentDataStream = indicesLookup.get(index.getName()).getParentDataStream();
+            if (parentDataStream != null) {
+                aliasDataStreams.add(parentDataStream);
+            }
+        }
+        return aliasDataStreams;
+    }
+
+    private static void resolveIndicesForDataStream(
+        Context context,
+        DataStream dataStream,
+        Set<Index> concreteIndicesResult,
+        IndexComponentSelector selector
+    ) {
+        if (shouldIncludeRegularIndices(context.getOptions(), selector)) {
             List<Index> indices = dataStream.getIndices();
             for (int i = 0, n = indices.size(); i < n; i++) {
                 Index index = indices.get(i);
@@ -530,55 +589,86 @@ private static void resolveIndicesForDataStream(Context context, DataStream data
                 }
             }
         }
-        if (shouldIncludeFailureIndices(context.getOptions())) {
-            // We short-circuit here, if failure indices are not allowed and they can be skipped
-            if (context.getOptions().allowFailureIndices() || context.getOptions().ignoreUnavailable() == false) {
-                List<Index> failureIndices = dataStream.getFailureIndices().getIndices();
-                for (int i = 0, n = failureIndices.size(); i < n; i++) {
-                    Index index = failureIndices.get(i);
-                    if (shouldTrackConcreteIndex(context, index)) {
-                        concreteIndicesResult.add(index);
-                    }
+        if (shouldIncludeFailureIndices(context.getOptions(), selector)) {
+            List<Index> failureIndices = dataStream.getFailureIndices().getIndices();
+            for (int i = 0, n = failureIndices.size(); i < n; i++) {
+                Index index = failureIndices.get(i);
+                if (shouldTrackConcreteIndex(context, index)) {
+                    concreteIndicesResult.add(index);
                 }
             }
         }
     }
 
-    private static void resolveWriteIndexForDataStreams(Context context, DataStream dataStream, Set<Index> concreteIndicesResult) {
-        if (shouldIncludeRegularIndices(context.getOptions())) {
+    private static void resolveWriteIndexForDataStreams(
+        Context context,
+        DataStream dataStream,
+        Set<Index> concreteIndicesResult,
+        IndexComponentSelector selector
+    ) {
+        if (shouldIncludeRegularIndices(context.getOptions(), selector)) {
             Index writeIndex = dataStream.getWriteIndex();
             if (addIndex(writeIndex, null, context)) {
                 concreteIndicesResult.add(writeIndex);
             }
         }
-        if (shouldIncludeFailureIndices(context.getOptions())) {
+        if (shouldIncludeFailureIndices(context.getOptions(), selector)) {
             Index failureStoreWriteIndex = dataStream.getFailureStoreWriteIndex();
             if (failureStoreWriteIndex != null && addIndex(failureStoreWriteIndex, null, context)) {
-                if (context.options.allowFailureIndices() == false) {
-                    throw new FailureIndexNotSupportedException(failureStoreWriteIndex);
-                }
                 concreteIndicesResult.add(failureStoreWriteIndex);
             }
         }
     }
 
-    private static boolean shouldIncludeRegularIndices(IndicesOptions indicesOptions) {
-        return DataStream.isFailureStoreFeatureFlagEnabled() == false || indicesOptions.includeRegularIndices();
+    private static boolean shouldIncludeRegularIndices(IndicesOptions indicesOptions, IndexComponentSelector expressionSelector) {
+        if (indicesOptions.allowSelectors()) {
+            return expressionSelector == null || expressionSelector.shouldIncludeData();
+        }
+        return true;
     }
 
-    private static boolean shouldIncludeFailureIndices(IndicesOptions indicesOptions) {
+    private static boolean shouldIncludeFailureIndices(IndicesOptions indicesOptions, IndexComponentSelector expressionSelector) {
         // We return failure indices regardless of whether the data stream actually has the `failureStoreEnabled` flag set to true.
-        return DataStream.isFailureStoreFeatureFlagEnabled() && indicesOptions.includeFailureIndices();
+        if (indicesOptions.allowSelectors()) {
+            return expressionSelector != null && expressionSelector.shouldIncludeFailures();
+        }
+        return false;
     }
 
-    private static boolean resolvesToMoreThanOneIndex(IndexAbstraction indexAbstraction, Context context) {
+    private static boolean resolvesToMoreThanOneIndex(IndexAbstraction indexAbstraction, Context context, ResolvedExpression expression) {
+        if (indexAbstraction.getType() == Type.ALIAS && indexAbstraction.isDataStreamRelated()) {
+            // We inline this logic instead of calling aliasDataStreams because we want to return as soon as we've identified that we have
+            // more than one index. No use resolving a colossal alias to all of its data streams if the first data stream we find puts us
+            // over the limit.
+            SortedMap<String, IndexAbstraction> indicesLookup = context.state.metadata().getIndicesLookup();
+            int count = 0;
+            Set<DataStream> visitedDataStreams = new HashSet<>();
+            List<Index> indices = indexAbstraction.getIndices();
+            for (int i = 0, n = indices.size(); i < n; i++) {
+                Index index = indices.get(i);
+                DataStream parentDataStream = indicesLookup.get(index.getName()).getParentDataStream();
+                if (parentDataStream != null && visitedDataStreams.add(parentDataStream)) {
+                    if (shouldIncludeRegularIndices(context.getOptions(), expression.selector())) {
+                        count += parentDataStream.getIndices().size();
+                    }
+                    if (shouldIncludeFailureIndices(context.getOptions(), expression.selector())) {
+                        count += parentDataStream.getFailureIndices().getIndices().size();
+                    }
+                    if (count > 1) {
+                        // Early out if we already have more than one index accounted
+                        return true;
+                    }
+                }
+            }
+            return false;
+        }
         if (indexAbstraction.getType() == Type.DATA_STREAM) {
             DataStream dataStream = (DataStream) indexAbstraction;
             int count = 0;
-            if (shouldIncludeRegularIndices(context.getOptions())) {
+            if (shouldIncludeRegularIndices(context.getOptions(), expression.selector())) {
                 count += dataStream.getIndices().size();
             }
-            if (shouldIncludeFailureIndices(context.getOptions())) {
+            if (shouldIncludeFailureIndices(context.getOptions(), expression.selector())) {
                 count += dataStream.getFailureIndices().getIndices().size();
             }
             return count > 1;
@@ -618,21 +708,9 @@ private static boolean shouldTrackConcreteIndex(Context context, Index index) {
             // Exclude this one as it's a net-new system index, and we explicitly don't want those.
             return false;
         }
-        IndicesOptions options = context.getOptions();
-        if (DataStream.isFailureStoreFeatureFlagEnabled() && context.options.allowFailureIndices() == false) {
-            DataStream parentDataStream = context.getState().metadata().getIndicesLookup().get(index.getName()).getParentDataStream();
-            if (parentDataStream != null && parentDataStream.isFailureStoreEnabled()) {
-                if (parentDataStream.isFailureStoreIndex(index.getName())) {
-                    if (options.ignoreUnavailable()) {
-                        return false;
-                    } else {
-                        throw new FailureIndexNotSupportedException(index);
-                    }
-                }
-            }
-        }
         final IndexMetadata imd = context.state.metadata().index(index);
         if (imd.getState() == IndexMetadata.State.CLOSE) {
+            IndicesOptions options = context.getOptions();
             if (options.forbidClosedIndices() && options.ignoreUnavailable() == false) {
                 throw new IndexClosedException(index);
             } else {
@@ -766,6 +844,37 @@ public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state)
         return state.metadata().hasIndexAbstraction(resolvedAliasOrIndex);
     }
 
+    /**
+     * @return True if the provided expression contains the <code>::</code> character sequence.
+     */
+    public static boolean hasSelectorSuffix(String expression) {
+        if (expression == null) {
+            return false;
+        }
+        return expression.contains(SelectorResolver.SELECTOR_SEPARATOR);
+    }
+
+    /**
+     * @return If the specified string is a selector expression then this method returns the base expression and its selector part.
+     */
+    public static Tuple<String, String> splitSelectorExpression(String expression) {
+        return SelectorResolver.splitSelectorExpression(expression, Tuple::new);
+    }
+
+    public static String combineSelector(String baseExpression, @Nullable IndexComponentSelector selectorExpression) {
+        Objects.requireNonNull(baseExpression, "baseExpression is null");
+        return selectorExpression == null || IndexComponentSelector.DATA.equals(selectorExpression)
+            ? baseExpression
+            : (baseExpression + SelectorResolver.SELECTOR_SEPARATOR + selectorExpression.getKey());
+    }
+
+    public static String combineSelectorExpression(String baseExpression, @Nullable String selectorExpression) {
+        Objects.requireNonNull(baseExpression, "baseExpression is null");
+        return selectorExpression == null || IndexComponentSelector.DATA.getKey().equals(selectorExpression)
+            ? baseExpression
+            : (baseExpression + SelectorResolver.SELECTOR_SEPARATOR + selectorExpression);
+    }
+
     /**
      * Resolve an array of expressions to the set of indices and aliases that these expressions match.
      */
@@ -850,15 +959,28 @@ public String[] indexAliases(
             throw new IndexNotFoundException(index);
         }
 
-        if (skipIdentity == false && resolvedExpressions.contains(new ResolvedExpression(index))) {
-            return null;
+        if (skipIdentity == false) {
+            if (resolvedExpressions.contains(new ResolvedExpression(index))) {
+                return null;
+            }
+            for (IndexComponentSelector selector : IndexComponentSelector.values()) {
+                if (resolvedExpressions.contains(new ResolvedExpression(index, selector))) {
+                    return null;
+                }
+            }
         }
 
         IndexAbstraction ia = state.metadata().getIndicesLookup().get(index);
         DataStream dataStream = ia.getParentDataStream();
         if (dataStream != null) {
-            if (skipIdentity == false && resolvedExpressions.contains(new ResolvedExpression(dataStream.getName()))) {
-                // skip the filters when the request targets the data stream name
+            if (dataStream.getFailureIndices().containsIndex(index)) {
+                // Alias filters are not applied against indices in an abstraction's failure component.
+                // They do not match the mapping of the data stream nor are the documents mapped for searching.
+                return null;
+            }
+
+            if (skipIdentity == false && resolvedExpressionsContainsAbstraction(resolvedExpressions, dataStream.getName())) {
+                // skip the filters when the request targets the data stream name + selector directly
                 return null;
             }
             Map<String, DataStreamAlias> dataStreamAliases = state.metadata().dataStreamAliases();
@@ -866,11 +988,12 @@ public String[] indexAliases(
             if (iterateIndexAliases(dataStreamAliases.size(), resolvedExpressions.size())) {
                 aliasesForDataStream = dataStreamAliases.values()
                     .stream()
-                    .filter(dataStreamAlias -> resolvedExpressions.contains(new ResolvedExpression(dataStreamAlias.getName())))
+                    .filter(dataStreamAlias -> resolvedExpressionsContainsAbstraction(resolvedExpressions, dataStreamAlias.getName()))
                     .filter(dataStreamAlias -> dataStreamAlias.getDataStreams().contains(dataStream.getName()))
                     .toList();
             } else {
                 aliasesForDataStream = resolvedExpressions.stream()
+                    .filter(expression -> expression.selector() == null || expression.selector().shouldIncludeData())
                     .map(ResolvedExpression::resource)
                     .map(dataStreamAliases::get)
                     .filter(dataStreamAlias -> dataStreamAlias != null && dataStreamAlias.getDataStreams().contains(dataStream.getName()))
@@ -900,7 +1023,8 @@ public String[] indexAliases(
                 // faster to iterate indexAliases
                 aliasCandidates = indexAliases.values()
                     .stream()
-                    .filter(aliasMetadata -> resolvedExpressions.contains(new ResolvedExpression(aliasMetadata.alias())))
+                    // Indices can only be referenced with a data selector, or a null selector if selectors are disabled
+                    .filter(aliasMetadata -> resolvedExpressionsContainsAbstraction(resolvedExpressions, aliasMetadata.alias()))
                     .toArray(AliasMetadata[]::new);
             } else {
                 // faster to iterate resolvedExpressions
@@ -931,6 +1055,12 @@ public String[] indexAliases(
         }
     }
 
+    private static boolean resolvedExpressionsContainsAbstraction(Set<ResolvedExpression> resolvedExpressions, String abstractionName) {
+        return resolvedExpressions.contains(new ResolvedExpression(abstractionName))
+            || resolvedExpressions.contains(new ResolvedExpression(abstractionName, IndexComponentSelector.DATA))
+            || resolvedExpressions.contains(new ResolvedExpression(abstractionName, IndexComponentSelector.ALL_APPLICABLE));
+    }
+
     /**
      * Resolves the search routing if in the expression aliases are used. If expressions point to concrete indices
      * or aliases with no routing defined the specified routing is used.
@@ -961,8 +1091,28 @@ public Map<String, Set<String>> resolveSearchRouting(ClusterState state, @Nullab
         for (ResolvedExpression resolvedExpression : resolvedExpressions) {
             IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(resolvedExpression.resource());
             if (indexAbstraction != null && indexAbstraction.getType() == Type.ALIAS) {
-                for (int i = 0, n = indexAbstraction.getIndices().size(); i < n; i++) {
-                    Index index = indexAbstraction.getIndices().get(i);
+                // Determine which set of indices to resolve for the alias based on the selector
+                List<Index> aliasIndices = null;
+                if (context.getOptions().allowSelectors()) {
+                    IndexComponentSelector selector = resolvedExpression.selector();
+                    if (shouldIncludeRegularIndices(context.getOptions(), selector)) {
+                        aliasIndices = new ArrayList<>(indexAbstraction.getIndices().size());
+                        aliasIndices.addAll(indexAbstraction.getIndices());
+                    }
+                    if (shouldIncludeFailureIndices(context.getOptions(), selector) && indexAbstraction.isDataStreamRelated()) {
+                        Set<DataStream> dataStreams = getAliasDataStreams(indexAbstraction, context.state.metadata().getIndicesLookup());
+                        aliasIndices = aliasIndices == null ? new ArrayList<>(dataStreams.size()) : aliasIndices;
+                        for (DataStream dataStream : dataStreams) {
+                            aliasIndices.addAll(dataStream.getFailureIndices().getIndices());
+                        }
+                    }
+                    aliasIndices = aliasIndices == null ? List.of() : aliasIndices;
+                } else {
+                    aliasIndices = indexAbstraction.getIndices();
+                }
+
+                for (int i = 0, n = aliasIndices.size(); i < n; i++) {
+                    Index index = aliasIndices.get(i);
                     String concreteIndex = index.getName();
                     if (norouting.contains(concreteIndex) == false) {
                         AliasMetadata aliasMetadata = state.metadata().index(concreteIndex).getAliases().get(indexAbstraction.getName());
@@ -990,15 +1140,27 @@ public Map<String, Set<String>> resolveSearchRouting(ClusterState state, @Nullab
                 if (dataStream.isAllowCustomRouting() == false) {
                     continue;
                 }
-                if (dataStream.getIndices() != null) {
-                    for (int i = 0, n = dataStream.getIndices().size(); i < n; i++) {
-                        Index index = dataStream.getIndices().get(i);
-                        String concreteIndex = index.getName();
-                        routings = collectRoutings(routings, paramRouting, norouting, concreteIndex);
+                if (shouldIncludeRegularIndices(context.getOptions(), resolvedExpression.selector())) {
+                    if (dataStream.getIndices() != null) {
+                        for (int i = 0, n = dataStream.getIndices().size(); i < n; i++) {
+                            Index index = dataStream.getIndices().get(i);
+                            String concreteIndex = index.getName();
+                            routings = collectRoutings(routings, paramRouting, norouting, concreteIndex);
+                        }
+                    }
+                }
+                if (shouldIncludeFailureIndices(context.getOptions(), resolvedExpression.selector())) {
+                    if (dataStream.getFailureIndices().getIndices() != null) {
+                        for (Index failureIndex : dataStream.getFailureIndices().getIndices()) {
+                            String concreteIndex = failureIndex.getName();
+                            routings = collectRoutings(routings, paramRouting, norouting, concreteIndex);
+                        }
                     }
                 }
             } else {
                 // Index
+                assert resolvedExpression.selector() == null || IndexComponentSelector.DATA.equals(resolvedExpression.selector())
+                    : "Concrete index is being resolved with a selector other than [data] which is illegal";
                 routings = collectRoutings(routings, paramRouting, norouting, resolvedExpression.resource());
             }
 
@@ -1053,31 +1215,31 @@ public static Map<String, Set<String>> resolveSearchRoutingAllIndices(Metadata m
      * @return true if the provided array maps to all indices, false otherwise
      */
     public static boolean isAllIndicesExpression(Collection<ResolvedExpression> aliasesOrIndices) {
-        return aliasesOrIndices == null || aliasesOrIndices.isEmpty() || isExplicitAllPatternExpression(aliasesOrIndices);
+        return isAllIndices(aliasesOrIndices, ResolvedExpression::resource);
     }
 
     /**
-     * Identifies whether the array containing index names given as argument explicitly refers to all indices
-     * The empty or null array doesn't explicitly map to all indices
+     * Identifies whether the array containing index names given as argument refers to all indices
+     * The empty or null array identifies all indices
      *
      * @param aliasesOrIndices the array containing index names
-     * @return true if the provided array explicitly maps to all indices, false otherwise
+     * @return true if the provided array maps to all indices, false otherwise
      */
-    static boolean isExplicitAllPatternExpression(Collection<ResolvedExpression> aliasesOrIndices) {
-        return aliasesOrIndices != null
-            && aliasesOrIndices.size() == 1
-            && Metadata.ALL.equals(aliasesOrIndices.iterator().next().resource());
+    public static boolean isAllIndices(Collection<String> aliasesOrIndices) {
+        return isAllIndices(aliasesOrIndices, Function.identity());
     }
 
     /**
-     * Identifies whether the array containing index names given as argument refers to all indices
+     * Identifies whether the array containing objects with index names given as argument refers to all indices
      * The empty or null array identifies all indices
      *
      * @param aliasesOrIndices the array containing index names
+     * @param resourceGetter allows for obtaining the index name from a generic object that contains an index expression
      * @return true if the provided array maps to all indices, false otherwise
+     * @param <T> any object that can contain an index expression in some form or another
      */
-    public static boolean isAllIndices(Collection<String> aliasesOrIndices) {
-        return aliasesOrIndices == null || aliasesOrIndices.isEmpty() || isExplicitAllPattern(aliasesOrIndices);
+    public static <T> boolean isAllIndices(Collection<T> aliasesOrIndices, Function<T, String> resourceGetter) {
+        return aliasesOrIndices == null || aliasesOrIndices.isEmpty() || isExplicitAllPattern(aliasesOrIndices, resourceGetter);
     }
 
     /**
@@ -1088,7 +1250,21 @@ public static boolean isAllIndices(Collection<String> aliasesOrIndices) {
      * @return true if the provided array explicitly maps to all indices, false otherwise
      */
     static boolean isExplicitAllPattern(Collection<String> aliasesOrIndices) {
-        return aliasesOrIndices != null && aliasesOrIndices.size() == 1 && Metadata.ALL.equals(aliasesOrIndices.iterator().next());
+        return isExplicitAllPattern(aliasesOrIndices, Function.identity());
+    }
+
+    /**
+     * Identifies whether the array containing index names given as argument explicitly refers to all indices
+     * The empty or null array doesn't explicitly map to all indices
+     *
+     * @param aliasesOrIndices the array containing index names
+     * @param resourceGetter function used to lazily convert the collection type into a string for checking equality with the ALL pattern
+     * @return true if the provided array explicitly maps to all indices, false otherwise
+     */
+    static <T> boolean isExplicitAllPattern(Collection<T> aliasesOrIndices, Function<T, String> resourceGetter) {
+        return aliasesOrIndices != null
+            && aliasesOrIndices.size() == 1
+            && Metadata.ALL.equals(resourceGetter.apply(aliasesOrIndices.iterator().next()));
     }
 
     /**
@@ -1146,7 +1322,7 @@ public Predicate<String> getNetNewSystemIndexPredicate() {
      * exception.
      */
     @Nullable
-    private static boolean ensureAliasOrIndexExists(Context context, String name) {
+    private static boolean ensureAliasOrIndexExists(Context context, String name, IndexComponentSelector selector) {
         boolean ignoreUnavailable = context.getOptions().ignoreUnavailable();
         IndexAbstraction indexAbstraction = context.getState().getMetadata().getIndicesLookup().get(name);
         if (indexAbstraction == null) {
@@ -1174,6 +1350,21 @@ private static boolean ensureAliasOrIndexExists(Context context, String name) {
                 throw infe;
             }
         }
+        if (context.options.allowSelectors()) {
+            // Ensure that the selectors are present and that they are compatible with the abstractions they are used with
+            assert selector != null : "Earlier logic should have parsed selectors or added the default selectors already";
+            // Check if ::failures has been explicitly requested, since requesting ::* for non-data-stream abstractions would just
+            // return their data components.
+            if (IndexComponentSelector.FAILURES.equals(selector) && indexAbstraction.isDataStreamRelated() == false) {
+                // If requested abstraction is not data stream related, then you cannot use ::failures
+                if (ignoreUnavailable) {
+                    return false;
+                } else {
+                    // Return the expression with the selector on it since the selector is the part that is incorrect
+                    throw notFoundException(combineSelector(name, selector));
+                }
+            }
+        }
         return true;
     }
 
@@ -1364,8 +1555,10 @@ private WildcardExpressionResolver() {
          * Returns all the indices, data streams, and aliases, considering the open/closed, system, and hidden context parameters.
          * Depending on the context, returns the names of the data streams themselves or their backing indices.
          */
-        public static Collection<ResolvedExpression> resolveAll(Context context) {
-            List<ResolvedExpression> concreteIndices = resolveEmptyOrTrivialWildcard(context);
+        public static Collection<ResolvedExpression> resolveAll(Context context, IndexComponentSelector selector) {
+            assert selector != null || context.options.allowSelectors() == false
+                : "selectors are enabled in this context, but a selector was not provided";
+            List<ResolvedExpression> concreteIndices = resolveEmptyOrTrivialWildcard(context, selector);
 
             if (context.includeDataStreams() == false && context.getOptions().ignoreAliases()) {
                 return concreteIndices;
@@ -1380,7 +1573,7 @@ public static Collection<ResolvedExpression> resolveAll(Context context) {
                 .filter(ia -> context.getOptions().expandWildcardsHidden() || ia.isHidden() == false)
                 .filter(ia -> shouldIncludeIfDataStream(ia, context) || shouldIncludeIfAlias(ia, context))
                 .filter(ia -> ia.isSystem() == false || context.systemIndexAccessPredicate.test(ia.getName()))
-                .forEach(ia -> resolved.addAll(expandToOpenClosed(context, ia)));
+                .forEach(ia -> resolved.addAll(expandToOpenClosed(context, ia, selector)));
 
             resolved.addAll(concreteIndices);
             return resolved;
@@ -1416,27 +1609,31 @@ private static IndexMetadata.State excludeState(IndicesOptions options) {
          * The {@param context} provides the current time-snapshot view of cluster state, as well as conditions
          * on whether to consider alias, data stream, system, and hidden resources.
          */
-        static Set<ResolvedExpression> matchWildcardToResources(Context context, String wildcardExpression) {
+        static Set<ResolvedExpression> matchWildcardToResources(
+            Context context,
+            String wildcardExpression,
+            IndexComponentSelector selector
+        ) {
             assert isWildcard(wildcardExpression);
             final SortedMap<String, IndexAbstraction> indicesLookup = context.getState().getMetadata().getIndicesLookup();
             Set<ResolvedExpression> matchedResources = new HashSet<>();
             // this applies an initial pre-filtering in the case where the expression is a common suffix wildcard, eg "test*"
             if (Regex.isSuffixMatchPattern(wildcardExpression)) {
                 for (IndexAbstraction ia : filterIndicesLookupForSuffixWildcard(indicesLookup, wildcardExpression).values()) {
-                    maybeAddToResult(context, wildcardExpression, ia, matchedResources);
+                    maybeAddToResult(context, wildcardExpression, ia, selector, matchedResources);
                 }
                 return matchedResources;
             }
             // In case of match all it fetches all index abstractions
             if (Regex.isMatchAllPattern(wildcardExpression)) {
                 for (IndexAbstraction ia : indicesLookup.values()) {
-                    maybeAddToResult(context, wildcardExpression, ia, matchedResources);
+                    maybeAddToResult(context, wildcardExpression, ia, selector, matchedResources);
                 }
                 return matchedResources;
             }
             for (IndexAbstraction indexAbstraction : indicesLookup.values()) {
                 if (Regex.simpleMatch(wildcardExpression, indexAbstraction.getName())) {
-                    maybeAddToResult(context, wildcardExpression, indexAbstraction, matchedResources);
+                    maybeAddToResult(context, wildcardExpression, indexAbstraction, selector, matchedResources);
                 }
             }
             return matchedResources;
@@ -1446,10 +1643,11 @@ private static void maybeAddToResult(
             Context context,
             String wildcardExpression,
             IndexAbstraction indexAbstraction,
+            @Nullable IndexComponentSelector selector,
             Set<ResolvedExpression> matchedResources
         ) {
             if (shouldExpandToIndexAbstraction(context, wildcardExpression, indexAbstraction)) {
-                matchedResources.addAll(expandToOpenClosed(context, indexAbstraction));
+                matchedResources.addAll(expandToOpenClosed(context, indexAbstraction, selector));
             }
         }
 
@@ -1505,30 +1703,53 @@ private static Map<String, IndexAbstraction> filterIndicesLookupForSuffixWildcar
          * Data streams and aliases are interpreted to refer to multiple indices,
          * then all index resources are filtered by their open/closed status.
          */
-        private static Set<ResolvedExpression> expandToOpenClosed(Context context, IndexAbstraction indexAbstraction) {
+        private static Set<ResolvedExpression> expandToOpenClosed(
+            Context context,
+            IndexAbstraction indexAbstraction,
+            IndexComponentSelector selector
+        ) {
             final IndexMetadata.State excludeState = excludeState(context.getOptions());
             Set<ResolvedExpression> resources = new HashSet<>();
             if (context.isPreserveAliases() && indexAbstraction.getType() == Type.ALIAS) {
-                resources.add(new ResolvedExpression(indexAbstraction.getName()));
+                expandToApplicableSelectors(indexAbstraction, selector, resources);
             } else if (context.isPreserveDataStreams() && indexAbstraction.getType() == Type.DATA_STREAM) {
-                resources.add(new ResolvedExpression(indexAbstraction.getName()));
+                expandToApplicableSelectors(indexAbstraction, selector, resources);
             } else {
-                if (shouldIncludeRegularIndices(context.getOptions())) {
+                if (shouldIncludeRegularIndices(context.getOptions(), selector)) {
                     for (int i = 0, n = indexAbstraction.getIndices().size(); i < n; i++) {
                         Index index = indexAbstraction.getIndices().get(i);
                         IndexMetadata indexMetadata = context.state.metadata().index(index);
                         if (indexMetadata.getState() != excludeState) {
-                            resources.add(new ResolvedExpression(index.getName()));
+                            resources.add(
+                                new ResolvedExpression(
+                                    index.getName(),
+                                    context.options.allowSelectors() ? IndexComponentSelector.DATA : null
+                                )
+                            );
                         }
                     }
                 }
-                if (indexAbstraction.getType() == Type.DATA_STREAM && shouldIncludeFailureIndices(context.getOptions())) {
-                    DataStream dataStream = (DataStream) indexAbstraction;
-                    for (int i = 0, n = dataStream.getFailureIndices().getIndices().size(); i < n; i++) {
-                        Index index = dataStream.getFailureIndices().getIndices().get(i);
-                        IndexMetadata indexMetadata = context.state.metadata().index(index);
-                        if (indexMetadata.getState() != excludeState) {
-                            resources.add(new ResolvedExpression(index.getName()));
+                if (shouldIncludeFailureIndices(context.getOptions(), selector)) {
+                    if (indexAbstraction.getType() == Type.ALIAS && indexAbstraction.isDataStreamRelated()) {
+                        Set<DataStream> aliasDataStreams = getAliasDataStreams(
+                            indexAbstraction,
+                            context.state.metadata().getIndicesLookup()
+                        );
+                        for (DataStream ds : aliasDataStreams) {
+                            List<Index> failureIndices = ds.getFailureIndices().getIndices();
+                            for (int i = 0; i < failureIndices.size(); i++) {
+                                Index index = failureIndices.get(i);
+                                resources.add(new ResolvedExpression(index.getName(), IndexComponentSelector.DATA));
+                            }
+                        }
+                    } else if (indexAbstraction.getType() == Type.DATA_STREAM) {
+                        DataStream dataStream = (DataStream) indexAbstraction;
+                        for (int i = 0, n = dataStream.getFailureIndices().getIndices().size(); i < n; i++) {
+                            Index index = dataStream.getFailureIndices().getIndices().get(i);
+                            IndexMetadata indexMetadata = context.state.metadata().index(index);
+                            if (indexMetadata.getState() != excludeState) {
+                                resources.add(new ResolvedExpression(index.getName(), IndexComponentSelector.DATA));
+                            }
                         }
                     }
                 }
@@ -1536,27 +1757,61 @@ private static Set<ResolvedExpression> expandToOpenClosed(Context context, Index
             return resources;
         }
 
-        private static List<ResolvedExpression> resolveEmptyOrTrivialWildcard(Context context) {
-            final String[] allIndices = resolveEmptyOrTrivialWildcardToAllIndices(context.getOptions(), context.getState().metadata());
-            if (context.systemIndexAccessLevel == SystemIndexAccessLevel.ALL) {
-                List<ResolvedExpression> result = new ArrayList<>(allIndices.length);
-                for (int i = 0; i < allIndices.length; i++) {
-                    result.add(new ResolvedExpression(allIndices[i]));
+        /**
+         * Adds the abstraction and selector to the results when preserving data streams and aliases at wildcard resolution. If a selector
+         * is provided, the result is only added if the selector is applicable to the abstraction provided. If
+         * {@link IndexComponentSelector#ALL_APPLICABLE} is given, the selectors are expanded only to those which are applicable to the
+         * provided abstraction.
+         * @param indexAbstraction abstraction to add
+         * @param selector The selector to add
+         * @param resources Result collector which is updated with all applicable resolved expressions for a given abstraction and selector
+         *                  pair.
+         */
+        private static void expandToApplicableSelectors(
+            IndexAbstraction indexAbstraction,
+            IndexComponentSelector selector,
+            Set<ResolvedExpression> resources
+        ) {
+            if (IndexComponentSelector.ALL_APPLICABLE.equals(selector)) {
+                resources.add(new ResolvedExpression(indexAbstraction.getName(), IndexComponentSelector.DATA));
+                if (indexAbstraction.isDataStreamRelated()) {
+                    resources.add(new ResolvedExpression(indexAbstraction.getName(), IndexComponentSelector.FAILURES));
                 }
-                return result;
+            } else if (selector == null || indexAbstraction.isDataStreamRelated() || selector.shouldIncludeFailures() == false) {
+                resources.add(new ResolvedExpression(indexAbstraction.getName(), selector));
+            }
+        }
+
+        private static List<ResolvedExpression> resolveEmptyOrTrivialWildcard(Context context, IndexComponentSelector selector) {
+            final String[] allIndices = resolveEmptyOrTrivialWildcardToAllIndices(
+                context.getOptions(),
+                context.getState().metadata(),
+                selector
+            );
+            List<String> indices;
+            if (context.systemIndexAccessLevel == SystemIndexAccessLevel.ALL) {
+                indices = List.of(allIndices);
             } else {
-                return resolveEmptyOrTrivialWildcardWithAllowedSystemIndices(context, allIndices);
+                indices = resolveEmptyOrTrivialWildcardWithAllowedSystemIndices(context, allIndices);
+            }
+            List<ResolvedExpression> result = new ArrayList<>(indices.size());
+            boolean allowSelectors = context.options.allowSelectors();
+            for (int i = 0; i < indices.size(); i++) {
+                if (allowSelectors) {
+                    // These only have values if the ::data selector is applicable, and they only support the ::data selector
+                    result.add(new ResolvedExpression(indices.get(i), IndexComponentSelector.DATA));
+                } else {
+                    result.add(new ResolvedExpression(indices.get(i)));
+                }
             }
+            return result;
         }
 
-        private static List<ResolvedExpression> resolveEmptyOrTrivialWildcardWithAllowedSystemIndices(
-            Context context,
-            String[] allIndices
-        ) {
-            List<ResolvedExpression> filteredIndices = new ArrayList<>(allIndices.length);
+        private static List<String> resolveEmptyOrTrivialWildcardWithAllowedSystemIndices(Context context, String[] allIndices) {
+            List<String> filteredIndices = new ArrayList<>(allIndices.length);
             for (int i = 0; i < allIndices.length; i++) {
                 if (shouldIncludeIndexAbstraction(context, allIndices[i])) {
-                    filteredIndices.add(new ResolvedExpression(allIndices[i]));
+                    filteredIndices.add(allIndices[i]);
                 }
             }
             return filteredIndices;
@@ -1575,8 +1830,12 @@ private static boolean shouldIncludeIndexAbstraction(Context context, String nam
             return SystemResourceAccess.isSystemIndexAbstractionAccessible(context, abstraction);
         }
 
-        private static String[] resolveEmptyOrTrivialWildcardToAllIndices(IndicesOptions options, Metadata metadata) {
-            if (shouldIncludeRegularIndices(options) == false) {
+        private static String[] resolveEmptyOrTrivialWildcardToAllIndices(
+            IndicesOptions options,
+            Metadata metadata,
+            IndexComponentSelector selector
+        ) {
+            if (selector != null && selector.shouldIncludeData() == false) {
                 return Strings.EMPTY_ARRAY;
             }
             if (options.expandWildcardsOpen() && options.expandWildcardsClosed() && options.expandWildcardsHidden()) {
@@ -1803,6 +2062,175 @@ private static String doResolveExpression(String expression, LongSupplier getTim
         }
     }
 
+    public static final class SelectorResolver {
+        public static final String SELECTOR_SEPARATOR = "::";
+
+        private SelectorResolver() {
+            // Utility class
+        }
+
+        /**
+         * Parses an index expression for a selector suffix. If a suffix is present and supported by the index options, the
+         * expression and its suffix are split apart and returned. If a suffix is not present on the expression, the default
+         * {@link IndexComponentSelector#DATA} selector will be returned. If suffixes are present but not supported by the
+         * index options, this will throw {@link IndexNotFoundException}. When suffixes are not allowed by the context, the
+         * selector returned will be null.
+         * @param options IndicesOptions object
+         * @param expression The expression to check for selectors
+         * @return A resolved expression, optionally paired with a selector if present and supported.
+         */
+        public static ResolvedExpression parseExpression(String expression, IndicesOptions options) {
+            return parseAndTransformSelector(expression, (baseExpression, selector) -> {
+                if (options.allowSelectors()) {
+                    IndexComponentSelector resolvedSelector = selector != null ? selector : IndexComponentSelector.DATA;
+                    return new ResolvedExpression(baseExpression, resolvedSelector);
+                } else {
+                    // Ensure there is no selector if the API doesn't allow it.
+                    ensureNoSelectorsProvided(expression, selector);
+                    return new ResolvedExpression(baseExpression);
+                }
+            });
+        }
+
+        /**
+         * Parses an index expression for selector suffixes. If a suffix is present and supported by the index options, the suffix is
+         * returned. If a suffix is not present on the expression, the default selector defined in the context is returned. If selectors
+         * are disabled in the options, this will return a null value. If a selector is provided when selectors are disabled in the options,
+         * this will throw {@link IndexNotFoundException}.
+         * @param context Context object
+         * @param matchAllExpression The match all expression given to the index request (e.g. `*`, `*::failures`, `_all::data`)
+         * @return The selector for this match all expression
+         */
+        public static IndexComponentSelector parseMatchAllToSelector(Context context, String matchAllExpression) {
+            return parseAndTransformSelector(matchAllExpression, (baseExpression, selector) -> {
+                if (context.options.allowSelectors()) {
+                    // if selector was not present in the expression, default to ::data
+                    return selector == null ? IndexComponentSelector.DATA : selector;
+                } else {
+                    // Ensure there is no selector if the API doesn't allow it.
+                    ensureNoSelectorsProvided(matchAllExpression, selector);
+                    return null;
+                }
+            });
+        }
+
+        /**
+         * Determines if the given expression matches the provided predicate. If the predicate evaluates to false for the expression, the
+         * expression is checked for selectors and the evaluation is repeated on just the index-part. If selectors are present when they
+         * should not be (according to the context object) this method will throw an exception.
+         * @param expression Index expression that may contain a selector suffix.
+         * @param context Context object.
+         * @param predicate Determines match criteria. May be called multiple times if expression contains a valid selector.
+         * @return true if the expression matches the predicate, with or without its selector suffix if one is present.
+         * @throws InvalidIndexNameException if the expression contains invalid selector syntax.
+         * @throws IllegalArgumentException if selectors were present on the expression, but they are not allowed in this context.
+         */
+        private static boolean selectorsValidatedAndMatchesPredicate(String expression, Context context, Predicate<String> predicate) {
+            if (expression == null) {
+                return false;
+            }
+            // We need to check if there was a selector present, and validate if it was allowed
+            return parseAndTransformSelector(expression, (base, selectors) -> {
+                if (context.options.allowSelectors() == false) {
+                    // Ensure there is no selector if the API doesn't allow it.
+                    ensureNoSelectorsProvided(expression, selectors);
+                }
+                return predicate.test(base);
+            });
+        }
+
+        /**
+         * Splits off a selector suffix from the expression and converts it to the corresponding {@link IndexComponentSelector} value. A
+         * provided function binds the results to the return type.
+         * @param expression The expression to parse and split apart
+         * @param bindFunction A function that is handed the remainder of the expression and any selector that was parsed off
+         *                       the expression.
+         * @return The result of the bind function
+         * @param <V> The type returned from the binding function
+         * @throws InvalidIndexNameException In the event that the selector syntax is used incorrectly.
+         */
+        private static <V> V parseAndTransformSelector(String expression, BiFunction<String, IndexComponentSelector, V> bindFunction) {
+            return splitSelectorExpression(expression, (expressionBase, suffix) -> {
+                if (suffix == null) {
+                    return bindFunction.apply(expressionBase, null);
+                } else {
+                    return bindFunction.apply(expressionBase, IndexComponentSelector.getByKey(suffix));
+                }
+            });
+        }
+
+        /**
+         * Splits off selector fragments from an index expression. Selectors are always expected to be the suffix of the expression, and
+         * must equal one of the supported selector keys (data, failures) or be the match-all wildcard (*) which maps to all selectors. The
+         * expression is then split into the base expression and the selector string. A provided function binds the results to the return
+         * type.
+         * @param expression The expression to parse and split apart
+         * @param bindFunction A function that is handed the remainder of the expression and any selector suffix that was parsed off
+         *                       the expression.
+         * @return The result of the bind function
+         * @param <V> The type returned from the binding function
+         * @throws InvalidIndexNameException In the event that the selector syntax is used incorrectly.
+         */
+        private static <V> V splitSelectorExpression(String expression, BiFunction<String, String, V> bindFunction) {
+            Objects.requireNonNull(expression, "expression cannot be null");
+            int lastDoubleColon = expression.lastIndexOf(SELECTOR_SEPARATOR);
+            if (lastDoubleColon >= 0) {
+                String suffix = expression.substring(lastDoubleColon + SELECTOR_SEPARATOR.length());
+                IndexComponentSelector selector = IndexComponentSelector.getByKey(suffix);
+                if (selector == null) {
+                    // Do some work to surface a helpful error message for likely errors
+                    if (Regex.isSimpleMatchPattern(suffix)) {
+                        throw new InvalidIndexNameException(
+                            expression,
+                            "Invalid usage of :: separator, ["
+                                + suffix
+                                + "] contains a wildcard, but only the match all wildcard [*] is supported in a selector"
+                        );
+                    } else {
+                        throw new InvalidIndexNameException(
+                            expression,
+                            "Invalid usage of :: separator, [" + suffix + "] is not a recognized selector"
+                        );
+                    }
+                }
+                String expressionBase = expression.substring(0, lastDoubleColon);
+                ensureNoMoreSelectorSeparators(expressionBase, expression);
+                return bindFunction.apply(expressionBase, suffix);
+            }
+            // Otherwise accept the default
+            return bindFunction.apply(expression, null);
+        }
+
+        /**
+         * Checks the selectors that have been returned from splitting an expression and throws an exception if any were present.
+         * @param expression Original expression
+         * @param selector Selectors to validate
+         * @throws IllegalArgumentException if selectors are present
+         */
+        private static void ensureNoSelectorsProvided(String expression, IndexComponentSelector selector) {
+            if (selector != null) {
+                throw new IllegalArgumentException(
+                    "Index component selectors are not supported in this context but found selector in expression [" + expression + "]"
+                );
+            }
+        }
+
+        /**
+         * Checks the remainder of an expression for any more selector separators and throws an exception if they are encountered.
+         * @param remainingExpression Remaining expression
+         * @param originalExpression Original expression to be used in the exception if invalid name is detected
+         * @throws InvalidIndexNameException if there are any more :: separators
+         */
+        private static void ensureNoMoreSelectorSeparators(String remainingExpression, String originalExpression) {
+            if (remainingExpression.contains(SELECTOR_SEPARATOR)) {
+                throw new InvalidIndexNameException(
+                    originalExpression,
+                    "Invalid usage of :: separator, only one :: separator is allowed per expression"
+                );
+            }
+        }
+    }
+
     /**
      * In this class we collect the system access relevant code. The helper methods provide the following functionalities:
      * - determining the access to a system index abstraction
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java
index a4fa139043e50..6a4f24818003e 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java
@@ -12,7 +12,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.CollectionUtil;
-import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.Diffable;
@@ -28,6 +27,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
@@ -39,6 +39,7 @@
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.core.Nullable;
@@ -116,25 +117,25 @@ public enum XContentContext {
     /**
      * Indicates that this custom metadata will be returned as part of an API call but will not be persisted
      */
-    public static EnumSet<XContentContext> API_ONLY = EnumSet.of(XContentContext.API);
+    public static final EnumSet<XContentContext> API_ONLY = EnumSet.of(XContentContext.API);
 
     /**
      * Indicates that this custom metadata will be returned as part of an API call and will be persisted between
      * node restarts, but will not be a part of a snapshot global state
      */
-    public static EnumSet<XContentContext> API_AND_GATEWAY = EnumSet.of(XContentContext.API, XContentContext.GATEWAY);
+    public static final EnumSet<XContentContext> API_AND_GATEWAY = EnumSet.of(XContentContext.API, XContentContext.GATEWAY);
 
     /**
      * Indicates that this custom metadata will be returned as part of an API call and stored as a part of
      * a snapshot global state, but will not be persisted between node restarts
      */
-    public static EnumSet<XContentContext> API_AND_SNAPSHOT = EnumSet.of(XContentContext.API, XContentContext.SNAPSHOT);
+    public static final EnumSet<XContentContext> API_AND_SNAPSHOT = EnumSet.of(XContentContext.API, XContentContext.SNAPSHOT);
 
     /**
      * Indicates that this custom metadata will be returned as part of an API call, stored as a part of
      * a snapshot global state, and will be persisted between node restarts
      */
-    public static EnumSet<XContentContext> ALL_CONTEXTS = EnumSet.allOf(XContentContext.class);
+    public static final EnumSet<XContentContext> ALL_CONTEXTS = EnumSet.allOf(XContentContext.class);
 
     /**
      * Custom metadata that persists (via XContent) across restarts. The deserialization method for each implementation must be registered
@@ -519,7 +520,7 @@ public Metadata withLastCommittedValues(
     /**
      * Creates a copy of this instance updated with the given {@link IndexMetadata} that must only contain changes to primary terms
      * and in-sync allocation ids relative to the existing entries. This method is only used by
-     * {@link org.elasticsearch.cluster.routing.allocation.IndexMetadataUpdater#applyChanges(Metadata, RoutingTable, TransportVersion)}.
+     * {@link org.elasticsearch.cluster.routing.allocation.IndexMetadataUpdater#applyChanges(Metadata, RoutingTable)}.
      * @param updates map of index name to {@link IndexMetadata}.
      * @return updated metadata instance
      */
@@ -1501,44 +1502,49 @@ public static Metadata fromXContent(XContentParser parser) throws IOException {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, CONTEXT_MODE_API));
-
-        return ChunkedToXContent.builder(params)
-            .append(
-                context == XContentContext.API
-                    ? (b, p) -> b.startObject("metadata")
-                    : (b, p) -> b.startObject("meta-data").field("version", version())
-            )
-            .append((b, p) -> {
-                b.field("cluster_uuid", clusterUUID);
-                b.field("cluster_uuid_committed", clusterUUIDCommitted);
-                b.startObject("cluster_coordination");
-                coordinationMetadata().toXContent(b, p);
-                return b.endObject();
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params p) {
+        XContentContext context = XContentContext.valueOf(p.param(CONTEXT_MODE_PARAM, CONTEXT_MODE_API));
+        final Iterator<? extends ToXContent> start = context == XContentContext.API
+            ? ChunkedToXContentHelper.startObject("metadata")
+            : Iterators.single((builder, params) -> builder.startObject("meta-data").field("version", version()));
+
+        final Iterator<? extends ToXContent> persistentSettings = context != XContentContext.API && persistentSettings().isEmpty() == false
+            ? Iterators.single((builder, params) -> {
+                builder.startObject("settings");
+                persistentSettings().toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
+                return builder.endObject();
             })
-            .execute(xb -> {
-                if (context != XContentContext.API && persistentSettings().isEmpty() == false) {
-                    xb.append((b, p) -> {
-                        b.startObject("settings");
-                        persistentSettings().toXContent(b, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
-                        return b.endObject();
-                    });
-                }
-            })
-            .object("templates", templates().values().iterator(), t -> (b, p) -> IndexTemplateMetadata.Builder.toXContentWithTypes(t, b, p))
-            .execute(xb -> {
-                if (context == XContentContext.API) {
-                    xb.xContentObject("indices", indices().values().iterator());
-                }
-            })
-            .forEach(customs.entrySet().iterator(), (b, e) -> {
-                if (e.getValue().context().contains(context)) {
-                    b.xContentObject(e.getKey(), e.getValue());
-                }
-            })
-            .xContentObject("reserved_state", reservedStateMetadata().values().iterator())
-            .append((b, p) -> b.endObject());
+            : Collections.emptyIterator();
+
+        final Iterator<? extends ToXContent> indices = context == XContentContext.API
+            ? ChunkedToXContentHelper.object("indices", indices().values().iterator())
+            : Collections.emptyIterator();
+
+        return Iterators.concat(start, Iterators.single((builder, params) -> {
+            builder.field("cluster_uuid", clusterUUID);
+            builder.field("cluster_uuid_committed", clusterUUIDCommitted);
+            builder.startObject("cluster_coordination");
+            coordinationMetadata().toXContent(builder, params);
+            return builder.endObject();
+        }),
+            persistentSettings,
+            ChunkedToXContentHelper.object(
+                "templates",
+                Iterators.map(
+                    templates().values().iterator(),
+                    template -> (builder, params) -> IndexTemplateMetadata.Builder.toXContentWithTypes(template, builder, params)
+                )
+            ),
+            indices,
+            Iterators.flatMap(
+                customs.entrySet().iterator(),
+                entry -> entry.getValue().context().contains(context)
+                    ? ChunkedToXContentHelper.object(entry.getKey(), entry.getValue().toXContentChunked(p))
+                    : Collections.emptyIterator()
+            ),
+            ChunkedToXContentHelper.object("reserved_state", reservedStateMetadata().values().iterator()),
+            ChunkedToXContentHelper.endObject()
+        );
     }
 
     public Map<String, MappingMetadata> getMappingsByHash() {
@@ -2486,7 +2492,6 @@ private static boolean assertContainsIndexIfDataStream(DataStream parent, IndexM
             assert parent == null
                 || parent.getIndices().stream().anyMatch(index -> indexMetadata.getIndex().getName().equals(index.getName()))
                 || (DataStream.isFailureStoreFeatureFlagEnabled()
-                    && parent.isFailureStoreEnabled()
                     && parent.getFailureIndices()
                         .getIndices()
                         .stream()
@@ -2512,7 +2517,7 @@ private static void collectDataStreams(
                 for (Index i : dataStream.getIndices()) {
                     indexToDataStreamLookup.put(i.getName(), dataStream);
                 }
-                if (DataStream.isFailureStoreFeatureFlagEnabled() && dataStream.isFailureStoreEnabled()) {
+                if (DataStream.isFailureStoreFeatureFlagEnabled()) {
                     for (Index i : dataStream.getFailureIndices().getIndices()) {
                         indexToDataStreamLookup.put(i.getName(), dataStream);
                     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java
index 0de87c7226380..254646f8e71a9 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java
@@ -205,7 +205,8 @@ static ClusterState createDataStream(
      * @param request The create data stream request
      * @param backingIndices List of backing indices. May be empty
      * @param writeIndex Write index for the data stream. If null, a new write index will be created.
-     * @param initializeFailureStore Whether the failure store should be initialized
+     * @param initializeFailureStore Whether the failure store should be initialized (N.B. if true, failure store index creation will be
+     *     performed regardless of whether the template indicates that the failure store is enabled)
      * @return Cluster state containing the new data stream
      */
     static ClusterState createDataStream(
@@ -265,12 +266,13 @@ static ClusterState createDataStream(
             ? MetadataIndexTemplateService.resolveDataStreamOptions(template, systemDataStreamDescriptor.getComponentTemplates())
             : MetadataIndexTemplateService.resolveDataStreamOptions(template, metadata.componentTemplates());
         final DataStreamOptions dataStreamOptions = dataStreamOptionsTemplate.mapAndGet(DataStreamOptions.Template::toDataStreamOptions);
-        var isFailureStoreEnabled = dataStreamOptions != null && dataStreamOptions.isFailureStoreEnabled();
 
         // If we need to create a failure store, do so first. Do not reroute during the creation since we will do
-        // that as part of creating the backing index if required.
+        // that as part of creating the backing index if required. N.B. This is done if initializeFailureStore,
+        // regardless of whether the template indicates that the failure store is enabled: it is the caller's
+        // responsibility to check that before setting.
         IndexMetadata failureStoreIndex = null;
-        if (isFailureStoreEnabled && initializeFailureStore) {
+        if (initializeFailureStore) {
             if (isSystem) {
                 throw new IllegalArgumentException("Failure stores are not supported on system data streams");
             }
@@ -308,8 +310,7 @@ static ClusterState createDataStream(
         }
         assert writeIndex != null;
         assert writeIndex.mapping() != null : "no mapping found for backing index [" + writeIndex.getIndex().getName() + "]";
-        assert isFailureStoreEnabled == false || initializeFailureStore == false || failureStoreIndex != null
-            : "failure store should have an initial index";
+        assert initializeFailureStore == false || failureStoreIndex != null : "failure store should have an initial index";
         assert failureStoreIndex == null || failureStoreIndex.mapping() != null
             : "no mapping found for failure store [" + failureStoreIndex.getIndex().getName() + "]";
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
index a99822c9de00c..e24d921ba78fd 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
@@ -69,9 +69,7 @@
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
-import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.query.SearchExecutionContext;
-import org.elasticsearch.index.shard.IndexLongFieldRange;
 import org.elasticsearch.indices.IndexCreationException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.InvalidIndexNameException;
@@ -238,6 +236,9 @@ public SystemIndices getSystemIndices() {
      * Validate the name for an index or alias against some static rules.
      */
     public static void validateIndexOrAliasName(String index, BiFunction<String, String, ? extends RuntimeException> exceptionCtor) {
+        if (index == null || index.isEmpty()) {
+            throw exceptionCtor.apply(index, "must not be empty");
+        }
         if (Strings.validFileName(index) == false) {
             throw exceptionCtor.apply(index, "must not contain the following characters " + Strings.INVALID_FILENAME_CHARS);
         }
@@ -529,8 +530,7 @@ private ClusterState applyCreateIndexWithTemporaryService(
                     temporaryIndexMeta.getSettings(),
                     temporaryIndexMeta.getRoutingNumShards(),
                     sourceMetadata,
-                    temporaryIndexMeta.isSystem(),
-                    currentState.getMinTransportVersion()
+                    temporaryIndexMeta.isSystem()
                 );
             } catch (Exception e) {
                 logger.info("failed to build index metadata [{}]", request.index());
@@ -1340,15 +1340,10 @@ static IndexMetadata buildIndexMetadata(
         Settings indexSettings,
         int routingNumShards,
         @Nullable IndexMetadata sourceMetadata,
-        boolean isSystem,
-        TransportVersion minClusterTransportVersion
+        boolean isSystem
     ) {
         IndexMetadata.Builder indexMetadataBuilder = createIndexMetadataBuilder(indexName, sourceMetadata, indexSettings, routingNumShards);
         indexMetadataBuilder.system(isSystem);
-        if (minClusterTransportVersion.before(TransportVersions.V_8_15_0)) {
-            // promote to UNKNOWN for older versions since they don't know how to handle event.ingested in cluster state
-            indexMetadataBuilder.eventIngestedRange(IndexLongFieldRange.UNKNOWN, minClusterTransportVersion);
-        }
         // now, update the mappings with the actual source
         Map<String, MappingMetadata> mappingsMetadata = new HashMap<>();
         DocumentMapper docMapper = documentMapperSupplier.get();
@@ -1590,7 +1585,7 @@ static void validateCloneIndex(
 
     private static final Set<String> UNMODIFIABLE_SETTINGS_DURING_RESIZE = Set.of(
         IndexSettings.MODE.getKey(),
-        SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(),
+        IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(),
         IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(),
         IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(),
         IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(),
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java
index 0c33878b01229..f4318eb017eb3 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java
@@ -885,7 +885,7 @@ static Tuple<ClusterState, List<IndexResult>> closeRoutingTable(
                 final IndexMetadata.Builder updatedMetadata = IndexMetadata.builder(indexMetadata).state(IndexMetadata.State.CLOSE);
                 metadata.put(
                     updatedMetadata.timestampRange(IndexLongFieldRange.NO_SHARDS)
-                        .eventIngestedRange(IndexLongFieldRange.NO_SHARDS, currentState.getMinTransportVersion())
+                        .eventIngestedRange(IndexLongFieldRange.NO_SHARDS)
                         .settingsVersion(indexMetadata.getSettingsVersion() + 1)
                         .settings(Settings.builder().put(indexMetadata.getSettings()).put(VERIFIED_BEFORE_CLOSE_SETTING.getKey(), true))
                 );
@@ -1120,6 +1120,7 @@ private ClusterState openIndices(final Index[] indices, final ClusterState curre
             final Metadata.Builder metadata = Metadata.builder(currentState.metadata());
             final ClusterBlocks.Builder blocks = ClusterBlocks.builder(currentState.blocks());
             final IndexVersion minIndexCompatibilityVersion = currentState.getNodes().getMinSupportedIndexVersion();
+            final IndexVersion minReadOnlyIndexCompatibilityVersion = currentState.getNodes().getMinReadOnlySupportedIndexVersion();
 
             for (IndexMetadata indexMetadata : indicesToOpen) {
                 final Index index = indexMetadata.getIndex();
@@ -1132,12 +1133,16 @@ private ClusterState openIndices(final Index[] indices, final ClusterState curre
                         .settingsVersion(indexMetadata.getSettingsVersion() + 1)
                         .settings(updatedSettings)
                         .timestampRange(IndexLongFieldRange.NO_SHARDS)
-                        .eventIngestedRange(IndexLongFieldRange.NO_SHARDS, currentState.getMinTransportVersion())
+                        .eventIngestedRange(IndexLongFieldRange.NO_SHARDS)
                         .build();
 
                     // The index might be closed because we couldn't import it due to an old incompatible
                     // version, so we need to verify its compatibility.
-                    newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(newIndexMetadata, minIndexCompatibilityVersion);
+                    newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(
+                        newIndexMetadata,
+                        minIndexCompatibilityVersion,
+                        minReadOnlyIndexCompatibilityVersion
+                    );
                     try {
                         indicesService.verifyIndexMetadata(newIndexMetadata, newIndexMetadata);
                     } catch (Exception e) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
index 7f8b87d2d3f48..569526ca1d9ba 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
@@ -1784,7 +1784,7 @@ private static void validateCompositeTemplate(
                     .put(
                         IndexMetadata.builder(temporaryIndexName)
                             // necessary to pass asserts in ClusterState constructor
-                            .eventIngestedRange(IndexLongFieldRange.UNKNOWN, state.getMinTransportVersion())
+                            .eventIngestedRange(IndexLongFieldRange.UNKNOWN)
                             .settings(finalResolvedSettings)
                     )
                     .build()
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java
index 272f321e386fc..50e0de92f75b4 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.cluster.SimpleDiffable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -191,7 +191,7 @@ public int hashCode() {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).xContentObjectFields(NODES_FIELD.getPreferredName(), nodes);
+        return ChunkedToXContentHelper.xContentObjectFields(NODES_FIELD.getPreferredName(), nodes);
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java
index a224b3ae82312..72448b56e3a07 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java
@@ -27,8 +27,8 @@
 import java.util.Iterator;
 import java.util.Objects;
 
+import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.chunk;
 import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.endObject;
-import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.singleChunk;
 import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.startObject;
 
 public class ShutdownShardMigrationStatus implements Writeable, ChunkedToXContentObject {
@@ -168,7 +168,7 @@ public ShardAllocationDecision getAllocationDecision() {
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         return Iterators.concat(
             startObject(),
-            singleChunk((builder, p) -> buildHeader(builder)),
+            chunk((builder, p) -> buildHeader(builder)),
             Objects.nonNull(allocationDecision)
                 ? Iterators.concat(startObject(NODE_ALLOCATION_DECISION_KEY), allocationDecision.toXContentChunked(params), endObject())
                 : Collections.emptyIterator(),
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java
index aa8b092ffcca0..ebd7a7db13f82 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java
@@ -28,6 +28,7 @@
 import java.util.Locale;
 import java.util.Objects;
 
+import static org.elasticsearch.TransportVersions.NODE_SHUTDOWN_EPHEMERAL_ID_ADDED;
 import static org.elasticsearch.core.Strings.format;
 
 /**
@@ -40,6 +41,7 @@ public class SingleNodeShutdownMetadata implements SimpleDiffable<SingleNodeShut
     public static final TransportVersion GRACE_PERIOD_ADDED_VERSION = TransportVersions.V_8_9_X;
 
     public static final ParseField NODE_ID_FIELD = new ParseField("node_id");
+    public static final ParseField NODE_EPHEMERAL_ID_FIELD = new ParseField("node_ephemeral_id");
     public static final ParseField TYPE_FIELD = new ParseField("type");
     public static final ParseField REASON_FIELD = new ParseField("reason");
     public static final String STARTED_AT_READABLE_FIELD = "shutdown_started";
@@ -53,18 +55,25 @@ public class SingleNodeShutdownMetadata implements SimpleDiffable<SingleNodeShut
         "node_shutdown_info",
         a -> new SingleNodeShutdownMetadata(
             (String) a[0],
-            Type.valueOf((String) a[1]),
-            (String) a[2],
-            (long) a[3],
-            (boolean) a[4],
-            (TimeValue) a[5],
-            (String) a[6],
-            (TimeValue) a[7]
+            (String) a[1],
+            Type.valueOf((String) a[2]),
+            (String) a[3],
+            (long) a[4],
+            (boolean) a[5],
+            (TimeValue) a[6],
+            (String) a[7],
+            (TimeValue) a[8]
         )
     );
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), NODE_ID_FIELD);
+        PARSER.declareField(
+            ConstructingObjectParser.optionalConstructorArg(),
+            (p, c) -> p.textOrNull(),
+            NODE_EPHEMERAL_ID_FIELD,
+            ObjectParser.ValueType.STRING_OR_NULL
+        );
         PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD);
         PARSER.declareString(ConstructingObjectParser.constructorArg(), REASON_FIELD);
         PARSER.declareLong(ConstructingObjectParser.constructorArg(), STARTED_AT_MILLIS_FIELD);
@@ -91,6 +100,8 @@ public static SingleNodeShutdownMetadata parse(XContentParser parser) {
     public static final TimeValue DEFAULT_RESTART_SHARD_ALLOCATION_DELAY = TimeValue.timeValueMinutes(5);
 
     private final String nodeId;
+    @Nullable
+    private final String nodeEphemeralId;
     private final Type type;
     private final String reason;
     private final long startedAtMillis;
@@ -110,6 +121,7 @@ public static SingleNodeShutdownMetadata parse(XContentParser parser) {
      */
     private SingleNodeShutdownMetadata(
         String nodeId,
+        @Nullable String nodeEphemeralId,
         Type type,
         String reason,
         long startedAtMillis,
@@ -119,6 +131,7 @@ private SingleNodeShutdownMetadata(
         @Nullable TimeValue gracePeriod
     ) {
         this.nodeId = Objects.requireNonNull(nodeId, "node ID must not be null");
+        this.nodeEphemeralId = nodeEphemeralId;
         this.type = Objects.requireNonNull(type, "shutdown type must not be null");
         this.reason = Objects.requireNonNull(reason, "shutdown reason must not be null");
         this.startedAtMillis = startedAtMillis;
@@ -157,6 +170,11 @@ private SingleNodeShutdownMetadata(
 
     public SingleNodeShutdownMetadata(StreamInput in) throws IOException {
         this.nodeId = in.readString();
+        if (in.getTransportVersion().onOrAfter(NODE_SHUTDOWN_EPHEMERAL_ID_ADDED)) {
+            this.nodeEphemeralId = in.readOptionalString();
+        } else {
+            this.nodeEphemeralId = null; // empty when talking to old nodes, meaning the persistent node id is the only differentiator
+        }
         this.type = in.readEnum(Type.class);
         this.reason = in.readString();
         this.startedAtMillis = in.readVLong();
@@ -181,6 +199,15 @@ public String getNodeId() {
         return nodeId;
     }
 
+    /**
+     * @return The ephemeral ID of the node this {@link SingleNodeShutdownMetadata} concerns, or
+     *  {@code null} if the ephemeral id is unknown.
+     */
+    @Nullable
+    public String getNodeEphemeralId() {
+        return nodeEphemeralId;
+    }
+
     /**
      * @return The type of shutdown this is (shutdown vs. permanent).
      */
@@ -241,6 +268,9 @@ public TimeValue getGracePeriod() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(nodeId);
+        if (out.getTransportVersion().onOrAfter(NODE_SHUTDOWN_EPHEMERAL_ID_ADDED)) {
+            out.writeOptionalString(nodeEphemeralId);
+        }
         if ((out.getTransportVersion().before(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION) && this.type == SingleNodeShutdownMetadata.Type.REPLACE)
             || (out.getTransportVersion().before(SIGTERM_ADDED_VERSION) && this.type == Type.SIGTERM)) {
             out.writeEnum(SingleNodeShutdownMetadata.Type.REMOVE);
@@ -264,6 +294,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         {
             builder.field(NODE_ID_FIELD.getPreferredName(), nodeId);
+            builder.field(NODE_EPHEMERAL_ID_FIELD.getPreferredName(), nodeEphemeralId);
             builder.field(TYPE_FIELD.getPreferredName(), type);
             builder.field(REASON_FIELD.getPreferredName(), reason);
             builder.timestampFieldsFromUnixEpochMillis(
@@ -295,6 +326,7 @@ public boolean equals(Object o) {
         return getStartedAtMillis() == that.getStartedAtMillis()
             && getNodeSeen() == that.getNodeSeen()
             && getNodeId().equals(that.getNodeId())
+            && Objects.equals(getNodeEphemeralId(), that.getNodeEphemeralId())
             && getType() == that.getType()
             && getReason().equals(that.getReason())
             && Objects.equals(getAllocationDelay(), that.getAllocationDelay())
@@ -306,6 +338,7 @@ && getReason().equals(that.getReason())
     public int hashCode() {
         return Objects.hash(
             getNodeId(),
+            getNodeEphemeralId(),
             getType(),
             getReason(),
             getStartedAtMillis(),
@@ -322,6 +355,8 @@ public String toString() {
         stringBuilder.append("{")
             .append("nodeId=[")
             .append(nodeId)
+            .append("], nodeEphemeralId=[")
+            .append(nodeEphemeralId)
             .append(']')
             .append(", type=[")
             .append(type)
@@ -350,6 +385,7 @@ public static Builder builder(SingleNodeShutdownMetadata original) {
             return builder();
         }
         return new Builder().setNodeId(original.getNodeId())
+            .setNodeEphemeralId(original.getNodeEphemeralId())
             .setType(original.getType())
             .setReason(original.getReason())
             .setStartedAtMillis(original.getStartedAtMillis())
@@ -359,6 +395,7 @@ public static Builder builder(SingleNodeShutdownMetadata original) {
 
     public static class Builder {
         private String nodeId;
+        private String nodeEphemeralId;
         private Type type;
         private String reason;
         private long startedAtMillis = -1;
@@ -378,6 +415,15 @@ public Builder setNodeId(String nodeId) {
             return this;
         }
 
+        /**
+         * @param nodeEphemeralId The node ephemeral ID this metadata refers to.
+         * @return This builder.
+         */
+        public Builder setNodeEphemeralId(String nodeEphemeralId) {
+            this.nodeEphemeralId = nodeEphemeralId;
+            return this;
+        }
+
         /**
          * @param type The type of shutdown.
          * @return This builder.
@@ -444,6 +490,7 @@ public SingleNodeShutdownMetadata build() {
 
             return new SingleNodeShutdownMetadata(
                 nodeId,
+                nodeEphemeralId,
                 type,
                 reason,
                 startedAtMillis,
diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
index 7bf367f99b929..46a83495ecfb9 100644
--- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
+++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
@@ -37,6 +37,7 @@
 import java.util.SortedSet;
 import java.util.TreeSet;
 
+import static org.elasticsearch.TransportVersions.NODE_VERSION_INFORMATION_WITH_MIN_READ_ONLY_INDEX_VERSION;
 import static org.elasticsearch.node.NodeRoleSettings.NODE_ROLES_SETTING;
 
 /**
@@ -325,7 +326,17 @@ public DiscoveryNode(StreamInput in) throws IOException {
             }
         }
         this.roles = Collections.unmodifiableSortedSet(roles);
-        versionInfo = new VersionInformation(Version.readVersion(in), IndexVersion.readVersion(in), IndexVersion.readVersion(in));
+        Version version = Version.readVersion(in);
+        IndexVersion minIndexVersion = IndexVersion.readVersion(in);
+        IndexVersion minReadOnlyIndexVersion;
+        if (in.getTransportVersion().onOrAfter(NODE_VERSION_INFORMATION_WITH_MIN_READ_ONLY_INDEX_VERSION)) {
+            minReadOnlyIndexVersion = IndexVersion.readVersion(in);
+        } else {
+            minReadOnlyIndexVersion = minIndexVersion;
+
+        }
+        IndexVersion maxIndexVersion = IndexVersion.readVersion(in);
+        versionInfo = new VersionInformation(version, minIndexVersion, minReadOnlyIndexVersion, maxIndexVersion);
         if (in.getTransportVersion().onOrAfter(EXTERNAL_ID_VERSION)) {
             this.externalId = readStringLiteral.read(in);
         } else {
@@ -360,6 +371,9 @@ public void writeTo(StreamOutput out) throws IOException {
         });
         Version.writeVersion(versionInfo.nodeVersion(), out);
         IndexVersion.writeVersion(versionInfo.minIndexVersion(), out);
+        if (out.getTransportVersion().onOrAfter(NODE_VERSION_INFORMATION_WITH_MIN_READ_ONLY_INDEX_VERSION)) {
+            IndexVersion.writeVersion(versionInfo.minReadOnlyIndexVersion(), out);
+        }
         IndexVersion.writeVersion(versionInfo.maxIndexVersion(), out);
         if (out.getTransportVersion().onOrAfter(EXTERNAL_ID_VERSION)) {
             out.writeString(externalId);
@@ -478,6 +492,10 @@ public IndexVersion getMinIndexVersion() {
         return versionInfo.minIndexVersion();
     }
 
+    public IndexVersion getMinReadOnlyIndexVersion() {
+        return versionInfo.minReadOnlyIndexVersion();
+    }
+
     public IndexVersion getMaxIndexVersion() {
         return versionInfo.maxIndexVersion();
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java
index 19f920fbe3c8a..e28786d8c8818 100644
--- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java
+++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java
@@ -153,7 +153,7 @@ public boolean isEnabledByDefault(Settings settings) {
     /**
      * Represents the role for a content node.
      */
-    public static DiscoveryNodeRole DATA_CONTENT_NODE_ROLE = new DiscoveryNodeRole("data_content", "s", true) {
+    public static final DiscoveryNodeRole DATA_CONTENT_NODE_ROLE = new DiscoveryNodeRole("data_content", "s", true) {
 
         @Override
         public boolean isEnabledByDefault(final Settings settings) {
@@ -164,7 +164,7 @@ public boolean isEnabledByDefault(final Settings settings) {
     /**
      * Represents the role for a hot node.
      */
-    public static DiscoveryNodeRole DATA_HOT_NODE_ROLE = new DiscoveryNodeRole("data_hot", "h", true) {
+    public static final DiscoveryNodeRole DATA_HOT_NODE_ROLE = new DiscoveryNodeRole("data_hot", "h", true) {
 
         @Override
         public boolean isEnabledByDefault(final Settings settings) {
@@ -175,7 +175,7 @@ public boolean isEnabledByDefault(final Settings settings) {
     /**
      * Represents the role for a warm node.
      */
-    public static DiscoveryNodeRole DATA_WARM_NODE_ROLE = new DiscoveryNodeRole("data_warm", "w", true) {
+    public static final DiscoveryNodeRole DATA_WARM_NODE_ROLE = new DiscoveryNodeRole("data_warm", "w", true) {
 
         @Override
         public boolean isEnabledByDefault(final Settings settings) {
@@ -186,7 +186,7 @@ public boolean isEnabledByDefault(final Settings settings) {
     /**
      * Represents the role for a cold node.
      */
-    public static DiscoveryNodeRole DATA_COLD_NODE_ROLE = new DiscoveryNodeRole("data_cold", "c", true) {
+    public static final DiscoveryNodeRole DATA_COLD_NODE_ROLE = new DiscoveryNodeRole("data_cold", "c", true) {
 
         @Override
         public boolean isEnabledByDefault(final Settings settings) {
@@ -197,7 +197,7 @@ public boolean isEnabledByDefault(final Settings settings) {
     /**
      * Represents the role for a frozen node.
      */
-    public static DiscoveryNodeRole DATA_FROZEN_NODE_ROLE = new DiscoveryNodeRole("data_frozen", "f", true) {
+    public static final DiscoveryNodeRole DATA_FROZEN_NODE_ROLE = new DiscoveryNodeRole("data_frozen", "f", true) {
 
         @Override
         public boolean isEnabledByDefault(final Settings settings) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
index 5e6dec7b68062..f733ab223fdd1 100644
--- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
+++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
@@ -69,6 +69,7 @@ public class DiscoveryNodes implements Iterable<DiscoveryNode>, SimpleDiffable<D
     private final Version minNodeVersion;
     private final IndexVersion maxDataNodeCompatibleIndexVersion;
     private final IndexVersion minSupportedIndexVersion;
+    private final IndexVersion minReadOnlySupportedIndexVersion;
 
     private final Map<String, Set<String>> tiersToNodeIds;
 
@@ -84,6 +85,7 @@ private DiscoveryNodes(
         Version minNodeVersion,
         IndexVersion maxDataNodeCompatibleIndexVersion,
         IndexVersion minSupportedIndexVersion,
+        IndexVersion minReadOnlySupportedIndexVersion,
         Map<String, Set<String>> tiersToNodeIds
     ) {
         this.nodeLeftGeneration = nodeLeftGeneration;
@@ -100,6 +102,8 @@ private DiscoveryNodes(
         this.maxNodeVersion = maxNodeVersion;
         this.maxDataNodeCompatibleIndexVersion = maxDataNodeCompatibleIndexVersion;
         this.minSupportedIndexVersion = minSupportedIndexVersion;
+        this.minReadOnlySupportedIndexVersion = minReadOnlySupportedIndexVersion;
+        assert minReadOnlySupportedIndexVersion.onOrBefore(minSupportedIndexVersion);
         assert (localNodeId == null) == (localNode == null);
         this.tiersToNodeIds = tiersToNodeIds;
     }
@@ -118,6 +122,7 @@ public DiscoveryNodes withMasterNodeId(@Nullable String masterNodeId) {
             minNodeVersion,
             maxDataNodeCompatibleIndexVersion,
             minSupportedIndexVersion,
+            minReadOnlySupportedIndexVersion,
             tiersToNodeIds
         );
     }
@@ -374,6 +379,13 @@ public IndexVersion getMinSupportedIndexVersion() {
         return minSupportedIndexVersion;
     }
 
+    /**
+     * Returns the minimum index version for read-only indices supported by all nodes in the cluster
+     */
+    public IndexVersion getMinReadOnlySupportedIndexVersion() {
+        return minReadOnlySupportedIndexVersion;
+    }
+
     /**
      * Return the node-left generation, which is the number of times the cluster membership has been updated by removing one or more nodes.
      * <p>
@@ -840,6 +852,7 @@ public DiscoveryNodes build() {
             Version maxNodeVersion = null;
             IndexVersion maxDataNodeCompatibleIndexVersion = null;
             IndexVersion minSupportedIndexVersion = null;
+            IndexVersion minReadOnlySupportedIndexVersion = null;
             for (Map.Entry<String, DiscoveryNode> nodeEntry : nodes.entrySet()) {
                 DiscoveryNode discoNode = nodeEntry.getValue();
                 Version version = discoNode.getVersion();
@@ -849,6 +862,7 @@ public DiscoveryNodes build() {
                 minNodeVersion = min(minNodeVersion, version);
                 maxNodeVersion = max(maxNodeVersion, version);
                 minSupportedIndexVersion = max(minSupportedIndexVersion, discoNode.getMinIndexVersion());
+                minReadOnlySupportedIndexVersion = max(minReadOnlySupportedIndexVersion, discoNode.getMinReadOnlyIndexVersion());
             }
 
             final long newNodeLeftGeneration;
@@ -881,6 +895,7 @@ public DiscoveryNodes build() {
                 Objects.requireNonNullElse(minNodeVersion, Version.CURRENT.minimumCompatibilityVersion()),
                 Objects.requireNonNullElse(maxDataNodeCompatibleIndexVersion, IndexVersion.current()),
                 Objects.requireNonNullElse(minSupportedIndexVersion, IndexVersions.MINIMUM_COMPATIBLE),
+                Objects.requireNonNullElse(minReadOnlySupportedIndexVersion, IndexVersions.MINIMUM_READONLY_COMPATIBLE),
                 computeTiersToNodesMap(dataNodes)
             );
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/node/VersionInformation.java b/server/src/main/java/org/elasticsearch/cluster/node/VersionInformation.java
index a4d0ff1eb55e4..852f31db69c92 100644
--- a/server/src/main/java/org/elasticsearch/cluster/node/VersionInformation.java
+++ b/server/src/main/java/org/elasticsearch/cluster/node/VersionInformation.java
@@ -18,20 +18,23 @@
 
 /**
  * Represents the versions of various aspects of an Elasticsearch node.
- * @param buildVersion      The node {@link BuildVersion}
- * @param minIndexVersion   The minimum {@link IndexVersion} supported by this node
- * @param maxIndexVersion   The maximum {@link IndexVersion} supported by this node
+ * @param buildVersion              The node {@link BuildVersion}
+ * @param minIndexVersion           The minimum {@link IndexVersion} supported by this node
+ * @param minReadOnlyIndexVersion   The minimum {@link IndexVersion} for read-only indices supported by this node
+ * @param maxIndexVersion           The maximum {@link IndexVersion} supported by this node
  */
 public record VersionInformation(
     BuildVersion buildVersion,
     Version nodeVersion,
     IndexVersion minIndexVersion,
+    IndexVersion minReadOnlyIndexVersion,
     IndexVersion maxIndexVersion
 ) {
 
     public static final VersionInformation CURRENT = new VersionInformation(
         BuildVersion.current(),
         IndexVersions.MINIMUM_COMPATIBLE,
+        IndexVersions.MINIMUM_READONLY_COMPATIBLE,
         IndexVersion.current()
     );
 
@@ -39,11 +42,18 @@ public record VersionInformation(
         Objects.requireNonNull(buildVersion);
         Objects.requireNonNull(nodeVersion);
         Objects.requireNonNull(minIndexVersion);
+        Objects.requireNonNull(minReadOnlyIndexVersion);
         Objects.requireNonNull(maxIndexVersion);
+        assert minReadOnlyIndexVersion.onOrBefore(minIndexVersion) : minReadOnlyIndexVersion + " > " + minIndexVersion;
     }
 
-    public VersionInformation(BuildVersion version, IndexVersion minIndexVersion, IndexVersion maxIndexVersion) {
-        this(version, Version.CURRENT, minIndexVersion, maxIndexVersion);
+    public VersionInformation(
+        BuildVersion version,
+        IndexVersion minIndexVersion,
+        IndexVersion minReadOnlyIndexVersion,
+        IndexVersion maxIndexVersion
+    ) {
+        this(version, Version.CURRENT, minIndexVersion, minReadOnlyIndexVersion, maxIndexVersion);
         /*
          * Whilst DiscoveryNode.getVersion exists, we need to be able to get a Version from VersionInfo
          * This needs to be consistent - on serverless, BuildVersion has an id of -1, which translates
@@ -57,7 +67,17 @@ public VersionInformation(BuildVersion version, IndexVersion minIndexVersion, In
 
     @Deprecated
     public VersionInformation(Version version, IndexVersion minIndexVersion, IndexVersion maxIndexVersion) {
-        this(BuildVersion.fromVersionId(version.id()), version, minIndexVersion, maxIndexVersion);
+        this(version, minIndexVersion, minIndexVersion, maxIndexVersion);
+    }
+
+    @Deprecated
+    public VersionInformation(
+        Version version,
+        IndexVersion minIndexVersion,
+        IndexVersion minReadOnlyIndexVersion,
+        IndexVersion maxIndexVersion
+    ) {
+        this(BuildVersion.fromVersionId(version.id()), version, minIndexVersion, minReadOnlyIndexVersion, maxIndexVersion);
     }
 
     @Deprecated
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java
index f42252df4ab7b..24b14a46c8782 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java
@@ -40,6 +40,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.OptionalInt;
 import java.util.Set;
 import java.util.function.IntConsumer;
 import java.util.function.IntSupplier;
@@ -53,8 +54,7 @@
  */
 public abstract class IndexRouting {
 
-    static final NodeFeature BOOLEAN_ROUTING_PATH = new NodeFeature("routing.boolean_routing_path");
-    static final NodeFeature MULTI_VALUE_ROUTING_PATH = new NodeFeature("routing.multi_value_routing_path");
+    static final NodeFeature LOGSB_ROUTE_ON_SORT_FIELDS = new NodeFeature("routing.logsb_route_on_sort_fields");
 
     /**
      * Build the routing from {@link IndexMetadata}.
@@ -165,7 +165,8 @@ private abstract static class IdAndRoutingOnly extends IndexRouting {
 
         @Override
         public void preProcess(IndexRequest indexRequest) {
-            // generate id if not already provided
+            // Generate id if not already provided.
+            // This is needed for routing, so it has to happen in pre-processing.
             final String id = indexRequest.id();
             if (id == null) {
                 if (shouldUseTimeBasedId(indexMode, creationVersion)) {
@@ -272,7 +273,9 @@ public void collectSearchShards(String routing, IntConsumer consumer) {
     public static class ExtractFromSource extends IndexRouting {
         private final Predicate<String> isRoutingPath;
         private final XContentParserConfiguration parserConfig;
+        private final IndexMode indexMode;
         private final boolean trackTimeSeriesRoutingHash;
+        private final boolean addIdWithRoutingHash;
         private int hash = Integer.MAX_VALUE;
 
         ExtractFromSource(IndexMetadata metadata) {
@@ -280,7 +283,10 @@ public static class ExtractFromSource extends IndexRouting {
             if (metadata.isRoutingPartitionedIndex()) {
                 throw new IllegalArgumentException("routing_partition_size is incompatible with routing_path");
             }
-            trackTimeSeriesRoutingHash = metadata.getCreationVersion().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID);
+            indexMode = metadata.getIndexMode();
+            trackTimeSeriesRoutingHash = indexMode == IndexMode.TIME_SERIES
+                && metadata.getCreationVersion().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID);
+            addIdWithRoutingHash = indexMode == IndexMode.LOGSDB;
             List<String> routingPaths = metadata.getRoutingPaths();
             isRoutingPath = Regex.simpleMatcher(routingPaths.toArray(String[]::new));
             this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(null, Set.copyOf(routingPaths), null, true);
@@ -292,8 +298,13 @@ public boolean matchesField(String fieldName) {
 
         @Override
         public void postProcess(IndexRequest indexRequest) {
+            // Update the request with the routing hash, if needed.
+            // This needs to happen in post-processing, after the routing hash is calculated.
             if (trackTimeSeriesRoutingHash) {
                 indexRequest.routing(TimeSeriesRoutingHashFieldMapper.encode(hash));
+            } else if (addIdWithRoutingHash) {
+                assert hash != Integer.MAX_VALUE;
+                indexRequest.autoGenerateTimeBasedId(OptionalInt.of(hash));
             }
         }
 
@@ -461,12 +472,15 @@ private int idToHash(String id) {
             try {
                 idBytes = Base64.getUrlDecoder().decode(id);
             } catch (IllegalArgumentException e) {
-                throw new ResourceNotFoundException("invalid id [{}] for index [{}] in time series mode", id, indexName);
+                throw new ResourceNotFoundException("invalid id [{}] for index [{}] in " + indexMode.getName() + " mode", id, indexName);
             }
             if (idBytes.length < 4) {
-                throw new ResourceNotFoundException("invalid id [{}] for index [{}] in time series mode", id, indexName);
+                throw new ResourceNotFoundException("invalid id [{}] for index [{}] in " + indexMode.getName() + " mode", id, indexName);
             }
-            return hashToShardId(ByteUtils.readIntLE(idBytes, 0));
+            // For TSDB, the hash is stored as the id prefix.
+            // For LogsDB with routing on sort fields, the routing hash is stored in the range[id.length - 9, id.length - 5] of the id,
+            // see IndexRequest#autoGenerateTimeBasedId.
+            return hashToShardId(ByteUtils.readIntLE(idBytes, addIdWithRoutingHash ? idBytes.length - 9 : 0));
         }
 
         @Override
@@ -480,7 +494,7 @@ public void collectSearchShards(String routing, IntConsumer consumer) {
         }
 
         private String error(String operation) {
-            return operation + " is not supported because the destination index [" + indexName + "] is in time series mode";
+            return operation + " is not supported because the destination index [" + indexName + "] is in " + indexMode.getName() + " mode";
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
index 5e2dbf1c5df5d..49da00eae8a5a 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
@@ -27,7 +27,6 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -125,32 +124,12 @@ public GroupShardsIterator<ShardIterator> searchShards(
                 nodeCounts
             );
             if (iterator != null) {
-                final List<ShardRouting> shardsThatCanHandleSearches;
-                if (isStateless) {
-                    shardsThatCanHandleSearches = statelessShardsThatHandleSearches(clusterState, iterator);
-                } else {
-                    shardsThatCanHandleSearches = statefulShardsThatHandleSearches(iterator);
-                }
-                set.add(new PlainShardIterator(iterator.shardId(), shardsThatCanHandleSearches));
+                set.add(PlainShardIterator.allSearchableShards(iterator));
             }
         }
         return GroupShardsIterator.sortAndCreate(new ArrayList<>(set));
     }
 
-    private static List<ShardRouting> statefulShardsThatHandleSearches(ShardIterator iterator) {
-        final List<ShardRouting> shardsThatCanHandleSearches = new ArrayList<>(iterator.size());
-        for (ShardRouting shardRouting : iterator) {
-            if (shardRouting.isSearchable()) {
-                shardsThatCanHandleSearches.add(shardRouting);
-            }
-        }
-        return shardsThatCanHandleSearches;
-    }
-
-    private static List<ShardRouting> statelessShardsThatHandleSearches(ClusterState clusterState, ShardIterator iterator) {
-        return iterator.getShardRoutings().stream().filter(ShardRouting::isSearchable).toList();
-    }
-
     public static ShardIterator getShards(ClusterState clusterState, ShardId shardId) {
         final IndexShardRoutingTable shard = clusterState.routingTable().shardRoutingTable(shardId);
         return shard.activeInitializingShardsRandomIt();
@@ -297,7 +276,7 @@ private static IndexMetadata indexMetadata(ClusterState clusterState, String ind
         return indexMetadata;
     }
 
-    public ShardId shardId(ClusterState clusterState, String index, String id, @Nullable String routing) {
+    public static ShardId shardId(ClusterState clusterState, String index, String id, @Nullable String routing) {
         IndexMetadata indexMetadata = indexMetadata(clusterState, index);
         return new ShardId(indexMetadata.getIndex(), IndexRouting.fromIndexMetadata(indexMetadata).getShard(id, routing));
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/PlainShardIterator.java b/server/src/main/java/org/elasticsearch/cluster/routing/PlainShardIterator.java
index b49f37c7440dd..a429cc040de3c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/PlainShardIterator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/PlainShardIterator.java
@@ -11,6 +11,7 @@
 
 import org.elasticsearch.index.shard.ShardId;
 
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -21,6 +22,20 @@ public class PlainShardIterator extends PlainShardsIterator implements ShardIter
 
     private final ShardId shardId;
 
+    public static PlainShardIterator allSearchableShards(ShardIterator shardIterator) {
+        return new PlainShardIterator(shardIterator.shardId(), shardsThatCanHandleSearches(shardIterator));
+    }
+
+    private static List<ShardRouting> shardsThatCanHandleSearches(ShardIterator iterator) {
+        final List<ShardRouting> shardsThatCanHandleSearches = new ArrayList<>(iterator.size());
+        for (ShardRouting shardRouting : iterator) {
+            if (shardRouting.isSearchable()) {
+                shardsThatCanHandleSearches.add(shardRouting);
+            }
+        }
+        return shardsThatCanHandleSearches;
+    }
+
     /**
      * Creates a {@link PlainShardIterator} instance that iterates over a subset of the given shards
      * this the a given <code>shardId</code>.
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java
index f8028ce7f9d68..461ac50e1efc8 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java
@@ -18,6 +18,11 @@ public class RoutingFeatures implements FeatureSpecification {
 
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(IndexRouting.BOOLEAN_ROUTING_PATH, IndexRouting.MULTI_VALUE_ROUTING_PATH);
+        return Set.of();
+    }
+
+    @Override
+    public Set<NodeFeature> getTestFeatures() {
+        return Set.of(IndexRouting.LOGSB_ROUTE_ON_SORT_FIELDS);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java
index 827cc378ef3a9..7bb97faa6b2d0 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
-import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -141,11 +140,7 @@ public static Iterator<ToXContent> nodeDecisionsToXContentChunked(List<NodeAlloc
             return Collections.emptyIterator();
         }
 
-        return Iterators.concat(
-            ChunkedToXContentHelper.startArray("node_allocation_decisions"),
-            nodeDecisions.iterator(),
-            ChunkedToXContentHelper.endArray()
-        );
+        return ChunkedToXContentHelper.array("node_allocation_decisions", nodeDecisions.iterator());
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java
index a61199e26191c..c0543a93ea9d0 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java
@@ -13,9 +13,9 @@
 import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ToXContent;
@@ -296,7 +296,7 @@ && getNodeDecisions().stream()
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         checkDecisionState();
-        return ChunkedToXContent.builder(params).append((builder, p) -> {
+        return Iterators.concat(Iterators.single((builder, p) -> {
             builder.field("can_allocate", getAllocationDecision());
             builder.field("allocate_explanation", getExplanation());
             if (targetNode != null) {
@@ -320,7 +320,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
                 );
             }
             return builder;
-        }).append(nodeDecisionsToXContentChunked(nodeDecisions));
+        }), nodeDecisionsToXContentChunked(nodeDecisions));
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
index 5d1e6741c5e22..2f2fd4ef453f6 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
@@ -158,9 +158,7 @@ public ClusterState applyStartedShards(ClusterState clusterState, List<ShardRout
     }
 
     private static ClusterState buildResultAndLogHealthChange(ClusterState oldState, RoutingAllocation allocation, String reason) {
-        final RoutingTable oldRoutingTable = oldState.routingTable();
-        final RoutingNodes newRoutingNodes = allocation.routingNodes();
-        final RoutingTable newRoutingTable = RoutingTable.of(newRoutingNodes);
+        final RoutingTable newRoutingTable = RoutingTable.of(allocation.routingNodes());
         final Metadata newMetadata = allocation.updateMetadataWithRoutingChanges(newRoutingTable);
         assert newRoutingTable.validate(newMetadata); // validates the routing table is coherent with the cluster state metadata
 
@@ -271,8 +269,7 @@ public ClusterState applyFailedShards(
     }
 
     /**
-     * unassigned an shards that are associated with nodes that are no longer part of the cluster, potentially promoting replicas
-     * if needed.
+     * Unassign any shards that are associated with nodes that are no longer part of the cluster, potentially promoting replicas if needed.
      */
     public ClusterState disassociateDeadNodes(ClusterState clusterState, boolean reroute, String reason) {
         RoutingAllocation allocation = createRoutingAllocation(clusterState, currentNanoTime());
@@ -284,7 +281,7 @@ public ClusterState disassociateDeadNodes(ClusterState clusterState, boolean rer
             clusterState = buildResultAndLogHealthChange(clusterState, allocation, reason);
         }
         if (reroute) {
-            return reroute(clusterState, reason, rerouteCompletionIsNotRequired());// this is not triggered by a user request
+            return reroute(clusterState, reason, rerouteCompletionIsNotRequired() /* this is not triggered by a user request */);
         } else {
             return clusterState;
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java
index b98e9050d2b4a..dfcc4da9dcc56 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java
@@ -19,35 +19,41 @@
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
+/**
+ * Exposes cluster allocation metrics. Constructs {@link NodeAllocationStats} per node, on demand.
+ */
 public class AllocationStatsService {
     private final ClusterService clusterService;
     private final ClusterInfoService clusterInfoService;
     private final Supplier<DesiredBalance> desiredBalanceSupplier;
-    private final NodeAllocationStatsProvider nodeAllocationStatsProvider;
+    private final NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator;
 
     public AllocationStatsService(
         ClusterService clusterService,
         ClusterInfoService clusterInfoService,
         ShardsAllocator shardsAllocator,
-        NodeAllocationStatsProvider nodeAllocationStatsProvider
+        NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator
     ) {
         this.clusterService = clusterService;
         this.clusterInfoService = clusterInfoService;
-        this.nodeAllocationStatsProvider = nodeAllocationStatsProvider;
+        this.nodeAllocationStatsAndWeightsCalculator = nodeAllocationStatsAndWeightsCalculator;
         this.desiredBalanceSupplier = shardsAllocator instanceof DesiredBalanceShardsAllocator allocator
             ? allocator::getDesiredBalance
             : () -> null;
     }
 
+    /**
+     * Returns a map of node IDs to node allocation stats.
+     */
     public Map<String, NodeAllocationStats> stats() {
-        var state = clusterService.state();
-        var stats = nodeAllocationStatsProvider.stats(
-            state.metadata(),
-            state.getRoutingNodes(),
+        var clusterState = clusterService.state();
+        var nodesStatsAndWeights = nodeAllocationStatsAndWeightsCalculator.nodesAllocationStatsAndWeights(
+            clusterState.metadata(),
+            clusterState.getRoutingNodes(),
             clusterInfoService.getClusterInfo(),
             desiredBalanceSupplier.get()
         );
-        return stats.entrySet()
+        return nodesStatsAndWeights.entrySet()
             .stream()
             .collect(
                 Collectors.toMap(
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java
index 1980cb17417d4..ec4bd4b2d077a 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java
@@ -10,7 +10,6 @@
 package org.elasticsearch.cluster.routing.allocation;
 
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
@@ -106,10 +105,9 @@ public void relocationCompleted(ShardRouting removedRelocationSource) {
      *
      * @param oldMetadata {@link Metadata} object from before the routing nodes was changed.
      * @param newRoutingTable {@link RoutingTable} object after routing changes were applied.
-     * @param minClusterTransportVersion minimum TransportVersion used between nodes of this cluster
      * @return adapted {@link Metadata}, potentially the original one if no change was needed.
      */
-    public Metadata applyChanges(Metadata oldMetadata, RoutingTable newRoutingTable, TransportVersion minClusterTransportVersion) {
+    public Metadata applyChanges(Metadata oldMetadata, RoutingTable newRoutingTable) {
         Map<Index, List<Map.Entry<ShardId, Updates>>> changesGroupedByIndex = shardChanges.entrySet()
             .stream()
             .collect(Collectors.groupingBy(e -> e.getKey().getIndex()));
@@ -122,14 +120,7 @@ public Metadata applyChanges(Metadata oldMetadata, RoutingTable newRoutingTable,
             for (Map.Entry<ShardId, Updates> shardEntry : indexChanges.getValue()) {
                 ShardId shardId = shardEntry.getKey();
                 Updates updates = shardEntry.getValue();
-                updatedIndexMetadata = updateInSyncAllocations(
-                    newRoutingTable,
-                    oldIndexMetadata,
-                    updatedIndexMetadata,
-                    shardId,
-                    updates,
-                    minClusterTransportVersion
-                );
+                updatedIndexMetadata = updateInSyncAllocations(newRoutingTable, oldIndexMetadata, updatedIndexMetadata, shardId, updates);
                 updatedIndexMetadata = updates.increaseTerm
                     ? updatedIndexMetadata.withIncrementedPrimaryTerm(shardId.id())
                     : updatedIndexMetadata;
@@ -150,8 +141,7 @@ private static IndexMetadata updateInSyncAllocations(
         IndexMetadata oldIndexMetadata,
         IndexMetadata updatedIndexMetadata,
         ShardId shardId,
-        Updates updates,
-        TransportVersion minClusterTransportVersion
+        Updates updates
     ) {
         assert Sets.haveEmptyIntersection(updates.addedAllocationIds, updates.removedAllocationIds)
             : "allocation ids cannot be both added and removed in the same allocation round, added ids: "
@@ -183,8 +173,7 @@ private static IndexMetadata updateInSyncAllocations(
                     allocationId = RecoverySource.ExistingStoreRecoverySource.FORCED_ALLOCATION_ID;
                     updatedIndexMetadata = updatedIndexMetadata.withTimestampRanges(
                         updatedIndexMetadata.getTimestampRange().removeShard(shardId.id(), oldIndexMetadata.getNumberOfShards()),
-                        updatedIndexMetadata.getEventIngestedRange().removeShard(shardId.id(), oldIndexMetadata.getNumberOfShards()),
-                        minClusterTransportVersion
+                        updatedIndexMetadata.getEventIngestedRange().removeShard(shardId.id(), oldIndexMetadata.getNumberOfShards())
                     );
                 } else {
                     assert recoverySource instanceof RecoverySource.SnapshotRecoverySource
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java
index 5dfac293de491..891818b8e68f7 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java
@@ -12,9 +12,9 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ToXContent;
 
@@ -260,7 +260,7 @@ public String getExplanation() {
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         checkDecisionState();
-        return ChunkedToXContent.builder(params).append((builder, p) -> {
+        return Iterators.concat(Iterators.single((builder, p) -> {
             if (targetNode != null) {
                 builder.startObject("target_node");
                 discoveryNodeToXContent(targetNode, true, builder);
@@ -289,7 +289,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
                 builder.field("move_explanation", getExplanation());
             }
             return builder;
-        }).append(nodeDecisionsToXContentChunked(nodeDecisions));
+        }), nodeDecisionsToXContentChunked(nodeDecisions));
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStats.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStats.java
index f7483a0734799..5ad6e7b7817c4 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStats.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStats.java
@@ -18,6 +18,15 @@
 
 import java.io.IOException;
 
+/**
+ * Point-in-time allocation stats for a particular node.
+ *
+ * @param shards count of shards on this node.
+ * @param undesiredShards count of shards that we want to move off of this node.
+ * @param forecastedIngestLoad the predicted near future total ingest load on this node.
+ * @param forecastedDiskUsage the predicted near future total disk usage on this node.
+ * @param currentDiskUsage the current total disk usage on this node.
+ */
 public record NodeAllocationStats(
     int shards,
     int undesiredShards,
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsProvider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsAndWeightsCalculator.java
similarity index 76%
rename from server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsProvider.java
rename to server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsAndWeightsCalculator.java
index 8368f5916ef91..f793ab8f9eb71 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsProvider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsAndWeightsCalculator.java
@@ -24,7 +24,10 @@
 
 import java.util.Map;
 
-public class NodeAllocationStatsProvider {
+/**
+ * Calculates the allocation weights and usage stats for each node: see {@link NodeAllocationStatsAndWeight} for details.
+ */
+public class NodeAllocationStatsAndWeightsCalculator {
     private final WriteLoadForecaster writeLoadForecaster;
 
     private volatile float indexBalanceFactor;
@@ -32,7 +35,10 @@ public class NodeAllocationStatsProvider {
     private volatile float writeLoadBalanceFactor;
     private volatile float diskUsageBalanceFactor;
 
-    public record NodeAllocationAndClusterBalanceStats(
+    /**
+     * Node shard allocation stats and the total node weight.
+     */
+    public record NodeAllocationStatsAndWeight(
         int shards,
         int undesiredShards,
         double forecastedIngestLoad,
@@ -41,7 +47,7 @@ public record NodeAllocationAndClusterBalanceStats(
         float currentNodeWeight
     ) {}
 
-    public NodeAllocationStatsProvider(WriteLoadForecaster writeLoadForecaster, ClusterSettings clusterSettings) {
+    public NodeAllocationStatsAndWeightsCalculator(WriteLoadForecaster writeLoadForecaster, ClusterSettings clusterSettings) {
         this.writeLoadForecaster = writeLoadForecaster;
         clusterSettings.initializeAndWatch(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING, value -> this.shardBalanceFactor = value);
         clusterSettings.initializeAndWatch(BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING, value -> this.indexBalanceFactor = value);
@@ -55,7 +61,10 @@ public NodeAllocationStatsProvider(WriteLoadForecaster writeLoadForecaster, Clus
         );
     }
 
-    public Map<String, NodeAllocationAndClusterBalanceStats> stats(
+    /**
+     * Returns a map of node IDs to {@link NodeAllocationStatsAndWeight}.
+     */
+    public Map<String, NodeAllocationStatsAndWeight> nodesAllocationStatsAndWeights(
         Metadata metadata,
         RoutingNodes routingNodes,
         ClusterInfo clusterInfo,
@@ -66,7 +75,7 @@ public Map<String, NodeAllocationAndClusterBalanceStats> stats(
         var avgWriteLoadPerNode = WeightFunction.avgWriteLoadPerNode(writeLoadForecaster, metadata, routingNodes);
         var avgDiskUsageInBytesPerNode = WeightFunction.avgDiskUsageInBytesPerNode(clusterInfo, metadata, routingNodes);
 
-        var stats = Maps.<String, NodeAllocationAndClusterBalanceStats>newMapWithExpectedSize(routingNodes.size());
+        var nodeAllocationStatsAndWeights = Maps.<String, NodeAllocationStatsAndWeight>newMapWithExpectedSize(routingNodes.size());
         for (RoutingNode node : routingNodes) {
             int shards = 0;
             int undesiredShards = 0;
@@ -75,9 +84,10 @@ public Map<String, NodeAllocationAndClusterBalanceStats> stats(
             long currentDiskUsage = 0;
             for (ShardRouting shardRouting : node) {
                 if (shardRouting.relocating()) {
+                    // Skip the shard if it is moving off this node. The node running recovery will count it.
                     continue;
                 }
-                shards++;
+                ++shards;
                 IndexMetadata indexMetadata = metadata.getIndexSafe(shardRouting.index());
                 if (isDesiredAllocation(desiredBalance, shardRouting) == false) {
                     undesiredShards++;
@@ -86,9 +96,8 @@ public Map<String, NodeAllocationAndClusterBalanceStats> stats(
                 forecastedWriteLoad += writeLoadForecaster.getForecastedWriteLoad(indexMetadata).orElse(0.0);
                 forecastedDiskUsage += Math.max(indexMetadata.getForecastedShardSizeInBytes().orElse(0), shardSize);
                 currentDiskUsage += shardSize;
-
             }
-            float currentNodeWeight = weightFunction.nodeWeight(
+            float currentNodeWeight = weightFunction.calculateNodeWeight(
                 shards,
                 avgShardsPerNode,
                 forecastedWriteLoad,
@@ -96,10 +105,12 @@ public Map<String, NodeAllocationAndClusterBalanceStats> stats(
                 currentDiskUsage,
                 avgDiskUsageInBytesPerNode
             );
-            stats.put(
+            nodeAllocationStatsAndWeights.put(
                 node.nodeId(),
-                new NodeAllocationAndClusterBalanceStats(
+                new NodeAllocationStatsAndWeight(
                     shards,
+                    // It's part of a public API contract for an 'undesired_shards' field that -1 will be returned if an allocator other
+                    // than the desired balance allocator is used.
                     desiredBalance != null ? undesiredShards : -1,
                     forecastedWriteLoad,
                     forecastedDiskUsage,
@@ -109,10 +120,13 @@ public Map<String, NodeAllocationAndClusterBalanceStats> stats(
             );
         }
 
-        return stats;
+        return nodeAllocationStatsAndWeights;
     }
 
-    private static boolean isDesiredAllocation(DesiredBalance desiredBalance, ShardRouting shardRouting) {
+    /**
+     * Checks whether a shard is currently allocated to a node that is wanted by the desired balance decision.
+     */
+    private static boolean isDesiredAllocation(@Nullable DesiredBalance desiredBalance, ShardRouting shardRouting) {
         if (desiredBalance == null) {
             return true;
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
index e0ada293e5508..1f6832649edf9 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
@@ -340,7 +340,7 @@ public RoutingChangesObserver changes() {
      * Returns updated {@link Metadata} based on the changes that were made to the routing nodes
      */
     public Metadata updateMetadataWithRoutingChanges(RoutingTable newRoutingTable) {
-        Metadata metadata = indexMetadataUpdater.applyChanges(metadata(), newRoutingTable, clusterState.getMinTransportVersion());
+        Metadata metadata = indexMetadataUpdater.applyChanges(metadata(), newRoutingTable);
         return resizeSourceIndexUpdater.applyChanges(metadata, newRoutingTable);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
index 8dd1f14564ce9..0ce1605234636 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
@@ -112,6 +112,7 @@ public class BalancedShardsAllocator implements ShardsAllocator {
         Property.NodeScope
     );
 
+    // TODO: deduplicate these fields, use the fields in NodeAllocationStatsAndWeightsCalculator instead.
     private volatile float indexBalanceFactor;
     private volatile float shardBalanceFactor;
     private volatile float writeLoadBalanceFactor;
@@ -168,7 +169,7 @@ private void collectAndRecordNodeWeightStats(Balancer balancer, WeightFunction w
         Map<DiscoveryNode, DesiredBalanceMetrics.NodeWeightStats> nodeLevelWeights = new HashMap<>();
         for (var entry : balancer.nodes.entrySet()) {
             var node = entry.getValue();
-            var nodeWeight = weightFunction.nodeWeight(
+            var nodeWeight = weightFunction.calculateNodeWeight(
                 node.numShards(),
                 balancer.avgShardsPerNode(),
                 node.writeLoad(),
@@ -263,7 +264,7 @@ public static class Balancer {
         private final RoutingAllocation allocation;
         private final RoutingNodes routingNodes;
         private final Metadata metadata;
-        private final WeightFunction weight;
+        private final WeightFunction weightFunction;
 
         private final float threshold;
         private final float avgShardsPerNode;
@@ -272,12 +273,17 @@ public static class Balancer {
         private final Map<String, ModelNode> nodes;
         private final NodeSorter sorter;
 
-        private Balancer(WriteLoadForecaster writeLoadForecaster, RoutingAllocation allocation, WeightFunction weight, float threshold) {
+        private Balancer(
+            WriteLoadForecaster writeLoadForecaster,
+            RoutingAllocation allocation,
+            WeightFunction weightFunction,
+            float threshold
+        ) {
             this.writeLoadForecaster = writeLoadForecaster;
             this.allocation = allocation;
             this.routingNodes = allocation.routingNodes();
             this.metadata = allocation.metadata();
-            this.weight = weight;
+            this.weightFunction = weightFunction;
             this.threshold = threshold;
             avgShardsPerNode = WeightFunction.avgShardPerNode(metadata, routingNodes);
             avgWriteLoadPerNode = WeightFunction.avgWriteLoadPerNode(writeLoadForecaster, metadata, routingNodes);
@@ -355,7 +361,7 @@ public double avgDiskUsageInBytesPerNode() {
          * to sort based on an index.
          */
         private NodeSorter newNodeSorter() {
-            return new NodeSorter(nodesArray(), weight, this);
+            return new NodeSorter(nodesArray(), weightFunction, this);
         }
 
         /**
@@ -435,7 +441,7 @@ private MoveDecision decideRebalance(final ShardRouting shard, Decision canRemai
 
             // balance the shard, if a better node can be found
             final String idxName = shard.getIndexName();
-            final float currentWeight = weight.weight(this, currentNode, idxName);
+            final float currentWeight = weightFunction.calculateNodeWeightWithIndex(this, currentNode, idxName);
             final AllocationDeciders deciders = allocation.deciders();
             Type rebalanceDecisionType = Type.NO;
             ModelNode targetNode = null;
@@ -451,7 +457,7 @@ private MoveDecision decideRebalance(final ShardRouting shard, Decision canRemai
                 // this is a comparison of the number of shards on this node to the number of shards
                 // that should be on each node on average (both taking the cluster as a whole into account
                 // as well as shards per index)
-                final float nodeWeight = weight.weight(this, node, idxName);
+                final float nodeWeight = weightFunction.calculateNodeWeightWithIndex(this, node, idxName);
                 // if the node we are examining has a worse (higher) weight than the node the shard is
                 // assigned to, then there is no way moving the shard to the node with the worse weight
                 // can make the balance of the cluster better, so we check for that here
@@ -1028,7 +1034,7 @@ private AllocateUnassignedDecision decideAllocateUnassigned(final ShardRouting s
                 }
 
                 // weight of this index currently on the node
-                float currentWeight = weight.weight(this, node, shard.getIndexName());
+                float currentWeight = weightFunction.calculateNodeWeightWithIndex(this, node, shard.getIndexName());
                 // moving the shard would not improve the balance, and we are not in explain mode, so short circuit
                 if (currentWeight > minWeight && explain == false) {
                     continue;
@@ -1319,7 +1325,7 @@ public void reset(String index) {
         }
 
         public float weight(ModelNode node) {
-            return function.weight(balancer, node, index);
+            return function.calculateNodeWeightWithIndex(balancer, node, index);
         }
 
         public float minWeightDelta() {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterBalanceStats.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterBalanceStats.java
index 410531b3fdf07..b1eb08255dccb 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterBalanceStats.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterBalanceStats.java
@@ -39,7 +39,7 @@ public record ClusterBalanceStats(
     Map<String, NodeBalanceStats> nodes
 ) implements Writeable, ToXContentObject {
 
-    public static ClusterBalanceStats EMPTY = new ClusterBalanceStats(0, 0, Map.of(), Map.of());
+    public static final ClusterBalanceStats EMPTY = new ClusterBalanceStats(0, 0, Map.of(), Map.of());
 
     public static ClusterBalanceStats createFrom(
         ClusterState clusterState,
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalance.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalance.java
index 406ca72868a40..16cbf41ee1bfa 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalance.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalance.java
@@ -19,9 +19,13 @@
 /**
  * The desired balance of the cluster, indicating which nodes should hold a copy of each shard.
  *
+ * @param lastConvergedIndex Identifies what input data the balancer computation round used to produce this {@link DesiredBalance}. See
+ *                           {@link DesiredBalanceInput#index()} for details. Each reroute request in the same master term is assigned a
+ *                           strictly increasing sequence number. A new master term restarts the index values from zero. The balancer,
+ *                           which runs async to reroute, uses the latest request's data to compute the desired balance.
  * @param assignments a set of the (persistent) node IDs to which each {@link ShardId} should be allocated
  * @param weightsPerNode The node weights calculated based on
- * {@link org.elasticsearch.cluster.routing.allocation.allocator.WeightFunction#nodeWeight}
+ * {@link org.elasticsearch.cluster.routing.allocation.allocator.WeightFunction#calculateNodeWeight}
  */
 public record DesiredBalance(
     long lastConvergedIndex,
@@ -31,8 +35,11 @@ public record DesiredBalance(
 ) {
 
     enum ComputationFinishReason {
+        /** Computation ran to completion */
         CONVERGED,
+        /** Computation exited and published early because a new cluster event occurred that affects computation */
         YIELD_TO_NEW_INPUT,
+        /** Computation stopped and published early to avoid delaying new shard assignment */
         STOP_EARLY
     }
 
@@ -44,6 +51,7 @@ public DesiredBalance(long lastConvergedIndex, Map<ShardId, ShardAssignment> ass
      * The placeholder value for {@link DesiredBalance} when the node stands down as master.
      */
     public static final DesiredBalance NOT_MASTER = new DesiredBalance(-2, Map.of());
+
     /**
      * The starting value for {@link DesiredBalance} when the node becomes the master.
      */
@@ -57,6 +65,10 @@ public static boolean hasChanges(DesiredBalance a, DesiredBalance b) {
         return Objects.equals(a.assignments, b.assignments) == false;
     }
 
+    /**
+     * Returns the sum of shard movements needed to reach the new desired balance. Doesn't count new shard copies as a move, nor removal or
+     * unassignment of a shard copy.
+     */
     public static int shardMovements(DesiredBalance old, DesiredBalance updated) {
         var intersection = Sets.intersection(old.assignments().keySet(), updated.assignments().keySet());
         int movements = 0;
@@ -70,8 +82,15 @@ public static int shardMovements(DesiredBalance old, DesiredBalance updated) {
         return movements;
     }
 
+    /**
+     * Returns the number of shard movements needed to reach the new shard assignment. Doesn't count new shard copies as a move, nor removal
+     * or unassignment of a shard copy.
+     */
     private static int shardMovements(ShardAssignment old, ShardAssignment updated) {
-        var movements = Math.min(0, old.assigned() - updated.assigned());// compensate newly started shards
+        // A shard move should retain the same number of assigned nodes, just swap out one node for another. We will compensate for newly
+        // started shards -- adding a shard copy is not a move -- by initializing the count with a negative value so that incrementing later
+        // for a new node zeros out.
+        var movements = Math.min(0, old.assigned() - updated.assigned());
         for (String nodeId : updated.nodeIds()) {
             if (old.nodeIds().contains(nodeId) == false) {
                 movements++;
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java
index 3b22221ea7db4..03630c284fa30 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java
@@ -415,11 +415,14 @@ boolean hasEnoughIterations(int currentIteration) {
     }
 
     private static Map<ShardId, ShardAssignment> collectShardAssignments(RoutingNodes routingNodes) {
-        final var entries = routingNodes.getAssignedShards().entrySet();
-        assert entries.stream().flatMap(t -> t.getValue().stream()).allMatch(ShardRouting::started) : routingNodes;
-        final Map<ShardId, ShardAssignment> res = Maps.newHashMapWithExpectedSize(entries.size());
-        for (var shardAndAssignments : entries) {
-            res.put(shardAndAssignments.getKey(), ShardAssignment.ofAssignedShards(shardAndAssignments.getValue()));
+        final var allAssignedShards = routingNodes.getAssignedShards().entrySet();
+        assert allAssignedShards.stream().flatMap(t -> t.getValue().stream()).allMatch(ShardRouting::started) : routingNodes;
+        final Map<ShardId, ShardAssignment> res = Maps.newHashMapWithExpectedSize(allAssignedShards.size());
+        for (var shardIdAndShardRoutings : allAssignedShards) {
+            res.put(
+                shardIdAndShardRoutings.getKey(),
+                ShardAssignment.createFromAssignedShardRoutingsList(shardIdAndShardRoutings.getValue())
+            );
         }
         return res;
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetrics.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetrics.java
index 9f6487bdc8abd..fddf9267cdbb1 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetrics.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetrics.java
@@ -10,7 +10,7 @@
 package org.elasticsearch.cluster.routing.allocation.allocator;
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsProvider.NodeAllocationAndClusterBalanceStats;
+import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator.NodeAllocationStatsAndWeight;
 import org.elasticsearch.telemetry.metric.DoubleWithAttributes;
 import org.elasticsearch.telemetry.metric.LongWithAttributes;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
@@ -20,6 +20,12 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 
+/**
+ * Maintains balancer metrics and makes them accessible to the {@link MeterRegistry} and APM reporting. Metrics are updated
+ * ({@link #updateMetrics}) or cleared ({@link #zeroAllMetrics}) as a result of cluster events and the metrics will be pulled for reporting
+ * via the MeterRegistry implementation. Only the master node reports metrics: see {@link #setNodeIsMaster}. When
+ * {@link #nodeIsMaster} is false, empty values are returned such that MeterRegistry ignores the metrics for reporting purposes.
+ */
 public class DesiredBalanceMetrics {
 
     public record AllocationStats(long unassignedShards, long totalAllocations, long undesiredAllocationsExcludingShuttingDownNodes) {}
@@ -69,13 +75,14 @@ public record NodeWeightStats(long shardCount, double diskUsageInBytes, double w
     private volatile long undesiredAllocations;
 
     private final AtomicReference<Map<DiscoveryNode, NodeWeightStats>> weightStatsPerNodeRef = new AtomicReference<>(Map.of());
-    private final AtomicReference<Map<DiscoveryNode, NodeAllocationAndClusterBalanceStats>> allocationStatsPerNodeRef =
-        new AtomicReference<>(Map.of());
+    private final AtomicReference<Map<DiscoveryNode, NodeAllocationStatsAndWeight>> allocationStatsPerNodeRef = new AtomicReference<>(
+        Map.of()
+    );
 
     public void updateMetrics(
         AllocationStats allocationStats,
         Map<DiscoveryNode, NodeWeightStats> weightStatsPerNode,
-        Map<DiscoveryNode, NodeAllocationAndClusterBalanceStats> nodeAllocationStats
+        Map<DiscoveryNode, NodeAllocationStatsAndWeight> nodeAllocationStats
     ) {
         assert allocationStats != null : "allocation stats cannot be null";
         assert weightStatsPerNode != null : "node balance weight stats cannot be null";
@@ -170,6 +177,10 @@ public DesiredBalanceMetrics(MeterRegistry meterRegistry) {
         );
     }
 
+    /**
+     * When {@link #nodeIsMaster} is set to true, the server will report APM metrics registered in this file. When set to false, empty
+     * values will be returned such that no APM metrics are sent from this server.
+     */
     public void setNodeIsMaster(boolean nodeIsMaster) {
         this.nodeIsMaster = nodeIsMaster;
     }
@@ -339,6 +350,10 @@ private List<DoubleWithAttributes> getUndesiredAllocationsRatioMetrics() {
         return List.of();
     }
 
+    /**
+     * Sets all the internal class fields to zero/empty. Typically used in conjunction with {@link #setNodeIsMaster}.
+     * This is best-effort because it is possible for {@link #updateMetrics} to race with this method.
+     */
     public void zeroAllMetrics() {
         unassignedShards = 0;
         totalAllocations = 0;
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java
index 2ee905634f760..909a7a7a99a61 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java
@@ -20,8 +20,8 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
-import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsProvider;
-import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsProvider.NodeAllocationAndClusterBalanceStats;
+import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator;
+import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator.NodeAllocationStatsAndWeight;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceMetrics.AllocationStats;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
@@ -54,6 +54,10 @@ public class DesiredBalanceReconciler {
 
     private static final Logger logger = LogManager.getLogger(DesiredBalanceReconciler.class);
 
+    /**
+     * The minimum interval that log messages will be written if the number of undesired shard allocations reaches the percentage of total
+     * shards set by {@link #UNDESIRED_ALLOCATIONS_LOG_THRESHOLD_SETTING}.
+     */
     public static final Setting<TimeValue> UNDESIRED_ALLOCATIONS_LOG_INTERVAL_SETTING = Setting.timeSetting(
         "cluster.routing.allocation.desired_balance.undesired_allocations.log_interval",
         TimeValue.timeValueHours(1),
@@ -62,6 +66,10 @@ public class DesiredBalanceReconciler {
         Setting.Property.NodeScope
     );
 
+    /**
+     * Warning log messages may be periodically written if the number of shards that are on undesired nodes reaches this percentage setting.
+     * Works together with {@link #UNDESIRED_ALLOCATIONS_LOG_INTERVAL_SETTING} to log on a periodic basis.
+     */
     public static final Setting<Double> UNDESIRED_ALLOCATIONS_LOG_THRESHOLD_SETTING = Setting.doubleSetting(
         "cluster.routing.allocation.desired_balance.undesired_allocations.threshold",
         0.1,
@@ -75,13 +83,13 @@ public class DesiredBalanceReconciler {
     private final NodeAllocationOrdering allocationOrdering = new NodeAllocationOrdering();
     private final NodeAllocationOrdering moveOrdering = new NodeAllocationOrdering();
     private final DesiredBalanceMetrics desiredBalanceMetrics;
-    private final NodeAllocationStatsProvider nodeAllocationStatsProvider;
+    private final NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator;
 
     public DesiredBalanceReconciler(
         ClusterSettings clusterSettings,
         ThreadPool threadPool,
         DesiredBalanceMetrics desiredBalanceMetrics,
-        NodeAllocationStatsProvider nodeAllocationStatsProvider
+        NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator
     ) {
         this.desiredBalanceMetrics = desiredBalanceMetrics;
         this.undesiredAllocationLogInterval = new FrequencyCappedAction(
@@ -93,9 +101,16 @@ public DesiredBalanceReconciler(
             UNDESIRED_ALLOCATIONS_LOG_THRESHOLD_SETTING,
             value -> this.undesiredAllocationsLogThreshold = value
         );
-        this.nodeAllocationStatsProvider = nodeAllocationStatsProvider;
+        this.nodeAllocationStatsAndWeightsCalculator = nodeAllocationStatsAndWeightsCalculator;
     }
 
+    /**
+     * Applies a desired shard allocation to the routing table by initializing and relocating shards in the cluster state.
+     *
+     * @param desiredBalance The new desired cluster shard allocation
+     * @param allocation Cluster state information with which to make decisions, contains routing table metadata that will be modified to
+     *                   reach the given desired balance.
+     */
     public void reconcile(DesiredBalance desiredBalance, RoutingAllocation allocation) {
         var nodeIds = allocation.routingNodes().getAllNodeIds();
         allocationOrdering.retainNodes(nodeIds);
@@ -159,20 +174,22 @@ void run() {
         }
 
         private void updateDesireBalanceMetrics(AllocationStats allocationStats) {
-            var stats = nodeAllocationStatsProvider.stats(
+            var nodesStatsAndWeights = nodeAllocationStatsAndWeightsCalculator.nodesAllocationStatsAndWeights(
                 allocation.metadata(),
                 allocation.routingNodes(),
                 allocation.clusterInfo(),
                 desiredBalance
             );
-            Map<DiscoveryNode, NodeAllocationAndClusterBalanceStats> nodeAllocationStats = new HashMap<>(stats.size());
-            for (var entry : stats.entrySet()) {
-                var node = allocation.nodes().get(entry.getKey());
+            Map<DiscoveryNode, NodeAllocationStatsAndWeight> filteredNodeAllocationStatsAndWeights = new HashMap<>(
+                nodesStatsAndWeights.size()
+            );
+            for (var nodeStatsAndWeight : nodesStatsAndWeights.entrySet()) {
+                var node = allocation.nodes().get(nodeStatsAndWeight.getKey());
                 if (node != null) {
-                    nodeAllocationStats.put(node, entry.getValue());
+                    filteredNodeAllocationStatsAndWeights.put(node, nodeStatsAndWeight.getValue());
                 }
             }
-            desiredBalanceMetrics.updateMetrics(allocationStats, desiredBalance.weightsPerNode(), nodeAllocationStats);
+            desiredBalanceMetrics.updateMetrics(allocationStats, desiredBalance.weightsPerNode(), filteredNodeAllocationStatsAndWeights);
         }
 
         private boolean allocateUnassignedInvariant() {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java
index 8408386b8da58..2c73a27ad3418 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java
@@ -17,8 +17,9 @@
 import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.AllocationService.RerouteStrategy;
-import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsProvider;
+import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.RoutingExplanations;
 import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
@@ -56,11 +57,30 @@ public class DesiredBalanceShardsAllocator implements ShardsAllocator {
 
     private final ShardsAllocator delegateAllocator;
     private final ThreadPool threadPool;
+    /**
+     * This is a callback to run {@link AllocationService#executeWithRoutingAllocation(ClusterState, String, RerouteStrategy)}, which
+     * produces a new ClusterState with the changes made by {@link DesiredBalanceReconciler#reconcile}. The {@link RerouteStrategy} provided
+     * to the callback calls into {@link #desiredBalanceReconciler} for the changes. The {@link #masterServiceTaskQueue} will publish the
+     * new cluster state after the cluster state is constructed by the {@link ReconcileDesiredBalanceExecutor}.
+     */
     private final DesiredBalanceReconcilerAction reconciler;
     private final DesiredBalanceComputer desiredBalanceComputer;
+    /**
+     * Reconciliation ({@link DesiredBalanceReconciler#reconcile(DesiredBalance, RoutingAllocation)}) takes the {@link DesiredBalance}
+     * output of {@link DesiredBalanceComputer#compute} and identifies how shards need to be added, moved or removed to go from the current
+     * cluster shard allocation to the new desired allocation.
+     */
     private final DesiredBalanceReconciler desiredBalanceReconciler;
     private final ContinuousComputation<DesiredBalanceInput> desiredBalanceComputation;
-    private final PendingListenersQueue queue;
+    /**
+     * Saves and runs listeners after DesiredBalance computations complete.
+     */
+    private final PendingListenersQueue pendingListenersQueue;
+    /**
+     * Each reroute request gets assigned a monotonically increasing sequence number. Many reroute requests may arrive before the balancer
+     * asynchronously runs a computation. The balancer will use the latest request and save this sequence number to track back to the
+     * request.
+     */
     private final AtomicLong indexGenerator = new AtomicLong(-1);
     private final ConcurrentLinkedQueue<List<MoveAllocationCommand>> pendingDesiredBalanceMoves = new ConcurrentLinkedQueue<>();
     private final MasterServiceTaskQueue<ReconcileDesiredBalanceTask> masterServiceTaskQueue;
@@ -89,7 +109,7 @@ public DesiredBalanceShardsAllocator(
         ClusterService clusterService,
         DesiredBalanceReconcilerAction reconciler,
         TelemetryProvider telemetryProvider,
-        NodeAllocationStatsProvider nodeAllocationStatsProvider
+        NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator
     ) {
         this(
             delegateAllocator,
@@ -98,7 +118,7 @@ public DesiredBalanceShardsAllocator(
             new DesiredBalanceComputer(clusterSettings, threadPool, delegateAllocator),
             reconciler,
             telemetryProvider,
-            nodeAllocationStatsProvider
+            nodeAllocationStatsAndWeightsCalculator
         );
     }
 
@@ -109,7 +129,7 @@ public DesiredBalanceShardsAllocator(
         DesiredBalanceComputer desiredBalanceComputer,
         DesiredBalanceReconcilerAction reconciler,
         TelemetryProvider telemetryProvider,
-        NodeAllocationStatsProvider nodeAllocationStatsProvider
+        NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator
     ) {
         this.desiredBalanceMetrics = new DesiredBalanceMetrics(telemetryProvider.getMeterRegistry());
         this.delegateAllocator = delegateAllocator;
@@ -120,7 +140,7 @@ public DesiredBalanceShardsAllocator(
             clusterService.getClusterSettings(),
             threadPool,
             desiredBalanceMetrics,
-            nodeAllocationStatsProvider
+            nodeAllocationStatsAndWeightsCalculator
         );
         this.desiredBalanceComputation = new ContinuousComputation<>(threadPool.generic()) {
 
@@ -199,7 +219,7 @@ public String toString() {
                 return "DesiredBalanceShardsAllocator#allocate";
             }
         };
-        this.queue = new PendingListenersQueue();
+        this.pendingListenersQueue = new PendingListenersQueue();
         this.masterServiceTaskQueue = clusterService.createTaskQueue(
             "reconcile-desired-balance",
             Priority.URGENT,
@@ -235,7 +255,7 @@ public void allocate(RoutingAllocation allocation, ActionListener<Void> listener
 
         var index = indexGenerator.incrementAndGet();
         logger.debug("Executing allocate for [{}]", index);
-        queue.add(index, listener);
+        pendingListenersQueue.add(index, listener);
         // This can only run on master, so unset not-master if exists
         if (currentDesiredBalanceRef.compareAndSet(DesiredBalance.NOT_MASTER, DesiredBalance.BECOME_MASTER_INITIAL)) {
             logger.debug("initialized desired balance for becoming master");
@@ -378,7 +398,7 @@ public DesiredBalanceStats getStats() {
     private void onNoLongerMaster() {
         if (indexGenerator.getAndSet(-1) != -1) {
             currentDesiredBalanceRef.set(DesiredBalance.NOT_MASTER);
-            queue.completeAllAsNotMaster();
+            pendingListenersQueue.completeAllAsNotMaster();
             pendingDesiredBalanceMoves.clear();
             desiredBalanceReconciler.clear();
             desiredBalanceMetrics.zeroAllMetrics();
@@ -428,7 +448,7 @@ private ClusterState applyBalance(
                     batchExecutionContext.initialState(),
                     createReconcileAllocationAction(latest.getTask().desiredBalance)
                 );
-                latest.success(() -> queue.complete(latest.getTask().desiredBalance.lastConvergedIndex()));
+                latest.success(() -> pendingListenersQueue.complete(latest.getTask().desiredBalance.lastConvergedIndex()));
                 return newState;
             }
         }
@@ -447,7 +467,7 @@ private static void discardSupersededTasks(
 
     // only for tests - in production, this happens after reconciliation
     protected final void completeToLastConvergedIndex() {
-        queue.complete(currentDesiredBalanceRef.get().lastConvergedIndex());
+        pendingListenersQueue.complete(currentDesiredBalanceRef.get().lastConvergedIndex());
     }
 
     private void recordTime(CounterMetric metric, Runnable action) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/PendingListenersQueue.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/PendingListenersQueue.java
index e1b58cf79ac09..5b14277f2c651 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/PendingListenersQueue.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/PendingListenersQueue.java
@@ -24,6 +24,10 @@
 import static org.elasticsearch.cluster.service.ClusterApplierService.CLUSTER_UPDATE_THREAD_NAME;
 import static org.elasticsearch.cluster.service.MasterService.MASTER_UPDATE_THREAD_NAME;
 
+/**
+ * Registers listeners with an `index` number ({@link #add(long, ActionListener)}) and then completes them whenever the latest index number
+ * is greater or equal to a listener's index value ({@link #complete(long)}).
+ */
 public class PendingListenersQueue {
 
     private static final Logger logger = LogManager.getLogger(PendingListenersQueue.class);
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardAssignment.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardAssignment.java
index 4fb9137cb4544..2bd1b9bb2bb64 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardAssignment.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardAssignment.java
@@ -17,6 +17,14 @@
 
 import static java.util.Collections.unmodifiableSet;
 
+/**
+ * Simple shard assignment summary of shard copies for a particular index shard.
+ *
+ * @param nodeIds The node IDs of nodes holding a shard copy.
+ * @param total The total number of shard copies.
+ * @param unassigned The number of unassigned shard copies.
+ * @param ignored The number of ignored shard copies.
+ */
 public record ShardAssignment(Set<String> nodeIds, int total, int unassigned, int ignored) {
 
     public ShardAssignment {
@@ -28,9 +36,13 @@ public int assigned() {
         return nodeIds.size();
     }
 
-    public static ShardAssignment ofAssignedShards(List<ShardRouting> routings) {
+    /**
+     * Helper method to instantiate a new ShardAssignment from a given list of ShardRouting instances. Assumes all shards are assigned.
+     */
+    public static ShardAssignment createFromAssignedShardRoutingsList(List<ShardRouting> routings) {
         var nodeIds = new LinkedHashSet<String>();
         for (ShardRouting routing : routings) {
+            assert routing.unassignedInfo() == null : "Expected assigned shard copies only, unassigned info: " + routing.unassignedInfo();
             nodeIds.add(routing.currentNodeId());
         }
         return new ShardAssignment(unmodifiableSet(nodeIds), routings.size(), 0, 0);
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/WeightFunction.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/WeightFunction.java
index 7203a92b147f6..66660ed7a8032 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/WeightFunction.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/WeightFunction.java
@@ -58,9 +58,9 @@ public WeightFunction(float shardBalance, float indexBalance, float writeLoadBal
         theta3 = diskUsageBalance / sum;
     }
 
-    float weight(BalancedShardsAllocator.Balancer balancer, BalancedShardsAllocator.ModelNode node, String index) {
+    float calculateNodeWeightWithIndex(BalancedShardsAllocator.Balancer balancer, BalancedShardsAllocator.ModelNode node, String index) {
         final float weightIndex = node.numShards(index) - balancer.avgShardsPerNode(index);
-        final float nodeWeight = nodeWeight(
+        final float nodeWeight = calculateNodeWeight(
             node.numShards(),
             balancer.avgShardsPerNode(),
             node.writeLoad(),
@@ -71,7 +71,7 @@ float weight(BalancedShardsAllocator.Balancer balancer, BalancedShardsAllocator.
         return nodeWeight + theta1 * weightIndex;
     }
 
-    public float nodeWeight(
+    public float calculateNodeWeight(
         int nodeNumShards,
         float avgShardsPerNode,
         double nodeWriteLoad,
diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
index c34d0d19988c8..05d4b29f8f28f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
@@ -158,6 +158,31 @@ public void run() {
         }
     }
 
+    private record TimedListener(ActionListener<Void> listener, Recorder recorder) implements ActionListener<Void> {
+
+        @Override
+        public void onResponse(Void response) {
+            try (Releasable ignored = recorder.record("listener.onResponse")) {
+                listener.onResponse(null);
+            } catch (Exception e) {
+                assert false : e;
+                logger.error("exception thrown by listener.onResponse", e);
+            }
+        }
+
+        @Override
+        public void onFailure(Exception e) {
+            assert e != null;
+            try (Releasable ignored = recorder.record("listener.onFailure")) {
+                listener.onFailure(e);
+            } catch (Exception inner) {
+                e.addSuppressed(inner);
+                assert false : e;
+                logger.error(() -> "exception thrown by listener.onFailure", e);
+            }
+        }
+    }
+
     @Override
     protected synchronized void doStop() {
         for (Map.Entry<TimeoutClusterStateListener, NotifyTimeout> onGoingTimeout : timeoutClusterStateListeners.entrySet()) {
@@ -394,12 +419,14 @@ private void runTask(String source, Function<ClusterState, ClusterState> updateF
 
         final long startTimeMillis = threadPool.relativeTimeInMillis();
         final Recorder stopWatch = new Recorder(threadPool, slowTaskThreadDumpTimeout);
+        final TimedListener timedListener = new TimedListener(clusterApplyListener, stopWatch);
         final ClusterState newClusterState;
         try {
             try (Releasable ignored = stopWatch.record("running task [" + source + ']')) {
                 newClusterState = updateFunction.apply(previousClusterState);
             }
         } catch (Exception e) {
+            timedListener.onFailure(e);
             TimeValue executionTime = getTimeSince(startTimeMillis);
             logger.trace(
                 () -> format(
@@ -412,15 +439,14 @@ private void runTask(String source, Function<ClusterState, ClusterState> updateF
                 e
             );
             warnAboutSlowTaskIfNeeded(executionTime, source, stopWatch);
-            clusterApplyListener.onFailure(e);
             return;
         }
 
         if (previousClusterState == newClusterState) {
+            timedListener.onResponse(null);
             TimeValue executionTime = getTimeSince(startTimeMillis);
             logger.debug("processing [{}]: took [{}] no change in cluster state", source, executionTime);
             warnAboutSlowTaskIfNeeded(executionTime, source, stopWatch);
-            clusterApplyListener.onResponse(null);
         } else {
             if (logger.isTraceEnabled()) {
                 logger.debug("cluster state updated, version [{}], source [{}]\n{}", newClusterState.version(), source, newClusterState);
@@ -430,6 +456,7 @@ private void runTask(String source, Function<ClusterState, ClusterState> updateF
             try {
                 setIsApplyingClusterState();
                 applyChanges(previousClusterState, newClusterState, source, stopWatch);
+                timedListener.onResponse(null);
                 TimeValue executionTime = getTimeSince(startTimeMillis);
                 logger.debug(
                     "processing [{}]: took [{}] done applying updated cluster state (version: {}, uuid: {})",
@@ -439,8 +466,11 @@ private void runTask(String source, Function<ClusterState, ClusterState> updateF
                     newClusterState.stateUUID()
                 );
                 warnAboutSlowTaskIfNeeded(executionTime, source, stopWatch);
-                clusterApplyListener.onResponse(null);
             } catch (Exception e) {
+                // failing to apply a cluster state with an exception indicates a bug in validation or in one of the appliers; if we
+                // continue we will retry with the same cluster state but that might not help.
+                assert applicationMayFail();
+                timedListener.onFailure(e);
                 TimeValue executionTime = getTimeSince(startTimeMillis);
                 if (logger.isTraceEnabled()) {
                     logger.warn(() -> format("""
@@ -460,10 +490,6 @@ private void runTask(String source, Function<ClusterState, ClusterState> updateF
                         e
                     );
                 }
-                // failing to apply a cluster state with an exception indicates a bug in validation or in one of the appliers; if we
-                // continue we will retry with the same cluster state but that might not help.
-                assert applicationMayFail();
-                clusterApplyListener.onFailure(e);
             } finally {
                 clearIsApplyingClusterState();
             }
diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterStateUpdateStats.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterStateUpdateStats.java
index 01ee60a29b341..75b33a7e20f59 100644
--- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterStateUpdateStats.java
+++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterStateUpdateStats.java
@@ -140,7 +140,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(failedNotificationElapsedMillis);
     }
 
-    public static ClusterStateUpdateStats EMPTY = new ClusterStateUpdateStats(
+    public static final ClusterStateUpdateStats EMPTY = new ClusterStateUpdateStats(
         0L,
         0L,
         0L,
diff --git a/server/src/main/java/org/elasticsearch/common/TimeBasedKOrderedUUIDGenerator.java b/server/src/main/java/org/elasticsearch/common/TimeBasedKOrderedUUIDGenerator.java
index 7ea58ee326a79..58ad3f5b47415 100644
--- a/server/src/main/java/org/elasticsearch/common/TimeBasedKOrderedUUIDGenerator.java
+++ b/server/src/main/java/org/elasticsearch/common/TimeBasedKOrderedUUIDGenerator.java
@@ -9,7 +9,10 @@
 
 package org.elasticsearch.common;
 
+import org.elasticsearch.common.util.ByteUtils;
+
 import java.nio.ByteBuffer;
+import java.util.OptionalInt;
 import java.util.function.Supplier;
 
 /**
@@ -28,6 +31,7 @@
  * The result is a compact base64-encoded string, optimized for efficient compression of the _id field in an inverted index.
  */
 public class TimeBasedKOrderedUUIDGenerator extends TimeBasedUUIDGenerator {
+    static final int SIZE_IN_BYTES = 15;
 
     public TimeBasedKOrderedUUIDGenerator(
         final Supplier<Long> timestampSupplier,
@@ -39,6 +43,10 @@ public TimeBasedKOrderedUUIDGenerator(
 
     @Override
     public String getBase64UUID() {
+        return getBase64UUID(OptionalInt.empty());
+    }
+
+    public String getBase64UUID(OptionalInt hash) {
         final int sequenceId = sequenceNumber.incrementAndGet() & 0x00FF_FFFF;
 
         // Calculate timestamp to ensure ordering and avoid backward movement in case of time shifts.
@@ -50,7 +58,7 @@ public String getBase64UUID() {
             sequenceId == 0 ? (lastTimestamp, currentTimeMillis) -> Math.max(lastTimestamp, currentTimeMillis) + 1 : Math::max
         );
 
-        final byte[] uuidBytes = new byte[15];
+        final byte[] uuidBytes = new byte[SIZE_IN_BYTES + (hash.isPresent() ? 4 : 0)];
         final ByteBuffer buffer = ByteBuffer.wrap(uuidBytes);
 
         buffer.put((byte) (timestamp >>> 40)); // changes every 35 years
@@ -64,6 +72,13 @@ public String getBase64UUID() {
         assert macAddress.length == 6;
         buffer.put(macAddress, 0, macAddress.length);
 
+        // Copy the hash value if provided
+        if (hash.isPresent()) {
+            byte[] hashBytes = new byte[4];
+            ByteUtils.writeIntLE(hash.getAsInt(), hashBytes, 0);
+            buffer.put(hashBytes, 0, hashBytes.length);
+        }
+
         buffer.put((byte) (sequenceId >>> 16));
 
         // From hereinafter everything is almost like random and does not compress well
diff --git a/server/src/main/java/org/elasticsearch/common/UUIDs.java b/server/src/main/java/org/elasticsearch/common/UUIDs.java
index ebcb375bc01bc..6b19fcddb87ca 100644
--- a/server/src/main/java/org/elasticsearch/common/UUIDs.java
+++ b/server/src/main/java/org/elasticsearch/common/UUIDs.java
@@ -11,6 +11,7 @@
 
 import org.elasticsearch.common.settings.SecureString;
 
+import java.util.OptionalInt;
 import java.util.Random;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
@@ -23,14 +24,14 @@ public class UUIDs {
     public static final Supplier<Long> DEFAULT_TIMESTAMP_SUPPLIER = System::currentTimeMillis;
     public static final Supplier<Integer> DEFAULT_SEQUENCE_ID_SUPPLIER = sequenceNumber::incrementAndGet;
     public static final Supplier<byte[]> DEFAULT_MAC_ADDRESS_SUPPLIER = MacAddressProvider::getSecureMungedAddress;
-    private static final UUIDGenerator RANDOM_UUID_GENERATOR = new RandomBasedUUIDGenerator();
-    private static final UUIDGenerator TIME_BASED_K_ORDERED_GENERATOR = new TimeBasedKOrderedUUIDGenerator(
+    private static final RandomBasedUUIDGenerator RANDOM_UUID_GENERATOR = new RandomBasedUUIDGenerator();
+    private static final TimeBasedKOrderedUUIDGenerator TIME_BASED_K_ORDERED_GENERATOR = new TimeBasedKOrderedUUIDGenerator(
         DEFAULT_TIMESTAMP_SUPPLIER,
         DEFAULT_SEQUENCE_ID_SUPPLIER,
         DEFAULT_MAC_ADDRESS_SUPPLIER
     );
 
-    private static final UUIDGenerator TIME_UUID_GENERATOR = new TimeBasedUUIDGenerator(
+    private static final TimeBasedUUIDGenerator TIME_UUID_GENERATOR = new TimeBasedUUIDGenerator(
         DEFAULT_TIMESTAMP_SUPPLIER,
         DEFAULT_SEQUENCE_ID_SUPPLIER,
         DEFAULT_MAC_ADDRESS_SUPPLIER
@@ -51,12 +52,8 @@ public static String base64UUID() {
         return TIME_UUID_GENERATOR.getBase64UUID();
     }
 
-    public static String base64TimeBasedKOrderedUUID() {
-        return TIME_BASED_K_ORDERED_GENERATOR.getBase64UUID();
-    }
-
-    public static String base64TimeBasedUUID() {
-        return TIME_UUID_GENERATOR.getBase64UUID();
+    public static String base64TimeBasedKOrderedUUIDWithHash(OptionalInt hash) {
+        return TIME_BASED_K_ORDERED_GENERATOR.getBase64UUID(hash);
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/common/blobstore/support/AbstractBlobContainer.java b/server/src/main/java/org/elasticsearch/common/blobstore/support/AbstractBlobContainer.java
index 635196dd7a52e..f62e4710c81d8 100644
--- a/server/src/main/java/org/elasticsearch/common/blobstore/support/AbstractBlobContainer.java
+++ b/server/src/main/java/org/elasticsearch/common/blobstore/support/AbstractBlobContainer.java
@@ -9,7 +9,6 @@
 
 package org.elasticsearch.common.blobstore.support;
 
-import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
 
@@ -24,17 +23,6 @@ protected AbstractBlobContainer(BlobPath path) {
         this.path = path;
     }
 
-    /**
-     * Temporary check that permits disabling CAS operations at runtime; TODO remove this when no longer needed
-     */
-    protected static boolean skipCas(ActionListener<?> listener) {
-        if ("true".equals(System.getProperty("test.repository_test_kit.skip_cas"))) {
-            listener.onFailure(new UnsupportedOperationException());
-            return true;
-        }
-        return false;
-    }
-
     @Override
     public BlobPath path() {
         return this.path;
diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
index c85df7c6be908..ef5b318a8b426 100644
--- a/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
+++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
@@ -39,7 +39,7 @@ public class DeprecationLogger {
      * Deprecation messages are logged at this level.
      * More serious that WARN by 1, but less serious than ERROR
      */
-    public static Level CRITICAL = Level.forName("CRITICAL", Level.WARN.intLevel() - 1);
+    public static final Level CRITICAL = Level.forName("CRITICAL", Level.WARN.intLevel() - 1);
     private static volatile List<String> skipTheseDeprecations = Collections.emptyList();
     private final Logger logger;
 
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
index a57b8b4d23cdb..bd48572a8bc11 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
@@ -70,6 +70,8 @@
 import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -87,6 +89,8 @@
 import java.util.Map;
 import java.util.Objects;
 
+import static org.apache.lucene.util.Version.LUCENE_10_0_0;
+
 public class Lucene {
     public static final String LATEST_CODEC = "Lucene100";
 
@@ -109,13 +113,6 @@ public class Lucene {
 
     private Lucene() {}
 
-    /**
-     * Reads the segments infos, failing if it fails to load
-     */
-    public static SegmentInfos readSegmentInfos(Directory directory) throws IOException {
-        return SegmentInfos.readLatestCommit(directory);
-    }
-
     /**
      * Returns an iterable that allows to iterate over all files in this segments info
      */
@@ -139,21 +136,45 @@ public static int getNumDocs(SegmentInfos info) {
         return numDocs;
     }
 
+    /**
+     * Reads the segments infos, failing if it fails to load
+     */
+    public static SegmentInfos readSegmentInfos(Directory directory) throws IOException {
+        return SegmentInfos.readLatestCommit(directory, IndexVersions.MINIMUM_READONLY_COMPATIBLE.luceneVersion().major);
+    }
+
     /**
      * Reads the segments infos from the given commit, failing if it fails to load
      */
     public static SegmentInfos readSegmentInfos(IndexCommit commit) throws IOException {
-        // Using commit.getSegmentsFileName() does NOT work here, have to
-        // manually create the segment filename
+        // Using commit.getSegmentsFileName() does NOT work here, have to manually create the segment filename
         String filename = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", commit.getGeneration());
-        return SegmentInfos.readCommit(commit.getDirectory(), filename);
+        return readSegmentInfos(filename, commit.getDirectory());
     }
 
     /**
      * Reads the segments infos from the given segments file name, failing if it fails to load
      */
     private static SegmentInfos readSegmentInfos(String segmentsFileName, Directory directory) throws IOException {
-        return SegmentInfos.readCommit(directory, segmentsFileName);
+        // TODO Use readCommit(Directory directory, String segmentFileName, int minSupportedMajorVersion) once Lucene 10.1 is available
+        // and remove the try-catch block for IndexFormatTooOldException
+        assert IndexVersion.current().luceneVersion().equals(LUCENE_10_0_0) : "remove the try-catch block below";
+        try {
+            return SegmentInfos.readCommit(directory, segmentsFileName);
+        } catch (IndexFormatTooOldException e) {
+            try {
+                // Temporary workaround until Lucene 10.1 is available: try to leverage min. read-only compatibility to read the last commit
+                // and then check if this is the commit we want. This should always work for the case we are interested in (archive and
+                // searchable snapshots indices in N-2 version) as no newer commit should be ever written.
+                var segmentInfos = readSegmentInfos(directory);
+                if (segmentsFileName.equals(segmentInfos.getSegmentsFileName())) {
+                    return segmentInfos;
+                }
+            } catch (Exception suppressed) {
+                e.addSuppressed(suppressed);
+            }
+            throw e;
+        }
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index a01571b8c237d..aecc750bd4e39 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -37,6 +37,8 @@
 import org.elasticsearch.cluster.coordination.MasterHistory;
 import org.elasticsearch.cluster.coordination.NoMasterBlockService;
 import org.elasticsearch.cluster.coordination.Reconfigurator;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
@@ -133,8 +135,12 @@
 import org.elasticsearch.transport.TransportSettings;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
+import java.util.Objects;
 import java.util.Set;
 import java.util.function.Predicate;
+import java.util.stream.Stream;
+
+import static java.util.stream.Collectors.toSet;
 
 /**
  * Encapsulates all valid cluster level settings.
@@ -205,7 +211,7 @@ public void apply(Settings value, Settings current, Settings previous) {
         }
     }
 
-    public static final Set<Setting<?>> BUILT_IN_CLUSTER_SETTINGS = Set.of(
+    public static final Set<Setting<?>> BUILT_IN_CLUSTER_SETTINGS = Stream.of(
         AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING,
         AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING,
         BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING,
@@ -616,6 +622,7 @@ public void apply(Settings value, Settings current, Settings previous) {
         TransportService.ENABLE_STACK_OVERFLOW_AVOIDANCE,
         DataStreamGlobalRetentionSettings.DATA_STREAMS_DEFAULT_RETENTION_SETTING,
         DataStreamGlobalRetentionSettings.DATA_STREAMS_MAX_RETENTION_SETTING,
-        ShardsAvailabilityHealthIndicatorService.REPLICA_UNASSIGNED_BUFFER_TIME
-    );
+        ShardsAvailabilityHealthIndicatorService.REPLICA_UNASSIGNED_BUFFER_TIME,
+        DataStream.isFailureStoreFeatureFlagEnabled() ? DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING : null
+    ).filter(Objects::nonNull).collect(toSet());
 }
diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
index fc8f128e92f32..3c1f53ca4a2c9 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -34,8 +34,8 @@
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.FsDirectoryFactory;
 import org.elasticsearch.index.store.Store;
@@ -182,13 +182,17 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         IndexSettings.LIFECYCLE_ORIGINATION_DATE_SETTING,
         IndexSettings.LIFECYCLE_PARSE_ORIGINATION_DATE_SETTING,
         IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING,
+        IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS,
+        IndexSettings.LOGSDB_SORT_ON_HOST_NAME,
+        IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD,
         IndexSettings.PREFER_ILM_SETTING,
         DataStreamFailureStoreDefinition.FAILURE_STORE_DEFINITION_VERSION_SETTING,
         FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING,
         IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING,
         IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING,
-        SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING,
+        IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING,
         IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING,
+        InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT,
 
         // validate that built-in similarities don't get redefined
         Setting.groupSetting("index.similarity.", (s) -> {
diff --git a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
index defaddb25eb47..a192dda84c4c0 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
@@ -120,7 +120,8 @@ public void writeTo(StreamOutput out) throws IOException {
     /** The version where lucene directory API changed from BE to LE. */
     public static final int LE_VERSION = 5;
     public static final int HIGHER_KDF_ITERATION_COUNT_VERSION = 6;
-    public static final int CURRENT_VERSION = HIGHER_KDF_ITERATION_COUNT_VERSION;
+    public static final int CIPHER_KEY_BITS_256_VERSION = 7;
+    public static final int CURRENT_VERSION = CIPHER_KEY_BITS_256_VERSION;
 
     /** The algorithm used to derive the cipher key from a password. */
     private static final String KDF_ALGO = "PBKDF2WithHmacSHA512";
@@ -128,14 +129,8 @@ public void writeTo(StreamOutput out) throws IOException {
     /** The number of iterations to derive the cipher key. */
     private static final int KDF_ITERS = 210000;
 
-    /**
-     * The number of bits for the cipher key.
-     *
-     * Note: The Oracle JDK 8 ships with a limited JCE policy that restricts key length for AES to 128 bits.
-     * This can be increased to 256 bits once minimum java 9 is the minimum java version.
-     * See http://www.oracle.com/technetwork/java/javase/terms/readme/jdk9-readme-3852447.html#jce
-     * */
-    private static final int CIPHER_KEY_BITS = 128;
+    /** The number of bits for the cipher key (256 bits are supported as of Java 9).*/
+    private static final int CIPHER_KEY_BITS = 256;
 
     /** The number of bits for the GCM tag. */
     private static final int GCM_TAG_BITS = 128;
@@ -156,6 +151,7 @@ public void writeTo(StreamOutput out) throws IOException {
     // 4: remove distinction between string/files, ES 6.8/7.1
     // 5: Lucene directory API changed to LE, ES 8.0
     // 6: increase KDF iteration count, ES 8.14
+    // 7: increase cipher key length to 256 bits, ES 9.0
 
     /** The metadata format version used to read the current keystore wrapper. */
     private final int formatVersion;
@@ -318,8 +314,9 @@ public boolean hasPassword() {
         return hasPassword;
     }
 
-    private static Cipher createCipher(int opmode, char[] password, byte[] salt, byte[] iv, int kdfIters) throws GeneralSecurityException {
-        PBEKeySpec keySpec = new PBEKeySpec(password, salt, kdfIters, CIPHER_KEY_BITS);
+    private static Cipher createCipher(int opmode, char[] password, byte[] salt, byte[] iv, int kdfIters, int cipherKeyBits)
+        throws GeneralSecurityException {
+        PBEKeySpec keySpec = new PBEKeySpec(password, salt, kdfIters, cipherKeyBits);
         SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(KDF_ALGO);
         SecretKey secretKey;
         try {
@@ -343,6 +340,11 @@ private static int getKdfIterationCountForVersion(int formatVersion) {
         return formatVersion < HIGHER_KDF_ITERATION_COUNT_VERSION ? 10000 : KDF_ITERS;
     }
 
+    private static int getCipherKeyBitsForVersion(int formatVersion) {
+        // cipher key length was increased in version 7; it was 128 bits in previous versions
+        return formatVersion < CIPHER_KEY_BITS_256_VERSION ? 128 : CIPHER_KEY_BITS;
+    }
+
     /**
      * Decrypts the underlying keystore data.
      *
@@ -371,7 +373,14 @@ public void decrypt(char[] password) throws GeneralSecurityException, IOExceptio
             throw new SecurityException("Keystore has been corrupted or tampered with", e);
         }
 
-        Cipher cipher = createCipher(Cipher.DECRYPT_MODE, password, salt, iv, getKdfIterationCountForVersion(formatVersion));
+        Cipher cipher = createCipher(
+            Cipher.DECRYPT_MODE,
+            password,
+            salt,
+            iv,
+            getKdfIterationCountForVersion(formatVersion),
+            getCipherKeyBitsForVersion(formatVersion)
+        );
         try (
             ByteArrayInputStream bytesStream = new ByteArrayInputStream(encryptedBytes);
             CipherInputStream cipherStream = new CipherInputStream(bytesStream, cipher);
@@ -409,11 +418,12 @@ private static byte[] readByteArray(DataInput input) throws IOException {
     }
 
     /** Encrypt the keystore entries and return the encrypted data. */
-    private byte[] encrypt(char[] password, byte[] salt, byte[] iv, int kdfIterationCount) throws GeneralSecurityException, IOException {
+    private byte[] encrypt(char[] password, byte[] salt, byte[] iv, int kdfIterationCount, int cipherKeyBits)
+        throws GeneralSecurityException, IOException {
         assert isLoaded();
 
         ByteArrayOutputStream bytes = new ByteArrayOutputStream();
-        Cipher cipher = createCipher(Cipher.ENCRYPT_MODE, password, salt, iv, kdfIterationCount);
+        Cipher cipher = createCipher(Cipher.ENCRYPT_MODE, password, salt, iv, kdfIterationCount, cipherKeyBits);
         try (
             CipherOutputStream cipherStream = new CipherOutputStream(bytes, cipher);
             DataOutputStream output = new DataOutputStream(cipherStream)
@@ -456,7 +466,13 @@ public synchronized void save(Path configDir, char[] password, boolean preserveP
             byte[] iv = new byte[12];
             random.nextBytes(iv);
             // encrypted data
-            byte[] encryptedBytes = encrypt(password, salt, iv, getKdfIterationCountForVersion(CURRENT_VERSION));
+            byte[] encryptedBytes = encrypt(
+                password,
+                salt,
+                iv,
+                getKdfIterationCountForVersion(CURRENT_VERSION),
+                getCipherKeyBitsForVersion(CURRENT_VERSION)
+            );
 
             // size of data block
             output.writeInt(4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length);
diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java
index aec9c108d898d..16c6844f46402 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java
@@ -1727,7 +1727,7 @@ public static <T extends Enum<T>> Setting<T> enumSetting(
      *
      * @param key the key for the setting
      * @param defaultValue the default value for this setting
-     * @param properties properties properties for this setting like scope, filtering...
+     * @param properties properties for this setting like scope, filtering...
      * @return the setting object
      */
     public static Setting<ByteSizeValue> memorySizeSetting(String key, ByteSizeValue defaultValue, Property... properties) {
diff --git a/server/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java b/server/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java
index 7b4f82c1072c6..376333cf7cdf7 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java
@@ -27,7 +27,7 @@ public final class SettingsFilter {
     /**
      * Can be used to specify settings filter that will be used to filter out matching settings in toXContent method
      */
-    public static String SETTINGS_FILTER_PARAM = "settings_filter";
+    public static final String SETTINGS_FILTER_PARAM = "settings_filter";
 
     private final Set<String> patterns;
     private final String patternString;
diff --git a/server/src/main/java/org/elasticsearch/common/time/EpochTime.java b/server/src/main/java/org/elasticsearch/common/time/EpochTime.java
index c53c9d0c03df3..51b28fd97e245 100644
--- a/server/src/main/java/org/elasticsearch/common/time/EpochTime.java
+++ b/server/src/main/java/org/elasticsearch/common/time/EpochTime.java
@@ -35,6 +35,25 @@ class EpochTime {
 
     private static final ValueRange POSITIVE_LONG_INTEGER_RANGE = ValueRange.of(0, Long.MAX_VALUE);
 
+    // TemporalField is only present in the presence of a rounded timestamp
+    private static final long ROUNDED_SIGN_PLACEHOLDER = -2;
+    private static final EpochField ROUNDED_SIGN_FIELD = new EpochField(
+        ChronoUnit.FOREVER,
+        ChronoUnit.FOREVER,
+        ValueRange.of(ROUNDED_SIGN_PLACEHOLDER, ROUNDED_SIGN_PLACEHOLDER)
+    ) {
+        // FIXME: what should this be?
+        @Override
+        public boolean isSupportedBy(TemporalAccessor temporal) {
+            return temporal.isSupported(ChronoField.INSTANT_SECONDS) && temporal.getLong(ChronoField.INSTANT_SECONDS) < 0;
+        }
+
+        @Override
+        public long getFrom(TemporalAccessor temporal) {
+            return ROUNDED_SIGN_PLACEHOLDER;
+        }
+    };
+
     // TemporalField is only present in the presence of a negative (potentially fractional) timestamp.
     private static final long NEGATIVE_SIGN_PLACEHOLDER = -1;
     private static final EpochField NEGATIVE_SIGN_FIELD = new EpochField(
@@ -161,6 +180,10 @@ public TemporalAccessor resolve(
             Long nanosOfMilli = fieldValues.remove(NANOS_OF_MILLI);
             long secondsAndMillis = fieldValues.remove(this);
 
+            // this flag indicates whether we were asked to round up and we defaulted to 999_999 nanos or nanos were given by the users
+            // specifically we do not wnat to confuse defaulted 999_999 nanos with user supplied 999_999 nanos
+            boolean roundUp = fieldValues.remove(ROUNDED_SIGN_FIELD) != null;
+
             long seconds;
             long nanos;
             if (isNegative != null) {
@@ -169,10 +192,18 @@ public TemporalAccessor resolve(
                 nanos = secondsAndMillis % 1000 * 1_000_000;
                 // `secondsAndMillis < 0` implies negative timestamp; so `nanos < 0`
                 if (nanosOfMilli != null) {
-                    // aggregate fractional part of the input; subtract b/c `nanos < 0`
-                    nanos -= nanosOfMilli;
+                    if (roundUp) {
+                        // these are not the nanos you think they are; these are "round up nanos" not the fractional part of the input
+                        // this is the case where we defaulted the value to 999_999 and the intention for rounding is that the value
+                        // moves closer to positive infinity
+                        nanos += nanosOfMilli;
+                    } else {
+                        // aggregate fractional part of the input; subtract b/c `nanos < 0`
+                        // this is the case where the user has supplied a nanos value and we'll want to shift toward negative infinity
+                        nanos -= nanosOfMilli;
+                    }
                 }
-                if (nanos != 0) {
+                if (nanos < 0) {
                     // nanos must be positive. B/c the timestamp is represented by the
                     // (seconds, nanos) tuple, seconds moves 1s toward negative-infinity
                     // and nanos moves 1s toward positive-infinity
@@ -235,38 +266,70 @@ public long getFrom(TemporalAccessor temporal) {
         .appendLiteral('.')
         .toFormatter(Locale.ROOT);
 
-    // this supports milliseconds
-    public static final DateTimeFormatter MILLISECONDS_FORMATTER1 = new DateTimeFormatterBuilder().optionalStart()
+    static final DateFormatter SECONDS_FORMATTER = new JavaDateFormatter(
+        "epoch_second",
+        new JavaTimeDateTimePrinter(SECONDS_FORMATTER1),
+        JavaTimeDateTimeParser.createRoundUpParserGenerator(builder -> builder.parseDefaulting(ChronoField.NANO_OF_SECOND, 999_999_999L)),
+        new JavaTimeDateTimeParser(SECONDS_FORMATTER1),
+        new JavaTimeDateTimeParser(SECONDS_FORMATTER2)
+    );
+
+    public static final DateTimeFormatter MILLISECONDS_FORMATTER_BASE = new DateTimeFormatterBuilder().optionalStart()
         .appendText(NEGATIVE_SIGN_FIELD, Map.of(-1L, "-")) // field is only created in the presence of a '-' char.
         .optionalEnd()
         .appendValue(UNSIGNED_MILLIS, 1, 19, SignStyle.NOT_NEGATIVE)
+        .toFormatter(Locale.ROOT);
+
+    // FIXME: clean these up and append one to the other
+    // this supports milliseconds
+    public static final DateTimeFormatter MILLISECONDS_FORMATTER = new DateTimeFormatterBuilder().append(MILLISECONDS_FORMATTER_BASE)
         .optionalStart()
         .appendFraction(NANOS_OF_MILLI, 0, 6, true)
         .optionalEnd()
         .toFormatter(Locale.ROOT);
 
-    // this supports milliseconds ending in dot
-    private static final DateTimeFormatter MILLISECONDS_FORMATTER2 = new DateTimeFormatterBuilder().optionalStart()
-        .appendText(NEGATIVE_SIGN_FIELD, Map.of(-1L, "-")) // field is only created in the presence of a '-' char.
-        .optionalEnd()
-        .appendValue(UNSIGNED_MILLIS, 1, 19, SignStyle.NOT_NEGATIVE)
-        .appendLiteral('.')
+    // this supports milliseconds
+    public static final DateTimeFormatter MILLISECONDS_PARSER_W_NANOS = new DateTimeFormatterBuilder().append(MILLISECONDS_FORMATTER_BASE)
+        .appendFraction(NANOS_OF_MILLI, 0, 6, true)
         .toFormatter(Locale.ROOT);
 
-    static final DateFormatter SECONDS_FORMATTER = new JavaDateFormatter(
-        "epoch_second",
-        new JavaTimeDateTimePrinter(SECONDS_FORMATTER1),
-        JavaTimeDateTimeParser.createRoundUpParserGenerator(builder -> builder.parseDefaulting(ChronoField.NANO_OF_SECOND, 999_999_999L)),
-        new JavaTimeDateTimeParser(SECONDS_FORMATTER1),
-        new JavaTimeDateTimeParser(SECONDS_FORMATTER2)
-    );
+    // we need an additional parser to detect the difference between user provided nanos and defaulted ones because of the necessity
+    // to parse the two differently in the round up case
+    public static final DateTimeFormatter MILLISECONDS_PARSER_WO_NANOS = new DateTimeFormatterBuilder().append(MILLISECONDS_FORMATTER_BASE)
+        .toFormatter(Locale.ROOT);
 
+    // we need an additional parser to detect the difference between user provided nanos and defaulted ones because of the necessity
+    // to parse the two differently in the round up case
+    public static final DateTimeFormatter MILLISECONDS_PARSER_WO_NANOS_ROUNDING = new DateTimeFormatterBuilder().append(
+        MILLISECONDS_FORMATTER_BASE
+    ).parseDefaulting(EpochTime.ROUNDED_SIGN_FIELD, -2L).parseDefaulting(EpochTime.NANOS_OF_MILLI, 999_999L).toFormatter(Locale.ROOT);
+
+    // this supports milliseconds ending in dot
+    private static final DateTimeFormatter MILLISECONDS_PARSER_ENDING_IN_PERIOD = new DateTimeFormatterBuilder().append(
+        MILLISECONDS_FORMATTER_BASE
+    ).appendLiteral('.').toFormatter(Locale.ROOT);
+
+    /*
+    We separately handle the rounded and non-rounded uses cases here with different parsers.  The reason is because of how we store and
+    handle negative milliseconds since the epoch.  If a user supplies nanoseconds as part of a negative millisecond since epoch value
+    then we need to round toward negative infinity.  However, in the case where nanos are not supplied, and we are requested to
+    round up we will default the value of nanos to 999_999 and need to delineate that this rounding was intended to push the value
+    toward positive infinity not negative infinity.  Differentiating these two cases during parsing requires a flag called out above
+    the ROUNDED_SIGN_FIELD flag.  In addition to this flag we need to know that we are in the "rounding up" state.  So any time we are
+    asked to round up we will force setting the ROUNDED_SIGN_FIELD flag and be able to detect that when parsing and
+    storing the time information and be able to make the correct decision to round toward positive infinity.
+     */
     static final DateFormatter MILLIS_FORMATTER = new JavaDateFormatter(
         "epoch_millis",
-        new JavaTimeDateTimePrinter(MILLISECONDS_FORMATTER1),
-        JavaTimeDateTimeParser.createRoundUpParserGenerator(builder -> builder.parseDefaulting(EpochTime.NANOS_OF_MILLI, 999_999L)),
-        new JavaTimeDateTimeParser(MILLISECONDS_FORMATTER1),
-        new JavaTimeDateTimeParser(MILLISECONDS_FORMATTER2)
+        new JavaTimeDateTimePrinter(MILLISECONDS_FORMATTER),
+        new JavaTimeDateTimeParser[] {
+            new JavaTimeDateTimeParser(MILLISECONDS_PARSER_WO_NANOS_ROUNDING),
+            new JavaTimeDateTimeParser(MILLISECONDS_PARSER_W_NANOS),
+            new JavaTimeDateTimeParser(MILLISECONDS_PARSER_ENDING_IN_PERIOD) },
+        new JavaTimeDateTimeParser[] {
+            new JavaTimeDateTimeParser(MILLISECONDS_PARSER_WO_NANOS),
+            new JavaTimeDateTimeParser(MILLISECONDS_PARSER_W_NANOS),
+            new JavaTimeDateTimeParser(MILLISECONDS_PARSER_ENDING_IN_PERIOD) }
     );
 
     private abstract static class EpochField implements TemporalField {
diff --git a/server/src/main/java/org/elasticsearch/common/time/JavaDateFormatter.java b/server/src/main/java/org/elasticsearch/common/time/JavaDateFormatter.java
index ae4845695734f..434db8254e06d 100644
--- a/server/src/main/java/org/elasticsearch/common/time/JavaDateFormatter.java
+++ b/server/src/main/java/org/elasticsearch/common/time/JavaDateFormatter.java
@@ -165,11 +165,40 @@ static DateFormatter combined(String input, List<DateFormatter> formatters) {
             input,
             printer,
             roundUpParsers.stream().flatMap(Arrays::stream).toArray(DateTimeParser[]::new),
-            parsers.stream().flatMap(Arrays::stream).toArray(DateTimeParser[]::new)
+            parsers.stream().flatMap(Arrays::stream).toArray(DateTimeParser[]::new),
+            false
         );
     }
 
-    private JavaDateFormatter(String format, DateTimePrinter printer, DateTimeParser[] roundupParsers, DateTimeParser[] parsers) {
+    JavaDateFormatter(String format, DateTimePrinter printer, DateTimeParser[] roundupParsers, DateTimeParser[] parsers) {
+        this(
+            format,
+            printer,
+            Arrays.copyOf(roundupParsers, roundupParsers.length, DateTimeParser[].class),
+            Arrays.copyOf(parsers, parsers.length, DateTimeParser[].class),
+            true
+        );
+    }
+
+    private JavaDateFormatter(
+        String format,
+        DateTimePrinter printer,
+        DateTimeParser[] roundupParsers,
+        DateTimeParser[] parsers,
+        boolean doValidate
+    ) {
+        if (doValidate) {
+            if (format.contains("||")) {
+                throw new IllegalArgumentException("This class cannot handle multiple format specifiers");
+            }
+            if (printer == null) {
+                throw new IllegalArgumentException("printer may not be null");
+            }
+            if (parsers.length == 0) {
+                throw new IllegalArgumentException("parsers need to be specified");
+            }
+            verifyPrinterParsers(printer, parsers);
+        }
         this.format = format;
         this.printer = printer;
         this.roundupParsers = roundupParsers;
@@ -247,7 +276,8 @@ private JavaDateFormatter mapParsers(UnaryOperator<DateTimePrinter> printerMappi
             format,
             printerMapping.apply(printer),
             mapParsers(parserMapping, this.roundupParsers),
-            mapParsers(parserMapping, this.parsers)
+            mapParsers(parserMapping, this.parsers),
+            false
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/common/util/CachedSupplier.java b/server/src/main/java/org/elasticsearch/common/util/CachedSupplier.java
index e5ab006bd3c14..f394d9d8367cb 100644
--- a/server/src/main/java/org/elasticsearch/common/util/CachedSupplier.java
+++ b/server/src/main/java/org/elasticsearch/common/util/CachedSupplier.java
@@ -19,7 +19,9 @@
 public final class CachedSupplier<T> implements Supplier<T> {
 
     private volatile Supplier<T> supplier;
-    private volatile T result;
+    // result does not need to be volatile as we only read it after reading that the supplier got nulled out. Since we null out the
+    // supplier after setting the result, total store order from an observed volatile write is sufficient to make a plain read safe.
+    private T result;
 
     public static <R> CachedSupplier<R> wrap(Supplier<R> supplier) {
         if (supplier instanceof CachedSupplier<R> c) {
@@ -38,14 +40,18 @@ public T get() {
         if (supplier == null) {
             return result;
         }
-        initResult();
-        return result;
+        return initResult();
     }
 
-    private synchronized void initResult() {
-        if (supplier != null) {
-            result = supplier.get();
+    private synchronized T initResult() {
+        var s = supplier;
+        if (s != null) {
+            T res = s.get();
+            result = res;
             supplier = null;
+            return res;
+        } else {
+            return result;
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
index 9120576815bac..b45b348376fdf 100644
--- a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
@@ -510,14 +510,14 @@ public void rejectedExecution(Runnable task, ThreadPoolExecutor executor) {
 
     public static class TaskTrackingConfig {
         // This is a random starting point alpha. TODO: revisit this with actual testing and/or make it configurable
-        public static double DEFAULT_EWMA_ALPHA = 0.3;
+        public static final double DEFAULT_EWMA_ALPHA = 0.3;
 
         private final boolean trackExecutionTime;
         private final boolean trackOngoingTasks;
         private final double ewmaAlpha;
 
-        public static TaskTrackingConfig DO_NOT_TRACK = new TaskTrackingConfig(false, false, DEFAULT_EWMA_ALPHA);
-        public static TaskTrackingConfig DEFAULT = new TaskTrackingConfig(true, false, DEFAULT_EWMA_ALPHA);
+        public static final TaskTrackingConfig DO_NOT_TRACK = new TaskTrackingConfig(false, false, DEFAULT_EWMA_ALPHA);
+        public static final TaskTrackingConfig DEFAULT = new TaskTrackingConfig(true, false, DEFAULT_EWMA_ALPHA);
 
         public TaskTrackingConfig(boolean trackOngoingTasks, double ewmaAlpha) {
             this(true, trackOngoingTasks, ewmaAlpha);
diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
index a9e13b86a5159..6841cb5bead0a 100644
--- a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
@@ -24,6 +24,8 @@
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.http.HttpTransportSettings;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.telemetry.tracing.TraceContext;
 
@@ -530,6 +532,17 @@ public String getHeader(String key) {
         return value;
     }
 
+    /**
+     * Returns the header for the given key or defaultValue if not present
+     */
+    public String getHeaderOrDefault(String key, String defaultValue) {
+        String value = getHeader(key);
+        if (value == null) {
+            return defaultValue;
+        }
+        return value;
+    }
+
     /**
      * Returns all of the request headers from the thread's context.<br>
      * <b>Be advised, headers might contain credentials.</b>
@@ -589,6 +602,14 @@ public void putHeader(Map<String, String> header) {
         threadLocal.set(threadLocal.get().putHeaders(header));
     }
 
+    public void setErrorTraceTransportHeader(RestRequest r) {
+        // set whether data nodes should send back stack trace based on the `error_trace` query parameter
+        if (r.paramAsBoolean("error_trace", RestController.ERROR_TRACE_DEFAULT)) {
+            // We only set it if error_trace is true (defaults to false) to avoid sending useless bytes
+            putHeader("error_trace", "true");
+        }
+    }
+
     /**
      * Puts a transient header object into this context
      */
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java
index 1970983654b88..e3a6c291c543e 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java
@@ -26,14 +26,6 @@
  */
 public interface ChunkedToXContent {
 
-    /**
-     * Returns a new {@link ChunkedToXContentBuilder} to construct a series of XContent chunks
-     * @param params    Params to use during generation for any nested {@code ChunkedToXContent} objects.
-     */
-    static ChunkedToXContentBuilder builder(ToXContent.Params params) {
-        return new ChunkedToXContentBuilder(params);
-    }
-
     /**
      * Create an iterator of {@link ToXContent} chunks for a REST response for the given {@link RestApiVersion}. Each chunk is serialized
      * with the same {@link XContentBuilder} and {@link ToXContent.Params}, which is also the same as the {@link ToXContent.Params} passed
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilder.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilder.java
deleted file mode 100644
index a3243ef3865a7..0000000000000
--- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilder.java
+++ /dev/null
@@ -1,477 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.common.xcontent;
-
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.xcontent.ToXContent;
-
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Objects;
-import java.util.function.BiConsumer;
-import java.util.function.Consumer;
-import java.util.function.Function;
-import java.util.stream.Stream;
-
-/**
- * A fluent builder to create {@code Iterator&lt;ToXContent&gt;} objects
- */
-public class ChunkedToXContentBuilder implements Iterator<ToXContent> {
-
-    private final ToXContent.Params params;
-    private final Stream.Builder<ToXContent> builder = Stream.builder();
-    private Iterator<ToXContent> iterator;
-
-    public ChunkedToXContentBuilder(ToXContent.Params params) {
-        this.params = params;
-    }
-
-    private void addChunk(ToXContent content) {
-        assert iterator == null : "Builder has been read, cannot add any more chunks";
-        builder.add(Objects.requireNonNull(content));
-    }
-
-    public ToXContent.Params params() {
-        return params;
-    }
-
-    private void startObject() {
-        addChunk((b, p) -> b.startObject());
-    }
-
-    private void startObject(String name) {
-        addChunk((b, p) -> b.startObject(name));
-    }
-
-    private void endObject() {
-        addChunk((b, p) -> b.endObject());
-    }
-
-    /**
-     * Creates an object, with the specified {@code contents}
-     */
-    public ChunkedToXContentBuilder xContentObject(ToXContent contents) {
-        addChunk((b, p) -> contents.toXContent(b.startObject(), p).endObject());
-        return this;
-    }
-
-    /**
-     * Creates an object named {@code name}, with the specified {@code contents}
-     */
-    public ChunkedToXContentBuilder xContentObject(String name, ToXContent contents) {
-        addChunk((b, p) -> contents.toXContent(b.startObject(name), p).endObject());
-        return this;
-    }
-
-    /**
-     * Creates an object, with the specified {@code contents}
-     */
-    public ChunkedToXContentBuilder xContentObject(ChunkedToXContent contents) {
-        startObject();
-        append(contents);
-        endObject();
-        return this;
-    }
-
-    /**
-     * Creates an object named {@code name}, with the specified {@code contents}
-     */
-    public ChunkedToXContentBuilder xContentObject(String name, ChunkedToXContent contents) {
-        startObject(name);
-        append(contents);
-        endObject();
-        return this;
-    }
-
-    /**
-     * Creates an object, with the specified {@code contents}
-     */
-    public ChunkedToXContentBuilder xContentObject(Iterator<? extends ToXContent> contents) {
-        startObject();
-        append(contents);
-        endObject();
-        return this;
-    }
-
-    /**
-     * Creates an object named {@code name}, with the specified {@code contents}
-     */
-    public ChunkedToXContentBuilder xContentObject(String name, Iterator<? extends ToXContent> contents) {
-        startObject(name);
-        append(contents);
-        endObject();
-        return this;
-    }
-
-    /**
-     * Creates an object with the contents of each field created from each entry in {@code map}
-     */
-    public ChunkedToXContentBuilder xContentObjectFields(Map<String, ? extends ToXContent> map) {
-        startObject();
-        map.forEach(this::field);
-        endObject();
-        return this;
-    }
-
-    /**
-     * Creates an object named {@code name}, with the contents of each field created from each entry in {@code map}
-     */
-    public ChunkedToXContentBuilder xContentObjectFields(String name, Map<String, ? extends ToXContent> map) {
-        startObject(name);
-        map.forEach(this::field);
-        endObject();
-        return this;
-    }
-
-    /**
-     * Creates an object with the contents of each field each another object created from each entry in {@code map}
-     */
-    public ChunkedToXContentBuilder xContentObjectFieldObjects(Map<String, ? extends ToXContent> map) {
-        startObject();
-        map.forEach(this::xContentObject);
-        endObject();
-        return this;
-    }
-
-    /**
-     * Creates an object named {@code name}, with the contents of each field each another object created from each entry in {@code map}
-     */
-    public ChunkedToXContentBuilder xContentObjectFieldObjects(String name, Map<String, ? extends ToXContent> map) {
-        startObject(name);
-        map.forEach(this::xContentObject);
-        endObject();
-        return this;
-    }
-
-    /**
-     * Creates an object, with the contents set by {@code contents}
-     */
-    public ChunkedToXContentBuilder object(Consumer<ChunkedToXContentBuilder> contents) {
-        startObject();
-        execute(contents);
-        endObject();
-        return this;
-    }
-
-    /**
-     * Creates an object named {@code name}, with the contents set by {@code contents}
-     */
-    public ChunkedToXContentBuilder object(String name, Consumer<ChunkedToXContentBuilder> contents) {
-        startObject(name);
-        execute(contents);
-        endObject();
-        return this;
-    }
-
-    /**
-     * Creates an object, with the contents set by calling {@code create} on each item returned by {@code items}
-     */
-    public <T> ChunkedToXContentBuilder object(Iterator<T> items, BiConsumer<ChunkedToXContentBuilder, ? super T> create) {
-        startObject();
-        forEach(items, create);
-        endObject();
-        return this;
-    }
-
-    /**
-     * Creates an object, with the contents set by appending together
-     * the return values of {@code create} called on each item returned by {@code items}
-     */
-    public <T> ChunkedToXContentBuilder object(Iterator<T> items, Function<? super T, ToXContent> create) {
-        startObject();
-        forEach(items, create);
-        endObject();
-        return this;
-    }
-
-    /**
-     * Creates an object named {@code name}, with the contents set by calling {@code create} on each item returned by {@code items}
-     */
-    public <T> ChunkedToXContentBuilder object(String name, Iterator<T> items, BiConsumer<ChunkedToXContentBuilder, ? super T> create) {
-        startObject(name);
-        forEach(items, create);
-        endObject();
-        return this;
-    }
-
-    /**
-     * Creates an object named {@code name}, with the contents set by appending together
-     * the return values of {@code create} called on each item returned by {@code items}
-     */
-    public <T> ChunkedToXContentBuilder object(String name, Iterator<T> items, Function<? super T, ToXContent> create) {
-        startObject(name);
-        forEach(items, create);
-        endObject();
-        return this;
-    }
-
-    /**
-     * Creates an object with the contents of each field set by {@code map}
-     */
-    public ChunkedToXContentBuilder object(Map<String, ?> map) {
-        startObject();
-        map.forEach(this::field);
-        endObject();
-        return this;
-    }
-
-    /**
-     * Creates an object named {@code name}, with the contents of each field set by {@code map}
-     */
-    public ChunkedToXContentBuilder object(String name, Map<String, ?> map) {
-        startObject(name);
-        map.forEach(this::field);
-        endObject();
-        return this;
-    }
-
-    private void startArray() {
-        addChunk((b, p) -> b.startArray());
-    }
-
-    private void startArray(String name) {
-        addChunk((b, p) -> b.startArray(name));
-    }
-
-    private void endArray() {
-        addChunk((b, p) -> b.endArray());
-    }
-
-    public ChunkedToXContentBuilder array(String name, String[] values) {
-        addChunk((b, p) -> b.array(name, values));
-        return this;
-    }
-
-    /**
-     * Creates an array, with the contents set by calling {@code create} on each item returned by {@code items}
-     */
-    public <T> ChunkedToXContentBuilder array(Iterator<T> items, BiConsumer<ChunkedToXContentBuilder, ? super T> create) {
-        startArray();
-        forEach(items, create);
-        endArray();
-        return this;
-    }
-
-    /**
-     * Creates an array with the contents set by appending together the contents of {@code items}
-     */
-    public ChunkedToXContentBuilder array(Iterator<? extends ToXContent> items) {
-        startArray();
-        items.forEachRemaining(this::append);
-        endArray();
-        return this;
-    }
-
-    /**
-     * Creates an array, with the contents set by appending together
-     * the return values of {@code create} called on each item returned by {@code items}
-     */
-    public <T> ChunkedToXContentBuilder array(Iterator<T> items, Function<? super T, ToXContent> create) {
-        startArray();
-        forEach(items, create);
-        endArray();
-        return this;
-    }
-
-    /**
-     * Creates an array named {@code name}, with the contents set by calling {@code create} on each item returned by {@code items}
-     */
-    public <T> ChunkedToXContentBuilder array(String name, Iterator<T> items, BiConsumer<ChunkedToXContentBuilder, ? super T> create) {
-        startArray(name);
-        forEach(items, create);
-        endArray();
-        return this;
-    }
-
-    /**
-     * Creates an array named {@code name}, with the contents set by appending together the contents of {@code items}
-     */
-    public ChunkedToXContentBuilder array(String name, Iterator<? extends ToXContent> items) {
-        startArray(name);
-        items.forEachRemaining(this::append);
-        endArray();
-        return this;
-    }
-
-    /**
-     * Creates an array named {@code name}, with the contents set by appending together
-     * the return values of {@code create} called on each item returned by {@code items}
-     */
-    public <T> ChunkedToXContentBuilder array(String name, Iterator<T> items, Function<? super T, ToXContent> create) {
-        startArray(name);
-        forEach(items, create);
-        endArray();
-        return this;
-    }
-
-    public ChunkedToXContentBuilder field(String name) {
-        addChunk((b, p) -> b.field(name));
-        return this;
-    }
-
-    public ChunkedToXContentBuilder field(String name, boolean value) {
-        addChunk((b, p) -> b.field(name, value));
-        return this;
-    }
-
-    public ChunkedToXContentBuilder field(String name, Boolean value) {
-        addChunk((b, p) -> b.field(name, value));
-        return this;
-    }
-
-    public ChunkedToXContentBuilder field(String name, int value) {
-        addChunk((b, p) -> b.field(name, value));
-        return this;
-    }
-
-    public ChunkedToXContentBuilder field(String name, Integer value) {
-        addChunk((b, p) -> b.field(name, value));
-        return this;
-    }
-
-    public ChunkedToXContentBuilder field(String name, long value) {
-        addChunk((b, p) -> b.field(name, value));
-        return this;
-    }
-
-    public ChunkedToXContentBuilder field(String name, Long value) {
-        addChunk((b, p) -> b.field(name, value));
-        return this;
-    }
-
-    public ChunkedToXContentBuilder field(String name, float value) {
-        addChunk((b, p) -> b.field(name, value));
-        return this;
-    }
-
-    public ChunkedToXContentBuilder field(String name, Float value) {
-        addChunk((b, p) -> b.field(name, value));
-        return this;
-    }
-
-    public ChunkedToXContentBuilder field(String name, double value) {
-        addChunk((b, p) -> b.field(name, value));
-        return this;
-    }
-
-    public ChunkedToXContentBuilder field(String name, Double value) {
-        addChunk((b, p) -> b.field(name, value));
-        return this;
-    }
-
-    public ChunkedToXContentBuilder field(String name, String value) {
-        addChunk((b, p) -> b.field(name, value));
-        return this;
-    }
-
-    public ChunkedToXContentBuilder field(String name, Enum<?> value) {
-        addChunk((b, p) -> b.field(name, value));
-        return this;
-    }
-
-    public ChunkedToXContentBuilder field(String name, ToXContent value) {
-        addChunk((b, p) -> b.field(name, value, p));
-        return this;
-    }
-
-    public ChunkedToXContentBuilder field(String name, ChunkedToXContent value) {
-        addChunk((b, p) -> b.field(name));
-        append(value);
-        return this;
-    }
-
-    public ChunkedToXContentBuilder field(String name, Object value) {
-        addChunk((b, p) -> b.field(name, value));
-        return this;
-    }
-
-    /**
-     * Passes this object to {@code consumer}.
-     * <p>
-     * This may seem superfluous, but it allows callers to 'break out' into more declarative code
-     * inside the lambda, whilst maintaining the top-level fluent method calls.
-     */
-    public ChunkedToXContentBuilder execute(Consumer<ChunkedToXContentBuilder> consumer) {
-        consumer.accept(this);
-        return this;
-    }
-
-    /**
-     * Adds chunks from an iterator. Each item is passed to {@code create} to add chunks to this builder.
-     */
-    public <T> ChunkedToXContentBuilder forEach(Iterator<T> items, BiConsumer<ChunkedToXContentBuilder, ? super T> create) {
-        items.forEachRemaining(t -> create.accept(this, t));
-        return this;
-    }
-
-    /**
-     * Adds chunks from an iterator. Each item is passed to {@code create}, and the resulting {@code ToXContent} objects
-     * are added to this builder in order.
-     */
-    public <T> ChunkedToXContentBuilder forEach(Iterator<T> items, Function<? super T, ToXContent> create) {
-        items.forEachRemaining(t -> append(create.apply(t)));
-        return this;
-    }
-
-    public ChunkedToXContentBuilder append(ToXContent xContent) {
-        if (xContent != ToXContent.EMPTY) {
-            addChunk(xContent);
-        }
-        return this;
-    }
-
-    public ChunkedToXContentBuilder append(ChunkedToXContent chunk) {
-        if (chunk != ChunkedToXContent.EMPTY) {
-            append(chunk.toXContentChunked(params));
-        }
-        return this;
-    }
-
-    public ChunkedToXContentBuilder append(Iterator<? extends ToXContent> xContents) {
-        xContents.forEachRemaining(this::append);
-        return this;
-    }
-
-    public ChunkedToXContentBuilder appendIfPresent(@Nullable ToXContent xContent) {
-        if (xContent != null) {
-            append(xContent);
-        }
-        return this;
-    }
-
-    public ChunkedToXContentBuilder appendIfPresent(@Nullable ChunkedToXContent chunk) {
-        if (chunk != null) {
-            append(chunk);
-        }
-        return this;
-    }
-
-    private Iterator<ToXContent> checkCreateIterator() {
-        if (iterator == null) {
-            iterator = builder.build().iterator();
-        }
-        return iterator;
-    }
-
-    @Override
-    public boolean hasNext() {
-        return checkCreateIterator().hasNext();
-    }
-
-    @Override
-    public ToXContent next() {
-        return checkCreateIterator().next();
-    }
-
-    @Override
-    public void forEachRemaining(Consumer<? super ToXContent> action) {
-        checkCreateIterator().forEachRemaining(action);
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java
index 6a5aa2943de92..7c7a1ea4a1891 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java
@@ -14,44 +14,67 @@
 
 import java.util.Collections;
 import java.util.Iterator;
+import java.util.Map;
+import java.util.function.Function;
 
 public enum ChunkedToXContentHelper {
     ;
 
     public static Iterator<ToXContent> startObject() {
-        return Iterators.single(((builder, params) -> builder.startObject()));
+        return Iterators.single((b, p) -> b.startObject());
     }
 
     public static Iterator<ToXContent> startObject(String name) {
-        return Iterators.single(((builder, params) -> builder.startObject(name)));
+        return Iterators.single((b, p) -> b.startObject(name));
     }
 
     public static Iterator<ToXContent> endObject() {
-        return Iterators.single(((builder, params) -> builder.endObject()));
+        return Iterators.single((b, p) -> b.endObject());
     }
 
     public static Iterator<ToXContent> startArray() {
-        return Iterators.single(((builder, params) -> builder.startArray()));
+        return Iterators.single((b, p) -> b.startArray());
     }
 
     public static Iterator<ToXContent> startArray(String name) {
-        return Iterators.single(((builder, params) -> builder.startArray(name)));
+        return Iterators.single((b, p) -> b.startArray(name));
     }
 
     public static Iterator<ToXContent> endArray() {
-        return Iterators.single(((builder, params) -> builder.endArray()));
+        return Iterators.single((b, p) -> b.endArray());
+    }
+
+    /**
+     * Defines an object named {@code name}, with the contents of each field set by {@code map}
+     */
+    public static Iterator<ToXContent> object(String name, Map<String, ?> map) {
+        return object(name, map, e -> (b, p) -> b.field(e.getKey(), e.getValue()));
+    }
+
+    /**
+     * Defines an object named {@code name}, with the contents of each field created from each entry in {@code map}
+     */
+    public static Iterator<ToXContent> xContentObjectFields(String name, Map<String, ? extends ToXContent> map) {
+        return object(name, map, e -> (b, p) -> e.getValue().toXContent(b.field(e.getKey()), p));
+    }
+
+    /**
+     * Defines an object named {@code name}, with the contents of each field each another object created from each entry in {@code map}
+     */
+    public static Iterator<ToXContent> xContentObjectFieldObjects(String name, Map<String, ? extends ToXContent> map) {
+        return object(name, map, e -> (b, p) -> e.getValue().toXContent(b.startObject(e.getKey()), p).endObject());
     }
 
     public static Iterator<ToXContent> field(String name, boolean value) {
-        return Iterators.single(((builder, params) -> builder.field(name, value)));
+        return Iterators.single((b, p) -> b.field(name, value));
     }
 
     public static Iterator<ToXContent> field(String name, long value) {
-        return Iterators.single(((builder, params) -> builder.field(name, value)));
+        return Iterators.single((b, p) -> b.field(name, value));
     }
 
     public static Iterator<ToXContent> field(String name, String value) {
-        return Iterators.single(((builder, params) -> builder.field(name, value)));
+        return Iterators.single((b, p) -> b.field(name, value));
     }
 
     public static Iterator<ToXContent> optionalField(String name, String value) {
@@ -62,6 +85,59 @@ public static Iterator<ToXContent> optionalField(String name, String value) {
         }
     }
 
+    /**
+     * Creates an Iterator to serialize a named field where the value is represented by a {@link ChunkedToXContentObject}.
+     * Chunked equivalent for {@code XContentBuilder field(String name, ToXContent value)}
+     * @param name name of the field
+     * @param value value for this field
+     * @param params params to propagate for XContent serialization
+     * @return Iterator composing field name and value serialization
+     */
+    public static Iterator<ToXContent> field(String name, ChunkedToXContentObject value, ToXContent.Params params) {
+        return Iterators.concat(Iterators.single((builder, innerParam) -> builder.field(name)), value.toXContentChunked(params));
+    }
+
+    public static Iterator<ToXContent> array(String name, Iterator<? extends ToXContent> contents) {
+        return Iterators.concat(startArray(name), contents, endArray());
+    }
+
+    /**
+     * Creates an Iterator to serialize a named field where the value is represented by an iterator of {@link ChunkedToXContentObject}.
+     * Chunked equivalent for {@code XContentBuilder array(String name, ToXContent value)}
+     * @param name name of the field
+     * @param contents values for this field
+     * @param params params to propagate for XContent serialization
+     * @return Iterator composing field name and value serialization
+     */
+    public static Iterator<ToXContent> array(String name, Iterator<? extends ChunkedToXContentObject> contents, ToXContent.Params params) {
+        return Iterators.concat(startArray(name), Iterators.flatMap(contents, c -> c.toXContentChunked(params)), endArray());
+    }
+
+    /**
+     * Defines an object named {@code name}, with the contents set by {@code iterator}
+     */
+    public static Iterator<ToXContent> object(String name, Iterator<? extends ToXContent> iterator) {
+        return Iterators.concat(startObject(name), iterator, endObject());
+    }
+
+    /**
+     * Defines an object named {@code name}, with the contents set by calling {@code toXContent} on each entry in {@code map}
+     */
+    public static <T> Iterator<ToXContent> object(String name, Map<String, T> map, Function<Map.Entry<String, T>, ToXContent> toXContent) {
+        return object(name, Iterators.map(map.entrySet().iterator(), toXContent));
+    }
+
+    /**
+     * Creates an Iterator of a single ToXContent object that serializes the given object as a single chunk. Just wraps {@link
+     * Iterators#single}, but still useful because it avoids any type ambiguity.
+     *
+     * @param item Item to wrap
+     * @return Singleton iterator for the given item.
+     */
+    public static Iterator<ToXContent> chunk(ToXContent item) {
+        return Iterators.single(item);
+    }
+
     /**
      * Creates an Iterator of a single ToXContent object that serializes the given object as a single chunk. Just wraps {@link
      * Iterators#single}, but still useful because it avoids any type ambiguity.
diff --git a/server/src/main/java/org/elasticsearch/discovery/HandshakingTransportAddressConnector.java b/server/src/main/java/org/elasticsearch/discovery/HandshakingTransportAddressConnector.java
index ce849c26ab780..98715127351aa 100644
--- a/server/src/main/java/org/elasticsearch/discovery/HandshakingTransportAddressConnector.java
+++ b/server/src/main/java/org/elasticsearch/discovery/HandshakingTransportAddressConnector.java
@@ -110,6 +110,7 @@ private void openProbeConnection(ActionListener<Transport.Connection> listener)
                     new VersionInformation(
                         Version.CURRENT.minimumCompatibilityVersion(),
                         IndexVersions.MINIMUM_COMPATIBLE,
+                        IndexVersions.MINIMUM_READONLY_COMPATIBLE,
                         IndexVersion.current()
                     )
                 ),
diff --git a/server/src/main/java/org/elasticsearch/env/BuildVersion.java b/server/src/main/java/org/elasticsearch/env/BuildVersion.java
index 7a6b27eab2330..5c3602283fef3 100644
--- a/server/src/main/java/org/elasticsearch/env/BuildVersion.java
+++ b/server/src/main/java/org/elasticsearch/env/BuildVersion.java
@@ -37,6 +37,12 @@
  */
 public abstract class BuildVersion implements ToXContentFragment, Writeable {
 
+    /**
+     * Checks if this version can operate properly in a cluster without features
+     * that are assumed in the currently running Elasticsearch.
+     */
+    public abstract boolean canRemoveAssumedFeatures();
+
     /**
      * Check whether this version is on or after a minimum threshold.
      *
diff --git a/server/src/main/java/org/elasticsearch/env/DefaultBuildVersion.java b/server/src/main/java/org/elasticsearch/env/DefaultBuildVersion.java
index a7e1a4fee341d..913a352debfb8 100644
--- a/server/src/main/java/org/elasticsearch/env/DefaultBuildVersion.java
+++ b/server/src/main/java/org/elasticsearch/env/DefaultBuildVersion.java
@@ -30,7 +30,7 @@
  */
 final class DefaultBuildVersion extends BuildVersion {
 
-    public static BuildVersion CURRENT = new DefaultBuildVersion(Version.CURRENT.id());
+    public static final BuildVersion CURRENT = new DefaultBuildVersion(Version.CURRENT.id());
 
     final Version version;
 
@@ -47,6 +47,17 @@ final class DefaultBuildVersion extends BuildVersion {
         this(in.readVInt());
     }
 
+    @Override
+    public boolean canRemoveAssumedFeatures() {
+        /*
+         * We can remove assumed features if the node version is the next major version.
+         * This is because the next major version can only form a cluster with the
+         * latest minor version of the previous major, so any features introduced before that point
+         * (that are marked as assumed in the running code version) are automatically met by that version.
+         */
+        return version.major == Version.CURRENT.major + 1;
+    }
+
     @Override
     public boolean onOrAfterMinimumCompatible() {
         return Version.CURRENT.minimumCompatibilityVersion().onOrBefore(version);
diff --git a/server/src/main/java/org/elasticsearch/features/FeatureInfrastructureFeatures.java b/server/src/main/java/org/elasticsearch/features/FeatureInfrastructureFeatures.java
index 76afb5eba8a47..b61d866b0f1bc 100644
--- a/server/src/main/java/org/elasticsearch/features/FeatureInfrastructureFeatures.java
+++ b/server/src/main/java/org/elasticsearch/features/FeatureInfrastructureFeatures.java
@@ -13,18 +13,9 @@
 
 /**
  * This class specifies features for the features functionality itself.
- * <p>
- * This adds a feature {@code features_supported} indicating that a node supports node features.
- * Nodes that do not support features won't have this feature in its feature set,
- * so this can be checked without needing to look at the node version.
  */
 public class FeatureInfrastructureFeatures implements FeatureSpecification {
 
-    @Override
-    public Set<NodeFeature> getFeatures() {
-        return Set.of(FeatureService.FEATURES_SUPPORTED);
-    }
-
     @Override
     public Set<NodeFeature> getTestFeatures() {
         return Set.of(FeatureService.TEST_FEATURES_ENABLED);
diff --git a/server/src/main/java/org/elasticsearch/features/FeatureService.java b/server/src/main/java/org/elasticsearch/features/FeatureService.java
index 9a0ac7cafc183..9f4ec6209c6f1 100644
--- a/server/src/main/java/org/elasticsearch/features/FeatureService.java
+++ b/server/src/main/java/org/elasticsearch/features/FeatureService.java
@@ -9,7 +9,10 @@
 
 package org.elasticsearch.features;
 
+import org.elasticsearch.cluster.ClusterFeatures;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
@@ -23,10 +26,6 @@
  */
 public class FeatureService {
 
-    /**
-     * A feature indicating that node features are supported.
-     */
-    public static final NodeFeature FEATURES_SUPPORTED = new NodeFeature("features_supported");
     public static final NodeFeature TEST_FEATURES_ENABLED = new NodeFeature("test_features_enabled");
 
     private static final Logger logger = LogManager.getLogger(FeatureService.class);
@@ -38,9 +37,7 @@ public class FeatureService {
      * as the local node's supported feature set
      */
     public FeatureService(List<? extends FeatureSpecification> specs) {
-
-        var featureData = FeatureData.createFromSpecifications(specs);
-        nodeFeatures = featureData.getNodeFeatures();
+        this.nodeFeatures = FeatureData.createFromSpecifications(specs).getNodeFeatures();
 
         logger.info("Registered local node features {}", nodeFeatures.keySet().stream().sorted().toList());
     }
@@ -53,11 +50,25 @@ public Map<String, NodeFeature> getNodeFeatures() {
         return nodeFeatures;
     }
 
+    /**
+     * Returns {@code true} if {@code node} can have assumed features.
+     */
+    public boolean featuresCanBeAssumedForNode(DiscoveryNode node) {
+        return ClusterFeatures.featuresCanBeAssumedForNode(node);
+    }
+
+    /**
+    * Returns {@code true} if one or more nodes in {@code nodes} can have assumed features.
+    */
+    public boolean featuresCanBeAssumedForNodes(DiscoveryNodes nodes) {
+        return ClusterFeatures.featuresCanBeAssumedForNodes(nodes);
+    }
+
     /**
      * Returns {@code true} if all nodes in {@code state} support feature {@code feature}.
      */
     @SuppressForbidden(reason = "We need basic feature information from cluster state")
     public boolean clusterHasFeature(ClusterState state, NodeFeature feature) {
-        return state.clusterFeatures().clusterHasFeature(feature);
+        return state.clusterFeatures().clusterHasFeature(state.nodes(), feature);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/features/FeatureSpecification.java b/server/src/main/java/org/elasticsearch/features/FeatureSpecification.java
index c37bc4488f109..5684e53bbd950 100644
--- a/server/src/main/java/org/elasticsearch/features/FeatureSpecification.java
+++ b/server/src/main/java/org/elasticsearch/features/FeatureSpecification.java
@@ -20,8 +20,14 @@
  * unless they also support that feature (this is known as the 'feature ratchet').
  * So once a feature is supported by a cluster, it will always be supported by that cluster in the future.
  * <p>
+ * The only situation where features can be removed is on a compatibility boundary (normally a new major version).
+ * Because a new major version can only form a cluster with the highest minor of the previous major,
+ * any features introduced before that point can be assumed to always be present on that cluster.
+ * The feature itself can be removed by marking it as assumed by {@link NodeFeature#assumedAfterNextCompatibilityBoundary()},
+ * in the last minor, and then removing it and associated conditions in the new major version.
+ * <p>
  * The feature information in cluster state should not normally be directly accessed.
- * All feature checks should be done through {@code FeatureService} to ensure that Elasticsearch's
+ * All feature checks should be done through {@link FeatureService} to ensure that Elasticsearch's
  * guarantees on the introduction of new functionality are followed;
  * that is, new functionality is not enabled until all nodes in the cluster support it.
  * <p>
@@ -31,7 +37,7 @@
  */
 public interface FeatureSpecification {
     /**
-     * Returns a set of regular features that this node supports.
+     * Returns a set of features that this node supports.
      */
     default Set<NodeFeature> getFeatures() {
         return Set.of();
diff --git a/server/src/main/java/org/elasticsearch/features/NodeFeature.java b/server/src/main/java/org/elasticsearch/features/NodeFeature.java
index 957308e805562..ad270540274b9 100644
--- a/server/src/main/java/org/elasticsearch/features/NodeFeature.java
+++ b/server/src/main/java/org/elasticsearch/features/NodeFeature.java
@@ -15,10 +15,17 @@
  * A feature published by a node.
  *
  * @param id        The feature id. Must be unique in the node.
+ * @param assumedAfterNextCompatibilityBoundary
+ *              {@code true} if this feature is removed at the next compatibility boundary (ie next major version),
+ *              and so should be assumed to be met by all nodes after that boundary, even if they don't publish it.
  */
-public record NodeFeature(String id) {
+public record NodeFeature(String id, boolean assumedAfterNextCompatibilityBoundary) {
 
     public NodeFeature {
         Objects.requireNonNull(id);
     }
+
+    public NodeFeature(String id) {
+        this(id, false);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
index bf2387453145d..6038a83130db5 100644
--- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
+++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
@@ -295,7 +295,11 @@ static Metadata upgradeMetadata(Metadata metadata, IndexMetadataVerifier indexMe
         boolean changed = false;
         final Metadata.Builder upgradedMetadata = Metadata.builder(metadata);
         for (IndexMetadata indexMetadata : metadata) {
-            IndexMetadata newMetadata = indexMetadataVerifier.verifyIndexMetadata(indexMetadata, IndexVersions.MINIMUM_COMPATIBLE);
+            IndexMetadata newMetadata = indexMetadataVerifier.verifyIndexMetadata(
+                indexMetadata,
+                IndexVersions.MINIMUM_COMPATIBLE,
+                IndexVersions.MINIMUM_READONLY_COMPATIBLE
+            );
             changed |= indexMetadata != newMetadata;
             upgradedMetadata.put(newMetadata, false);
         }
diff --git a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
index a15fef653dabe..9359f6e377ef4 100644
--- a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
+++ b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
@@ -125,6 +125,7 @@ public ClusterState execute(ClusterState currentState) {
                         currentState.routingTable()
                     );
                     IndexVersion minIndexCompatibilityVersion = currentState.nodes().getMinSupportedIndexVersion();
+                    IndexVersion minReadOnlyIndexCompatibilityVersion = currentState.nodes().getMinReadOnlySupportedIndexVersion();
                     IndexVersion maxIndexCompatibilityVersion = currentState.nodes().getMaxDataNodeCompatibleIndexVersion();
                     boolean importNeeded = false;
                     StringBuilder sb = new StringBuilder();
@@ -176,7 +177,11 @@ public ClusterState execute(ClusterState currentState) {
                         try {
                             // The dangled index might be from an older version, we need to make sure it's compatible
                             // with the current version.
-                            newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(indexMetadata, minIndexCompatibilityVersion);
+                            newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(
+                                indexMetadata,
+                                minIndexCompatibilityVersion,
+                                minReadOnlyIndexCompatibilityVersion
+                            );
                             newIndexMetadata = IndexMetadata.builder(newIndexMetadata)
                                 .settings(
                                     Settings.builder()
diff --git a/server/src/main/java/org/elasticsearch/health/Diagnosis.java b/server/src/main/java/org/elasticsearch/health/Diagnosis.java
index b1af4a1c383da..41301e2d52a53 100644
--- a/server/src/main/java/org/elasticsearch/health/Diagnosis.java
+++ b/server/src/main/java/org/elasticsearch/health/Diagnosis.java
@@ -10,12 +10,14 @@
 package org.elasticsearch.health;
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
-import org.elasticsearch.common.xcontent.ChunkedToXContentBuilder;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ToXContent;
 
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
@@ -76,20 +78,22 @@ public Resource(Collection<DiscoveryNode> nodes) {
         }
 
         @Override
-        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-            var builder = ChunkedToXContent.builder(params);
+        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
+            final Iterator<? extends ToXContent> valuesIterator;
             if (nodes != null) {
-                return builder.array(type.displayValue, nodes.iterator(), node -> (b, p) -> {
-                    b.startObject();
-                    b.field(ID_FIELD, node.getId());
+                valuesIterator = Iterators.map(nodes.iterator(), node -> (builder, params) -> {
+                    builder.startObject();
+                    builder.field(ID_FIELD, node.getId());
                     if (node.getName() != null) {
-                        b.field(NAME_FIELD, node.getName());
+                        builder.field(NAME_FIELD, node.getName());
                     }
-                    return b.endObject();
+                    builder.endObject();
+                    return builder;
                 });
             } else {
-                return builder.array(type.displayValue, values.toArray(String[]::new));
+                valuesIterator = Iterators.map(values.iterator(), value -> (builder, params) -> builder.value(value));
             }
+            return ChunkedToXContentHelper.array(type.displayValue, valuesIterator);
         }
 
         @Override
@@ -140,18 +144,30 @@ public String getUniqueId() {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).object(ob -> {
-            ob.append((b, p) -> {
-                b.field("id", definition.getUniqueId());
-                b.field("cause", definition.cause);
-                b.field("action", definition.action);
-                b.field("help_url", definition.helpURL);
-                return b;
-            });
-            if (affectedResources != null && affectedResources.isEmpty() == false) {
-                ob.object("affected_resources", affectedResources.iterator(), ChunkedToXContentBuilder::append);
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
+        final Iterator<? extends ToXContent> resourcesIterator;
+        if (affectedResources == null) {
+            resourcesIterator = Collections.emptyIterator();
+        } else {
+            resourcesIterator = Iterators.flatMap(affectedResources.iterator(), s -> s.toXContentChunked(outerParams));
+        }
+        return Iterators.concat(Iterators.single((ToXContent) (builder, params) -> {
+            builder.startObject();
+            builder.field("id", definition.getUniqueId());
+            builder.field("cause", definition.cause);
+            builder.field("action", definition.action);
+            builder.field("help_url", definition.helpURL);
+
+            if (affectedResources != null && affectedResources.size() > 0) {
+                builder.startObject("affected_resources");
+            }
+            return builder;
+        }), resourcesIterator, Iterators.single((builder, params) -> {
+            if (affectedResources != null && affectedResources.size() > 0) {
+                builder.endObject();
             }
-        });
+            builder.endObject();
+            return builder;
+        }));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/health/HealthIndicatorResult.java b/server/src/main/java/org/elasticsearch/health/HealthIndicatorResult.java
index 1a84abd9f7c16..6944ac74c8115 100644
--- a/server/src/main/java/org/elasticsearch/health/HealthIndicatorResult.java
+++ b/server/src/main/java/org/elasticsearch/health/HealthIndicatorResult.java
@@ -9,11 +9,11 @@
 
 package org.elasticsearch.health;
 
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
-import org.elasticsearch.common.xcontent.ChunkedToXContentBuilder;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.xcontent.ToXContent;
 
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
@@ -26,22 +26,33 @@ public record HealthIndicatorResult(
     List<Diagnosis> diagnosisList
 ) implements ChunkedToXContentObject {
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).object(ob -> {
-            ob.append((b, p) -> {
-                b.field("status", status.xContentValue());
-                b.field("symptom", symptom);
-                if (details != null && HealthIndicatorDetails.EMPTY.equals(details) == false) {
-                    b.field("details", details, p);
-                }
-                if (impacts != null && impacts.isEmpty() == false) {
-                    b.field("impacts", impacts);
-                }
-                return b;
-            });
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
+        final Iterator<? extends ToXContent> diagnosisIterator;
+        if (diagnosisList == null) {
+            diagnosisIterator = Collections.emptyIterator();
+        } else {
+            diagnosisIterator = Iterators.flatMap(diagnosisList.iterator(), s -> s.toXContentChunked(outerParams));
+        }
+        return Iterators.concat(Iterators.single((ToXContent) (builder, params) -> {
+            builder.startObject();
+            builder.field("status", status.xContentValue());
+            builder.field("symptom", symptom);
+            if (details != null && HealthIndicatorDetails.EMPTY.equals(details) == false) {
+                builder.field("details", details, params);
+            }
+            if (impacts != null && impacts.isEmpty() == false) {
+                builder.field("impacts", impacts);
+            }
+            if (diagnosisList != null && diagnosisList.isEmpty() == false) {
+                builder.startArray("diagnosis");
+            }
+            return builder;
+        }), diagnosisIterator, Iterators.single((builder, params) -> {
             if (diagnosisList != null && diagnosisList.isEmpty() == false) {
-                ob.array("diagnosis", diagnosisList.iterator(), ChunkedToXContentBuilder::append);
+                builder.endArray();
             }
-        });
+            builder.endObject();
+            return builder;
+        }));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java b/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java
index aab9e972cba73..113e789727f0a 100644
--- a/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java
+++ b/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.RunOnce;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.health.metadata.HealthMetadata;
 import org.elasticsearch.health.node.action.HealthNodeNotDiscoveredException;
 import org.elasticsearch.health.node.selection.HealthNode;
@@ -62,7 +61,6 @@ public class LocalHealthMonitor implements ClusterStateListener {
     private final ClusterService clusterService;
     private final ThreadPool threadPool;
     private final Client client;
-    private final FeatureService featureService;
 
     private volatile TimeValue monitorInterval;
     private volatile boolean enabled;
@@ -88,7 +86,6 @@ private LocalHealthMonitor(
         ClusterService clusterService,
         ThreadPool threadPool,
         Client client,
-        FeatureService featureService,
         List<HealthTracker<?>> healthTrackers
     ) {
         this.threadPool = threadPool;
@@ -96,7 +93,6 @@ private LocalHealthMonitor(
         this.enabled = HealthNodeTaskExecutor.ENABLED_SETTING.get(settings);
         this.clusterService = clusterService;
         this.client = client;
-        this.featureService = featureService;
         this.healthTrackers = healthTrackers;
     }
 
@@ -105,17 +101,9 @@ public static LocalHealthMonitor create(
         ClusterService clusterService,
         ThreadPool threadPool,
         Client client,
-        FeatureService featureService,
         List<HealthTracker<?>> healthTrackers
     ) {
-        LocalHealthMonitor localHealthMonitor = new LocalHealthMonitor(
-            settings,
-            clusterService,
-            threadPool,
-            client,
-            featureService,
-            healthTrackers
-        );
+        LocalHealthMonitor localHealthMonitor = new LocalHealthMonitor(settings, clusterService, threadPool, client, healthTrackers);
         localHealthMonitor.registerListeners();
         return localHealthMonitor;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/Index.java b/server/src/main/java/org/elasticsearch/index/Index.java
index b6aa8cf19a7e2..d587d8ef2feef 100644
--- a/server/src/main/java/org/elasticsearch/index/Index.java
+++ b/server/src/main/java/org/elasticsearch/index/Index.java
@@ -30,7 +30,7 @@ public class Index implements Writeable, ToXContentObject {
 
     public static final Index[] EMPTY_ARRAY = new Index[0];
 
-    public static Comparator<Index> COMPARE_BY_NAME = Comparator.comparing(Index::getName);
+    public static final Comparator<Index> COMPARE_BY_NAME = Comparator.comparing(Index::getName);
 
     private static final String INDEX_UUID_KEY = "index_uuid";
     private static final String INDEX_NAME_KEY = "index_name";
diff --git a/server/src/main/java/org/elasticsearch/health/HealthFeatures.java b/server/src/main/java/org/elasticsearch/index/IndexFeatures.java
similarity index 65%
rename from server/src/main/java/org/elasticsearch/health/HealthFeatures.java
rename to server/src/main/java/org/elasticsearch/index/IndexFeatures.java
index 091dbc0eae742..f940e87e51391 100644
--- a/server/src/main/java/org/elasticsearch/health/HealthFeatures.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexFeatures.java
@@ -7,19 +7,24 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.health;
+package org.elasticsearch.index;
 
 import org.elasticsearch.features.FeatureSpecification;
 import org.elasticsearch.features.NodeFeature;
 
 import java.util.Set;
 
-public class HealthFeatures implements FeatureSpecification {
-
-    public static final NodeFeature SUPPORTS_EXTENDED_REPOSITORY_INDICATOR = new NodeFeature("health.extended_repository_indicator");
+public class IndexFeatures implements FeatureSpecification {
 
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(SUPPORTS_EXTENDED_REPOSITORY_INDICATOR);
+        return Set.of();
+    }
+
+    public static final NodeFeature LOGSDB_NO_HOST_NAME_FIELD = new NodeFeature("index.logsdb_no_host_name_field");
+
+    @Override
+    public Set<NodeFeature> getTestFeatures() {
+        return Set.of(LOGSDB_NO_HOST_NAME_FIELD);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java
index f5f923f3657f8..7287a0bf307b9 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexMode.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java
@@ -63,7 +63,8 @@ public enum IndexMode {
     STANDARD("standard") {
         @Override
         void validateWithOtherSettings(Map<Setting<?>, Object> settings) {
-            IndexMode.validateTimeSeriesSettings(settings);
+            validateRoutingPathSettings(settings);
+            validateTimeSeriesSettings(settings);
         }
 
         @Override
@@ -177,7 +178,7 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma
 
         @Override
         public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) {
-            return DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
+            return DEFAULT_MAPPING_TIMESTAMP;
         }
 
         @Override
@@ -235,7 +236,11 @@ public SourceFieldMapper.Mode defaultSourceMode() {
     LOGSDB("logsdb") {
         @Override
         void validateWithOtherSettings(Map<Setting<?>, Object> settings) {
-            IndexMode.validateTimeSeriesSettings(settings);
+            validateTimeSeriesSettings(settings);
+            var setting = settings.get(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS);
+            if (setting.equals(Boolean.FALSE)) {
+                validateRoutingPathSettings(settings);
+            }
         }
 
         @Override
@@ -255,9 +260,9 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma
 
         @Override
         public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) {
-            return indexSettings != null && indexSettings.getIndexSortConfig().hasPrimarySortOnField(HOST_NAME)
-                ? DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME
-                : DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
+            return indexSettings != null && indexSettings.logsdbAddHostNameField()
+                ? DEFAULT_MAPPING_TIMESTAMP_HOSTNAME
+                : DEFAULT_MAPPING_TIMESTAMP;
         }
 
         @Override
@@ -387,10 +392,13 @@ public SourceFieldMapper.Mode defaultSourceMode() {
         }
     };
 
-    private static final String HOST_NAME = "host.name";
+    static final String HOST_NAME = "host.name";
 
-    private static void validateTimeSeriesSettings(Map<Setting<?>, Object> settings) {
+    private static void validateRoutingPathSettings(Map<Setting<?>, Object> settings) {
         settingRequiresTimeSeries(settings, IndexMetadata.INDEX_ROUTING_PATH);
+    }
+
+    private static void validateTimeSeriesSettings(Map<Setting<?>, Object> settings) {
         settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_START_TIME);
         settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_END_TIME);
     }
@@ -424,14 +432,14 @@ private static CompressedXContent createDefaultMapping(boolean includeHostName)
         });
     }
 
-    private static final CompressedXContent DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
+    private static final CompressedXContent DEFAULT_MAPPING_TIMESTAMP;
 
-    private static final CompressedXContent DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME;
+    private static final CompressedXContent DEFAULT_MAPPING_TIMESTAMP_HOSTNAME;
 
     static {
         try {
-            DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING = createDefaultMapping(false);
-            DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME = createDefaultMapping(true);
+            DEFAULT_MAPPING_TIMESTAMP = createDefaultMapping(false);
+            DEFAULT_MAPPING_TIMESTAMP_HOSTNAME = createDefaultMapping(true);
         } catch (IOException e) {
             throw new AssertionError(e);
         }
@@ -450,6 +458,7 @@ private static CompressedXContent createDefaultMapping(boolean includeHostName)
                 IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING,
                 IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING,
                 IndexMetadata.INDEX_ROUTING_PATH,
+                IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS,
                 IndexSettings.TIME_SERIES_START_TIME,
                 IndexSettings.TIME_SERIES_END_TIME
             ),
@@ -614,10 +623,7 @@ public Settings getAdditionalIndexSettings(
                 }
             }
             if (indexMode == LOOKUP) {
-                return Settings.builder()
-                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                    .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-all")
-                    .build();
+                return Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build();
             } else {
                 return Settings.EMPTY;
             }
diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java
index 2168ad1df5d2f..7d63a0432cdbc 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexModule.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java
@@ -203,8 +203,9 @@ public IndexModule(
         this.engineFactory = Objects.requireNonNull(engineFactory);
         // Need to have a mutable arraylist for plugins to add listeners to it
         this.searchOperationListeners = new ArrayList<>(searchOperationListeners);
-        this.searchOperationListeners.add(new SearchSlowLog(indexSettings, slowLogFieldProvider));
-        this.indexOperationListeners.add(new IndexingSlowLog(indexSettings, slowLogFieldProvider));
+        SlowLogFields slowLogFields = slowLogFieldProvider.create(indexSettings);
+        this.searchOperationListeners.add(new SearchSlowLog(indexSettings, slowLogFields));
+        this.indexOperationListeners.add(new IndexingSlowLog(indexSettings, slowLogFields));
         this.directoryFactories = Collections.unmodifiableMap(directoryFactories);
         this.allowExpensiveQueries = allowExpensiveQueries;
         this.expressionResolver = expressionResolver;
diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
index b15828c5594ae..284140460a437 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
@@ -52,7 +51,6 @@
 import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING;
 import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING;
 import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING;
-import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING;
 
 /**
  * This class encapsulates all index level settings and handles settings updates.
@@ -655,6 +653,74 @@ public Iterator<Setting<?>> settings() {
         Property.Final
     );
 
+    /**
+     * Returns <code>true</code> if TSDB encoding is enabled. The default is <code>true</code>
+     */
+    public boolean isES87TSDBCodecEnabled() {
+        return es87TSDBCodecEnabled;
+    }
+
+    public static final Setting<Boolean> LOGSDB_ROUTE_ON_SORT_FIELDS = Setting.boolSetting(
+        "index.logsdb.route_on_sort_fields",
+        false,
+        Property.IndexScope,
+        Property.Final
+    );
+
+    public static final Setting<Boolean> LOGSDB_ADD_HOST_NAME_FIELD = Setting.boolSetting(
+        "index.logsdb.add_host_name_field",
+        false,
+        Property.IndexScope,
+        Property.PrivateIndex,
+        Property.Final
+    );
+
+    public static final Setting<Boolean> LOGSDB_SORT_ON_HOST_NAME = Setting.boolSetting(
+        "index.logsdb.sort_on_host_name",
+        false,
+        Property.IndexScope,
+        Property.PrivateIndex,
+        Property.Final
+    );
+
+    /**
+     * The {@link IndexMode "mode"} of the index.
+     */
+    public static final Setting<IndexMode> MODE = Setting.enumSetting(
+        IndexMode.class,
+        "index.mode",
+        IndexMode.STANDARD,
+        new Setting.Validator<>() {
+            @Override
+            public void validate(IndexMode value) {}
+
+            @Override
+            public void validate(IndexMode value, Map<Setting<?>, Object> settings) {
+                value.validateWithOtherSettings(settings);
+            }
+
+            @Override
+            public Iterator<Setting<?>> settings() {
+                return IndexMode.VALIDATE_WITH_SETTINGS.iterator();
+            }
+        },
+        Property.IndexScope,
+        Property.Final,
+        Property.ServerlessPublic
+    );
+
+    public static final Setting<SourceFieldMapper.Mode> INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting(
+        SourceFieldMapper.Mode.class,
+        settings -> {
+            final IndexMode indexMode = IndexSettings.MODE.get(settings);
+            return indexMode.defaultSourceMode().name();
+        },
+        "index.mapping.source.mode",
+        value -> {},
+        Setting.Property.Final,
+        Setting.Property.IndexScope
+    );
+
     public static final Setting<Boolean> RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting(
         "index.recovery.use_synthetic_source",
         false,
@@ -685,29 +751,11 @@ public void validate(Boolean enabled, Map<Setting<?>, Object> settings) {
                         );
                     }
                 }
-
-                // Verify that all nodes can handle this setting
-                var version = (IndexVersion) settings.get(SETTING_INDEX_VERSION_CREATED);
-                if (version.before(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY)
-                    && version.between(
-                        IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BACKPORT,
-                        IndexVersions.UPGRADE_TO_LUCENE_10_0_0
-                    ) == false) {
-                    throw new IllegalArgumentException(
-                        String.format(
-                            Locale.ROOT,
-                            "The setting [%s] is unavailable on this cluster because some nodes are running older "
-                                + "versions that do not support it. Please upgrade all nodes to the latest version "
-                                + "and try again.",
-                            RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey()
-                        )
-                    );
-                }
             }
 
             @Override
             public Iterator<Setting<?>> settings() {
-                List<Setting<?>> res = List.of(INDEX_MAPPER_SOURCE_MODE_SETTING, SETTING_INDEX_VERSION_CREATED, MODE);
+                List<Setting<?>> res = List.of(INDEX_MAPPER_SOURCE_MODE_SETTING, MODE);
                 return res.iterator();
             }
         },
@@ -715,39 +763,6 @@ public Iterator<Setting<?>> settings() {
         Property.Final
     );
 
-    /**
-     * Returns <code>true</code> if TSDB encoding is enabled. The default is <code>true</code>
-     */
-    public boolean isES87TSDBCodecEnabled() {
-        return es87TSDBCodecEnabled;
-    }
-
-    /**
-     * The {@link IndexMode "mode"} of the index.
-     */
-    public static final Setting<IndexMode> MODE = Setting.enumSetting(
-        IndexMode.class,
-        "index.mode",
-        IndexMode.STANDARD,
-        new Setting.Validator<>() {
-            @Override
-            public void validate(IndexMode value) {}
-
-            @Override
-            public void validate(IndexMode value, Map<Setting<?>, Object> settings) {
-                value.validateWithOtherSettings(settings);
-            }
-
-            @Override
-            public Iterator<Setting<?>> settings() {
-                return IndexMode.VALIDATE_WITH_SETTINGS.iterator();
-            }
-        },
-        Property.IndexScope,
-        Property.Final,
-        Property.ServerlessPublic
-    );
-
     /**
      * Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use.
      * TODO: Remove in 9.0
@@ -801,8 +816,6 @@ private static String getIgnoreAboveDefaultValue(final Settings settings) {
         }
     }
 
-    public static final NodeFeature IGNORE_ABOVE_INDEX_LEVEL_SETTING = new NodeFeature("mapper.ignore_above_index_level_setting");
-
     private final Index index;
     private final IndexVersion version;
     private final Logger logger;
@@ -843,6 +856,9 @@ private static String getIgnoreAboveDefaultValue(final Settings settings) {
     private final boolean softDeleteEnabled;
     private volatile long softDeleteRetentionOperations;
     private final boolean es87TSDBCodecEnabled;
+    private final boolean logsdbRouteOnSortFields;
+    private final boolean logsdbSortOnHostName;
+    private final boolean logsdbAddHostNameField;
 
     private volatile long retentionLeaseMillis;
 
@@ -953,6 +969,20 @@ public boolean isDefaultAllowUnmappedFields() {
         return defaultAllowUnmappedFields;
     }
 
+    /**
+     * Returns <code>true</code> if routing on sort fields is enabled for LogsDB. The default is <code>false</code>
+     */
+    public boolean logsdbRouteOnSortFields() {
+        return logsdbRouteOnSortFields;
+    }
+
+    /**
+     * Returns <code>true</code> if the index is in logsdb mode and needs a [host.name] keyword field. The default is <code>false</code>
+     */
+    public boolean logsdbAddHostNameField() {
+        return logsdbAddHostNameField;
+    }
+
     /**
      * Creates a new {@link IndexSettings} instance. The given node settings will be merged with the settings in the metadata
      * while index level settings will overwrite node settings.
@@ -1045,11 +1075,35 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         indexRouting = IndexRouting.fromIndexMetadata(indexMetadata);
         sourceKeepMode = scopedSettings.get(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING);
         es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING);
+        logsdbRouteOnSortFields = scopedSettings.get(LOGSDB_ROUTE_ON_SORT_FIELDS);
+        logsdbSortOnHostName = scopedSettings.get(LOGSDB_SORT_ON_HOST_NAME);
+        logsdbAddHostNameField = scopedSettings.get(LOGSDB_ADD_HOST_NAME_FIELD);
         skipIgnoredSourceWrite = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING);
         skipIgnoredSourceRead = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING);
         indexMappingSourceMode = scopedSettings.get(INDEX_MAPPER_SOURCE_MODE_SETTING);
         recoverySourceEnabled = RecoverySettings.INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(nodeSettings);
         recoverySourceSyntheticEnabled = scopedSettings.get(RECOVERY_USE_SYNTHETIC_SOURCE_SETTING);
+        if (recoverySourceSyntheticEnabled) {
+            if (DiscoveryNode.isStateless(settings)) {
+                throw new IllegalArgumentException("synthetic recovery source is only allowed in stateful");
+            }
+            // Verify that all nodes can handle this setting
+            if (version.before(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY)
+                && version.between(
+                    IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BACKPORT,
+                    IndexVersions.UPGRADE_TO_LUCENE_10_0_0
+                ) == false) {
+                throw new IllegalArgumentException(
+                    String.format(
+                        Locale.ROOT,
+                        "The setting [%s] is unavailable on this cluster because some nodes are running older "
+                            + "versions that do not support it. Please upgrade all nodes to the latest version "
+                            + "and try again.",
+                        RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey()
+                    )
+                );
+            }
+        }
 
         scopedSettings.addSettingsUpdateConsumer(
             MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING,
diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java
index 2811c7493a277..94d1cc5182457 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java
@@ -13,7 +13,8 @@
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.SortedNumericSortField;
 import org.apache.lucene.search.SortedSetSortField;
-import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.common.logging.DeprecationCategory;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -53,6 +54,8 @@
 **/
 public final class IndexSortConfig {
 
+    private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(IndexSortConfig.class);
+
     /**
      * The list of field names
      */
@@ -99,12 +102,23 @@ public final class IndexSortConfig {
         Setting.Property.ServerlessPublic
     );
 
-    public static final FieldSortSpec[] TIME_SERIES_SORT;
-
+    public static final FieldSortSpec[] TIME_SERIES_SORT, TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_BWC_SORT;
     static {
         FieldSortSpec timeStampSpec = new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH);
         timeStampSpec.order = SortOrder.DESC;
         TIME_SERIES_SORT = new FieldSortSpec[] { new FieldSortSpec(TimeSeriesIdFieldMapper.NAME), timeStampSpec };
+        TIMESTAMP_SORT = new FieldSortSpec[] { timeStampSpec };
+
+        FieldSortSpec hostnameSpec = new FieldSortSpec(IndexMode.HOST_NAME);
+        hostnameSpec.order = SortOrder.ASC;
+        hostnameSpec.missingValue = "_last";
+        hostnameSpec.mode = MultiValueMode.MIN;
+        HOSTNAME_TIMESTAMP_SORT = new FieldSortSpec[] { hostnameSpec, timeStampSpec };
+
+        // Older indexes use ascending ordering for host name and timestamp.
+        HOSTNAME_TIMESTAMP_BWC_SORT = new FieldSortSpec[] {
+            new FieldSortSpec(IndexMode.HOST_NAME),
+            new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH) };
     }
 
     private static String validateMissingValue(String missing) {
@@ -134,32 +148,50 @@ private static MultiValueMode parseMultiValueMode(String value) {
 
     // visible for tests
     final FieldSortSpec[] sortSpecs;
+    private final IndexVersion indexCreatedVersion;
+    private final String indexName;
     private final IndexMode indexMode;
 
     public IndexSortConfig(IndexSettings indexSettings) {
         final Settings settings = indexSettings.getSettings();
+        this.indexCreatedVersion = indexSettings.getIndexVersionCreated();
+        this.indexName = indexSettings.getIndex().getName();
         this.indexMode = indexSettings.getMode();
 
-        if (this.indexMode == IndexMode.TIME_SERIES) {
-            this.sortSpecs = TIME_SERIES_SORT;
+        if (indexMode == IndexMode.TIME_SERIES) {
+            sortSpecs = TIME_SERIES_SORT;
             return;
         }
 
         List<String> fields = INDEX_SORT_FIELD_SETTING.get(settings);
-        if (this.indexMode == IndexMode.LOGSDB && fields.isEmpty()) {
-            fields = List.of("host.name", DataStream.TIMESTAMP_FIELD_NAME);
+        if (indexMode == IndexMode.LOGSDB && INDEX_SORT_FIELD_SETTING.exists(settings) == false) {
+            if (INDEX_SORT_ORDER_SETTING.exists(settings)) {
+                var order = INDEX_SORT_ORDER_SETTING.get(settings);
+                throw new IllegalArgumentException("index.sort.fields:" + fields + " index.sort.order:" + order + ", size mismatch");
+            }
+            if (INDEX_SORT_MODE_SETTING.exists(settings)) {
+                var mode = INDEX_SORT_MODE_SETTING.get(settings);
+                throw new IllegalArgumentException("index.sort.fields:" + fields + " index.sort.mode:" + mode + ", size mismatch");
+            }
+            if (INDEX_SORT_MISSING_SETTING.exists(settings)) {
+                var missing = INDEX_SORT_MISSING_SETTING.get(settings);
+                throw new IllegalArgumentException("index.sort.fields:" + fields + " index.sort.missing:" + missing + ", size mismatch");
+            }
+            var version = indexSettings.getIndexVersionCreated();
+            if (version.onOrAfter(IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME)
+                || version.between(IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0)) {
+                sortSpecs = (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(settings)) ? HOSTNAME_TIMESTAMP_SORT : TIMESTAMP_SORT;
+            } else {
+                sortSpecs = HOSTNAME_TIMESTAMP_BWC_SORT;
+            }
+            return;
         }
-        this.sortSpecs = fields.stream().map(FieldSortSpec::new).toArray(FieldSortSpec[]::new);
+        sortSpecs = fields.stream().map(FieldSortSpec::new).toArray(FieldSortSpec[]::new);
 
         if (INDEX_SORT_ORDER_SETTING.exists(settings)) {
             List<SortOrder> orders = INDEX_SORT_ORDER_SETTING.get(settings);
-            if (this.indexMode == IndexMode.LOGSDB && orders.isEmpty()) {
-                orders = List.of(SortOrder.DESC, SortOrder.DESC);
-            }
             if (orders.size() != sortSpecs.length) {
-                throw new IllegalArgumentException(
-                    "index.sort.field:" + fields + " index.sort.order:" + orders.toString() + ", size mismatch"
-                );
+                throw new IllegalArgumentException("index.sort.field:" + fields + " index.sort.order:" + orders + ", size mismatch");
             }
             for (int i = 0; i < sortSpecs.length; i++) {
                 sortSpecs[i].order = orders.get(i);
@@ -168,9 +200,6 @@ public IndexSortConfig(IndexSettings indexSettings) {
 
         if (INDEX_SORT_MODE_SETTING.exists(settings)) {
             List<MultiValueMode> modes = INDEX_SORT_MODE_SETTING.get(settings);
-            if (this.indexMode == IndexMode.LOGSDB && modes.isEmpty()) {
-                modes = List.of(MultiValueMode.MIN, MultiValueMode.MIN);
-            }
             if (modes.size() != sortSpecs.length) {
                 throw new IllegalArgumentException("index.sort.field:" + fields + " index.sort.mode:" + modes + ", size mismatch");
             }
@@ -181,9 +210,6 @@ public IndexSortConfig(IndexSettings indexSettings) {
 
         if (INDEX_SORT_MISSING_SETTING.exists(settings)) {
             List<String> missingValues = INDEX_SORT_MISSING_SETTING.get(settings);
-            if (this.indexMode == IndexMode.LOGSDB && missingValues.isEmpty()) {
-                missingValues = List.of("_first", "_first");
-            }
             if (missingValues.size() != sortSpecs.length) {
                 throw new IllegalArgumentException(
                     "index.sort.field:" + fields + " index.sort.missing:" + missingValues + ", size mismatch"
@@ -230,7 +256,22 @@ public Sort buildIndexSort(
                 throw new IllegalArgumentException(err);
             }
             if (Objects.equals(ft.name(), sortSpec.field) == false) {
-                throw new IllegalArgumentException("Cannot use alias [" + sortSpec.field + "] as an index sort field");
+                if (this.indexCreatedVersion.onOrAfter(IndexVersions.V_7_13_0)) {
+                    throw new IllegalArgumentException("Cannot use alias [" + sortSpec.field + "] as an index sort field");
+                } else {
+                    DEPRECATION_LOGGER.warn(
+                        DeprecationCategory.MAPPINGS,
+                        "index-sort-aliases",
+                        "Index sort for index ["
+                            + indexName
+                            + "] defined on field ["
+                            + sortSpec.field
+                            + "] which resolves to field ["
+                            + ft.name()
+                            + "]. "
+                            + "You will not be able to define an index sort over aliased fields in new indexes"
+                    );
+                }
             }
             boolean reverse = sortSpec.order == null ? false : (sortSpec.order == SortOrder.DESC);
             MultiValueMode mode = sortSpec.mode;
diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java
index fd321f6256194..69ebcd4ba3fe6 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java
@@ -12,7 +12,6 @@
 import org.apache.lucene.util.Version;
 import org.elasticsearch.ReleaseVersions;
 import org.elasticsearch.core.Assertions;
-import org.elasticsearch.core.UpdateForV9;
 
 import java.lang.reflect.Field;
 import java.text.ParseException;
@@ -25,6 +24,7 @@
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.function.IntFunction;
+import java.util.stream.Collectors;
 
 @SuppressWarnings("deprecation")
 public class IndexVersions {
@@ -58,7 +58,6 @@ private static Version parseUnchecked(String version) {
         }
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_FOUNDATIONS) // remove the index versions with which v9 will not need to interact
     public static final IndexVersion ZERO = def(0, Version.LATEST);
 
     public static final IndexVersion V_7_0_0 = def(7_00_00_99, parseUnchecked("8.0.0"));
@@ -134,11 +133,15 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion V8_DEPRECATE_SOURCE_MODE_MAPPER = def(8_521_00_0, Version.LUCENE_9_12_0);
     public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BACKPORT = def(8_522_00_0, Version.LUCENE_9_12_0);
     public static final IndexVersion UPGRADE_TO_LUCENE_9_12_1 = def(8_523_00_0, parseUnchecked("9.12.1"));
+    public static final IndexVersion INFERENCE_METADATA_FIELDS_BACKPORT = def(8_524_00_0, parseUnchecked("9.12.1"));
+    public static final IndexVersion LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT = def(8_525_00_0, parseUnchecked("9.12.1"));
     public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = def(9_001_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID = def(9_002_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion DEPRECATE_SOURCE_MODE_MAPPER = def(9_003_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY = def(9_004_00_0, Version.LUCENE_10_0_0);
+    public static final IndexVersion INFERENCE_METADATA_FIELDS = def(9_005_00_0, Version.LUCENE_10_0_0);
+    public static final IndexVersion LOGSB_OPTIONAL_SORTING_ON_HOST_NAME = def(9_006_00_0, Version.LUCENE_10_0_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _
@@ -244,10 +247,12 @@ static NavigableMap<Integer, IndexVersion> getAllVersionIds(Class<?> cls) {
         return Collections.unmodifiableNavigableMap(builder);
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
-    // We can simplify this once we've removed all references to index versions earlier than MINIMUM_COMPATIBLE
+    static Collection<IndexVersion> getAllWriteVersions() {
+        return VERSION_IDS.values().stream().filter(v -> v.onOrAfter(IndexVersions.MINIMUM_COMPATIBLE)).collect(Collectors.toSet());
+    }
+
     static Collection<IndexVersion> getAllVersions() {
-        return VERSION_IDS.values().stream().filter(v -> v.onOrAfter(MINIMUM_COMPATIBLE)).toList();
+        return VERSION_IDS.values();
     }
 
     static final IntFunction<String> VERSION_LOOKUP = ReleaseVersions.generateVersionsLookup(IndexVersions.class, LATEST_DEFINED.id());
diff --git a/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java b/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
index 3ae4c0eb82ad0..5a7990a4e70c5 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
@@ -103,7 +103,7 @@ public final class IndexingSlowLog implements IndexingOperationListener {
      * <em>characters</em> of the source.
      */
     private int maxSourceCharsToLog;
-    private final SlowLogFieldProvider slowLogFieldProvider;
+    private final SlowLogFields slowLogFields;
 
     /**
      * Reads how much of the source to log. The user can specify any value they
@@ -125,8 +125,8 @@ public final class IndexingSlowLog implements IndexingOperationListener {
         Property.IndexScope
     );
 
-    IndexingSlowLog(IndexSettings indexSettings, SlowLogFieldProvider slowLogFieldProvider) {
-        this.slowLogFieldProvider = slowLogFieldProvider;
+    IndexingSlowLog(IndexSettings indexSettings, SlowLogFields slowLogFields) {
+        this.slowLogFields = slowLogFields;
         this.index = indexSettings.getIndex();
 
         indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING, this::setReformat);
@@ -179,47 +179,19 @@ public void postIndex(ShardId shardId, Engine.Index indexOperation, Engine.Index
             final long tookInNanos = result.getTook();
             if (indexWarnThreshold >= 0 && tookInNanos > indexWarnThreshold) {
                 indexLogger.warn(
-                    IndexingSlowLogMessage.of(
-                        this.slowLogFieldProvider.indexSlowLogFields(),
-                        index,
-                        doc,
-                        tookInNanos,
-                        reformat,
-                        maxSourceCharsToLog
-                    )
+                    IndexingSlowLogMessage.of(this.slowLogFields.indexFields(), index, doc, tookInNanos, reformat, maxSourceCharsToLog)
                 );
             } else if (indexInfoThreshold >= 0 && tookInNanos > indexInfoThreshold) {
                 indexLogger.info(
-                    IndexingSlowLogMessage.of(
-                        this.slowLogFieldProvider.indexSlowLogFields(),
-                        index,
-                        doc,
-                        tookInNanos,
-                        reformat,
-                        maxSourceCharsToLog
-                    )
+                    IndexingSlowLogMessage.of(this.slowLogFields.indexFields(), index, doc, tookInNanos, reformat, maxSourceCharsToLog)
                 );
             } else if (indexDebugThreshold >= 0 && tookInNanos > indexDebugThreshold) {
                 indexLogger.debug(
-                    IndexingSlowLogMessage.of(
-                        this.slowLogFieldProvider.indexSlowLogFields(),
-                        index,
-                        doc,
-                        tookInNanos,
-                        reformat,
-                        maxSourceCharsToLog
-                    )
+                    IndexingSlowLogMessage.of(this.slowLogFields.indexFields(), index, doc, tookInNanos, reformat, maxSourceCharsToLog)
                 );
             } else if (indexTraceThreshold >= 0 && tookInNanos > indexTraceThreshold) {
                 indexLogger.trace(
-                    IndexingSlowLogMessage.of(
-                        this.slowLogFieldProvider.indexSlowLogFields(),
-                        index,
-                        doc,
-                        tookInNanos,
-                        reformat,
-                        maxSourceCharsToLog
-                    )
+                    IndexingSlowLogMessage.of(this.slowLogFields.indexFields(), index, doc, tookInNanos, reformat, maxSourceCharsToLog)
                 );
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java b/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java
index e4836a391bfec..81e7cff862e32 100644
--- a/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java
+++ b/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java
@@ -45,7 +45,7 @@ public final class SearchSlowLog implements SearchOperationListener {
     private static final Logger queryLogger = LogManager.getLogger(INDEX_SEARCH_SLOWLOG_PREFIX + ".query");
     private static final Logger fetchLogger = LogManager.getLogger(INDEX_SEARCH_SLOWLOG_PREFIX + ".fetch");
 
-    private final SlowLogFieldProvider slowLogFieldProvider;
+    private final SlowLogFields slowLogFields;
 
     public static final Setting<Boolean> INDEX_SEARCH_SLOWLOG_INCLUDE_USER_SETTING = Setting.boolSetting(
         INDEX_SEARCH_SLOWLOG_PREFIX + ".include.user",
@@ -126,9 +126,8 @@ public final class SearchSlowLog implements SearchOperationListener {
 
     private static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("pretty", "false"));
 
-    public SearchSlowLog(IndexSettings indexSettings, SlowLogFieldProvider slowLogFieldProvider) {
-        slowLogFieldProvider.init(indexSettings);
-        this.slowLogFieldProvider = slowLogFieldProvider;
+    public SearchSlowLog(IndexSettings indexSettings, SlowLogFields slowLogFields) {
+        this.slowLogFields = slowLogFields;
         indexSettings.getScopedSettings()
             .addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING, this::setQueryWarnThreshold);
         this.queryWarnThreshold = indexSettings.getValue(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING).nanos();
@@ -159,26 +158,26 @@ public SearchSlowLog(IndexSettings indexSettings, SlowLogFieldProvider slowLogFi
     @Override
     public void onQueryPhase(SearchContext context, long tookInNanos) {
         if (queryWarnThreshold >= 0 && tookInNanos > queryWarnThreshold) {
-            queryLogger.warn(SearchSlowLogMessage.of(this.slowLogFieldProvider.searchSlowLogFields(), context, tookInNanos));
+            queryLogger.warn(SearchSlowLogMessage.of(this.slowLogFields.searchFields(), context, tookInNanos));
         } else if (queryInfoThreshold >= 0 && tookInNanos > queryInfoThreshold) {
-            queryLogger.info(SearchSlowLogMessage.of(this.slowLogFieldProvider.searchSlowLogFields(), context, tookInNanos));
+            queryLogger.info(SearchSlowLogMessage.of(this.slowLogFields.searchFields(), context, tookInNanos));
         } else if (queryDebugThreshold >= 0 && tookInNanos > queryDebugThreshold) {
-            queryLogger.debug(SearchSlowLogMessage.of(this.slowLogFieldProvider.searchSlowLogFields(), context, tookInNanos));
+            queryLogger.debug(SearchSlowLogMessage.of(this.slowLogFields.searchFields(), context, tookInNanos));
         } else if (queryTraceThreshold >= 0 && tookInNanos > queryTraceThreshold) {
-            queryLogger.trace(SearchSlowLogMessage.of(this.slowLogFieldProvider.searchSlowLogFields(), context, tookInNanos));
+            queryLogger.trace(SearchSlowLogMessage.of(this.slowLogFields.searchFields(), context, tookInNanos));
         }
     }
 
     @Override
     public void onFetchPhase(SearchContext context, long tookInNanos) {
         if (fetchWarnThreshold >= 0 && tookInNanos > fetchWarnThreshold) {
-            fetchLogger.warn(SearchSlowLogMessage.of(this.slowLogFieldProvider.searchSlowLogFields(), context, tookInNanos));
+            fetchLogger.warn(SearchSlowLogMessage.of(this.slowLogFields.searchFields(), context, tookInNanos));
         } else if (fetchInfoThreshold >= 0 && tookInNanos > fetchInfoThreshold) {
-            fetchLogger.info(SearchSlowLogMessage.of(this.slowLogFieldProvider.searchSlowLogFields(), context, tookInNanos));
+            fetchLogger.info(SearchSlowLogMessage.of(this.slowLogFields.searchFields(), context, tookInNanos));
         } else if (fetchDebugThreshold >= 0 && tookInNanos > fetchDebugThreshold) {
-            fetchLogger.debug(SearchSlowLogMessage.of(this.slowLogFieldProvider.searchSlowLogFields(), context, tookInNanos));
+            fetchLogger.debug(SearchSlowLogMessage.of(this.slowLogFields.searchFields(), context, tookInNanos));
         } else if (fetchTraceThreshold >= 0 && tookInNanos > fetchTraceThreshold) {
-            fetchLogger.trace(SearchSlowLogMessage.of(this.slowLogFieldProvider.searchSlowLogFields(), context, tookInNanos));
+            fetchLogger.trace(SearchSlowLogMessage.of(this.slowLogFields.searchFields(), context, tookInNanos));
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/SlowLogFieldProvider.java b/server/src/main/java/org/elasticsearch/index/SlowLogFieldProvider.java
index c61d4d4c85a86..e93edccc83b15 100644
--- a/server/src/main/java/org/elasticsearch/index/SlowLogFieldProvider.java
+++ b/server/src/main/java/org/elasticsearch/index/SlowLogFieldProvider.java
@@ -9,28 +9,14 @@
 
 package org.elasticsearch.index;
 
-import java.util.Map;
-
 /**
  * Interface for providing additional fields to the slow log from a plugin.
  * Intended to be loaded through SPI.
  */
 public interface SlowLogFieldProvider {
     /**
-     * Initialize field provider with index level settings to be able to listen for updates and set initial values
+     * Create a field provider with index level settings to be able to listen for updates and set initial values
      * @param indexSettings settings for the index
      */
-    void init(IndexSettings indexSettings);
-
-    /**
-     * Slow log fields for indexing events
-     * @return map of field name to value
-     */
-    Map<String, String> indexSlowLogFields();
-
-    /**
-     * Slow log fields for search events
-     * @return map of field name to value
-     */
-    Map<String, String> searchSlowLogFields();
+    SlowLogFields create(IndexSettings indexSettings);
 }
diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/Features.java b/server/src/main/java/org/elasticsearch/index/SlowLogFields.java
similarity index 50%
rename from plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/Features.java
rename to server/src/main/java/org/elasticsearch/index/SlowLogFields.java
index 51a2d2bbe1d40..e018e3a4d6bb7 100644
--- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/Features.java
+++ b/server/src/main/java/org/elasticsearch/index/SlowLogFields.java
@@ -7,21 +7,24 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.index.mapper.annotatedtext;
+package org.elasticsearch.index;
 
-import org.elasticsearch.features.FeatureSpecification;
-import org.elasticsearch.features.NodeFeature;
-
-import java.util.Set;
+import java.util.Map;
 
 /**
- * Provides features for annotated text mapper.
+ * Fields for the slow log. These may be different each call depending on the state of the system.
  */
-public class Features implements FeatureSpecification {
-    @Override
-    public Set<NodeFeature> getFeatures() {
-        return Set.of(
-            AnnotatedTextFieldMapper.SYNTHETIC_SOURCE_SUPPORT // Added in 8.15
-        );
-    }
+public interface SlowLogFields {
+
+    /**
+     * Slow log fields for indexing events
+     * @return map of field name to value
+     */
+    Map<String, String> indexFields();
+
+    /**
+     * Slow log fields for search events
+     * @return map of field name to value
+     */
+    Map<String, String> searchFields();
 }
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsWriter.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsWriter.java
index 02dda6a4a9da1..932925ea423ba 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsWriter.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsWriter.java
@@ -443,21 +443,27 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex(
         float cDotC
     ) throws IOException {
         long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES);
-        final IndexOutput tempQuantizedVectorData = segmentWriteState.directory.createTempOutput(
-            binarizedVectorData.getName(),
-            "temp",
-            segmentWriteState.context
-        );
-        final IndexOutput tempScoreQuantizedVectorData = segmentWriteState.directory.createTempOutput(
-            binarizedVectorData.getName(),
-            "score_temp",
-            segmentWriteState.context
-        );
         IndexInput binarizedDataInput = null;
         IndexInput binarizedScoreDataInput = null;
+        IndexOutput tempQuantizedVectorData = null;
+        IndexOutput tempScoreQuantizedVectorData = null;
         boolean success = false;
         OptimizedScalarQuantizer quantizer = new OptimizedScalarQuantizer(fieldInfo.getVectorSimilarityFunction());
         try {
+            // Since we are opening two files, it's possible that one or the other fails to open
+            // we open them within the try to ensure they are cleaned
+            tempQuantizedVectorData = segmentWriteState.directory.createTempOutput(
+                binarizedVectorData.getName(),
+                "temp",
+                segmentWriteState.context
+            );
+            tempScoreQuantizedVectorData = segmentWriteState.directory.createTempOutput(
+                binarizedVectorData.getName(),
+                "score_temp",
+                segmentWriteState.context
+            );
+            final String tempQuantizedVectorDataName = tempQuantizedVectorData.getName();
+            final String tempScoreQuantizedVectorDataName = tempScoreQuantizedVectorData.getName();
             FloatVectorValues floatVectorValues = KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState);
             if (fieldInfo.getVectorSimilarityFunction() == COSINE) {
                 floatVectorValues = new NormalizedFloatVectorValues(floatVectorValues);
@@ -516,8 +522,8 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex(
                 IOUtils.close(finalBinarizedDataInput, finalBinarizedScoreDataInput);
                 IOUtils.deleteFilesIgnoringExceptions(
                     segmentWriteState.directory,
-                    tempQuantizedVectorData.getName(),
-                    tempScoreQuantizedVectorData.getName()
+                    tempQuantizedVectorDataName,
+                    tempScoreQuantizedVectorDataName
                 );
             });
         } finally {
@@ -528,11 +534,12 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex(
                     binarizedDataInput,
                     binarizedScoreDataInput
                 );
-                IOUtils.deleteFilesIgnoringExceptions(
-                    segmentWriteState.directory,
-                    tempQuantizedVectorData.getName(),
-                    tempScoreQuantizedVectorData.getName()
-                );
+                if (tempQuantizedVectorData != null) {
+                    IOUtils.deleteFilesIgnoringExceptions(segmentWriteState.directory, tempQuantizedVectorData.getName());
+                }
+                if (tempScoreQuantizedVectorData != null) {
+                    IOUtils.deleteFilesIgnoringExceptions(segmentWriteState.directory, tempScoreQuantizedVectorData.getName());
+                }
             }
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 0a470e86ef856..8d3d1bde316ea 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -661,7 +661,8 @@ private Translog openTranslog(
             translogDeletionPolicy,
             globalCheckpointSupplier,
             engineConfig.getPrimaryTermSupplier(),
-            persistedSequenceNumberConsumer
+            persistedSequenceNumberConsumer,
+            TranslogOperationAsserter.withEngineConfig(engineConfig)
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java b/server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java
index f21a3c06ab015..08508103181ed 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java
@@ -77,12 +77,13 @@ public LuceneSyntheticSourceChangesSnapshot(
         IndexVersion indexVersionCreated
     ) throws IOException {
         super(engineSearcher, searchBatchSize, fromSeqNo, toSeqNo, requiredFullRange, accessStats, indexVersionCreated);
-        assert mappingLookup.isSourceSynthetic();
+        // a MapperService#updateMapping(...) of empty index may not have been invoked and then mappingLookup is empty
+        assert engineSearcher.getDirectoryReader().maxDoc() == 0 || mappingLookup.isSourceSynthetic()
+            : "either an empty index or synthetic source must be enabled for proper functionality.";
         // ensure we can buffer at least one document
         this.maxMemorySizeInBytes = maxMemorySizeInBytes > 0 ? maxMemorySizeInBytes : 1;
         this.sourceLoader = mappingLookup.newSourceLoader(null, SourceFieldMetrics.NOOP);
         Set<String> storedFields = sourceLoader.requiredStoredFields();
-        assert mappingLookup.isSourceSynthetic() : "synthetic source must be enabled for proper functionality.";
         this.storedFieldLoader = StoredFieldLoader.create(false, storedFields);
         this.lastSeenSeqNo = fromSeqNo - 1;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
index 49e0ae0587085..044e6f6712c77 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
@@ -166,7 +166,8 @@ public void trimUnreferencedTranslogFiles() {
                         translogDeletionPolicy,
                         engineConfig.getGlobalCheckpointSupplier(),
                         engineConfig.getPrimaryTermSupplier(),
-                        seqNo -> {}
+                        seqNo -> {},
+                        TranslogOperationAsserter.DEFAULT
                     )
                 ) {
                     translog.trimUnreferencedReaders();
diff --git a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
index 1d032c1f400ef..010fc1bd9e411 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
@@ -220,7 +220,7 @@ protected DirectoryReader open(IndexCommit commit) throws IOException {
         assert Transports.assertNotTransportThread("opening index commit of a read-only engine");
         DirectoryReader directoryReader = DirectoryReader.open(
             commit,
-            org.apache.lucene.util.Version.MIN_SUPPORTED_MAJOR,
+            IndexVersions.MINIMUM_READONLY_COMPATIBLE.luceneVersion().major,
             engineConfig.getLeafSorter()
         );
         if (lazilyLoadSoftDeletes) {
@@ -267,7 +267,8 @@ private static TranslogStats translogStats(final EngineConfig config, final Segm
                 translogDeletionPolicy,
                 config.getGlobalCheckpointSupplier(),
                 config.getPrimaryTermSupplier(),
-                seqNo -> {}
+                seqNo -> {},
+                TranslogOperationAsserter.DEFAULT
             )
         ) {
             return translog.stats();
@@ -575,7 +576,8 @@ public void advanceMaxSeqNoOfUpdatesOrDeletes(long maxSeqNoOfUpdatesOnPrimary) {
 
     protected DirectoryReader openDirectory(Directory directory) throws IOException {
         assert Transports.assertNotTransportThread("opening directory reader of a read-only engine");
-        final DirectoryReader reader = DirectoryReader.open(directory);
+        var commit = Lucene.getIndexCommit(Lucene.readSegmentInfos(directory), directory);
+        final DirectoryReader reader = DirectoryReader.open(commit, IndexVersions.MINIMUM_READONLY_COMPATIBLE.luceneVersion().major, null);
         if (lazilyLoadSoftDeletes) {
             return new LazySoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD);
         } else {
diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java
index 0f772b49bf92b..0928b4500e6da 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java
@@ -99,10 +99,6 @@ private static UnsupportedOperationException unsupported() {
         return new UnsupportedOperationException();
     }
 
-    public TranslogLeafReader getLeafReader() {
-        return leafReader;
-    }
-
     @Override
     protected DirectoryReader doOpenIfChanged() {
         throw unsupported();
@@ -143,6 +139,45 @@ public CacheHelper getReaderCacheHelper() {
         return leafReader.getReaderCacheHelper();
     }
 
+    static DirectoryReader createInMemoryReader(
+        ShardId shardId,
+        EngineConfig engineConfig,
+        Directory directory,
+        DocumentParser documentParser,
+        MappingLookup mappingLookup,
+        Translog.Index operation
+    ) {
+        final ParsedDocument parsedDocs = documentParser.parseDocument(
+            new SourceToParse(operation.id(), operation.source(), XContentHelper.xContentType(operation.source()), operation.routing()),
+            mappingLookup
+        );
+
+        parsedDocs.updateSeqID(operation.seqNo(), operation.primaryTerm());
+        parsedDocs.version().setLongValue(operation.version());
+        // To guarantee indexability, we configure the analyzer and codec using the main engine configuration
+        final IndexWriterConfig writeConfig = new IndexWriterConfig(engineConfig.getAnalyzer()).setOpenMode(
+            IndexWriterConfig.OpenMode.CREATE
+        ).setCodec(engineConfig.getCodec());
+        try (IndexWriter writer = new IndexWriter(directory, writeConfig)) {
+            writer.addDocument(parsedDocs.rootDoc());
+            final DirectoryReader reader = open(writer);
+            if (reader.leaves().size() != 1 || reader.leaves().get(0).reader().numDocs() != 1) {
+                reader.close();
+                throw new IllegalStateException(
+                    "Expected a single document segment; "
+                        + "but ["
+                        + reader.leaves().size()
+                        + " segments with "
+                        + reader.leaves().get(0).reader().numDocs()
+                        + " documents"
+                );
+            }
+            return reader;
+        } catch (IOException e) {
+            throw new EngineException(shardId, "failed to create an in-memory segment for get [" + operation.id() + "]", e);
+        }
+    }
+
     private static class TranslogLeafReader extends LeafReader {
 
         private static final FieldInfo FAKE_SOURCE_FIELD = new FieldInfo(
@@ -244,7 +279,8 @@ private LeafReader getDelegate() {
                     ensureOpen();
                     reader = delegate.get();
                     if (reader == null) {
-                        reader = createInMemoryLeafReader();
+                        var indexReader = createInMemoryReader(shardId, engineConfig, directory, documentParser, mappingLookup, operation);
+                        reader = indexReader.leaves().get(0).reader();
                         final LeafReader existing = delegate.getAndSet(reader);
                         assert existing == null;
                         onSegmentCreated.run();
@@ -254,39 +290,6 @@ private LeafReader getDelegate() {
             return reader;
         }
 
-        private LeafReader createInMemoryLeafReader() {
-            assert Thread.holdsLock(this);
-            final ParsedDocument parsedDocs = documentParser.parseDocument(
-                new SourceToParse(operation.id(), operation.source(), XContentHelper.xContentType(operation.source()), operation.routing()),
-                mappingLookup
-            );
-
-            parsedDocs.updateSeqID(operation.seqNo(), operation.primaryTerm());
-            parsedDocs.version().setLongValue(operation.version());
-            // To guarantee indexability, we configure the analyzer and codec using the main engine configuration
-            final IndexWriterConfig writeConfig = new IndexWriterConfig(engineConfig.getAnalyzer()).setOpenMode(
-                IndexWriterConfig.OpenMode.CREATE
-            ).setCodec(engineConfig.getCodec());
-            try (IndexWriter writer = new IndexWriter(directory, writeConfig)) {
-                writer.addDocument(parsedDocs.rootDoc());
-                final DirectoryReader reader = open(writer);
-                if (reader.leaves().size() != 1 || reader.leaves().get(0).reader().numDocs() != 1) {
-                    reader.close();
-                    throw new IllegalStateException(
-                        "Expected a single document segment; "
-                            + "but ["
-                            + reader.leaves().size()
-                            + " segments with "
-                            + reader.leaves().get(0).reader().numDocs()
-                            + " documents"
-                    );
-                }
-                return reader.leaves().get(0).reader();
-            } catch (IOException e) {
-                throw new EngineException(shardId, "failed to create an in-memory segment for get [" + operation.id() + "]", e);
-            }
-        }
-
         @Override
         public CacheHelper getCoreCacheHelper() {
             return getDelegate().getCoreCacheHelper();
@@ -440,7 +443,7 @@ private void readStoredFieldsDirectly(StoredFieldVisitor visitor) throws IOExcep
                 SourceFieldMapper mapper = mappingLookup.getMapping().getMetadataMapperByClass(SourceFieldMapper.class);
                 if (mapper != null) {
                     try {
-                        sourceBytes = mapper.applyFilters(sourceBytes, null);
+                        sourceBytes = mapper.applyFilters(mappingLookup, sourceBytes, null);
                     } catch (IOException e) {
                         throw new IOException("Failed to reapply filters after reading from translog", e);
                     }
diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogOperationAsserter.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogOperationAsserter.java
new file mode 100644
index 0000000000000..00de13b8e8d8e
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogOperationAsserter.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.engine;
+
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.elasticsearch.index.cache.query.TrivialQueryCachingPolicy;
+import org.elasticsearch.index.mapper.DocumentParser;
+import org.elasticsearch.index.mapper.MappingLookup;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.translog.Translog;
+
+import java.io.IOException;
+
+/**
+ *
+ * A utility class to assert that translog operations with the same sequence number
+ * in the same generation are either identical or equivalent when synthetic sources are used.
+ */
+public abstract class TranslogOperationAsserter {
+    public static final TranslogOperationAsserter DEFAULT = new TranslogOperationAsserter() {
+    };
+
+    private TranslogOperationAsserter() {
+
+    }
+
+    public static TranslogOperationAsserter withEngineConfig(EngineConfig engineConfig) {
+        return new TranslogOperationAsserter() {
+            @Override
+            public boolean assertSameIndexOperation(Translog.Index o1, Translog.Index o2) throws IOException {
+                if (super.assertSameIndexOperation(o1, o2)) {
+                    return true;
+                }
+                if (engineConfig.getIndexSettings().isRecoverySourceSyntheticEnabled()) {
+                    return super.assertSameIndexOperation(synthesizeSource(engineConfig, o1), o2)
+                        || super.assertSameIndexOperation(o1, synthesizeSource(engineConfig, o2));
+                }
+                return false;
+            }
+        };
+    }
+
+    static Translog.Index synthesizeSource(EngineConfig engineConfig, Translog.Index op) throws IOException {
+        final ShardId shardId = engineConfig.getShardId();
+        final MappingLookup mappingLookup = engineConfig.getMapperService().mappingLookup();
+        final DocumentParser documentParser = engineConfig.getMapperService().documentParser();
+        try (
+            var directory = new ByteBuffersDirectory();
+            var reader = TranslogDirectoryReader.createInMemoryReader(shardId, engineConfig, directory, documentParser, mappingLookup, op)
+        ) {
+            final Engine.Searcher searcher = new Engine.Searcher(
+                "assert_translog",
+                reader,
+                new BM25Similarity(),
+                null,
+                TrivialQueryCachingPolicy.NEVER,
+                () -> {}
+            );
+            try (
+                LuceneSyntheticSourceChangesSnapshot snapshot = new LuceneSyntheticSourceChangesSnapshot(
+                    mappingLookup,
+                    searcher,
+                    LuceneSyntheticSourceChangesSnapshot.DEFAULT_BATCH_SIZE,
+                    Integer.MAX_VALUE,
+                    op.seqNo(),
+                    op.seqNo(),
+                    true,
+                    false,
+                    engineConfig.getIndexSettings().getIndexVersionCreated()
+                )
+            ) {
+                final Translog.Operation normalized = snapshot.next();
+                assert normalized != null : "expected one operation; got zero";
+                return (Translog.Index) normalized;
+            }
+        }
+    }
+
+    public boolean assertSameIndexOperation(Translog.Index o1, Translog.Index o2) throws IOException {
+        return Translog.Index.equalsWithoutAutoGeneratedTimestamp(o1, o2);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/get/GetResult.java b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
index 3c504d400c7c6..49e0ccffa17cd 100644
--- a/server/src/main/java/org/elasticsearch/index/get/GetResult.java
+++ b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.mapper.IgnoredFieldMapper;
 import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
@@ -42,7 +41,6 @@
 import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
 
 public class GetResult implements Writeable, Iterable<DocumentField>, ToXContentObject {
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(GetResult.class);
 
     public static final String _INDEX = "_index";
     public static final String _ID = "_id";
diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java
index 43b5d2c7d3f78..9ed9fc6dabf3d 100644
--- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java
+++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java
@@ -9,7 +9,9 @@
 
 package org.elasticsearch.index.get;
 
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.search.IndexSearcher;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.document.DocumentField;
@@ -26,6 +28,7 @@
 import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader;
 import org.elasticsearch.index.fieldvisitor.StoredFieldLoader;
 import org.elasticsearch.index.mapper.IgnoredFieldMapper;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperMetrics;
@@ -39,6 +42,7 @@
 import org.elasticsearch.index.shard.MultiEngineGet;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.lookup.SourceFilter;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -190,9 +194,13 @@ public GetResult getFromTranslog(
     }
 
     public GetResult getForUpdate(String id, long ifSeqNo, long ifPrimaryTerm) throws IOException {
+        return getForUpdate(id, ifSeqNo, ifPrimaryTerm, new String[] { RoutingFieldMapper.NAME });
+    }
+
+    public GetResult getForUpdate(String id, long ifSeqNo, long ifPrimaryTerm, String[] gFields) throws IOException {
         return get(
             id,
-            new String[] { RoutingFieldMapper.NAME },
+            gFields,
             true,
             Versions.MATCH_ANY,
             VersionType.INTERNAL,
@@ -288,11 +296,18 @@ private GetResult innerGetFetch(
         boolean forceSyntheticSource
     ) throws IOException {
         assert get.exists() : "method should only be called if document could be retrieved";
-
         // check first if stored fields to be loaded don't contain an object field
         MappingLookup mappingLookup = mapperService.mappingLookup();
+        final IndexVersion indexVersion = indexSettings.getIndexVersionCreated();
+        final Set<String> storedFieldSet = new HashSet<>();
+        boolean hasInferenceMetadataFields = false;
         if (storedFields != null) {
             for (String field : storedFields) {
+                if (field.equals(InferenceMetadataFieldsMapper.NAME)
+                    && InferenceMetadataFieldsMapper.isEnabled(indexShard.mapperService().mappingLookup())) {
+                    hasInferenceMetadataFields = true;
+                    continue;
+                }
                 Mapper fieldMapper = mappingLookup.getMapper(field);
                 if (fieldMapper == null) {
                     if (mappingLookup.objectMappers().get(field) != null) {
@@ -300,6 +315,7 @@ private GetResult innerGetFetch(
                         throw new IllegalArgumentException("field [" + field + "] isn't a leaf field");
                     }
                 }
+                storedFieldSet.add(field);
             }
         }
 
@@ -313,8 +329,8 @@ private GetResult innerGetFetch(
                 () -> mappingLookup.getMapping().syntheticFieldLoader(sourceFilter),
                 mapperMetrics.sourceFieldMetrics()
             )
-            : mappingLookup.newSourceLoader(fetchSourceContext.filter(), mapperMetrics.sourceFieldMetrics());
-        StoredFieldLoader storedFieldLoader = buildStoredFieldLoader(storedFields, fetchSourceContext, loader);
+            : mappingLookup.newSourceLoader(sourceFilter, mapperMetrics.sourceFieldMetrics());
+        StoredFieldLoader storedFieldLoader = buildStoredFieldLoader(storedFieldSet, fetchSourceContext, loader);
         LeafStoredFieldLoader leafStoredFieldLoader = storedFieldLoader.getLoader(docIdAndVersion.reader.getContext(), null);
         try {
             leafStoredFieldLoader.advanceTo(docIdAndVersion.docId);
@@ -323,7 +339,6 @@ private GetResult innerGetFetch(
         }
 
         // put stored fields into result objects
-        final IndexVersion indexVersion = indexSettings.getIndexVersionCreated();
         if (leafStoredFieldLoader.storedFields().isEmpty() == false) {
             Set<String> needed = new HashSet<>();
             if (storedFields != null) {
@@ -372,6 +387,19 @@ private GetResult innerGetFetch(
         if (mapperService.mappingLookup().isSourceEnabled() && fetchSourceContext.fetchSource()) {
             Source source = loader.leaf(docIdAndVersion.reader, new int[] { docIdAndVersion.docId })
                 .source(leafStoredFieldLoader, docIdAndVersion.docId);
+
+            SourceFilter filter = fetchSourceContext.filter();
+            if (filter != null) {
+                source = source.filter(filter);
+            }
+
+            if (hasInferenceMetadataFields) {
+                /**
+                 * Adds the {@link InferenceMetadataFieldsMapper#NAME} field from the document fields
+                 * to the original _source if it has been requested.
+                 */
+                source = addInferenceMetadataFields(mapperService, docIdAndVersion.reader.getContext(), docIdAndVersion.docId, source);
+            }
             sourceBytes = source.internalSourceRef();
         }
 
@@ -404,18 +432,38 @@ private static DocumentField loadIgnoredMetadataField(final DocIdAndVersion docI
         return new DocumentField(IgnoredFieldMapper.NAME, ignoredValues);
     }
 
-    private static StoredFieldLoader buildStoredFieldLoader(String[] fields, FetchSourceContext fetchSourceContext, SourceLoader loader) {
-        Set<String> fieldsToLoad = new HashSet<>();
-        if (fields != null && fields.length > 0) {
-            Collections.addAll(fieldsToLoad, fields);
+    private static Source addInferenceMetadataFields(MapperService mapperService, LeafReaderContext readerContext, int docId, Source source)
+        throws IOException {
+        var mappingLookup = mapperService.mappingLookup();
+        var inferenceMetadata = (InferenceMetadataFieldsMapper) mappingLookup.getMapping()
+            .getMetadataMapperByName(InferenceMetadataFieldsMapper.NAME);
+        if (inferenceMetadata == null || mapperService.mappingLookup().inferenceFields().isEmpty()) {
+            return source;
         }
+        var inferenceLoader = inferenceMetadata.fieldType()
+            .valueFetcher(mappingLookup, mapperService.getBitSetProducer(), new IndexSearcher(readerContext.reader()));
+        inferenceLoader.setNextReader(readerContext);
+        List<Object> values = inferenceLoader.fetchValues(source, docId, List.of());
+        if (values.size() == 1) {
+            var newSource = source.source();
+            newSource.put(InferenceMetadataFieldsMapper.NAME, values.get(0));
+            return Source.fromMap(newSource, source.sourceContentType());
+        }
+        return source;
+    }
+
+    private static StoredFieldLoader buildStoredFieldLoader(
+        Set<String> fields,
+        FetchSourceContext fetchSourceContext,
+        SourceLoader loader
+    ) {
         if (fetchSourceContext.fetchSource()) {
-            fieldsToLoad.addAll(loader.requiredStoredFields());
+            fields.addAll(loader.requiredStoredFields());
         } else {
-            if (fieldsToLoad.isEmpty()) {
+            if (fields.isEmpty()) {
                 return StoredFieldLoader.empty();
             }
         }
-        return StoredFieldLoader.create(fetchSourceContext.fetchSource(), fieldsToLoad);
+        return StoredFieldLoader.create(fetchSourceContext.fetchSource(), fields);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java
index 02a3ae11524e3..318e877c7ebb9 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java
@@ -8,9 +8,18 @@
  */
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.geo.Orientation;
+import org.elasticsearch.geometry.Rectangle;
+import org.elasticsearch.geometry.utils.WellKnownBinary;
+import org.elasticsearch.lucene.spatial.CoordinateEncoder;
+import org.elasticsearch.lucene.spatial.GeometryDocValueReader;
 
+import java.io.IOException;
+import java.nio.ByteOrder;
 import java.util.Map;
 import java.util.function.Function;
 
@@ -69,6 +78,82 @@ protected Object nullValueAsSource(T nullValue) {
             // we don't support null value fors shapes
             return nullValue;
         }
+
+        @Override
+        public BlockLoader blockLoader(BlockLoaderContext blContext) {
+            return blContext.fieldExtractPreference() == FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS && isBoundsExtractionSupported()
+                ? new BoundsBlockLoader(name(), coordinateEncoder())
+                : blockLoaderFromSource(blContext);
+        }
+
+        protected abstract boolean isBoundsExtractionSupported();
+
+        protected abstract CoordinateEncoder coordinateEncoder();
+
+        // Visible for testing
+        static class BoundsBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader {
+            private final String fieldName;
+            private final CoordinateEncoder encoder;
+
+            BoundsBlockLoader(String fieldName, CoordinateEncoder encoder) {
+                this.fieldName = fieldName;
+                this.encoder = encoder;
+            }
+
+            @Override
+            public BlockLoader.AllReader reader(LeafReaderContext context) throws IOException {
+                return new BlockLoader.AllReader() {
+                    @Override
+                    public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs) throws IOException {
+                        var binaryDocValues = context.reader().getBinaryDocValues(fieldName);
+                        var reader = new GeometryDocValueReader();
+                        try (var builder = factory.bytesRefs(docs.count())) {
+                            for (int i = 0; i < docs.count(); i++) {
+                                read(binaryDocValues, docs.get(i), reader, builder);
+                            }
+                            return builder.build();
+                        }
+                    }
+
+                    @Override
+                    public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException {
+                        var binaryDocValues = context.reader().getBinaryDocValues(fieldName);
+                        var reader = new GeometryDocValueReader();
+                        read(binaryDocValues, docId, reader, (BytesRefBuilder) builder);
+                    }
+
+                    private void read(BinaryDocValues binaryDocValues, int doc, GeometryDocValueReader reader, BytesRefBuilder builder)
+                        throws IOException {
+                        if (binaryDocValues.advanceExact(doc) == false) {
+                            builder.appendNull();
+                            return;
+                        }
+                        reader.reset(binaryDocValues.binaryValue());
+                        var extent = reader.getExtent();
+                        // This is rather silly: an extent is already encoded as ints, but we convert it to Rectangle to
+                        // preserve its properties as a WKB shape, only to convert it back to ints when we compute the
+                        // aggregation. An obvious optimization would be to avoid this back-and-forth conversion.
+                        var rectangle = new Rectangle(
+                            encoder.decodeX(extent.minX()),
+                            encoder.decodeX(extent.maxX()),
+                            encoder.decodeY(extent.maxY()),
+                            encoder.decodeY(extent.minY())
+                        );
+                        builder.appendBytesRef(new BytesRef(WellKnownBinary.toWKB(rectangle, ByteOrder.LITTLE_ENDIAN)));
+                    }
+
+                    @Override
+                    public boolean canReuse(int startingDocID) {
+                        return true;
+                    }
+                };
+            }
+
+            @Override
+            public BlockLoader.Builder builder(BlockLoader.BlockFactory factory, int expectedCount) {
+                return factory.bytesRefs(expectedCount);
+            }
+        }
     }
 
     protected Explicit<Boolean> coerce;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
index f74d58093a7f5..77de1654cf4ba 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.FieldDataContext;
@@ -62,8 +61,6 @@ public class BooleanFieldMapper extends FieldMapper {
 
     public static final String CONTENT_TYPE = "boolean";
 
-    static final NodeFeature BOOLEAN_DIMENSION = new NodeFeature("mapper.boolean_dimension");
-
     public static class Values {
         public static final BytesRef TRUE = new BytesRef("T");
         public static final BytesRef FALSE = new BytesRef("F");
@@ -182,11 +179,8 @@ private FieldValues<Boolean> scriptValues() {
         }
     }
 
-    private static final IndexVersion MINIMUM_COMPATIBILITY_VERSION = IndexVersion.fromId(5000099);
-
-    public static final TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.scriptCompiler(), IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated()),
-        MINIMUM_COMPATIBILITY_VERSION
+    public static final TypeParser PARSER = createTypeParserWithLegacySupport(
+        (n, c) -> new Builder(n, c.scriptCompiler(), IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated())
     );
 
     public static final class BooleanFieldType extends TermBasedFieldType {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
index bb229c795a83e..f0c679d4f4994 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
@@ -18,10 +18,11 @@
 import org.apache.lucene.search.suggest.document.RegexCompletionQuery;
 import org.apache.lucene.search.suggest.document.SuggestField;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.query.SearchExecutionContext;
@@ -156,8 +157,6 @@ public static class Builder extends FieldMapper.Builder {
         private final NamedAnalyzer defaultAnalyzer;
         private final IndexVersion indexVersionCreated;
 
-        private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(Builder.class);
-
         /**
          * @param name of the completion field to build
          */
@@ -210,11 +209,29 @@ public CompletionFieldMapper build(MapperBuilderContext context) {
 
         private void checkCompletionContextsLimit() {
             if (this.contexts.getValue() != null && this.contexts.getValue().size() > COMPLETION_CONTEXTS_LIMIT) {
-                throw new IllegalArgumentException(
-                    "Limit of completion field contexts [" + COMPLETION_CONTEXTS_LIMIT + "] has been exceeded"
-                );
+                if (indexVersionCreated.onOrAfter(IndexVersions.V_8_0_0)) {
+                    throw new IllegalArgumentException(
+                        "Limit of completion field contexts [" + COMPLETION_CONTEXTS_LIMIT + "] has been exceeded"
+                    );
+                } else {
+                    deprecationLogger.warn(
+                        DeprecationCategory.MAPPINGS,
+                        "excessive_completion_contexts",
+                        "You have defined more than ["
+                            + COMPLETION_CONTEXTS_LIMIT
+                            + "] completion contexts"
+                            + " in the mapping for field ["
+                            + leafName()
+                            + "]. "
+                            + "The maximum allowed number of completion contexts in a mapping will be limited to "
+                            + "["
+                            + COMPLETION_CONTEXTS_LIMIT
+                            + "] starting in version [8.0]."
+                    );
+                }
             }
         }
+
     }
 
     public static final Set<String> ALLOWED_CONTENT_FIELD_NAMES = Set.of(
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
index 87e4ce5f90479..fb17f1c7ebaf8 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
@@ -25,6 +25,8 @@
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.geo.ShapeRelation;
+import org.elasticsearch.common.logging.DeprecationCategory;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.DateFormatters;
@@ -34,6 +36,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
@@ -76,6 +79,7 @@
 /** A {@link FieldMapper} for dates. */
 public final class DateFieldMapper extends FieldMapper {
 
+    private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(DateFieldMapper.class);
     private static final Logger logger = LogManager.getLogger(DateFieldMapper.class);
 
     public static final String CONTENT_TYPE = "date";
@@ -342,7 +346,20 @@ private Long parseNullValue(DateFieldType fieldType) {
             try {
                 return fieldType.parse(nullValue.getValue());
             } catch (Exception e) {
-                throw new MapperParsingException("Error parsing [null_value] on field [" + leafName() + "]: " + e.getMessage(), e);
+                if (indexCreatedVersion.onOrAfter(IndexVersions.V_8_0_0)) {
+                    throw new MapperParsingException("Error parsing [null_value] on field [" + leafName() + "]: " + e.getMessage(), e);
+                } else {
+                    DEPRECATION_LOGGER.warn(
+                        DeprecationCategory.MAPPINGS,
+                        "date_mapper_null_field",
+                        "Error parsing ["
+                            + nullValue.getValue()
+                            + "] as date in [null_value] on field ["
+                            + leafName()
+                            + "]); [null_value] will be ignored"
+                    );
+                    return null;
+                }
             }
         }
 
@@ -381,9 +398,7 @@ public DateFieldMapper build(MapperBuilderContext context) {
         }
     }
 
-    private static final IndexVersion MINIMUM_COMPATIBILITY_VERSION = IndexVersion.fromId(5000099);
-
-    public static final TypeParser MILLIS_PARSER = new TypeParser((n, c) -> {
+    public static final TypeParser MILLIS_PARSER = createTypeParserWithLegacySupport((n, c) -> {
         boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
         return new Builder(
             n,
@@ -393,9 +408,9 @@ public DateFieldMapper build(MapperBuilderContext context) {
             ignoreMalformedByDefault,
             c.indexVersionCreated()
         );
-    }, MINIMUM_COMPATIBILITY_VERSION);
+    });
 
-    public static final TypeParser NANOS_PARSER = new TypeParser((n, c) -> {
+    public static final TypeParser NANOS_PARSER = createTypeParserWithLegacySupport((n, c) -> {
         boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
         return new Builder(
             n,
@@ -405,7 +420,7 @@ public DateFieldMapper build(MapperBuilderContext context) {
             ignoreMalformedByDefault,
             c.indexVersionCreated()
         );
-    }, MINIMUM_COMPATIBILITY_VERSION);
+    });
 
     public static final class DateFieldType extends MappedFieldType {
         final DateFormatter dateTimeFormatter;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
index a99fa3f93679b..cf0c355a22e65 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
@@ -12,7 +12,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexSortConfig;
 import org.elasticsearch.index.IndexVersion;
@@ -21,8 +21,6 @@
 import java.util.List;
 
 public class DocumentMapper {
-    static final NodeFeature INDEX_SORTING_ON_NESTED = new NodeFeature("mapper.index_sorting_on_nested");
-
     private final String type;
     private final CompressedXContent mappingSource;
     private final MappingLookup mappingLookup;
@@ -177,18 +175,16 @@ public void validate(IndexSettings settings, boolean checkLimits) {
         }
         List<String> routingPaths = settings.getIndexMetadata().getRoutingPaths();
         for (String path : routingPaths) {
-            for (String match : mappingLookup.getMatchingFieldNames(path)) {
-                mappingLookup.getFieldType(match).validateMatchedRoutingPath(path);
+            if (settings.getMode() == IndexMode.TIME_SERIES) {
+                for (String match : mappingLookup.getMatchingFieldNames(path)) {
+                    mappingLookup.getFieldType(match).validateMatchedRoutingPath(path);
+                }
             }
             for (String objectName : mappingLookup.objectMappers().keySet()) {
                 // object type is not allowed in the routing paths
                 if (path.equals(objectName)) {
                     throw new IllegalArgumentException(
-                        "All fields that match routing_path must be configured with [time_series_dimension: true] "
-                            + "or flattened fields with a list of dimensions in [time_series_dimensions] "
-                            + "and without the [script] parameter. ["
-                            + objectName
-                            + "] was [object]."
+                        "All fields that match routing_path must be flattened fields. [" + objectName + "] was [object]."
                     );
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
index 7238127571fed..5bbecbf117dba 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
@@ -57,7 +57,6 @@
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
-import java.util.stream.Stream;
 
 import static org.elasticsearch.core.Strings.format;
 
@@ -444,7 +443,11 @@ protected void doXContentBody(XContentBuilder builder, Params params) throws IOE
 
     @Override
     public int getTotalFieldsCount() {
-        return 1 + Stream.of(builderParams.multiFields.mappers).mapToInt(FieldMapper::getTotalFieldsCount).sum();
+        int sum = 1;
+        for (FieldMapper mapper : builderParams.multiFields.mappers) {
+            sum += mapper.getTotalFieldsCount();
+        }
+        return sum;
     }
 
     public Map<String, NamedAnalyzer> indexAnalyzers() {
@@ -1618,6 +1621,12 @@ public static BiConsumer<String, MappingParserContext> notFromDynamicTemplates(S
         };
     }
 
+    private static final IndexVersion MINIMUM_LEGACY_COMPATIBILITY_VERSION = IndexVersion.fromId(5000099);
+
+    public static TypeParser createTypeParserWithLegacySupport(BiFunction<String, MappingParserContext, Builder> builderFunction) {
+        return new TypeParser(builderFunction, MINIMUM_LEGACY_COMPATIBILITY_VERSION);
+    }
+
     /**
      * TypeParser implementation that automatically handles parsing
      */
@@ -1632,14 +1641,14 @@ public static final class TypeParser implements Mapper.TypeParser {
          * @param builderFunction a function that produces a Builder from a name and parsercontext
          */
         public TypeParser(BiFunction<String, MappingParserContext, Builder> builderFunction) {
-            this(builderFunction, (n, c) -> {}, IndexVersions.MINIMUM_COMPATIBLE);
+            this(builderFunction, (n, c) -> {}, IndexVersions.MINIMUM_READONLY_COMPATIBLE);
         }
 
         /**
-         * Variant of {@link #TypeParser(BiFunction)} that allows to defining a minimumCompatibilityVersion to
+         * Variant of {@link #TypeParser(BiFunction)} that allows to define a minimumCompatibilityVersion to
          * allow parsing mapping definitions of legacy indices (see {@link Mapper.TypeParser#supportsVersion(IndexVersion)}).
          */
-        public TypeParser(BiFunction<String, MappingParserContext, Builder> builderFunction, IndexVersion minimumCompatibilityVersion) {
+        private TypeParser(BiFunction<String, MappingParserContext, Builder> builderFunction, IndexVersion minimumCompatibilityVersion) {
             this(builderFunction, (n, c) -> {}, minimumCompatibilityVersion);
         }
 
@@ -1647,14 +1656,14 @@ public TypeParser(
             BiFunction<String, MappingParserContext, Builder> builderFunction,
             BiConsumer<String, MappingParserContext> contextValidator
         ) {
-            this(builderFunction, contextValidator, IndexVersions.MINIMUM_COMPATIBLE);
+            this(builderFunction, contextValidator, IndexVersions.MINIMUM_READONLY_COMPATIBLE);
         }
 
         public TypeParser(
             BiFunction<String, MappingParserContext, Builder> builderFunction,
             List<BiConsumer<String, MappingParserContext>> contextValidator
         ) {
-            this(builderFunction, (n, c) -> contextValidator.forEach(v -> v.accept(n, c)), IndexVersions.MINIMUM_COMPATIBLE);
+            this(builderFunction, (n, c) -> contextValidator.forEach(v -> v.accept(n, c)), IndexVersions.MINIMUM_READONLY_COMPATIBLE);
         }
 
         private TypeParser(
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java
index 161e1ba84aa40..4ebcbbf8a068b 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java
@@ -228,17 +228,14 @@ public FieldMapper build(MapperBuilderContext context) {
 
     }
 
-    private static final IndexVersion MINIMUM_COMPATIBILITY_VERSION = IndexVersion.fromId(5000099);
-
-    public static TypeParser PARSER = new TypeParser(
+    public static final TypeParser PARSER = createTypeParserWithLegacySupport(
         (n, c) -> new Builder(
             n,
             c.scriptCompiler(),
             IGNORE_MALFORMED_SETTING.get(c.getSettings()),
             c.indexVersionCreated(),
             c.getIndexSettings().getMode()
-        ),
-        MINIMUM_COMPATIBILITY_VERSION
+        )
     );
 
     private final Builder builder;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java
index 7e2bebfd403cb..bdb3d97d4c187 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java
@@ -56,7 +56,6 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper {
 
     public static final TypeParser PARSER = new FixedTypeParser(context -> new IgnoredSourceFieldMapper(context.getIndexSettings()));
 
-    static final NodeFeature TRACK_IGNORED_SOURCE = new NodeFeature("mapper.track_ignored_source");
     static final NodeFeature DONT_EXPAND_DOTS_IN_IGNORED_SOURCE = new NodeFeature("mapper.ignored_source.dont_expand_dots");
     static final NodeFeature IGNORED_SOURCE_AS_TOP_LEVEL_METADATA_ARRAY_FIELD = new NodeFeature(
         "mapper.ignored_source_as_top_level_metadata_array_field"
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java
index 6dbd89ba43d63..9708753926e1d 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java
@@ -13,7 +13,6 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.regex.Regex;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -31,8 +30,6 @@
 
 public class IndexModeFieldMapper extends MetadataFieldMapper {
 
-    static final NodeFeature QUERYING_INDEX_MODE = new NodeFeature("mapper.query_index_mode");
-
     public static final String NAME = "_index_mode";
 
     public static final String CONTENT_TYPE = "_index_mode";
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/InferenceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/InferenceFieldMapper.java
index 249ef5004e59c..f7c6eef7dfd49 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/InferenceFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/InferenceFieldMapper.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
 import org.elasticsearch.inference.InferenceService;
 
-import java.util.Map;
 import java.util.Set;
 
 /**
@@ -26,12 +25,4 @@ public interface InferenceFieldMapper {
      * @param sourcePaths The source path that populates the input for the field (before inference)
      */
     InferenceFieldMetadata getMetadata(Set<String> sourcePaths);
-
-    /**
-     * Get the field's original value (i.e. the value the user specified) from the provided source.
-     *
-     * @param sourceAsMap The source as a map
-     * @return The field's original value, or {@code null} if none was provided
-     */
-    Object getOriginalValue(Map<String, Object> sourceAsMap);
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/InferenceMetadataFieldsMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/InferenceMetadataFieldsMapper.java
new file mode 100644
index 0000000000000..80fee58e93110
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/mapper/InferenceMetadataFieldsMapper.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.BitSetProducer;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexVersions;
+import org.elasticsearch.index.query.SearchExecutionContext;
+
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * An abstract {@link MetadataFieldMapper} used as a placeholder for implementation
+ * in the inference module. It is required by {@link SourceFieldMapper} to identify
+ * the field name for removal from _source.
+ */
+public abstract class InferenceMetadataFieldsMapper extends MetadataFieldMapper {
+    /**
+     * Internal index setting to control the format used for semantic text fields.
+     * Determines whether to use the legacy format (default: true).
+     * This setting is immutable and can only be defined at index creation
+     * to ensure the internal format remains consistent throughout the index's lifecycle.
+     */
+    public static final Setting<Boolean> USE_LEGACY_SEMANTIC_TEXT_FORMAT = Setting.boolSetting(
+        "index.mapping.semantic_text.use_legacy_format",
+        // don't use the new format by default yet
+        true,
+        Setting.Property.Final,
+        Setting.Property.IndexScope,
+        Setting.Property.InternalIndex
+    );
+
+    public static final String NAME = "_inference_fields";
+    public static final String CONTENT_TYPE = "_inference_fields";
+
+    protected InferenceMetadataFieldsMapper(MappedFieldType inferenceFieldType) {
+        super(inferenceFieldType);
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+
+    @Override
+    public InferenceMetadataFieldType fieldType() {
+        return (InferenceMetadataFieldType) super.fieldType();
+    }
+
+    public abstract static class InferenceMetadataFieldType extends MappedFieldType {
+        public InferenceMetadataFieldType() {
+            super(NAME, false, false, false, TextSearchInfo.NONE, Map.of());
+        }
+
+        /**
+         * Returns a {@link ValueFetcher} without requiring the construction of a full {@link SearchExecutionContext}.
+         */
+        public abstract ValueFetcher valueFetcher(
+            MappingLookup mappingLookup,
+            Function<Query, BitSetProducer> bitSetCache,
+            IndexSearcher searcher
+        );
+    }
+
+    /**
+     * Checks if the {@link InferenceMetadataFieldsMapper} is enabled for the given {@link Settings}.
+     *
+     * This indicates whether the new format for semantic text fields is active.
+     * The new format is enabled if:
+     * 1. The index version is on or after {@link IndexVersions#INFERENCE_METADATA_FIELDS}, and
+     * 2. The legacy semantic text format is disabled.
+     *
+     * @param settings the index settings to evaluate
+     * @return {@code true} if the new format is enabled; {@code false} otherwise
+     */
+    public static boolean isEnabled(Settings settings) {
+        var version = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings);
+        if (version.before(IndexVersions.INFERENCE_METADATA_FIELDS)
+            && version.between(IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0) == false) {
+            return false;
+        }
+        return USE_LEGACY_SEMANTIC_TEXT_FORMAT.get(settings) == false;
+    }
+
+    /**
+     * Checks if the {@link InferenceMetadataFieldsMapper} is enabled based on the provided {@link Mapping}.
+     *
+     * This indicates whether the new format for semantic text fields is active by verifying the existence
+     * of the {@link InferenceMetadataFieldsMapper} in the mapping's metadata.
+     *
+     * @param mappingLookup the mapping to evaluate
+     * @return {@code true} if the {@link InferenceMetadataFieldsMapper} is present; {@code false} otherwise
+     */
+    public static boolean isEnabled(MappingLookup mappingLookup) {
+        return mappingLookup != null
+            && mappingLookup.getMapping()
+                .getMetadataMapperByName(InferenceMetadataFieldsMapper.NAME) instanceof InferenceMetadataFieldsMapper;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
index 09f44f139d8bc..0204a2b10013a 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
@@ -23,11 +23,14 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.automaton.CompiledAutomaton;
+import org.elasticsearch.common.logging.DeprecationCategory;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
@@ -59,6 +62,8 @@
  */
 public class IpFieldMapper extends FieldMapper {
 
+    private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(IpFieldMapper.class);
+
     public static final String CONTENT_TYPE = "ip";
 
     private static IpFieldMapper toType(FieldMapper in) {
@@ -129,7 +134,20 @@ private InetAddress parseNullValue() {
             try {
                 return InetAddresses.forString(nullValueAsString);
             } catch (Exception e) {
-                throw new MapperParsingException("Error parsing [null_value] on field [" + leafName() + "]: " + e.getMessage(), e);
+                if (indexCreatedVersion.onOrAfter(IndexVersions.V_8_0_0)) {
+                    throw new MapperParsingException("Error parsing [null_value] on field [" + leafName() + "]: " + e.getMessage(), e);
+                } else {
+                    DEPRECATION_LOGGER.warn(
+                        DeprecationCategory.MAPPINGS,
+                        "ip_mapper_null_field",
+                        "Error parsing ["
+                            + nullValue.getValue()
+                            + "] as IP in [null_value] on field ["
+                            + leafName()
+                            + "]); [null_value] will be ignored"
+                    );
+                    return null;
+                }
             }
         }
 
@@ -186,12 +204,10 @@ public IpFieldMapper build(MapperBuilderContext context) {
 
     }
 
-    private static final IndexVersion MINIMUM_COMPATIBILITY_VERSION = IndexVersion.fromId(5000099);
-
-    public static final TypeParser PARSER = new TypeParser((n, c) -> {
+    public static final TypeParser PARSER = createTypeParserWithLegacySupport((n, c) -> {
         boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
         return new Builder(n, c.scriptCompiler(), ignoreMalformedByDefault, c.indexVersionCreated());
-    }, MINIMUM_COMPATIBILITY_VERSION);
+    });
 
     public static final class IpFieldType extends SimpleMappedFieldType {
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
index 32aa422b18bcc..bdcf9bf98279f 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
@@ -38,7 +38,6 @@
 import org.elasticsearch.common.lucene.search.AutomatonQueries;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
@@ -89,9 +88,6 @@ public final class KeywordFieldMapper extends FieldMapper {
 
     public static final String CONTENT_TYPE = "keyword";
 
-    static final NodeFeature KEYWORD_DIMENSION_IGNORE_ABOVE = new NodeFeature("mapper.keyword_dimension_ignore_above");
-    static final NodeFeature KEYWORD_NORMALIZER_SYNTHETIC_SOURCE = new NodeFeature("mapper.keyword_normalizer_synthetic_source");
-
     public static class Defaults {
         public static final FieldType FIELD_TYPE;
 
@@ -104,7 +100,7 @@ public static class Defaults {
             FIELD_TYPE = freezeAndDeduplicateFieldType(ft);
         }
 
-        public static TextSearchInfo TEXT_SEARCH_INFO = new TextSearchInfo(
+        public static final TextSearchInfo TEXT_SEARCH_INFO = new TextSearchInfo(
             FIELD_TYPE,
             null,
             Lucene.KEYWORD_ANALYZER,
@@ -385,9 +381,7 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
         }
     }
 
-    private static final IndexVersion MINIMUM_COMPATIBILITY_VERSION = IndexVersion.fromId(5000099);
-
-    public static final TypeParser PARSER = new TypeParser(Builder::new, MINIMUM_COMPATIBILITY_VERSION);
+    public static final TypeParser PARSER = createTypeParserWithLegacySupport(Builder::new);
 
     public static final class KeywordFieldType extends StringFieldType {
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
index 35722be20b9be..4b68e20673572 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
@@ -677,10 +677,15 @@ public enum FieldExtractPreference {
          * Load the field from doc-values into a BlockLoader supporting doc-values.
          */
         DOC_VALUES,
+        /**
+         *  Loads the field by extracting the extent from the binary encoded representation
+         */
+        EXTRACT_SPATIAL_BOUNDS,
         /**
          * No preference. Leave the choice of where to load the field from up to the FieldType.
          */
-        NONE
+        NONE;
+
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java
index 6bc63bdbcceaf..bafa74b662f00 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.util.StringLiteralDeduplicator;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
@@ -30,8 +29,6 @@
 
 public abstract class Mapper implements ToXContentFragment, Iterable<Mapper> {
 
-    public static final NodeFeature SYNTHETIC_SOURCE_KEEP_FEATURE = new NodeFeature("mapper.synthetic_source_keep");
-
     public static final String SYNTHETIC_SOURCE_KEEP_PARAM = "synthetic_source_keep";
 
     // Only relevant for synthetic source mode.
@@ -107,9 +104,8 @@ public abstract static class Builder {
 
         private String leafName;
 
-        @SuppressWarnings("this-escape")
         protected Builder(String leafName) {
-            setLeafName(leafName);
+            this.leafName = leafName;
         }
 
         public final String leafName() {
@@ -120,7 +116,7 @@ public final String leafName() {
         public abstract Mapper build(MapperBuilderContext context);
 
         void setLeafName(String leafName) {
-            this.leafName = internFieldName(leafName);
+            this.leafName = leafName;
         }
     }
 
@@ -131,7 +127,7 @@ public interface TypeParser {
          * Whether we can parse this type on indices with the given index created version.
          */
         default boolean supportsVersion(IndexVersion indexCreatedVersion) {
-            return indexCreatedVersion.onOrAfter(IndexVersions.MINIMUM_COMPATIBLE);
+            return indexCreatedVersion.onOrAfter(IndexVersions.MINIMUM_READONLY_COMPATIBLE);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
index 276d3e151361c..8e669a91fd9ea 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
@@ -11,9 +11,6 @@
 
 import org.elasticsearch.features.FeatureSpecification;
 import org.elasticsearch.features.NodeFeature;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper;
-import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 
 import java.util.Set;
 
@@ -28,33 +25,7 @@ public class MapperFeatures implements FeatureSpecification {
 
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(
-            BWC_WORKAROUND_9_0,
-            IgnoredSourceFieldMapper.TRACK_IGNORED_SOURCE,
-            PassThroughObjectMapper.PASS_THROUGH_PRIORITY,
-            RangeFieldMapper.NULL_VALUES_OFF_BY_ONE_FIX,
-            SourceFieldMapper.SYNTHETIC_SOURCE_FALLBACK,
-            DenseVectorFieldMapper.INT4_QUANTIZATION,
-            DenseVectorFieldMapper.BIT_VECTORS,
-            DocumentMapper.INDEX_SORTING_ON_NESTED,
-            KeywordFieldMapper.KEYWORD_DIMENSION_IGNORE_ABOVE,
-            IndexModeFieldMapper.QUERYING_INDEX_MODE,
-            NodeMappingStats.SEGMENT_LEVEL_FIELDS_STATS,
-            BooleanFieldMapper.BOOLEAN_DIMENSION,
-            ObjectMapper.SUBOBJECTS_AUTO,
-            ObjectMapper.SUBOBJECTS_AUTO_FIXES,
-            KeywordFieldMapper.KEYWORD_NORMALIZER_SYNTHETIC_SOURCE,
-            SourceFieldMapper.SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX,
-            Mapper.SYNTHETIC_SOURCE_KEEP_FEATURE,
-            SourceFieldMapper.SYNTHETIC_SOURCE_WITH_COPY_TO_AND_DOC_VALUES_FALSE_SUPPORT,
-            SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_FIX,
-            FlattenedFieldMapper.IGNORE_ABOVE_SUPPORT,
-            IndexSettings.IGNORE_ABOVE_INDEX_LEVEL_SETTING,
-            SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX,
-            TimeSeriesRoutingHashFieldMapper.TS_ROUTING_HASH_FIELD_PARSES_BYTES_REF,
-            FlattenedFieldMapper.IGNORE_ABOVE_WITH_ARRAYS_SUPPORT,
-            DenseVectorFieldMapper.BBQ_FORMAT
-        );
+        return Set.of(BWC_WORKAROUND_9_0);
     }
 
     public static final NodeFeature CONSTANT_KEYWORD_SYNTHETIC_SOURCE_WRITE_FIX = new NodeFeature(
@@ -77,7 +48,8 @@ public Set<NodeFeature> getTestFeatures() {
             DocumentParser.FIX_PARSING_SUBOBJECTS_FALSE_DYNAMIC_FALSE,
             CONSTANT_KEYWORD_SYNTHETIC_SOURCE_WRITE_FIX,
             META_FETCH_FIELDS_ERROR_CODE_CHANGED,
-            SPARSE_VECTOR_STORE_SUPPORT
+            SPARSE_VECTOR_STORE_SUPPORT,
+            SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE
         );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperMetrics.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperMetrics.java
index fa87dd449dba8..d107957179d6e 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperMetrics.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperMetrics.java
@@ -14,5 +14,5 @@
  * Main purpose of this class is to avoid verbosity of passing individual metric instances around.
  */
 public record MapperMetrics(SourceFieldMetrics sourceFieldMetrics) {
-    public static MapperMetrics NOOP = new MapperMetrics(SourceFieldMetrics.NOOP);
+    public static final MapperMetrics NOOP = new MapperMetrics(SourceFieldMetrics.NOOP);
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java
index ea94576f5c536..44f7def74ec0e 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java
@@ -57,9 +57,13 @@ public MapperRegistry(
      */
     public Mapper.TypeParser getMapperParser(String type, IndexVersion indexVersionCreated) {
         Mapper.TypeParser parser = mapperParsers.get(type);
-        if (indexVersionCreated.isLegacyIndexVersion() && (parser == null || parser.supportsVersion(indexVersionCreated) == false)) {
-            return PlaceHolderFieldMapper.PARSER.apply(type);
+        if (indexVersionCreated.isLegacyIndexVersion()) {
+            if (parser == null || parser.supportsVersion(indexVersionCreated) == false) {
+                return PlaceHolderFieldMapper.PARSER.apply(type);
+            }
+            return parser;
         } else {
+            assert parser == null || parser.supportsVersion(indexVersionCreated);
             return parser;
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index 1673b1719d8bf..fb4f86c3cba98 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -171,6 +171,7 @@ public boolean isAutoUpdate() {
     private final IndexVersion indexVersionCreated;
     private final MapperRegistry mapperRegistry;
     private final Supplier<MappingParserContext> mappingParserContextSupplier;
+    private final Function<Query, BitSetProducer> bitSetProducer;
     private final MapperMetrics mapperMetrics;
 
     private volatile DocumentMapper mapper;
@@ -245,6 +246,7 @@ public MapperService(
             this::getMetadataMappers,
             this::resolveDocumentType
         );
+        this.bitSetProducer = bitSetProducer;
         this.mapperMetrics = mapperMetrics;
     }
 
@@ -826,6 +828,10 @@ public MapperRegistry getMapperRegistry() {
         return mapperRegistry;
     }
 
+    public Function<Query, BitSetProducer> getBitSetProducer() {
+        return bitSetProducer;
+    }
+
     public MapperMetrics getMapperMetrics() {
         return mapperMetrics;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java
index 1278ebf0a393a..7ec175799ddec 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java
@@ -81,7 +81,7 @@ public CompressedXContent toCompressedXContent() {
     /**
      * Returns the root object for the current mapping
      */
-    RootObjectMapper getRoot() {
+    public RootObjectMapper getRoot() {
         return root;
     }
 
@@ -106,7 +106,7 @@ public <T extends MetadataFieldMapper> T getMetadataMapperByClass(Class<T> clazz
         return (T) metadataMappersMap.get(clazz);
     }
 
-    MetadataFieldMapper getMetadataMapperByName(String mapperName) {
+    public MetadataFieldMapper getMetadataMapperByName(String mapperName) {
         return metadataMappersByName.get(mapperName);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java
index 31aa787c3f758..033742b3b57fc 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java
@@ -10,13 +10,17 @@
 package org.elasticsearch.index.mapper;
 
 import org.elasticsearch.common.Explicit;
+import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 
 /**
@@ -132,6 +136,8 @@ public final MetadataFieldMapper build(MapperBuilderContext context) {
             return build();
         }
 
+        private static final Set<String> UNSUPPORTED_PARAMETERS_8_6_0 = Set.of("type", "fields", "copy_to", "boost");
+
         public final void parseMetadataField(String name, MappingParserContext parserContext, Map<String, Object> fieldNode) {
             final Parameter<?>[] params = getParameters();
             Map<String, Parameter<?>> paramsMap = Maps.newHashMapWithExpectedSize(params.length);
@@ -144,6 +150,22 @@ public final void parseMetadataField(String name, MappingParserContext parserCon
                 final Object propNode = entry.getValue();
                 Parameter<?> parameter = paramsMap.get(propName);
                 if (parameter == null) {
+                    IndexVersion indexVersionCreated = parserContext.indexVersionCreated();
+                    if (indexVersionCreated.before(IndexVersions.UPGRADE_TO_LUCENE_10_0_0)
+                        && UNSUPPORTED_PARAMETERS_8_6_0.contains(propName)) {
+                        if (indexVersionCreated.onOrAfter(IndexVersions.V_8_6_0)) {
+                            // silently ignore type, and a few other parameters: sadly we've been doing this for a long time
+                            deprecationLogger.warn(
+                                DeprecationCategory.API,
+                                propName,
+                                "Parameter [{}] has no effect on metadata field [{}] and will be removed in future",
+                                propName,
+                                name
+                            );
+                        }
+                        iterator.remove();
+                        continue;
+                    }
                     throw new MapperParsingException("unknown parameter [" + propName + "] on metadata field [" + name + "]");
                 }
                 parameter.parse(name, parserContext, propNode);
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java
index 03818f7b5c83f..08b0fb32a10c5 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java
@@ -245,6 +245,10 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) {
         this.indexSettings = indexSettings;
     }
 
+    public IndexSettings indexSettings() {
+        return indexSettings;
+    }
+
     public Query parentTypeFilter() {
         return parentTypeFilter;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java b/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java
index 10b0856540399..0987c6dfb8c8b 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -28,8 +27,6 @@
  */
 public class NodeMappingStats implements Writeable, ToXContentFragment {
 
-    public static final NodeFeature SEGMENT_LEVEL_FIELDS_STATS = new NodeFeature("mapper.segment_level_fields_stats");
-
     private static final class Fields {
         static final String MAPPINGS = "mappings";
         static final String TOTAL_COUNT = "total_count";
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
index 8c21dfea31b9a..1b8fb3dfc101c 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
@@ -89,8 +89,6 @@ private static NumberFieldMapper toType(FieldMapper in) {
         return (NumberFieldMapper) in;
     }
 
-    private static final IndexVersion MINIMUM_COMPATIBILITY_VERSION = IndexVersion.fromId(5000099);
-
     public static final class Builder extends FieldMapper.DimensionBuilder {
 
         private final Parameter<Boolean> indexed;
@@ -1378,16 +1376,8 @@ private boolean isOutOfRange(Object value) {
         NumberType(String name, NumericType numericType) {
             this.name = name;
             this.numericType = numericType;
-            this.parser = new TypeParser(
-                (n, c) -> new Builder(
-                    n,
-                    this,
-                    c.scriptCompiler(),
-                    c.getSettings(),
-                    c.indexVersionCreated(),
-                    c.getIndexSettings().getMode()
-                ),
-                MINIMUM_COMPATIBILITY_VERSION
+            this.parser = createTypeParserWithLegacySupport(
+                (n, c) -> new Builder(n, this, c.scriptCompiler(), c.getSettings(), c.indexVersionCreated(), c.getIndexSettings().getMode())
             );
         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
index 46b70193ba0e8..86ce4fbb74837 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
@@ -52,9 +51,6 @@ public class ObjectMapper extends Mapper {
 
     public static final String CONTENT_TYPE = "object";
     static final String STORE_ARRAY_SOURCE_PARAM = "store_array_source";
-    static final NodeFeature SUBOBJECTS_AUTO = new NodeFeature("mapper.subobjects_auto");
-    // No-op. All uses of this feature were reverted but node features can't be removed.
-    static final NodeFeature SUBOBJECTS_AUTO_FIXES = new NodeFeature("mapper.subobjects_auto_fixes");
 
     /**
      * Enhances the previously boolean option for subobjects support with an intermediate mode `auto` that uses
@@ -263,7 +259,11 @@ public ObjectMapper build(MapperBuilderContext context) {
 
     @Override
     public int getTotalFieldsCount() {
-        return 1 + mappers.values().stream().mapToInt(Mapper::getTotalFieldsCount).sum();
+        int sum = 1;
+        for (Mapper mapper : mappers.values()) {
+            sum += mapper.getTotalFieldsCount();
+        }
+        return sum;
     }
 
     public static class TypeParser implements Mapper.TypeParser {
@@ -855,7 +855,7 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
 
     }
 
-    ObjectMapper findParentMapper(String leafFieldPath) {
+    public ObjectMapper findParentMapper(String leafFieldPath) {
         var pathComponents = leafFieldPath.split("\\.");
         int startPathComponent = 0;
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java
index decc6d40a2f8e..fbf8dd4538037 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java
@@ -10,7 +10,6 @@
 package org.elasticsearch.index.mapper;
 
 import org.elasticsearch.common.Explicit;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -39,8 +38,6 @@ public class PassThroughObjectMapper extends ObjectMapper {
     public static final String CONTENT_TYPE = "passthrough";
     public static final String PRIORITY_PARAM_NAME = "priority";
 
-    static final NodeFeature PASS_THROUGH_PRIORITY = new NodeFeature("mapper.pass_through_priority");
-
     public static class Builder extends ObjectMapper.Builder {
 
         // Controls whether subfields are configured as time-series dimensions.
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
index 6ca30304201b2..461ad74a9434d 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
@@ -51,7 +51,6 @@
 
 /** A {@link FieldMapper} for indexing numeric and date ranges, and creating queries */
 public class RangeFieldMapper extends FieldMapper {
-    public static final NodeFeature NULL_VALUES_OFF_BY_ONE_FIX = new NodeFeature("mapper.range.null_values_off_by_one_fix");
     public static final NodeFeature DATE_RANGE_INDEXING_FIX = new NodeFeature("mapper.range.date_range_indexing_fix");
 
     public static final boolean DEFAULT_INCLUDE_UPPER = true;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
index ce983e8a327c9..2fe82b4eacfc5 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
@@ -538,6 +538,6 @@ private static boolean processField(
 
     @Override
     public int getTotalFieldsCount() {
-        return mappers.values().stream().mapToInt(Mapper::getTotalFieldsCount).sum() + runtimeFields.size();
+        return super.getTotalFieldsCount() - 1 + runtimeFields.size();
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
index 85f4217811a84..44c0193206efc 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.core.Nullable;
@@ -41,21 +40,11 @@
 import java.util.Locale;
 
 public class SourceFieldMapper extends MetadataFieldMapper {
-    public static final NodeFeature SYNTHETIC_SOURCE_FALLBACK = new NodeFeature("mapper.source.synthetic_source_fallback");
-    public static final NodeFeature SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX = new NodeFeature(
-        "mapper.source.synthetic_source_stored_fields_advance_fix"
-    );
-    public static final NodeFeature SYNTHETIC_SOURCE_WITH_COPY_TO_AND_DOC_VALUES_FALSE_SUPPORT = new NodeFeature(
-        "mapper.source.synthetic_source_with_copy_to_and_doc_values_false"
-    );
-    public static final NodeFeature SYNTHETIC_SOURCE_COPY_TO_FIX = new NodeFeature("mapper.source.synthetic_source_copy_to_fix");
-    public static final NodeFeature SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX = new NodeFeature(
-        "mapper.source.synthetic_source_copy_to_inside_objects_fix"
-    );
     public static final NodeFeature REMOVE_SYNTHETIC_SOURCE_ONLY_VALIDATION = new NodeFeature(
         "mapper.source.remove_synthetic_source_only_validation"
     );
     public static final NodeFeature SOURCE_MODE_FROM_INDEX_SETTING = new NodeFeature("mapper.source.mode_from_index_setting");
+    public static final NodeFeature SYNTHETIC_RECOVERY_SOURCE = new NodeFeature("mapper.synthetic_recovery_source");
 
     public static final String NAME = "_source";
     public static final String RECOVERY_SOURCE_NAME = "_recovery_source";
@@ -66,11 +55,6 @@ public class SourceFieldMapper extends MetadataFieldMapper {
 
     public static final String LOSSY_PARAMETERS_ALLOWED_SETTING_NAME = "index.lossy.source-mapping-parameters";
 
-    public static final Setting<Mode> INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting(SourceFieldMapper.Mode.class, settings -> {
-        final IndexMode indexMode = IndexSettings.MODE.get(settings);
-        return indexMode.defaultSourceMode().name();
-    }, "index.mapping.source.mode", value -> {}, Setting.Property.Final, Setting.Property.IndexScope);
-
     public static final String DEPRECATION_WARNING = "Configuring source mode in mappings is deprecated and will be removed "
         + "in future versions. Use [index.mapping.source.mode] index setting instead.";
 
@@ -260,8 +244,8 @@ public SourceFieldMapper build() {
         private Mode resolveSourceMode() {
             // If the `index.mapping.source.mode` exists it takes precedence to determine the source mode for `_source`
             // otherwise the mode is determined according to `_source.mode`.
-            if (INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings)) {
-                return INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings);
+            if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings)) {
+                return IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings);
             }
 
             // If `_source.mode` is not set we need to apply a default according to index mode.
@@ -293,7 +277,7 @@ private static SourceFieldMapper resolveStaticInstance(final Mode sourceMode) {
             return DEFAULT;
         }
 
-        final Mode settingSourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(c.getSettings());
+        final Mode settingSourceMode = IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(c.getSettings());
         // Needed for bwc so that "mode" is not serialized in case of standard index with stored source.
         if (indexMode == IndexMode.STANDARD && settingSourceMode == Mode.STORED) {
             return DEFAULT;
@@ -404,7 +388,7 @@ public boolean isComplete() {
     public void preParse(DocumentParserContext context) throws IOException {
         BytesReference originalSource = context.sourceToParse().source();
         XContentType contentType = context.sourceToParse().getXContentType();
-        final BytesReference adaptedSource = applyFilters(originalSource, contentType);
+        final BytesReference adaptedSource = applyFilters(context.mappingLookup(), originalSource, contentType);
 
         if (adaptedSource != null) {
             final BytesRef ref = adaptedSource.toBytesRef();
@@ -432,13 +416,32 @@ public void preParse(DocumentParserContext context) throws IOException {
     }
 
     @Nullable
-    public BytesReference applyFilters(@Nullable BytesReference originalSource, @Nullable XContentType contentType) throws IOException {
-        if (stored() == false) {
+    public BytesReference applyFilters(
+        @Nullable MappingLookup mappingLookup,
+        @Nullable BytesReference originalSource,
+        @Nullable XContentType contentType
+    ) throws IOException {
+        if (stored() == false || originalSource == null) {
             return null;
         }
-        if (originalSource != null && sourceFilter != null) {
+        var modSourceFilter = sourceFilter;
+        if (mappingLookup != null
+            && InferenceMetadataFieldsMapper.isEnabled(mappingLookup)
+            && mappingLookup.inferenceFields().isEmpty() == false) {
+            /**
+             * Removes {@link InferenceMetadataFieldsMapper} content from _source.
+             * This content is re-generated at query time (if requested) using stored fields and doc values.
+             */
+            String[] modExcludes = new String[excludes != null ? excludes.length + 1 : 1];
+            if (excludes != null) {
+                System.arraycopy(excludes, 0, modExcludes, 0, excludes.length);
+            }
+            modExcludes[modExcludes.length - 1] = InferenceMetadataFieldsMapper.NAME;
+            modSourceFilter = new SourceFilter(includes, modExcludes);
+        }
+        if (modSourceFilter != null) {
             // Percolate and tv APIs may not set the source and that is ok, because these APIs will not index any data
-            return Source.fromBytes(originalSource, contentType).filter(sourceFilter).internalSourceRef();
+            return Source.fromBytes(originalSource, contentType).filter(modSourceFilter).internalSourceRef();
         } else {
             return originalSource;
         }
@@ -459,11 +462,11 @@ public boolean isSynthetic() {
     }
 
     public static boolean isSynthetic(IndexSettings indexSettings) {
-        return INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC;
+        return IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC;
     }
 
     public static boolean isStored(IndexSettings indexSettings) {
-        return INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == Mode.STORED;
+        return IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == Mode.STORED;
     }
 
     public boolean isDisabled() {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
index cf75f1ddf3b94..b7323f2c0e3e4 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
@@ -480,11 +480,8 @@ public TextFieldMapper build(MapperBuilderContext context) {
         }
     }
 
-    private static final IndexVersion MINIMUM_COMPATIBILITY_VERSION = IndexVersion.fromId(5000099);
-
-    public static final TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), SourceFieldMapper.isSynthetic(c.getIndexSettings())),
-        MINIMUM_COMPATIBILITY_VERSION
+    public static final TypeParser PARSER = createTypeParserWithLegacySupport(
+        (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), SourceFieldMapper.isSynthetic(c.getIndexSettings()))
     );
 
     private static class PhraseWrappedAnalyzer extends AnalyzerWrapper {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java
index 6336ed831b78d..4a6ba5d1fa800 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.ByteUtils;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.fielddata.FieldData;
@@ -47,9 +46,8 @@ public class TimeSeriesRoutingHashFieldMapper extends MetadataFieldMapper {
     public static final TimeSeriesRoutingHashFieldMapper INSTANCE = new TimeSeriesRoutingHashFieldMapper();
 
     public static final TypeParser PARSER = new FixedTypeParser(c -> c.getIndexSettings().getMode().timeSeriesRoutingHashFieldMapper());
-    static final NodeFeature TS_ROUTING_HASH_FIELD_PARSES_BYTES_REF = new NodeFeature("tsdb.ts_routing_hash_doc_value_parse_byte_ref");
 
-    public static DocValueFormat TS_ROUTING_HASH_DOC_VALUE_FORMAT = TimeSeriesRoutingHashFieldType.DOC_VALUE_FORMAT;
+    public static final DocValueFormat TS_ROUTING_HASH_DOC_VALUE_FORMAT = TimeSeriesRoutingHashFieldType.DOC_VALUE_FORMAT;
 
     static final class TimeSeriesRoutingHashFieldType extends MappedFieldType {
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java
index 93a2157b2338a..7ef12f6dd30d2 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java
@@ -38,7 +38,6 @@
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.FieldDataContext;
@@ -52,6 +51,7 @@
 import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.DynamicFieldType;
 import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
@@ -111,9 +111,6 @@
  */
 public final class FlattenedFieldMapper extends FieldMapper {
 
-    public static final NodeFeature IGNORE_ABOVE_SUPPORT = new NodeFeature("flattened.ignore_above_support");
-    public static final NodeFeature IGNORE_ABOVE_WITH_ARRAYS_SUPPORT = new NodeFeature("mapper.flattened.ignore_above_with_arrays_support");
-
     public static final String CONTENT_TYPE = "flattened";
     public static final String KEYED_FIELD_SUFFIX = "._keyed";
     public static final String KEYED_IGNORED_VALUES_FIELD_SUFFIX = "._keyed._ignored";
@@ -670,7 +667,7 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme
         private final boolean isDimension;
         private final int ignoreAbove;
 
-        public RootFlattenedFieldType(
+        RootFlattenedFieldType(
             String name,
             boolean indexed,
             boolean hasDocValues,
@@ -682,7 +679,7 @@ public RootFlattenedFieldType(
             this(name, indexed, hasDocValues, meta, splitQueriesOnWhitespace, eagerGlobalOrdinals, Collections.emptyList(), ignoreAbove);
         }
 
-        public RootFlattenedFieldType(
+        RootFlattenedFieldType(
             String name,
             boolean indexed,
             boolean hasDocValues,
@@ -806,6 +803,10 @@ public MappedFieldType getChildFieldType(String childPath) {
             return new KeyedFlattenedFieldType(name(), childPath, this);
         }
 
+        public MappedFieldType getKeyedFieldType() {
+            return new KeywordFieldMapper.KeywordFieldType(name() + KEYED_FIELD_SUFFIX);
+        }
+
         @Override
         public boolean isDimension() {
             return isDimension;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java
index 950fef95772fb..53f68fb6edeef 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java
@@ -55,7 +55,7 @@
  * }`
  *
  */
-class FlattenedFieldSyntheticWriterHelper {
+public class FlattenedFieldSyntheticWriterHelper {
 
     private record Prefix(List<String> prefix) {
 
@@ -225,17 +225,17 @@ public boolean equals(Object obj) {
         }
     }
 
-    interface SortedKeyedValues {
+    public interface SortedKeyedValues {
         BytesRef next() throws IOException;
     }
 
     private final SortedKeyedValues sortedKeyedValues;
 
-    FlattenedFieldSyntheticWriterHelper(final SortedKeyedValues sortedKeyedValues) {
+    public FlattenedFieldSyntheticWriterHelper(final SortedKeyedValues sortedKeyedValues) {
         this.sortedKeyedValues = sortedKeyedValues;
     }
 
-    void write(final XContentBuilder b) throws IOException {
+    public void write(final XContentBuilder b) throws IOException {
         KeyValue curr = new KeyValue(sortedKeyedValues.next());
         KeyValue prev = KeyValue.EMPTY;
         final List<String> values = new ArrayList<>();
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
index 8c6e874ff577f..5edff48577efc 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
@@ -38,7 +38,6 @@
 import org.apache.lucene.util.VectorUtil;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.codec.vectors.ES813FlatVectorFormat;
@@ -102,16 +101,12 @@
 public class DenseVectorFieldMapper extends FieldMapper {
     public static final String COSINE_MAGNITUDE_FIELD_SUFFIX = "._magnitude";
     private static final float EPS = 1e-3f;
-    static final int BBQ_MIN_DIMS = 64;
+    public static final int BBQ_MIN_DIMS = 64;
 
     public static boolean isNotUnitVector(float magnitude) {
         return Math.abs(magnitude - 1.0f) > EPS;
     }
 
-    public static final NodeFeature INT4_QUANTIZATION = new NodeFeature("mapper.vectors.int4_quantization");
-    public static final NodeFeature BIT_VECTORS = new NodeFeature("mapper.vectors.bit_vectors");
-    public static final NodeFeature BBQ_FORMAT = new NodeFeature("mapper.vectors.bbq");
-
     public static final IndexVersion MAGNITUDE_STORED_INDEX_VERSION = IndexVersions.V_7_5_0;
     public static final IndexVersion INDEXED_BY_DEFAULT_INDEX_VERSION = IndexVersions.FIRST_DETACHED_INDEX_VERSION;
     public static final IndexVersion NORMALIZE_COSINE = IndexVersions.NORMALIZED_VECTOR_COSINE;
@@ -119,12 +114,13 @@ public static boolean isNotUnitVector(float magnitude) {
     public static final IndexVersion LITTLE_ENDIAN_FLOAT_STORED_INDEX_VERSION = IndexVersions.V_8_9_0;
 
     public static final String CONTENT_TYPE = "dense_vector";
-    public static short MAX_DIMS_COUNT = 4096; // maximum allowed number of dimensions
-    public static int MAX_DIMS_COUNT_BIT = 4096 * Byte.SIZE; // maximum allowed number of dimensions
+    public static final short MAX_DIMS_COUNT = 4096; // maximum allowed number of dimensions
+    public static final int MAX_DIMS_COUNT_BIT = 4096 * Byte.SIZE; // maximum allowed number of dimensions
 
-    public static short MIN_DIMS_FOR_DYNAMIC_FLOAT_MAPPING = 128; // minimum number of dims for floats to be dynamically mapped to vector
+    public static final short MIN_DIMS_FOR_DYNAMIC_FLOAT_MAPPING = 128; // minimum number of dims for floats to be dynamically mapped to
+                                                                        // vector
     public static final int MAGNITUDE_BYTES = 4;
-    public static final int NUM_CANDS_OVERSAMPLE_LIMIT = 10_000; // Max oversample allowed for k and num_candidates
+    public static final int OVERSAMPLE_LIMIT = 10_000; // Max oversample allowed
 
     private static DenseVectorFieldMapper toType(FieldMapper in) {
         return (DenseVectorFieldMapper) in;
@@ -486,8 +482,12 @@ private VectorData parseHexEncodedVector(
             }
 
             @Override
-            VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity)
-                throws IOException {
+            public VectorData parseKnnVector(
+                DocumentParserContext context,
+                int dims,
+                IntBooleanConsumer dimChecker,
+                VectorSimilarity similarity
+            ) throws IOException {
                 XContentParser.Token token = context.parser().currentToken();
                 return switch (token) {
                     case START_ARRAY -> parseVectorArray(context, dims, dimChecker, similarity);
@@ -517,17 +517,17 @@ public void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFie
             }
 
             @Override
-            int getNumBytes(int dimensions) {
+            public int getNumBytes(int dimensions) {
                 return dimensions;
             }
 
             @Override
-            ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) {
+            public ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) {
                 return ByteBuffer.wrap(new byte[numBytes]);
             }
 
             @Override
-            int parseDimensionCount(DocumentParserContext context) throws IOException {
+            public int parseDimensionCount(DocumentParserContext context) throws IOException {
                 XContentParser.Token currentToken = context.parser().currentToken();
                 return switch (currentToken) {
                     case START_ARRAY -> {
@@ -691,8 +691,12 @@ && isNotUnitVector(squaredMagnitude)) {
             }
 
             @Override
-            VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity)
-                throws IOException {
+            public VectorData parseKnnVector(
+                DocumentParserContext context,
+                int dims,
+                IntBooleanConsumer dimChecker,
+                VectorSimilarity similarity
+            ) throws IOException {
                 int index = 0;
                 float squaredMagnitude = 0;
                 float[] vector = new float[dims];
@@ -711,12 +715,12 @@ VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanCon
             }
 
             @Override
-            int getNumBytes(int dimensions) {
+            public int getNumBytes(int dimensions) {
                 return dimensions * Float.BYTES;
             }
 
             @Override
-            ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) {
+            public ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) {
                 return indexVersion.onOrAfter(LITTLE_ENDIAN_FLOAT_STORED_INDEX_VERSION)
                     ? ByteBuffer.wrap(new byte[numBytes]).order(ByteOrder.LITTLE_ENDIAN)
                     : ByteBuffer.wrap(new byte[numBytes]);
@@ -889,8 +893,12 @@ private VectorData parseHexEncodedVector(DocumentParserContext context, IntBoole
             }
 
             @Override
-            VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity)
-                throws IOException {
+            public VectorData parseKnnVector(
+                DocumentParserContext context,
+                int dims,
+                IntBooleanConsumer dimChecker,
+                VectorSimilarity similarity
+            ) throws IOException {
                 XContentParser.Token token = context.parser().currentToken();
                 return switch (token) {
                     case START_ARRAY -> parseVectorArray(context, dims, dimChecker, similarity);
@@ -920,18 +928,18 @@ public void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFie
             }
 
             @Override
-            int getNumBytes(int dimensions) {
+            public int getNumBytes(int dimensions) {
                 assert dimensions % Byte.SIZE == 0;
                 return dimensions / Byte.SIZE;
             }
 
             @Override
-            ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) {
+            public ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) {
                 return ByteBuffer.wrap(new byte[numBytes]);
             }
 
             @Override
-            int parseDimensionCount(DocumentParserContext context) throws IOException {
+            public int parseDimensionCount(DocumentParserContext context) throws IOException {
                 XContentParser.Token currentToken = context.parser().currentToken();
                 return switch (currentToken) {
                     case START_ARRAY -> {
@@ -974,16 +982,16 @@ public void checkDimensions(Integer dvDims, int qvDims) {
 
         abstract void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException;
 
-        abstract VectorData parseKnnVector(
+        public abstract VectorData parseKnnVector(
             DocumentParserContext context,
             int dims,
             IntBooleanConsumer dimChecker,
             VectorSimilarity similarity
         ) throws IOException;
 
-        abstract int getNumBytes(int dimensions);
+        public abstract int getNumBytes(int dimensions);
 
-        abstract ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes);
+        public abstract ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes);
 
         public abstract void checkVectorBounds(float[] vector);
 
@@ -1001,7 +1009,7 @@ public void checkDimensions(Integer dvDims, int qvDims) {
             }
         }
 
-        int parseDimensionCount(DocumentParserContext context) throws IOException {
+        public int parseDimensionCount(DocumentParserContext context) throws IOException {
             int index = 0;
             for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
                 index++;
@@ -1088,7 +1096,7 @@ public static ElementType fromString(String name) {
         }
     }
 
-    static final Map<String, ElementType> namesToElementType = Map.of(
+    public static final Map<String, ElementType> namesToElementType = Map.of(
         ElementType.BYTE.toString(),
         ElementType.BYTE,
         ElementType.FLOAT.toString(),
@@ -2006,9 +2014,9 @@ && isNotUnitVector(squaredMagnitude)) {
 
         public Query createKnnQuery(
             VectorData queryVector,
-            Integer k,
+            int k,
             int numCands,
-            Float numCandsFactor,
+            Float oversample,
             Query filter,
             Float similarityThreshold,
             BitSetProducer parentFilter
@@ -2024,7 +2032,7 @@ public Query createKnnQuery(
                     queryVector.asFloatVector(),
                     k,
                     numCands,
-                    numCandsFactor,
+                    oversample,
                     filter,
                     similarityThreshold,
                     parentFilter
@@ -2034,12 +2042,16 @@ public Query createKnnQuery(
         }
 
         private boolean needsRescore(Float rescoreOversample) {
-            return rescoreOversample != null && (indexOptions != null && indexOptions.type != null && indexOptions.type.isQuantized());
+            return rescoreOversample != null && isQuantized();
+        }
+
+        private boolean isQuantized() {
+            return indexOptions != null && indexOptions.type != null && indexOptions.type.isQuantized();
         }
 
         private Query createKnnBitQuery(
             byte[] queryVector,
-            Integer k,
+            int k,
             int numCands,
             Query filter,
             Float similarityThreshold,
@@ -2061,7 +2073,7 @@ private Query createKnnBitQuery(
 
         private Query createKnnByteQuery(
             byte[] queryVector,
-            Integer k,
+            int k,
             int numCands,
             Query filter,
             Float similarityThreshold,
@@ -2088,9 +2100,9 @@ private Query createKnnByteQuery(
 
         private Query createKnnFloatQuery(
             float[] queryVector,
-            Integer k,
+            int k,
             int numCands,
-            Float numCandsFactor,
+            Float oversample,
             Query filter,
             Float similarityThreshold,
             BitSetProducer parentFilter
@@ -2111,18 +2123,17 @@ && isNotUnitVector(squaredMagnitude)) {
                 }
             }
 
-            Integer adjustedK = k;
-            int adjustedNumCands = numCands;
-            if (needsRescore(numCandsFactor)) {
-                // Get all candidates, get top k as part of rescoring
-                adjustedK = null;
-                // numCands * numCandsFactor <= NUM_CANDS_OVERSAMPLE_LIMIT. Adjust otherwise.
-                adjustedNumCands = Math.min((int) Math.ceil(numCands * numCandsFactor), NUM_CANDS_OVERSAMPLE_LIMIT);
+            int adjustedK = k;
+            boolean rescore = needsRescore(oversample);
+            if (rescore) {
+                // Will get k * oversample for rescoring, and get the top k
+                adjustedK = Math.min((int) Math.ceil(k * oversample), OVERSAMPLE_LIMIT);
+                numCands = Math.max(adjustedK, numCands);
             }
             Query knnQuery = parentFilter != null
-                ? new ESDiversifyingChildrenFloatKnnVectorQuery(name(), queryVector, filter, adjustedK, adjustedNumCands, parentFilter)
-                : new ESKnnFloatVectorQuery(name(), queryVector, adjustedK, adjustedNumCands, filter);
-            if (needsRescore(numCandsFactor)) {
+                ? new ESDiversifyingChildrenFloatKnnVectorQuery(name(), queryVector, filter, adjustedK, numCands, parentFilter)
+                : new ESKnnFloatVectorQuery(name(), queryVector, adjustedK, numCands, filter);
+            if (rescore) {
                 knnQuery = new RescoreKnnVectorQuery(
                     name(),
                     queryVector,
@@ -2500,9 +2511,10 @@ public String fieldName() {
     }
 
     /**
-     * @FunctionalInterface for a function that takes a int and boolean
+     * Interface for a function that takes a int and boolean
      */
-    interface IntBooleanConsumer {
+    @FunctionalInterface
+    public interface IntBooleanConsumer {
         void accept(int value, boolean isComplete);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsDVLeafFieldData.java
deleted file mode 100644
index 0125d0249ec2b..0000000000000
--- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsDVLeafFieldData.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.index.mapper.vectors;
-
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.LeafReader;
-import org.elasticsearch.index.fielddata.LeafFieldData;
-import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
-import org.elasticsearch.script.field.DocValuesScriptFieldFactory;
-import org.elasticsearch.script.field.vectors.BitRankVectorsDocValuesField;
-import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField;
-import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField;
-
-import java.io.IOException;
-
-final class RankVectorsDVLeafFieldData implements LeafFieldData {
-    private final LeafReader reader;
-    private final String field;
-    private final DenseVectorFieldMapper.ElementType elementType;
-    private final int dims;
-
-    RankVectorsDVLeafFieldData(LeafReader reader, String field, DenseVectorFieldMapper.ElementType elementType, int dims) {
-        this.reader = reader;
-        this.field = field;
-        this.elementType = elementType;
-        this.dims = dims;
-    }
-
-    @Override
-    public DocValuesScriptFieldFactory getScriptFieldFactory(String name) {
-        try {
-            BinaryDocValues values = DocValues.getBinary(reader, field);
-            BinaryDocValues magnitudeValues = DocValues.getBinary(reader, field + RankVectorsFieldMapper.VECTOR_MAGNITUDES_SUFFIX);
-            return switch (elementType) {
-                case BYTE -> new ByteRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims);
-                case FLOAT -> new FloatRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims);
-                case BIT -> new BitRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims);
-            };
-        } catch (IOException e) {
-            throw new IllegalStateException("Cannot load doc values for multi-vector field!", e);
-        }
-    }
-
-    @Override
-    public SortedBinaryDocValues getBytesValues() {
-        throw new UnsupportedOperationException("String representation of doc values for multi-vector fields is not supported");
-    }
-
-    @Override
-    public long ramBytesUsed() {
-        return 0;
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java
index b4de73e3b62ce..c73f4fef23361 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java
@@ -78,6 +78,11 @@ public Builder(String name) {
             super(name);
         }
 
+        public Builder setStored(boolean value) {
+            stored.setValue(value);
+            return this;
+        }
+
         @Override
         protected Parameter<?>[] getParameters() {
             return new Parameter<?>[] { stored, meta };
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorSimilarityFloatValueSource.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorSimilarityFloatValueSource.java
index 74a7dbe168e6b..80cbec1e2f6c2 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorSimilarityFloatValueSource.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorSimilarityFloatValueSource.java
@@ -18,8 +18,6 @@
 import org.apache.lucene.search.DoubleValues;
 import org.apache.lucene.search.DoubleValuesSource;
 import org.apache.lucene.search.IndexSearcher;
-import org.elasticsearch.search.profile.query.QueryProfiler;
-import org.elasticsearch.search.vectors.QueryProfilerProvider;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -29,12 +27,11 @@
  * DoubleValuesSource that is used to calculate scores according to a similarity function for a KnnFloatVectorField, using the
  * original vector values stored in the index
  */
-public class VectorSimilarityFloatValueSource extends DoubleValuesSource implements QueryProfilerProvider {
+public class VectorSimilarityFloatValueSource extends DoubleValuesSource {
 
     private final String field;
     private final float[] target;
     private final VectorSimilarityFunction vectorSimilarityFunction;
-    private long vectorOpsCount;
 
     public VectorSimilarityFloatValueSource(String field, float[] target, VectorSimilarityFunction vectorSimilarityFunction) {
         this.field = field;
@@ -52,7 +49,6 @@ public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws
         return new DoubleValues() {
             @Override
             public double doubleValue() throws IOException {
-                vectorOpsCount++;
                 return vectorSimilarityFunction.compare(target, vectorValues.vectorValue(iterator.index()));
             }
 
@@ -73,11 +69,6 @@ public DoubleValuesSource rewrite(IndexSearcher reader) throws IOException {
         return this;
     }
 
-    @Override
-    public void profile(QueryProfiler queryProfiler) {
-        queryProfiler.addVectorOpsCount(vectorOpsCount);
-    }
-
     @Override
     public int hashCode() {
         return Objects.hash(field, Arrays.hashCode(target), vectorSimilarityFunction);
diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
index b63cf232eca9b..458ff598737fa 100644
--- a/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
@@ -41,9 +41,9 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
     public static final String NAME = "span_near";
 
     /** Default for flag controlling whether matches are required to be in-order */
-    public static boolean DEFAULT_IN_ORDER = true;
+    public static final boolean DEFAULT_IN_ORDER = true;
     /** Default slop value, this is the same that lucene {@link SpanNearQuery} uses if no slop is provided */
-    public static int DEFAULT_SLOP = 0;
+    public static final int DEFAULT_SLOP = 0;
 
     private static final ParseField SLOP_FIELD = new ParseField("slop");
     private static final ParseField CLAUSES_FIELD = new ParseField("clauses");
diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java
index 6d4b2dd4ab1f5..88f1ab1ba5c2e 100644
--- a/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java
@@ -13,12 +13,10 @@
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.DeprecationCategory;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.lucene.search.function.RandomScoreFunction;
 import org.elasticsearch.common.lucene.search.function.ScoreFunction;
-import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
@@ -30,7 +28,6 @@
  * A function that computes a random score for the matched documents
  */
 public class RandomScoreFunctionBuilder extends ScoreFunctionBuilder<RandomScoreFunctionBuilder> {
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RandomScoreFunctionBuilder.class);
 
     public static final String NAME = "random_score";
     private String field;
@@ -140,17 +137,7 @@ protected ScoreFunction doToFunction(SearchExecutionContext context) {
             // DocID-based random score generation
             return new RandomScoreFunction(hash(context.nowInMillis()), salt, null);
         } else {
-            String fieldName;
-            if (field == null) {
-                deprecationLogger.warn(
-                    DeprecationCategory.QUERIES,
-                    "seed_requires_field",
-                    "As of version 7.0 Elasticsearch will require that a [field] parameter is provided when a [seed] is set"
-                );
-                fieldName = IdFieldMapper.NAME;
-            } else {
-                fieldName = field;
-            }
+            final String fieldName = Objects.requireNonNullElse(field, SeqNoFieldMapper.NAME);
             if (context.isFieldMapped(fieldName) == false) {
                 if (context.hasMappings() == false) {
                     // no mappings: the index is empty anyway
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
index a1d37a8c2cb97..e71156e6dda18 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
@@ -350,7 +349,7 @@ public boolean getShouldStoreResult() {
      * Set scroll timeout for {@link SearchRequest}
      */
     public Self setScroll(TimeValue keepAlive) {
-        searchRequest.scroll(new Scroll(keepAlive));
+        searchRequest.scroll(keepAlive);
         return self();
     }
 
@@ -358,7 +357,7 @@ public Self setScroll(TimeValue keepAlive) {
      * Get scroll timeout
      */
     public TimeValue getScrollTime() {
-        return searchRequest.scroll().keepAlive();
+        return searchRequest.scroll();
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/server/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
index 59ba8cd2a6d00..8d97ae5aadf4a 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
@@ -353,7 +353,7 @@ public static class Status implements Task.Status, SuccessfullyProcessed {
         public static final String THROTTLED_UNTIL_HR_FIELD = "throttled_until";
         public static final String SLICES_FIELD = "slices";
 
-        public static Set<String> FIELDS_SET = new HashSet<>();
+        public static final Set<String> FIELDS_SET = new HashSet<>();
         static {
             FIELDS_SET.add(SLICE_ID_FIELD);
             FIELDS_SET.add(TOTAL_FIELD);
@@ -774,7 +774,7 @@ public static class StatusOrException implements Writeable, ToXContentObject {
         private final Status status;
         private final Exception exception;
 
-        public static Set<String> EXPECTED_EXCEPTION_FIELDS = new HashSet<>();
+        public static final Set<String> EXPECTED_EXCEPTION_FIELDS = new HashSet<>();
         static {
             EXPECTED_EXCEPTION_FIELDS.add("type");
             EXPECTED_EXCEPTION_FIELDS.add("reason");
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java b/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
index 1327721a88427..68582138d3f61 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
@@ -78,7 +78,7 @@ public void doStart(RejectAwareActionListener<Response> searchListener) {
     protected void doStartNextScroll(String scrollId, TimeValue extraKeepAlive, RejectAwareActionListener<Response> searchListener) {
         SearchScrollRequest request = new SearchScrollRequest();
         // Add the wait time into the scroll timeout so it won't timeout while we wait for throttling
-        request.scrollId(scrollId).scroll(timeValueNanos(firstSearchRequest.scroll().keepAlive().nanos() + extraKeepAlive.nanos()));
+        request.scrollId(scrollId).scroll(timeValueNanos(firstSearchRequest.scroll().nanos() + extraKeepAlive.nanos()));
         client.searchScroll(request, wrapListener(searchListener));
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java
index 5f135c674ba1a..20d05216d62cc 100644
--- a/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java
+++ b/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java
@@ -43,6 +43,9 @@
 import org.elasticsearch.index.mapper.PlaceHolderFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.mapper.TextSearchInfo;
+import org.elasticsearch.index.query.MatchBoolPrefixQueryBuilder;
+import org.elasticsearch.index.query.MatchPhrasePrefixQueryBuilder;
+import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.query.ZeroTermsQueryOption;
 import org.elasticsearch.lucene.analysis.miscellaneous.DisableGraphAttribute;
@@ -63,24 +66,26 @@ public enum Type implements Writeable {
         /**
          * The text is analyzed and terms are added to a boolean query.
          */
-        BOOLEAN(0),
+        BOOLEAN(0, org.elasticsearch.index.query.MatchQueryBuilder.NAME),
         /**
          * The text is analyzed and used as a phrase query.
          */
-        PHRASE(1),
+        PHRASE(1, MatchPhraseQueryBuilder.NAME),
         /**
          * The text is analyzed and used in a phrase query, with the last term acting as a prefix.
          */
-        PHRASE_PREFIX(2),
+        PHRASE_PREFIX(2, MatchPhrasePrefixQueryBuilder.NAME),
         /**
          * The text is analyzed, terms are added to a boolean query with the last term acting as a prefix.
          */
-        BOOLEAN_PREFIX(3);
+        BOOLEAN_PREFIX(3, MatchBoolPrefixQueryBuilder.NAME);
 
         private final int ordinal;
+        private final String queryName;
 
-        Type(int ordinal) {
+        Type(int ordinal, String queryName) {
             this.ordinal = ordinal;
+            this.queryName = queryName;
         }
 
         public static Type readFromStream(StreamInput in) throws IOException {
@@ -93,6 +98,10 @@ public static Type readFromStream(StreamInput in) throws IOException {
             throw new ElasticsearchException("unknown serialized type [" + ord + "]");
         }
 
+        public String getQueryName() {
+            return queryName;
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeVInt(this.ordinal);
@@ -207,11 +216,23 @@ public Query parse(Type type, String fieldName, Object value) throws IOException
             IllegalArgumentException iae;
             if (fieldType instanceof PlaceHolderFieldMapper.PlaceHolderFieldType) {
                 iae = new IllegalArgumentException(
-                    "Field [" + fieldType.name() + "] of type [" + fieldType.typeName() + "] in legacy index does not support match queries"
+                    "Field ["
+                        + fieldType.name()
+                        + "] of type ["
+                        + fieldType.typeName()
+                        + "] in legacy index does not support "
+                        + type.getQueryName()
+                        + " queries"
                 );
             } else {
                 iae = new IllegalArgumentException(
-                    "Field [" + fieldType.name() + "] of type [" + fieldType.typeName() + "] does not support match queries"
+                    "Field ["
+                        + fieldType.name()
+                        + "] of type ["
+                        + fieldType.typeName()
+                        + "] does not support "
+                        + type.getQueryName()
+                        + " queries"
                 );
             }
             if (lenient) {
diff --git a/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java
index d237a03335337..7efff7f69124a 100644
--- a/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java
+++ b/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java
@@ -268,18 +268,19 @@ private Map<String, Float> extractMultiFields(String field, boolean quoted) {
             if (allFields && this.field != null && this.field.equals(field)) {
                 // "*" is the default field
                 extractedFields = fieldsAndWeights;
+            } else {
+                boolean multiFields = Regex.isSimpleMatchPattern(field);
+                // Filters unsupported fields if a pattern is requested
+                // Filters metadata fields if all fields are requested
+                extractedFields = resolveMappingField(
+                    context,
+                    field,
+                    1.0f,
+                    allFields == false,
+                    multiFields == false,
+                    quoted ? quoteFieldSuffix : null
+                );
             }
-            boolean multiFields = Regex.isSimpleMatchPattern(field);
-            // Filters unsupported fields if a pattern is requested
-            // Filters metadata fields if all fields are requested
-            extractedFields = resolveMappingField(
-                context,
-                field,
-                1.0f,
-                allFields == false,
-                multiFields == false,
-                quoted ? quoteFieldSuffix : null
-            );
         } else if (quoted && quoteFieldSuffix != null) {
             extractedFields = resolveMappingFields(context, fieldsAndWeights, quoteFieldSuffix);
         } else {
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
index 5707abc220d3f..54904323a25ad 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
@@ -39,8 +39,8 @@ public class GlobalCheckpointSyncAction extends TransportReplicationAction<
     GlobalCheckpointSyncAction.Request,
     ReplicationResponse> {
 
-    public static String ACTION_NAME = "indices:admin/seq_no/global_checkpoint_sync";
-    public static ActionType<ReplicationResponse> TYPE = new ActionType<>(ACTION_NAME);
+    public static final String ACTION_NAME = "indices:admin/seq_no/global_checkpoint_sync";
+    public static final ActionType<ReplicationResponse> TYPE = new ActionType<>(ACTION_NAME);
 
     @Inject
     public GlobalCheckpointSyncAction(
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeases.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeases.java
index b6b6e709f4f17..76b8d814ee8f3 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeases.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeases.java
@@ -118,7 +118,7 @@ public RetentionLease get(final String id) {
      * Represents an empty an un-versioned retention lease collection. This is used when no retention lease collection is found in the
      * commit point
      */
-    public static RetentionLeases EMPTY = new RetentionLeases(1, 0, Collections.emptyList());
+    public static final RetentionLeases EMPTY = new RetentionLeases(1, 0, Collections.emptyList());
 
     /**
      * Constructs a new retention lease collection with the specified version and underlying collection of retention leases.
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 966764d2797c9..ec260a40452b5 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -189,6 +189,7 @@
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.index.seqno.RetentionLeaseActions.RETAIN_ALL;
 import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
+import static org.elasticsearch.index.shard.IndexShard.PrimaryPermitCheck.CHECK_PRIMARY_MODE;
 
 public class IndexShard extends AbstractIndexShardComponent implements IndicesClusterStateService.Shard {
 
@@ -2206,7 +2207,7 @@ private Engine createEngine(EngineConfig config) {
      * Asserts that the latest Lucene commit contains expected information about sequence numbers or ES version.
      */
     private boolean assertLastestCommitUserData() throws IOException {
-        final SegmentInfos segmentCommitInfos = SegmentInfos.readLatestCommit(store.directory());
+        final SegmentInfos segmentCommitInfos = store.readLastCommittedSegmentsInfo();
         final Map<String, String> userData = segmentCommitInfos.getUserData();
         // Ensure sequence numbers are present in commit data
         assert userData.containsKey(SequenceNumbers.LOCAL_CHECKPOINT_KEY) : "commit point doesn't contains a local checkpoint";
@@ -3568,24 +3569,48 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) {
         );
     }
 
+    /**
+     * Check to run before running the primary permit operation
+     */
+    public enum PrimaryPermitCheck {
+        CHECK_PRIMARY_MODE,
+        /**
+         * IMPORTANT: Currently intented to be used only for acquiring primary permits during the recovery of hollow shards.
+         * Don't disable primary mode checks unless you're really sure.
+         */
+        NONE
+    }
+
     /**
      * Acquire a primary operation permit whenever the shard is ready for indexing. If a permit is directly available, the provided
      * ActionListener will be called on the calling thread. During relocation hand-off, permit acquisition can be delayed. The provided
      * ActionListener will then be called using the provided executor.
-     *
      */
     public void acquirePrimaryOperationPermit(ActionListener<Releasable> onPermitAcquired, Executor executorOnDelay) {
-        acquirePrimaryOperationPermit(onPermitAcquired, executorOnDelay, false);
+        acquirePrimaryOperationPermit(onPermitAcquired, executorOnDelay, false, CHECK_PRIMARY_MODE);
     }
 
     public void acquirePrimaryOperationPermit(
         ActionListener<Releasable> onPermitAcquired,
         Executor executorOnDelay,
         boolean forceExecution
+    ) {
+        acquirePrimaryOperationPermit(onPermitAcquired, executorOnDelay, forceExecution, CHECK_PRIMARY_MODE);
+    }
+
+    public void acquirePrimaryOperationPermit(
+        ActionListener<Releasable> onPermitAcquired,
+        Executor executorOnDelay,
+        boolean forceExecution,
+        PrimaryPermitCheck primaryPermitCheck
     ) {
         verifyNotClosed();
         assert shardRouting.primary() : "acquirePrimaryOperationPermit should only be called on primary shard: " + shardRouting;
-        indexShardOperationPermits.acquire(wrapPrimaryOperationPermitListener(onPermitAcquired), executorOnDelay, forceExecution);
+        indexShardOperationPermits.acquire(
+            wrapPrimaryOperationPermitListener(primaryPermitCheck, onPermitAcquired),
+            executorOnDelay,
+            forceExecution
+        );
     }
 
     public boolean isPrimaryMode() {
@@ -3593,33 +3618,51 @@ public boolean isPrimaryMode() {
         return replicationTracker.isPrimaryMode();
     }
 
+    public void acquireAllPrimaryOperationsPermits(final ActionListener<Releasable> onPermitAcquired, final TimeValue timeout) {
+        acquireAllPrimaryOperationsPermits(onPermitAcquired, timeout, CHECK_PRIMARY_MODE);
+    }
+
     /**
      * Acquire all primary operation permits. Once all permits are acquired, the provided ActionListener is called.
      * It is the responsibility of the caller to close the {@link Releasable}.
      */
-    public void acquireAllPrimaryOperationsPermits(final ActionListener<Releasable> onPermitAcquired, final TimeValue timeout) {
+    public void acquireAllPrimaryOperationsPermits(
+        final ActionListener<Releasable> onPermitAcquired,
+        final TimeValue timeout,
+        final PrimaryPermitCheck primaryPermitCheck
+    ) {
         verifyNotClosed();
         assert shardRouting.primary() : "acquireAllPrimaryOperationsPermits should only be called on primary shard: " + shardRouting;
 
-        asyncBlockOperations(wrapPrimaryOperationPermitListener(onPermitAcquired), timeout.duration(), timeout.timeUnit());
+        asyncBlockOperations(
+            wrapPrimaryOperationPermitListener(primaryPermitCheck, onPermitAcquired),
+            timeout.duration(),
+            timeout.timeUnit()
+        );
     }
 
     /**
-     * Wraps the action to run on a primary after acquiring permit. This wrapping is used to check if the shard is in primary mode before
-     * executing the action.
+     * Wraps the action to run on a primary after acquiring permit.
      *
+     * @param primaryPermitCheck check to run before the primary mode operation
      * @param listener the listener to wrap
      * @return the wrapped listener
      */
-    private ActionListener<Releasable> wrapPrimaryOperationPermitListener(final ActionListener<Releasable> listener) {
-        return listener.delegateFailure((l, r) -> {
-            if (isPrimaryMode()) {
-                l.onResponse(r);
-            } else {
-                r.close();
-                l.onFailure(new ShardNotInPrimaryModeException(shardId, state));
-            }
-        });
+    private ActionListener<Releasable> wrapPrimaryOperationPermitListener(
+        final PrimaryPermitCheck primaryPermitCheck,
+        final ActionListener<Releasable> listener
+    ) {
+        return switch (primaryPermitCheck) {
+            case CHECK_PRIMARY_MODE -> listener.delegateFailure((l, r) -> {
+                if (isPrimaryMode()) {
+                    l.onResponse(r);
+                } else {
+                    r.close();
+                    l.onFailure(new ShardNotInPrimaryModeException(shardId, state));
+                }
+            });
+            case NONE -> listener;
+        };
     }
 
     private void asyncBlockOperations(ActionListener<Releasable> onPermitAcquired, long timeout, TimeUnit timeUnit) {
@@ -3657,7 +3700,7 @@ public void runUnderPrimaryPermit(final Runnable runnable, final Consumer<Except
                 runnable.run();
             }
         }, onFailure);
-        acquirePrimaryOperationPermit(onPermitAcquired, executorOnDelay);
+        acquirePrimaryOperationPermit(onPermitAcquired, executorOnDelay, false, CHECK_PRIMARY_MODE);
     }
 
     private <E extends Exception> void bumpPrimaryTerm(
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java b/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java
index b0a4d333ba77f..62e456d95f467 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java
@@ -34,6 +34,7 @@ public static class Stats implements Writeable, ToXContentFragment {
         private long indexTimeInMillis;
         private long indexCurrent;
         private long indexFailedCount;
+        private long indexFailedDueToVersionConflictCount;
         private long deleteCount;
         private long deleteTimeInMillis;
         private long deleteCurrent;
@@ -50,6 +51,9 @@ public Stats(StreamInput in) throws IOException {
             indexTimeInMillis = in.readVLong();
             indexCurrent = in.readVLong();
             indexFailedCount = in.readVLong();
+            if (in.getTransportVersion().onOrAfter(TransportVersions.TRACK_INDEX_FAILED_DUE_TO_VERSION_CONFLICT_METRIC)) {
+                indexFailedDueToVersionConflictCount = in.readVLong();
+            }
             deleteCount = in.readVLong();
             deleteTimeInMillis = in.readVLong();
             deleteCurrent = in.readVLong();
@@ -67,6 +71,7 @@ public Stats(
             long indexTimeInMillis,
             long indexCurrent,
             long indexFailedCount,
+            long indexFailedDueToVersionConflictCount,
             long deleteCount,
             long deleteTimeInMillis,
             long deleteCurrent,
@@ -80,6 +85,7 @@ public Stats(
             this.indexTimeInMillis = indexTimeInMillis;
             this.indexCurrent = indexCurrent;
             this.indexFailedCount = indexFailedCount;
+            this.indexFailedDueToVersionConflictCount = indexFailedDueToVersionConflictCount;
             this.deleteCount = deleteCount;
             this.deleteTimeInMillis = deleteTimeInMillis;
             this.deleteCurrent = deleteCurrent;
@@ -96,6 +102,7 @@ public void add(Stats stats) {
             indexTimeInMillis += stats.indexTimeInMillis;
             indexCurrent += stats.indexCurrent;
             indexFailedCount += stats.indexFailedCount;
+            indexFailedDueToVersionConflictCount += stats.indexFailedDueToVersionConflictCount;
 
             deleteCount += stats.deleteCount;
             deleteTimeInMillis += stats.deleteTimeInMillis;
@@ -124,6 +131,13 @@ public long getIndexFailedCount() {
             return indexFailedCount;
         }
 
+        /**
+         * The number of indexing operations that failed because of a version conflict (a subset of all index failed operations)
+         */
+        public long getIndexFailedDueToVersionConflictCount() {
+            return indexFailedDueToVersionConflictCount;
+        }
+
         /**
          * The total amount of time spend on executing index operations.
          */
@@ -191,6 +205,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeVLong(indexTimeInMillis);
             out.writeVLong(indexCurrent);
             out.writeVLong(indexFailedCount);
+            if (out.getTransportVersion().onOrAfter(TransportVersions.TRACK_INDEX_FAILED_DUE_TO_VERSION_CONFLICT_METRIC)) {
+                out.writeVLong(indexFailedDueToVersionConflictCount);
+            }
             out.writeVLong(deleteCount);
             out.writeVLong(deleteTimeInMillis);
             out.writeVLong(deleteCurrent);
@@ -209,6 +226,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.humanReadableField(Fields.INDEX_TIME_IN_MILLIS, Fields.INDEX_TIME, getIndexTime());
             builder.field(Fields.INDEX_CURRENT, indexCurrent);
             builder.field(Fields.INDEX_FAILED, indexFailedCount);
+            builder.field(Fields.INDEX_FAILED_DUE_TO_VERSION_CONFLICT, indexFailedDueToVersionConflictCount);
 
             builder.field(Fields.DELETE_TOTAL, deleteCount);
             builder.humanReadableField(Fields.DELETE_TIME_IN_MILLIS, Fields.DELETE_TIME, getDeleteTime());
@@ -232,6 +250,7 @@ public boolean equals(Object o) {
                 && indexTimeInMillis == that.indexTimeInMillis
                 && indexCurrent == that.indexCurrent
                 && indexFailedCount == that.indexFailedCount
+                && indexFailedDueToVersionConflictCount == that.indexFailedDueToVersionConflictCount
                 && deleteCount == that.deleteCount
                 && deleteTimeInMillis == that.deleteTimeInMillis
                 && deleteCurrent == that.deleteCurrent
@@ -249,6 +268,7 @@ public int hashCode() {
                 indexTimeInMillis,
                 indexCurrent,
                 indexFailedCount,
+                indexFailedDueToVersionConflictCount,
                 deleteCount,
                 deleteTimeInMillis,
                 deleteCurrent,
@@ -323,12 +343,12 @@ public int hashCode() {
 
     static final class Fields {
         static final String INDEXING = "indexing";
-        static final String TYPES = "types";
         static final String INDEX_TOTAL = "index_total";
         static final String INDEX_TIME = "index_time";
         static final String INDEX_TIME_IN_MILLIS = "index_time_in_millis";
         static final String INDEX_CURRENT = "index_current";
         static final String INDEX_FAILED = "index_failed";
+        static final String INDEX_FAILED_DUE_TO_VERSION_CONFLICT = "index_failed_due_to_version_conflict";
         static final String DELETE_TOTAL = "delete_total";
         static final String DELETE_TIME = "delete_time";
         static final String DELETE_TIME_IN_MILLIS = "delete_time_in_millis";
diff --git a/server/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java b/server/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java
index d0bb7629c3969..13d270ba36786 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java
@@ -9,9 +9,11 @@
 
 package org.elasticsearch.index.shard;
 
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.metrics.MeanMetric;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.engine.VersionConflictEngineException;
 
 import java.util.concurrent.TimeUnit;
 
@@ -78,6 +80,9 @@ public void postIndex(ShardId shardId, Engine.Index index, Exception ex) {
         if (index.origin().isRecovery() == false) {
             totalStats.indexCurrent.dec();
             totalStats.indexFailed.inc();
+            if (ExceptionsHelper.unwrapCause(ex) instanceof VersionConflictEngineException) {
+                totalStats.indexFailedDueToVersionConflicts.inc();
+            }
         }
     }
 
@@ -124,6 +129,7 @@ static class StatsHolder {
         private final MeanMetric deleteMetric = new MeanMetric();
         private final CounterMetric indexCurrent = new CounterMetric();
         private final CounterMetric indexFailed = new CounterMetric();
+        private final CounterMetric indexFailedDueToVersionConflicts = new CounterMetric();
         private final CounterMetric deleteCurrent = new CounterMetric();
         private final CounterMetric noopUpdates = new CounterMetric();
 
@@ -140,6 +146,7 @@ IndexingStats.Stats stats(
                 TimeUnit.NANOSECONDS.toMillis(totalIndexingTimeInNanos),
                 indexCurrent.count(),
                 indexFailed.count(),
+                indexFailedDueToVersionConflicts.count(),
                 deleteMetric.count(),
                 TimeUnit.NANOSECONDS.toMillis(deleteMetric.sum()),
                 deleteCurrent.count(),
diff --git a/server/src/main/java/org/elasticsearch/index/store/Store.java b/server/src/main/java/org/elasticsearch/index/store/Store.java
index 322064f09cf77..64bbd15198b4b 100644
--- a/server/src/main/java/org/elasticsearch/index/store/Store.java
+++ b/server/src/main/java/org/elasticsearch/index/store/Store.java
@@ -819,12 +819,12 @@ public record MetadataSnapshot(Map<String, StoreFileMetadata> fileMetadataMap, M
 
         public static final MetadataSnapshot EMPTY = new MetadataSnapshot(emptyMap(), emptyMap(), 0L);
 
-        static MetadataSnapshot loadFromIndexCommit(IndexCommit commit, Directory directory, Logger logger) throws IOException {
+        static MetadataSnapshot loadFromIndexCommit(@Nullable IndexCommit commit, Directory directory, Logger logger) throws IOException {
             final long numDocs;
             final Map<String, StoreFileMetadata> metadataByFile = new HashMap<>();
             final Map<String, String> commitUserData;
             try {
-                final SegmentInfos segmentCommitInfos = Store.readSegmentsInfo(commit, directory);
+                final SegmentInfos segmentCommitInfos = readSegmentsInfo(commit, directory);
                 numDocs = Lucene.getNumDocs(segmentCommitInfos);
                 commitUserData = Map.copyOf(segmentCommitInfos.getUserData());
                 // we don't know which version was used to write so we take the max version.
@@ -1449,7 +1449,6 @@ public void bootstrapNewHistory() throws IOException {
      * @see SequenceNumbers#MAX_SEQ_NO
      */
     public void bootstrapNewHistory(long localCheckpoint, long maxSeqNo) throws IOException {
-        assert indexSettings.getIndexMetadata().isSearchableSnapshot() == false;
         metadataLock.writeLock().lock();
         try (IndexWriter writer = newTemporaryAppendingIndexWriter(directory, null)) {
             final Map<String, String> map = new HashMap<>();
@@ -1573,7 +1572,6 @@ private IndexWriter newTemporaryEmptyIndexWriter(final Directory dir, final Vers
     }
 
     private IndexWriterConfig newTemporaryIndexWriterConfig() {
-        assert indexSettings.getIndexMetadata().isSearchableSnapshot() == false;
         // this config is only used for temporary IndexWriter instances, used to initialize the index or update the commit data,
         // so we don't want any merges to happen
         var iwc = indexWriterConfigWithNoMerging(null).setSoftDeletesField(Lucene.SOFT_DELETES_FIELD).setCommitOnClose(false);
diff --git a/server/src/main/java/org/elasticsearch/index/translog/Translog.java b/server/src/main/java/org/elasticsearch/index/translog/Translog.java
index 75f38ed5f7342..b1a203616b120 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.engine.TranslogOperationAsserter;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.Uid;
@@ -123,6 +124,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
     private final TranslogDeletionPolicy deletionPolicy;
     private final LongConsumer persistedSequenceNumberConsumer;
     private final OperationListener operationListener;
+    private final TranslogOperationAsserter operationAsserter;
 
     /**
      * Creates a new Translog instance. This method will create a new transaction log unless the given {@link TranslogGeneration} is
@@ -150,7 +152,8 @@ public Translog(
         TranslogDeletionPolicy deletionPolicy,
         final LongSupplier globalCheckpointSupplier,
         final LongSupplier primaryTermSupplier,
-        final LongConsumer persistedSequenceNumberConsumer
+        final LongConsumer persistedSequenceNumberConsumer,
+        final TranslogOperationAsserter operationAsserter
     ) throws IOException {
         super(config.getShardId(), config.getIndexSettings());
         this.config = config;
@@ -158,6 +161,7 @@ public Translog(
         this.primaryTermSupplier = primaryTermSupplier;
         this.persistedSequenceNumberConsumer = persistedSequenceNumberConsumer;
         this.operationListener = config.getOperationListener();
+        this.operationAsserter = operationAsserter;
         this.deletionPolicy = deletionPolicy;
         this.translogUUID = translogUUID;
         this.bigArrays = config.getBigArrays();
@@ -586,6 +590,7 @@ TranslogWriter createWriter(
                 bigArrays,
                 diskIoBufferPool,
                 operationListener,
+                operationAsserter,
                 config.fsync()
             );
         } catch (final IOException e) {
@@ -971,7 +976,7 @@ public TranslogDeletionPolicy getDeletionPolicy() {
 
     public record Location(long generation, long translogLocation, int size) implements Comparable<Location> {
 
-        public static Location EMPTY = new Location(0, 0, 0);
+        public static final Location EMPTY = new Location(0, 0, 0);
 
         @Override
         public String toString() {
@@ -1269,17 +1274,8 @@ public boolean equals(Object o) {
                 return false;
             }
 
-            Index index = (Index) o;
-
-            if (version != index.version
-                || seqNo != index.seqNo
-                || primaryTerm != index.primaryTerm
-                || id.equals(index.id) == false
-                || autoGeneratedIdTimestamp != index.autoGeneratedIdTimestamp
-                || source.equals(index.source) == false) {
-                return false;
-            }
-            return Objects.equals(routing, index.routing);
+            Index other = (Index) o;
+            return autoGeneratedIdTimestamp == other.autoGeneratedIdTimestamp && equalsWithoutAutoGeneratedTimestamp(this, other);
         }
 
         @Override
@@ -1315,6 +1311,15 @@ public long getAutoGeneratedIdTimestamp() {
             return autoGeneratedIdTimestamp;
         }
 
+        public static boolean equalsWithoutAutoGeneratedTimestamp(Translog.Index o1, Translog.Index o2) {
+            return o1.version == o2.version
+                && o1.seqNo == o2.seqNo
+                && o1.primaryTerm == o2.primaryTerm
+                && o1.id.equals(o2.id)
+                && o1.source.equals(o2.source)
+                && Objects.equals(o1.routing, o2.routing);
+        }
+
     }
 
     public static final class Delete extends Operation {
@@ -1962,6 +1967,7 @@ public static String createEmptyTranslog(
             BigArrays.NON_RECYCLING_INSTANCE,
             DiskIoBufferPool.INSTANCE,
             TranslogConfig.NOOP_OPERATION_LISTENER,
+            TranslogOperationAsserter.DEFAULT,
             true
         );
         writer.close();
diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
index 3dda44ff5a6db..8cf631b660b1e 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.engine.TranslogOperationAsserter;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.ShardId;
 
@@ -39,7 +40,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.LongConsumer;
@@ -69,6 +69,7 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
     // callback that's called whenever an operation with a given sequence number is successfully persisted.
     private final LongConsumer persistedSequenceNumberConsumer;
     private final OperationListener operationListener;
+    private final TranslogOperationAsserter operationAsserter;
     private final boolean fsync;
 
     protected final AtomicBoolean closed = new AtomicBoolean(false);
@@ -108,6 +109,7 @@ private TranslogWriter(
         BigArrays bigArrays,
         DiskIoBufferPool diskIoBufferPool,
         OperationListener operationListener,
+        TranslogOperationAsserter operationAsserter,
         boolean fsync
     ) throws IOException {
         super(initialCheckpoint.generation, channel, path, header);
@@ -136,6 +138,7 @@ private TranslogWriter(
         this.seenSequenceNumbers = Assertions.ENABLED ? new HashMap<>() : null;
         this.tragedy = tragedy;
         this.operationListener = operationListener;
+        this.operationAsserter = operationAsserter;
         this.fsync = fsync;
         this.lastModifiedTimeCache = new LastModifiedTimeCache(-1, -1, -1);
     }
@@ -157,6 +160,7 @@ public static TranslogWriter create(
         BigArrays bigArrays,
         DiskIoBufferPool diskIoBufferPool,
         OperationListener operationListener,
+        TranslogOperationAsserter operationAsserter,
         boolean fsync
     ) throws IOException {
         final Path checkpointFile = file.getParent().resolve(Translog.CHECKPOINT_FILE_NAME);
@@ -201,6 +205,7 @@ public static TranslogWriter create(
                 bigArrays,
                 diskIoBufferPool,
                 operationListener,
+                operationAsserter,
                 fsync
             );
         } catch (Exception exception) {
@@ -276,25 +281,16 @@ private synchronized boolean assertNoSeqNumberConflict(long seqNo, BytesReferenc
                 Translog.Operation prvOp = Translog.readOperation(
                     new BufferedChecksumStreamInput(previous.v1().streamInput(), "assertion")
                 );
-                // TODO: We haven't had timestamp for Index operations in Lucene yet, we need to loosen this check without timestamp.
                 final boolean sameOp;
                 if (newOp instanceof final Translog.Index o2 && prvOp instanceof final Translog.Index o1) {
-                    sameOp = Objects.equals(o1.id(), o2.id())
-                        && Objects.equals(o1.source(), o2.source())
-                        && Objects.equals(o1.routing(), o2.routing())
-                        && o1.primaryTerm() == o2.primaryTerm()
-                        && o1.seqNo() == o2.seqNo()
-                        && o1.version() == o2.version();
+                    sameOp = operationAsserter.assertSameIndexOperation(o1, o2);
                 } else if (newOp instanceof final Translog.Delete o1 && prvOp instanceof final Translog.Delete o2) {
-                    sameOp = Objects.equals(o1.id(), o2.id())
-                        && o1.primaryTerm() == o2.primaryTerm()
-                        && o1.seqNo() == o2.seqNo()
-                        && o1.version() == o2.version();
+                    sameOp = o1.equals(o2);
                 } else {
                     sameOp = false;
                 }
-                if (sameOp == false) {
-                    throw new AssertionError(
+                assert sameOp
+                    : new AssertionError(
                         "seqNo ["
                             + seqNo
                             + "] was processed twice in generation ["
@@ -307,7 +303,6 @@ private synchronized boolean assertNoSeqNumberConflict(long seqNo, BytesReferenc
                             + "]",
                         previous.v2()
                     );
-                }
             }
         } else {
             seenSequenceNumbers.put(
diff --git a/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java b/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java
index 7f97edb2ff695..6ef1e2eff785b 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.engine.TranslogOperationAsserter;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.RemoveCorruptedShardDataCommand;
 import org.elasticsearch.index.shard.ShardPath;
@@ -171,7 +172,8 @@ private static boolean isTranslogClean(ShardPath shardPath, ClusterState cluster
                     translogDeletionPolicy,
                     () -> translogGlobalCheckpoint,
                     () -> primaryTerm,
-                    seqNo -> {}
+                    seqNo -> {},
+                    TranslogOperationAsserter.DEFAULT
                 );
                 Translog.Snapshot snapshot = translog.newSnapshot(0, Long.MAX_VALUE)
             ) {
diff --git a/server/src/main/java/org/elasticsearch/indices/ExecutorNames.java b/server/src/main/java/org/elasticsearch/indices/ExecutorNames.java
index e1bf11227592c..2bfc642111d87 100644
--- a/server/src/main/java/org/elasticsearch/indices/ExecutorNames.java
+++ b/server/src/main/java/org/elasticsearch/indices/ExecutorNames.java
@@ -20,7 +20,7 @@ public record ExecutorNames(String threadPoolForGet, String threadPoolForSearch,
     /**
      * The thread pools for a typical system index.
      */
-    public static ExecutorNames DEFAULT_SYSTEM_INDEX_THREAD_POOLS = new ExecutorNames(
+    public static final ExecutorNames DEFAULT_SYSTEM_INDEX_THREAD_POOLS = new ExecutorNames(
         ThreadPool.Names.SYSTEM_READ,
         ThreadPool.Names.SYSTEM_READ,
         ThreadPool.Names.SYSTEM_WRITE
@@ -29,7 +29,7 @@ public record ExecutorNames(String threadPoolForGet, String threadPoolForSearch,
     /**
      * The thread pools for a typical system data stream. These are also the usual thread pools for non-system indices and data streams.
      */
-    public static ExecutorNames DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS = new ExecutorNames(
+    public static final ExecutorNames DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS = new ExecutorNames(
         ThreadPool.Names.GET,
         ThreadPool.Names.SEARCH,
         ThreadPool.Names.WRITE
@@ -38,7 +38,7 @@ public record ExecutorNames(String threadPoolForGet, String threadPoolForSearch,
     /**
      * The thread pools that should be used for critical system index operations.
      */
-    public static ExecutorNames CRITICAL_SYSTEM_INDEX_THREAD_POOLS = new ExecutorNames(
+    public static final ExecutorNames CRITICAL_SYSTEM_INDEX_THREAD_POOLS = new ExecutorNames(
         ThreadPool.Names.SYSTEM_CRITICAL_READ,
         ThreadPool.Names.SYSTEM_CRITICAL_READ,
         ThreadPool.Names.SYSTEM_CRITICAL_WRITE
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
index 3dc25b058b1d6..09be98630d5c4 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
@@ -67,7 +67,6 @@
 import org.elasticsearch.index.mapper.VersionFieldMapper;
 import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
-import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper;
 import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper;
 import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction;
 import org.elasticsearch.index.seqno.RetentionLeaseSyncAction;
@@ -211,9 +210,6 @@ public static Map<String, Mapper.TypeParser> getMappers(List<MapperPlugin> mappe
 
         mappers.put(DenseVectorFieldMapper.CONTENT_TYPE, DenseVectorFieldMapper.PARSER);
         mappers.put(SparseVectorFieldMapper.CONTENT_TYPE, SparseVectorFieldMapper.PARSER);
-        if (RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()) {
-            mappers.put(RankVectorsFieldMapper.CONTENT_TYPE, RankVectorsFieldMapper.PARSER);
-        }
 
         for (MapperPlugin mapperPlugin : mapperPlugins) {
             for (Map.Entry<String, Mapper.TypeParser> entry : mapperPlugin.getMappers().entrySet()) {
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java
index a5765a1a707d2..f22a99cb27faf 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -264,6 +264,7 @@ public class IndicesService extends AbstractLifecycleComponent
     private final PostRecoveryMerger postRecoveryMerger;
     private final List<SearchOperationListener> searchOperationListeners;
     private final QueryRewriteInterceptor queryRewriteInterceptor;
+    final SlowLogFieldProvider slowLogFieldProvider; // pkg-private for testingå
 
     @Override
     protected void doStart() {
@@ -383,6 +384,7 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon
         this.timestampFieldMapperService = new TimestampFieldMapperService(settings, threadPool, this);
         this.postRecoveryMerger = new PostRecoveryMerger(settings, threadPool.executor(ThreadPool.Names.FORCE_MERGE), this::getShardOrNull);
         this.searchOperationListeners = builder.searchOperationListener;
+        this.slowLogFieldProvider = builder.slowLogFieldProvider;
     }
 
     private static final String DANGLING_INDICES_UPDATE_THREAD_NAME = "DanglingIndices#updateTask";
@@ -753,7 +755,7 @@ private synchronized IndexService createIndexService(
             () -> allowExpensiveQueries,
             indexNameExpressionResolver,
             recoveryStateFactories,
-            loadSlowLogFieldProvider(),
+            slowLogFieldProvider,
             mapperMetrics,
             searchOperationListeners
         );
@@ -833,7 +835,7 @@ public synchronized MapperService createIndexMapperServiceForValidation(IndexMet
             () -> allowExpensiveQueries,
             indexNameExpressionResolver,
             recoveryStateFactories,
-            loadSlowLogFieldProvider(),
+            slowLogFieldProvider,
             mapperMetrics,
             searchOperationListeners
         );
@@ -1439,31 +1441,6 @@ int numPendingDeletes(Index index) {
         }
     }
 
-    // pkg-private for testing
-    SlowLogFieldProvider loadSlowLogFieldProvider() {
-        List<? extends SlowLogFieldProvider> slowLogFieldProviders = pluginsService.loadServiceProviders(SlowLogFieldProvider.class);
-        return new SlowLogFieldProvider() {
-            @Override
-            public void init(IndexSettings indexSettings) {
-                slowLogFieldProviders.forEach(provider -> provider.init(indexSettings));
-            }
-
-            @Override
-            public Map<String, String> indexSlowLogFields() {
-                return slowLogFieldProviders.stream()
-                    .flatMap(provider -> provider.indexSlowLogFields().entrySet().stream())
-                    .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
-            }
-
-            @Override
-            public Map<String, String> searchSlowLogFields() {
-                return slowLogFieldProviders.stream()
-                    .flatMap(provider -> provider.searchSlowLogFields().entrySet().stream())
-                    .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
-            }
-        };
-    }
-
     /**
      * Checks if all pending deletes have completed. Used by tests to ensure we don't check directory contents
      * while deletion still ongoing. * The reason is that, on Windows, browsing the directory contents can interfere
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java b/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java
index f0f0f453e3be8..d88bbfa3eba17 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java
@@ -23,6 +23,8 @@
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.gateway.MetaStateService;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.SlowLogFieldProvider;
+import org.elasticsearch.index.SlowLogFields;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.engine.EngineFactory;
 import org.elasticsearch.index.mapper.MapperMetrics;
@@ -79,6 +81,22 @@ public class IndicesServiceBuilder {
     MapperMetrics mapperMetrics;
     List<SearchOperationListener> searchOperationListener = List.of();
     QueryRewriteInterceptor queryRewriteInterceptor = null;
+    SlowLogFieldProvider slowLogFieldProvider = new SlowLogFieldProvider() {
+        @Override
+        public SlowLogFields create(IndexSettings indexSettings) {
+            return new SlowLogFields() {
+                @Override
+                public Map<String, String> indexFields() {
+                    return Map.of();
+                }
+
+                @Override
+                public Map<String, String> searchFields() {
+                    return Map.of();
+                }
+            };
+        }
+    };
 
     public IndicesServiceBuilder settings(Settings settings) {
         this.settings = settings;
@@ -191,6 +209,11 @@ public IndicesServiceBuilder searchOperationListeners(List<SearchOperationListen
         return this;
     }
 
+    public IndicesServiceBuilder slowLogFieldProvider(SlowLogFieldProvider slowLogFieldProvider) {
+        this.slowLogFieldProvider = slowLogFieldProvider;
+        return this;
+    }
+
     public IndicesService build() {
         Objects.requireNonNull(settings);
         Objects.requireNonNull(pluginsService);
@@ -216,6 +239,7 @@ public IndicesService build() {
         Objects.requireNonNull(snapshotCommitSuppliers);
         Objects.requireNonNull(mapperMetrics);
         Objects.requireNonNull(searchOperationListener);
+        Objects.requireNonNull(slowLogFieldProvider);
 
         // collect engine factory providers from plugins
         engineFactoryProviders = pluginsService.filterPlugins(EnginePlugin.class)
diff --git a/server/src/main/java/org/elasticsearch/inference/EmptyTaskSettings.java b/server/src/main/java/org/elasticsearch/inference/EmptyTaskSettings.java
index cba0282f7fed8..5b3ee08d205d5 100644
--- a/server/src/main/java/org/elasticsearch/inference/EmptyTaskSettings.java
+++ b/server/src/main/java/org/elasticsearch/inference/EmptyTaskSettings.java
@@ -24,7 +24,7 @@
 public record EmptyTaskSettings() implements TaskSettings {
     public static final String NAME = "empty_task_settings";
 
-    public static EmptyTaskSettings INSTANCE = new EmptyTaskSettings();
+    public static final EmptyTaskSettings INSTANCE = new EmptyTaskSettings();
 
     public EmptyTaskSettings(StreamInput in) {
         this();
diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceServiceConfiguration.java b/server/src/main/java/org/elasticsearch/inference/InferenceServiceConfiguration.java
index c8bd4f2e27e8b..41cf339c751d1 100644
--- a/server/src/main/java/org/elasticsearch/inference/InferenceServiceConfiguration.java
+++ b/server/src/main/java/org/elasticsearch/inference/InferenceServiceConfiguration.java
@@ -25,10 +25,12 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 
@@ -37,75 +39,88 @@
  */
 public class InferenceServiceConfiguration implements Writeable, ToXContentObject {
 
-    private final String provider;
-    private final List<TaskSettingsConfiguration> taskTypes;
-    private final Map<String, SettingsConfiguration> configuration;
+    private final String service;
+    private final String name;
+    private final EnumSet<TaskType> taskTypes;
+    private final Map<String, SettingsConfiguration> configurations;
 
     /**
      * Constructs a new {@link InferenceServiceConfiguration} instance with specified properties.
      *
-     * @param provider       The name of the service provider.
-     * @param taskTypes      A list of {@link TaskSettingsConfiguration} supported by the service provider.
-     * @param configuration  The configuration of the service provider, defined by {@link SettingsConfiguration}.
+     * @param service        The name of the service provider.
+     * @param name           The user-friendly name of the service provider.
+     * @param taskTypes      A list of {@link TaskType} supported by the service provider.
+     * @param configurations  The configuration of the service provider, defined by {@link SettingsConfiguration}.
      */
     private InferenceServiceConfiguration(
-        String provider,
-        List<TaskSettingsConfiguration> taskTypes,
-        Map<String, SettingsConfiguration> configuration
+        String service,
+        String name,
+        EnumSet<TaskType> taskTypes,
+        Map<String, SettingsConfiguration> configurations
     ) {
-        this.provider = provider;
+        this.service = service;
+        this.name = name;
         this.taskTypes = taskTypes;
-        this.configuration = configuration;
+        this.configurations = configurations;
     }
 
     public InferenceServiceConfiguration(StreamInput in) throws IOException {
-        this.provider = in.readString();
-        this.taskTypes = in.readCollectionAsList(TaskSettingsConfiguration::new);
-        this.configuration = in.readMap(SettingsConfiguration::new);
+        this.service = in.readString();
+        this.name = in.readString();
+        this.taskTypes = in.readEnumSet(TaskType.class);
+        this.configurations = in.readMap(SettingsConfiguration::new);
     }
 
-    static final ParseField PROVIDER_FIELD = new ParseField("provider");
+    static final ParseField SERVICE_FIELD = new ParseField("service");
+    static final ParseField NAME_FIELD = new ParseField("name");
     static final ParseField TASK_TYPES_FIELD = new ParseField("task_types");
-    static final ParseField CONFIGURATION_FIELD = new ParseField("configuration");
+    static final ParseField CONFIGURATIONS_FIELD = new ParseField("configurations");
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<InferenceServiceConfiguration, Void> PARSER = new ConstructingObjectParser<>(
         "inference_service_configuration",
         true,
         args -> {
-            List<String> taskTypes = (ArrayList<String>) args[1];
-            return new InferenceServiceConfiguration.Builder().setProvider((String) args[0])
-                .setTaskTypes((List<TaskSettingsConfiguration>) args[1])
-                .setConfiguration((Map<String, SettingsConfiguration>) args[2])
+            List<String> taskTypes = (ArrayList<String>) args[2];
+            return new InferenceServiceConfiguration.Builder().setService((String) args[0])
+                .setName((String) args[1])
+                .setTaskTypes(EnumSet.copyOf(taskTypes.stream().map(TaskType::fromString).collect(Collectors.toList())))
+                .setConfigurations((Map<String, SettingsConfiguration>) args[3])
                 .build();
         }
     );
 
     static {
-        PARSER.declareString(constructorArg(), PROVIDER_FIELD);
-        PARSER.declareObjectArray(constructorArg(), (p, c) -> TaskSettingsConfiguration.fromXContent(p), TASK_TYPES_FIELD);
-        PARSER.declareObject(constructorArg(), (p, c) -> p.map(), CONFIGURATION_FIELD);
+        PARSER.declareString(constructorArg(), SERVICE_FIELD);
+        PARSER.declareString(constructorArg(), NAME_FIELD);
+        PARSER.declareStringArray(constructorArg(), TASK_TYPES_FIELD);
+        PARSER.declareObject(constructorArg(), (p, c) -> p.map(), CONFIGURATIONS_FIELD);
     }
 
-    public String getProvider() {
-        return provider;
+    public String getService() {
+        return service;
     }
 
-    public List<TaskSettingsConfiguration> getTaskTypes() {
+    public String getName() {
+        return name;
+    }
+
+    public EnumSet<TaskType> getTaskTypes() {
         return taskTypes;
     }
 
-    public Map<String, SettingsConfiguration> getConfiguration() {
-        return configuration;
+    public Map<String, SettingsConfiguration> getConfigurations() {
+        return configurations;
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         {
-            builder.field(PROVIDER_FIELD.getPreferredName(), provider);
+            builder.field(SERVICE_FIELD.getPreferredName(), service);
+            builder.field(NAME_FIELD.getPreferredName(), name);
             builder.field(TASK_TYPES_FIELD.getPreferredName(), taskTypes);
-            builder.field(CONFIGURATION_FIELD.getPreferredName(), configuration);
+            builder.field(CONFIGURATIONS_FIELD.getPreferredName(), configurations);
         }
         builder.endObject();
         return builder;
@@ -125,17 +140,19 @@ public static InferenceServiceConfiguration fromXContentBytes(BytesReference sou
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(provider);
+        out.writeString(service);
+        out.writeString(name);
         out.writeCollection(taskTypes);
-        out.writeMapValues(configuration);
+        out.writeMapValues(configurations);
     }
 
     public Map<String, Object> toMap() {
         Map<String, Object> map = new HashMap<>();
 
-        map.put(PROVIDER_FIELD.getPreferredName(), provider);
+        map.put(SERVICE_FIELD.getPreferredName(), service);
+        map.put(NAME_FIELD.getPreferredName(), name);
         map.put(TASK_TYPES_FIELD.getPreferredName(), taskTypes);
-        map.put(CONFIGURATION_FIELD.getPreferredName(), configuration);
+        map.put(CONFIGURATIONS_FIELD.getPreferredName(), configurations);
 
         return map;
     }
@@ -145,39 +162,46 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         InferenceServiceConfiguration that = (InferenceServiceConfiguration) o;
-        return provider.equals(that.provider)
+        return service.equals(that.service)
+            && name.equals(that.name)
             && Objects.equals(taskTypes, that.taskTypes)
-            && Objects.equals(configuration, that.configuration);
+            && Objects.equals(configurations, that.configurations);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(provider, taskTypes, configuration);
+        return Objects.hash(service, name, taskTypes, configurations);
     }
 
     public static class Builder {
 
-        private String provider;
-        private List<TaskSettingsConfiguration> taskTypes;
-        private Map<String, SettingsConfiguration> configuration;
+        private String service;
+        private String name;
+        private EnumSet<TaskType> taskTypes;
+        private Map<String, SettingsConfiguration> configurations;
+
+        public Builder setService(String service) {
+            this.service = service;
+            return this;
+        }
 
-        public Builder setProvider(String provider) {
-            this.provider = provider;
+        public Builder setName(String name) {
+            this.name = name;
             return this;
         }
 
-        public Builder setTaskTypes(List<TaskSettingsConfiguration> taskTypes) {
+        public Builder setTaskTypes(EnumSet<TaskType> taskTypes) {
             this.taskTypes = taskTypes;
             return this;
         }
 
-        public Builder setConfiguration(Map<String, SettingsConfiguration> configuration) {
-            this.configuration = configuration;
+        public Builder setConfigurations(Map<String, SettingsConfiguration> configurations) {
+            this.configurations = configurations;
             return this;
         }
 
         public InferenceServiceConfiguration build() {
-            return new InferenceServiceConfiguration(provider, taskTypes, configuration);
+            return new InferenceServiceConfiguration(service, name, taskTypes, configurations);
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/inference/SettingsConfiguration.java b/server/src/main/java/org/elasticsearch/inference/SettingsConfiguration.java
index fb97e62f01b19..188b8a7e82b57 100644
--- a/server/src/main/java/org/elasticsearch/inference/SettingsConfiguration.java
+++ b/server/src/main/java/org/elasticsearch/inference/SettingsConfiguration.java
@@ -16,11 +16,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDependency;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption;
-import org.elasticsearch.inference.configuration.SettingsConfigurationValidation;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -32,9 +28,7 @@
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
@@ -47,120 +41,62 @@
  */
 public class SettingsConfiguration implements Writeable, ToXContentObject {
 
-    @Nullable
-    private final String category;
     @Nullable
     private final Object defaultValue;
     @Nullable
-    private final List<SettingsConfigurationDependency> dependsOn;
-    @Nullable
-    private final SettingsConfigurationDisplayType display;
+    private final String description;
     private final String label;
-    @Nullable
-    private final List<SettingsConfigurationSelectOption> options;
-    @Nullable
-    private final Integer order;
-    @Nullable
-    private final String placeholder;
     private final boolean required;
     private final boolean sensitive;
-    @Nullable
-    private final String tooltip;
-    @Nullable
+    private final boolean updatable;
     private final SettingsConfigurationFieldType type;
-    @Nullable
-    private final List<String> uiRestrictions;
-    @Nullable
-    private final List<SettingsConfigurationValidation> validations;
-    @Nullable
-    private final Object value;
 
     /**
      * Constructs a new {@link SettingsConfiguration} instance with specified properties.
      *
-     * @param category       The category of the configuration field.
      * @param defaultValue   The default value for the configuration.
-     * @param dependsOn      A list of {@link SettingsConfigurationDependency} indicating dependencies on other configurations.
-     * @param display        The display type, defined by {@link SettingsConfigurationDisplayType}.
+     * @param description    A description of the configuration.
      * @param label          The display label associated with the config field.
-     * @param options        A list of {@link SettingsConfigurationSelectOption} for selectable options.
-     * @param order          The order in which this configuration appears.
-     * @param placeholder    A placeholder text for the configuration field.
      * @param required       A boolean indicating whether the configuration is required.
      * @param sensitive      A boolean indicating whether the configuration contains sensitive information.
-     * @param tooltip        A tooltip text providing additional information about the configuration.
+     * @param updatable      A boolean indicating whether the configuration can be updated.
      * @param type           The type of the configuration field, defined by {@link SettingsConfigurationFieldType}.
-     * @param uiRestrictions A list of UI restrictions in string format.
-     * @param validations    A list of {@link SettingsConfigurationValidation} for validating the configuration.
-     * @param value          The current value of the configuration.
      */
     private SettingsConfiguration(
-        String category,
         Object defaultValue,
-        List<SettingsConfigurationDependency> dependsOn,
-        SettingsConfigurationDisplayType display,
+        String description,
         String label,
-        List<SettingsConfigurationSelectOption> options,
-        Integer order,
-        String placeholder,
         boolean required,
         boolean sensitive,
-        String tooltip,
-        SettingsConfigurationFieldType type,
-        List<String> uiRestrictions,
-        List<SettingsConfigurationValidation> validations,
-        Object value
+        boolean updatable,
+        SettingsConfigurationFieldType type
     ) {
-        this.category = category;
         this.defaultValue = defaultValue;
-        this.dependsOn = dependsOn;
-        this.display = display;
+        this.description = description;
         this.label = label;
-        this.options = options;
-        this.order = order;
-        this.placeholder = placeholder;
         this.required = required;
         this.sensitive = sensitive;
-        this.tooltip = tooltip;
+        this.updatable = updatable;
         this.type = type;
-        this.uiRestrictions = uiRestrictions;
-        this.validations = validations;
-        this.value = value;
     }
 
     public SettingsConfiguration(StreamInput in) throws IOException {
-        this.category = in.readString();
         this.defaultValue = in.readGenericValue();
-        this.dependsOn = in.readOptionalCollectionAsList(SettingsConfigurationDependency::new);
-        this.display = in.readEnum(SettingsConfigurationDisplayType.class);
+        this.description = in.readOptionalString();
         this.label = in.readString();
-        this.options = in.readOptionalCollectionAsList(SettingsConfigurationSelectOption::new);
-        this.order = in.readOptionalInt();
-        this.placeholder = in.readOptionalString();
         this.required = in.readBoolean();
         this.sensitive = in.readBoolean();
-        this.tooltip = in.readOptionalString();
+        this.updatable = in.readBoolean();
         this.type = in.readEnum(SettingsConfigurationFieldType.class);
-        this.uiRestrictions = in.readOptionalStringCollectionAsList();
-        this.validations = in.readOptionalCollectionAsList(SettingsConfigurationValidation::new);
-        this.value = in.readGenericValue();
     }
 
-    static final ParseField CATEGORY_FIELD = new ParseField("category");
     static final ParseField DEFAULT_VALUE_FIELD = new ParseField("default_value");
-    static final ParseField DEPENDS_ON_FIELD = new ParseField("depends_on");
-    static final ParseField DISPLAY_FIELD = new ParseField("display");
+    static final ParseField DESCRIPTION_FIELD = new ParseField("description");
     static final ParseField LABEL_FIELD = new ParseField("label");
-    static final ParseField OPTIONS_FIELD = new ParseField("options");
-    static final ParseField ORDER_FIELD = new ParseField("order");
-    static final ParseField PLACEHOLDER_FIELD = new ParseField("placeholder");
     static final ParseField REQUIRED_FIELD = new ParseField("required");
     static final ParseField SENSITIVE_FIELD = new ParseField("sensitive");
-    static final ParseField TOOLTIP_FIELD = new ParseField("tooltip");
+    static final ParseField UPDATABLE_FIELD = new ParseField("updatable");
     static final ParseField TYPE_FIELD = new ParseField("type");
-    static final ParseField UI_RESTRICTIONS_FIELD = new ParseField("ui_restrictions");
-    static final ParseField VALIDATIONS_FIELD = new ParseField("validations");
-    static final ParseField VALUE_FIELD = new ParseField("value");
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<SettingsConfiguration, Void> PARSER = new ConstructingObjectParser<>(
@@ -168,27 +104,18 @@ public SettingsConfiguration(StreamInput in) throws IOException {
         true,
         args -> {
             int i = 0;
-            return new SettingsConfiguration.Builder().setCategory((String) args[i++])
-                .setDefaultValue(args[i++])
-                .setDependsOn((List<SettingsConfigurationDependency>) args[i++])
-                .setDisplay((SettingsConfigurationDisplayType) args[i++])
+            return new SettingsConfiguration.Builder().setDefaultValue(args[i++])
+                .setDescription((String) args[i++])
                 .setLabel((String) args[i++])
-                .setOptions((List<SettingsConfigurationSelectOption>) args[i++])
-                .setOrder((Integer) args[i++])
-                .setPlaceholder((String) args[i++])
                 .setRequired((Boolean) args[i++])
                 .setSensitive((Boolean) args[i++])
-                .setTooltip((String) args[i++])
+                .setUpdatable((Boolean) args[i++])
                 .setType((SettingsConfigurationFieldType) args[i++])
-                .setUiRestrictions((List<String>) args[i++])
-                .setValidations((List<SettingsConfigurationValidation>) args[i++])
-                .setValue(args[i])
                 .build();
         }
     );
 
     static {
-        PARSER.declareString(optionalConstructorArg(), CATEGORY_FIELD);
         PARSER.declareField(optionalConstructorArg(), (p, c) -> {
             if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
                 return p.text();
@@ -201,68 +128,31 @@ public SettingsConfiguration(StreamInput in) throws IOException {
             }
             throw new XContentParseException("Unsupported token [" + p.currentToken() + "]");
         }, DEFAULT_VALUE_FIELD, ObjectParser.ValueType.VALUE);
-        PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> SettingsConfigurationDependency.fromXContent(p), DEPENDS_ON_FIELD);
-        PARSER.declareField(
-            optionalConstructorArg(),
-            (p, c) -> SettingsConfigurationDisplayType.displayType(p.text()),
-            DISPLAY_FIELD,
-            ObjectParser.ValueType.STRING_OR_NULL
-        );
+        PARSER.declareStringOrNull(optionalConstructorArg(), DESCRIPTION_FIELD);
         PARSER.declareString(constructorArg(), LABEL_FIELD);
-        PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> SettingsConfigurationSelectOption.fromXContent(p), OPTIONS_FIELD);
-        PARSER.declareInt(optionalConstructorArg(), ORDER_FIELD);
-        PARSER.declareStringOrNull(optionalConstructorArg(), PLACEHOLDER_FIELD);
         PARSER.declareBoolean(optionalConstructorArg(), REQUIRED_FIELD);
         PARSER.declareBoolean(optionalConstructorArg(), SENSITIVE_FIELD);
-        PARSER.declareStringOrNull(optionalConstructorArg(), TOOLTIP_FIELD);
+        PARSER.declareBoolean(optionalConstructorArg(), UPDATABLE_FIELD);
         PARSER.declareField(
             optionalConstructorArg(),
             (p, c) -> p.currentToken() == XContentParser.Token.VALUE_NULL ? null : SettingsConfigurationFieldType.fieldType(p.text()),
             TYPE_FIELD,
             ObjectParser.ValueType.STRING_OR_NULL
         );
-        PARSER.declareStringArray(optionalConstructorArg(), UI_RESTRICTIONS_FIELD);
-        PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> SettingsConfigurationValidation.fromXContent(p), VALIDATIONS_FIELD);
-        PARSER.declareField(
-            optionalConstructorArg(),
-            (p, c) -> parseConfigurationValue(p),
-            VALUE_FIELD,
-            ObjectParser.ValueType.VALUE_OBJECT_ARRAY
-        );
-    }
-
-    public String getCategory() {
-        return category;
     }
 
     public Object getDefaultValue() {
         return defaultValue;
     }
 
-    public List<SettingsConfigurationDependency> getDependsOn() {
-        return dependsOn;
-    }
-
-    public SettingsConfigurationDisplayType getDisplay() {
-        return display;
+    public String getDescription() {
+        return description;
     }
 
     public String getLabel() {
         return label;
     }
 
-    public List<SettingsConfigurationSelectOption> getOptions() {
-        return options;
-    }
-
-    public Integer getOrder() {
-        return order;
-    }
-
-    public String getPlaceholder() {
-        return placeholder;
-    }
-
     public boolean isRequired() {
         return required;
     }
@@ -271,26 +161,14 @@ public boolean isSensitive() {
         return sensitive;
     }
 
-    public String getTooltip() {
-        return tooltip;
+    public boolean isUpdatable() {
+        return updatable;
     }
 
     public SettingsConfigurationFieldType getType() {
         return type;
     }
 
-    public List<String> getUiRestrictions() {
-        return uiRestrictions;
-    }
-
-    public List<SettingsConfigurationValidation> getValidations() {
-        return validations;
-    }
-
-    public Object getValue() {
-        return value;
-    }
-
     /**
      * Parses a configuration value from a parser context.
      * This method can parse strings, numbers, booleans, objects, and null values, matching the types commonly
@@ -319,47 +197,20 @@ public static Object parseConfigurationValue(XContentParser p) throws IOExceptio
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         {
-            if (category != null) {
-                builder.field(CATEGORY_FIELD.getPreferredName(), category);
-            }
-            builder.field(DEFAULT_VALUE_FIELD.getPreferredName(), defaultValue);
-            if (dependsOn != null) {
-                builder.xContentList(DEPENDS_ON_FIELD.getPreferredName(), dependsOn);
-            } else {
-                builder.xContentList(DEPENDS_ON_FIELD.getPreferredName(), new ArrayList<>());
+            if (defaultValue != null) {
+                builder.field(DEFAULT_VALUE_FIELD.getPreferredName(), defaultValue);
             }
-            if (display != null) {
-                builder.field(DISPLAY_FIELD.getPreferredName(), display.toString());
+            if (description != null) {
+                builder.field(DESCRIPTION_FIELD.getPreferredName(), description);
             }
             builder.field(LABEL_FIELD.getPreferredName(), label);
-            if (options != null) {
-                builder.xContentList(OPTIONS_FIELD.getPreferredName(), options);
-            }
-            if (order != null) {
-                builder.field(ORDER_FIELD.getPreferredName(), order);
-            }
-            if (placeholder != null) {
-                builder.field(PLACEHOLDER_FIELD.getPreferredName(), placeholder);
-            }
             builder.field(REQUIRED_FIELD.getPreferredName(), required);
             builder.field(SENSITIVE_FIELD.getPreferredName(), sensitive);
-            if (tooltip != null) {
-                builder.field(TOOLTIP_FIELD.getPreferredName(), tooltip);
-            }
+            builder.field(UPDATABLE_FIELD.getPreferredName(), updatable);
+
             if (type != null) {
                 builder.field(TYPE_FIELD.getPreferredName(), type.toString());
             }
-            if (uiRestrictions != null) {
-                builder.stringListField(UI_RESTRICTIONS_FIELD.getPreferredName(), uiRestrictions);
-            } else {
-                builder.stringListField(UI_RESTRICTIONS_FIELD.getPreferredName(), new ArrayList<>());
-            }
-            if (validations != null) {
-                builder.xContentList(VALIDATIONS_FIELD.getPreferredName(), validations);
-            } else {
-                builder.xContentList(VALIDATIONS_FIELD.getPreferredName(), new ArrayList<>());
-            }
-            builder.field(VALUE_FIELD.getPreferredName(), value);
         }
         builder.endObject();
         return builder;
@@ -379,57 +230,29 @@ public static SettingsConfiguration fromXContentBytes(BytesReference source, XCo
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(category);
         out.writeGenericValue(defaultValue);
-        out.writeOptionalCollection(dependsOn);
-        out.writeEnum(display);
+        out.writeOptionalString(description);
         out.writeString(label);
-        out.writeOptionalCollection(options);
-        out.writeOptionalInt(order);
-        out.writeOptionalString(placeholder);
         out.writeBoolean(required);
         out.writeBoolean(sensitive);
-        out.writeOptionalString(tooltip);
+        out.writeBoolean(updatable);
         out.writeEnum(type);
-        out.writeOptionalStringCollection(uiRestrictions);
-        out.writeOptionalCollection(validations);
-        out.writeGenericValue(value);
     }
 
     public Map<String, Object> toMap() {
         Map<String, Object> map = new HashMap<>();
 
-        Optional.ofNullable(category).ifPresent(c -> map.put(CATEGORY_FIELD.getPreferredName(), c));
         map.put(DEFAULT_VALUE_FIELD.getPreferredName(), defaultValue);
-
-        Optional.ofNullable(dependsOn)
-            .ifPresent(d -> map.put(DEPENDS_ON_FIELD.getPreferredName(), d.stream().map(SettingsConfigurationDependency::toMap).toList()));
-
-        Optional.ofNullable(display).ifPresent(d -> map.put(DISPLAY_FIELD.getPreferredName(), d.toString()));
+        Optional.ofNullable(description).ifPresent(t -> map.put(DESCRIPTION_FIELD.getPreferredName(), t));
 
         map.put(LABEL_FIELD.getPreferredName(), label);
 
-        Optional.ofNullable(options)
-            .ifPresent(o -> map.put(OPTIONS_FIELD.getPreferredName(), o.stream().map(SettingsConfigurationSelectOption::toMap).toList()));
-
-        Optional.ofNullable(order).ifPresent(o -> map.put(ORDER_FIELD.getPreferredName(), o));
-
-        Optional.ofNullable(placeholder).ifPresent(p -> map.put(PLACEHOLDER_FIELD.getPreferredName(), p));
-
         map.put(REQUIRED_FIELD.getPreferredName(), required);
         map.put(SENSITIVE_FIELD.getPreferredName(), sensitive);
-
-        Optional.ofNullable(tooltip).ifPresent(t -> map.put(TOOLTIP_FIELD.getPreferredName(), t));
+        map.put(UPDATABLE_FIELD.getPreferredName(), updatable);
 
         Optional.ofNullable(type).ifPresent(t -> map.put(TYPE_FIELD.getPreferredName(), t.toString()));
 
-        Optional.ofNullable(uiRestrictions).ifPresent(u -> map.put(UI_RESTRICTIONS_FIELD.getPreferredName(), u));
-
-        Optional.ofNullable(validations)
-            .ifPresent(v -> map.put(VALIDATIONS_FIELD.getPreferredName(), v.stream().map(SettingsConfigurationValidation::toMap).toList()));
-
-        map.put(VALUE_FIELD.getPreferredName(), value);
-
         return map;
     }
 
@@ -440,77 +263,35 @@ public boolean equals(Object o) {
         SettingsConfiguration that = (SettingsConfiguration) o;
         return required == that.required
             && sensitive == that.sensitive
-            && Objects.equals(category, that.category)
+            && updatable == that.updatable
             && Objects.equals(defaultValue, that.defaultValue)
-            && Objects.equals(dependsOn, that.dependsOn)
-            && display == that.display
+            && Objects.equals(description, that.description)
             && Objects.equals(label, that.label)
-            && Objects.equals(options, that.options)
-            && Objects.equals(order, that.order)
-            && Objects.equals(placeholder, that.placeholder)
-            && Objects.equals(tooltip, that.tooltip)
-            && type == that.type
-            && Objects.equals(uiRestrictions, that.uiRestrictions)
-            && Objects.equals(validations, that.validations)
-            && Objects.equals(value, that.value);
+            && type == that.type;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(
-            category,
-            defaultValue,
-            dependsOn,
-            display,
-            label,
-            options,
-            order,
-            placeholder,
-            required,
-            sensitive,
-            tooltip,
-            type,
-            uiRestrictions,
-            validations,
-            value
-        );
+        return Objects.hash(defaultValue, description, label, required, sensitive, updatable, type);
     }
 
     public static class Builder {
 
-        private String category;
         private Object defaultValue;
-        private List<SettingsConfigurationDependency> dependsOn;
-        private SettingsConfigurationDisplayType display;
+        private String description;
         private String label;
-        private List<SettingsConfigurationSelectOption> options;
-        private Integer order;
-        private String placeholder;
         private boolean required;
         private boolean sensitive;
-        private String tooltip;
+        private boolean updatable;
         private SettingsConfigurationFieldType type;
-        private List<String> uiRestrictions;
-        private List<SettingsConfigurationValidation> validations;
-        private Object value;
-
-        public Builder setCategory(String category) {
-            this.category = category;
-            return this;
-        }
 
         public Builder setDefaultValue(Object defaultValue) {
             this.defaultValue = defaultValue;
             return this;
         }
 
-        public Builder setDependsOn(List<SettingsConfigurationDependency> dependsOn) {
-            this.dependsOn = dependsOn;
-            return this;
-        }
-
-        public Builder setDisplay(SettingsConfigurationDisplayType display) {
-            this.display = display;
+        public Builder setDescription(String description) {
+            this.description = description;
             return this;
         }
 
@@ -519,21 +300,6 @@ public Builder setLabel(String label) {
             return this;
         }
 
-        public Builder setOptions(List<SettingsConfigurationSelectOption> options) {
-            this.options = options;
-            return this;
-        }
-
-        public Builder setOrder(Integer order) {
-            this.order = order;
-            return this;
-        }
-
-        public Builder setPlaceholder(String placeholder) {
-            this.placeholder = placeholder;
-            return this;
-        }
-
         public Builder setRequired(Boolean required) {
             this.required = Objects.requireNonNullElse(required, false);
             return this;
@@ -544,8 +310,8 @@ public Builder setSensitive(Boolean sensitive) {
             return this;
         }
 
-        public Builder setTooltip(String tooltip) {
-            this.tooltip = tooltip;
+        public Builder setUpdatable(Boolean updatable) {
+            this.updatable = Objects.requireNonNullElse(updatable, false);
             return this;
         }
 
@@ -554,39 +320,8 @@ public Builder setType(SettingsConfigurationFieldType type) {
             return this;
         }
 
-        public Builder setUiRestrictions(List<String> uiRestrictions) {
-            this.uiRestrictions = uiRestrictions;
-            return this;
-        }
-
-        public Builder setValidations(List<SettingsConfigurationValidation> validations) {
-            this.validations = validations;
-            return this;
-        }
-
-        public Builder setValue(Object value) {
-            this.value = value;
-            return this;
-        }
-
         public SettingsConfiguration build() {
-            return new SettingsConfiguration(
-                category,
-                defaultValue,
-                dependsOn,
-                display,
-                label,
-                options,
-                order,
-                placeholder,
-                required,
-                sensitive,
-                tooltip,
-                type,
-                uiRestrictions,
-                validations,
-                value
-            );
+            return new SettingsConfiguration(defaultValue, description, label, required, sensitive, updatable, type);
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/inference/TaskSettingsConfiguration.java b/server/src/main/java/org/elasticsearch/inference/TaskSettingsConfiguration.java
deleted file mode 100644
index 150532f138e8d..0000000000000
--- a/server/src/main/java/org/elasticsearch/inference/TaskSettingsConfiguration.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.inference;
-
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.XContentParserConfiguration;
-import org.elasticsearch.xcontent.XContentType;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-
-/**
- * Represents the configuration field settings for a specific task type inference provider.
- */
-public class TaskSettingsConfiguration implements Writeable, ToXContentObject {
-
-    private final TaskType taskType;
-    private final Map<String, SettingsConfiguration> configuration;
-
-    /**
-     * Constructs a new {@link TaskSettingsConfiguration} instance with specified properties.
-     *
-     * @param taskType       The {@link TaskType} this configuration describes.
-     * @param configuration  The configuration of the task, defined by {@link SettingsConfiguration}.
-     */
-    private TaskSettingsConfiguration(TaskType taskType, Map<String, SettingsConfiguration> configuration) {
-        this.taskType = taskType;
-        this.configuration = configuration;
-    }
-
-    public TaskSettingsConfiguration(StreamInput in) throws IOException {
-        this.taskType = in.readEnum(TaskType.class);
-        this.configuration = in.readMap(SettingsConfiguration::new);
-    }
-
-    static final ParseField TASK_TYPE_FIELD = new ParseField("task_type");
-    static final ParseField CONFIGURATION_FIELD = new ParseField("configuration");
-
-    @SuppressWarnings("unchecked")
-    private static final ConstructingObjectParser<TaskSettingsConfiguration, Void> PARSER = new ConstructingObjectParser<>(
-        "task_configuration",
-        true,
-        args -> {
-            return new TaskSettingsConfiguration.Builder().setTaskType(TaskType.fromString((String) args[0]))
-                .setConfiguration((Map<String, SettingsConfiguration>) args[1])
-                .build();
-        }
-    );
-
-    static {
-        PARSER.declareString(constructorArg(), TASK_TYPE_FIELD);
-        PARSER.declareObject(constructorArg(), (p, c) -> p.map(), CONFIGURATION_FIELD);
-    }
-
-    public TaskType getTaskType() {
-        return taskType;
-    }
-
-    public Map<String, SettingsConfiguration> getConfiguration() {
-        return configuration;
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        {
-            builder.field(TASK_TYPE_FIELD.getPreferredName(), taskType);
-            builder.field(CONFIGURATION_FIELD.getPreferredName(), configuration);
-        }
-        builder.endObject();
-        return builder;
-    }
-
-    public static TaskSettingsConfiguration fromXContent(XContentParser parser) throws IOException {
-        return PARSER.parse(parser, null);
-    }
-
-    public static TaskSettingsConfiguration fromXContentBytes(BytesReference source, XContentType xContentType) {
-        try (XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, source, xContentType)) {
-            return TaskSettingsConfiguration.fromXContent(parser);
-        } catch (IOException e) {
-            throw new ElasticsearchParseException("failed to parse task configuration", e);
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeEnum(taskType);
-        out.writeMapValues(configuration);
-    }
-
-    public Map<String, Object> toMap() {
-        Map<String, Object> map = new HashMap<>();
-
-        map.put(TASK_TYPE_FIELD.getPreferredName(), taskType);
-        map.put(CONFIGURATION_FIELD.getPreferredName(), configuration);
-
-        return map;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        TaskSettingsConfiguration that = (TaskSettingsConfiguration) o;
-        return Objects.equals(taskType, that.taskType) && Objects.equals(configuration, that.configuration);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(taskType, configuration);
-    }
-
-    public static class Builder {
-
-        private TaskType taskType;
-        private Map<String, SettingsConfiguration> configuration;
-
-        public Builder setTaskType(TaskType taskType) {
-            this.taskType = taskType;
-            return this;
-        }
-
-        public Builder setConfiguration(Map<String, SettingsConfiguration> configuration) {
-            this.configuration = configuration;
-            return this;
-        }
-
-        public TaskSettingsConfiguration build() {
-            return new TaskSettingsConfiguration(taskType, configuration);
-        }
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/inference/TaskType.java b/server/src/main/java/org/elasticsearch/inference/TaskType.java
index fcb8ea7213795..f319157828dfa 100644
--- a/server/src/main/java/org/elasticsearch/inference/TaskType.java
+++ b/server/src/main/java/org/elasticsearch/inference/TaskType.java
@@ -31,7 +31,7 @@ public boolean isAnyOrSame(TaskType other) {
         }
     };
 
-    public static String NAME = "task_type";
+    public static final String NAME = "task_type";
 
     public static TaskType fromString(String name) {
         return valueOf(name.trim().toUpperCase(Locale.ROOT));
diff --git a/server/src/main/java/org/elasticsearch/inference/UnifiedCompletionRequest.java b/server/src/main/java/org/elasticsearch/inference/UnifiedCompletionRequest.java
index e596be626b518..ae6a3bb10bd31 100644
--- a/server/src/main/java/org/elasticsearch/inference/UnifiedCompletionRequest.java
+++ b/server/src/main/java/org/elasticsearch/inference/UnifiedCompletionRequest.java
@@ -111,20 +111,24 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalFloat(topP);
     }
 
-    public record Message(Content content, String role, @Nullable String name, @Nullable String toolCallId, List<ToolCall> toolCalls)
+    public record Message(Content content, String role, @Nullable String toolCallId, @Nullable List<ToolCall> toolCalls)
         implements
             Writeable {
 
         @SuppressWarnings("unchecked")
         static final ConstructingObjectParser<Message, Void> PARSER = new ConstructingObjectParser<>(
             Message.class.getSimpleName(),
-            args -> new Message((Content) args[0], (String) args[1], (String) args[2], (String) args[3], (List<ToolCall>) args[4])
+            args -> new Message((Content) args[0], (String) args[1], (String) args[2], (List<ToolCall>) args[3])
         );
 
         static {
-            PARSER.declareField(constructorArg(), (p, c) -> parseContent(p), new ParseField("content"), ObjectParser.ValueType.VALUE_ARRAY);
+            PARSER.declareField(
+                optionalConstructorArg(),
+                (p, c) -> parseContent(p),
+                new ParseField("content"),
+                ObjectParser.ValueType.VALUE_ARRAY
+            );
             PARSER.declareString(constructorArg(), new ParseField("role"));
-            PARSER.declareString(optionalConstructorArg(), new ParseField("name"));
             PARSER.declareString(optionalConstructorArg(), new ParseField("tool_call_id"));
             PARSER.declareObjectArray(optionalConstructorArg(), ToolCall.PARSER::apply, new ParseField("tool_calls"));
         }
@@ -143,19 +147,17 @@ private static Content parseContent(XContentParser parser) throws IOException {
 
         public Message(StreamInput in) throws IOException {
             this(
-                in.readNamedWriteable(Content.class),
+                in.readOptionalNamedWriteable(Content.class),
                 in.readString(),
                 in.readOptionalString(),
-                in.readOptionalString(),
                 in.readOptionalCollectionAsList(ToolCall::new)
             );
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeNamedWriteable(content);
+            out.writeOptionalNamedWriteable(content);
             out.writeString(role);
-            out.writeOptionalString(name);
             out.writeOptionalString(toolCallId);
             out.writeOptionalCollection(toolCalls);
         }
diff --git a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationDependency.java b/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationDependency.java
deleted file mode 100644
index d319d1a395f85..0000000000000
--- a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationDependency.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.inference.configuration;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ObjectParser;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParseException;
-import org.elasticsearch.xcontent.XContentParser;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-
-/**
- * Represents a dependency within a connector configuration, defining a specific field and its associated value.
- * This class is used to encapsulate configuration dependencies in a structured format.
- */
-public class SettingsConfigurationDependency implements Writeable, ToXContentObject {
-
-    private final String field;
-    private final Object value;
-
-    /**
-     * Constructs a new instance of SettingsConfigurationDependency.
-     *
-     * @param field The name of the field in the service dependency.
-     * @param value The value associated with the field.
-     */
-    public SettingsConfigurationDependency(String field, Object value) {
-        this.field = field;
-        this.value = value;
-    }
-
-    public SettingsConfigurationDependency(StreamInput in) throws IOException {
-        this.field = in.readString();
-        this.value = in.readGenericValue();
-    }
-
-    private static final ParseField FIELD_FIELD = new ParseField("field");
-    private static final ParseField VALUE_FIELD = new ParseField("value");
-
-    private static final ConstructingObjectParser<SettingsConfigurationDependency, Void> PARSER = new ConstructingObjectParser<>(
-        "service_configuration_dependency",
-        true,
-        args -> new SettingsConfigurationDependency.Builder().setField((String) args[0]).setValue(args[1]).build()
-    );
-
-    static {
-        PARSER.declareString(constructorArg(), FIELD_FIELD);
-        PARSER.declareField(constructorArg(), (p, c) -> {
-            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
-                return p.text();
-            } else if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) {
-                return p.numberValue();
-            } else if (p.currentToken() == XContentParser.Token.VALUE_BOOLEAN) {
-                return p.booleanValue();
-            } else if (p.currentToken() == XContentParser.Token.VALUE_NULL) {
-                return null;
-            }
-            throw new XContentParseException("Unsupported token [" + p.currentToken() + "]");
-        }, VALUE_FIELD, ObjectParser.ValueType.VALUE);
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        {
-            builder.field(FIELD_FIELD.getPreferredName(), field);
-            builder.field(VALUE_FIELD.getPreferredName(), value);
-        }
-        builder.endObject();
-        return builder;
-    }
-
-    public Map<String, Object> toMap() {
-        Map<String, Object> map = new HashMap<>();
-        map.put(FIELD_FIELD.getPreferredName(), field);
-        map.put(VALUE_FIELD.getPreferredName(), value);
-        return map;
-    }
-
-    public static SettingsConfigurationDependency fromXContent(XContentParser parser) throws IOException {
-        return PARSER.parse(parser, null);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(field);
-        out.writeGenericValue(value);
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        SettingsConfigurationDependency that = (SettingsConfigurationDependency) o;
-        return Objects.equals(field, that.field) && Objects.equals(value, that.value);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(field, value);
-    }
-
-    public static class Builder {
-
-        private String field;
-        private Object value;
-
-        public Builder setField(String field) {
-            this.field = field;
-            return this;
-        }
-
-        public Builder setValue(Object value) {
-            this.value = value;
-            return this;
-        }
-
-        public SettingsConfigurationDependency build() {
-            return new SettingsConfigurationDependency(field, value);
-        }
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationDisplayType.java b/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationDisplayType.java
deleted file mode 100644
index e072238a52d01..0000000000000
--- a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationDisplayType.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.inference.configuration;
-
-import java.util.Locale;
-
-public enum SettingsConfigurationDisplayType {
-    TEXT,
-    TEXTBOX,
-    TEXTAREA,
-    NUMERIC,
-    TOGGLE,
-    DROPDOWN;
-
-    @Override
-    public String toString() {
-        return name().toLowerCase(Locale.ROOT);
-    }
-
-    public static SettingsConfigurationDisplayType displayType(String type) {
-        for (SettingsConfigurationDisplayType displayType : SettingsConfigurationDisplayType.values()) {
-            if (displayType.name().equalsIgnoreCase(type)) {
-                return displayType;
-            }
-        }
-        throw new IllegalArgumentException("Unknown " + SettingsConfigurationDisplayType.class.getSimpleName() + " [" + type + "].");
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationSelectOption.java b/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationSelectOption.java
deleted file mode 100644
index 8ad8d561da58e..0000000000000
--- a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationSelectOption.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.inference.configuration;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ObjectParser;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParseException;
-import org.elasticsearch.xcontent.XContentParser;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-
-public class SettingsConfigurationSelectOption implements Writeable, ToXContentObject {
-    private final String label;
-    private final Object value;
-
-    private SettingsConfigurationSelectOption(String label, Object value) {
-        this.label = label;
-        this.value = value;
-    }
-
-    public SettingsConfigurationSelectOption(StreamInput in) throws IOException {
-        this.label = in.readString();
-        this.value = in.readGenericValue();
-    }
-
-    private static final ParseField LABEL_FIELD = new ParseField("label");
-    private static final ParseField VALUE_FIELD = new ParseField("value");
-
-    private static final ConstructingObjectParser<SettingsConfigurationSelectOption, Void> PARSER = new ConstructingObjectParser<>(
-        "service_configuration_select_option",
-        true,
-        args -> new SettingsConfigurationSelectOption.Builder().setLabel((String) args[0]).setValue(args[1]).build()
-    );
-
-    static {
-        PARSER.declareString(constructorArg(), LABEL_FIELD);
-        PARSER.declareField(constructorArg(), (p, c) -> {
-            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
-                return p.text();
-            } else if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) {
-                return p.numberValue();
-            }
-            throw new XContentParseException("Unsupported token [" + p.currentToken() + "]");
-        }, VALUE_FIELD, ObjectParser.ValueType.VALUE);
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        {
-            builder.field(LABEL_FIELD.getPreferredName(), label);
-            builder.field(VALUE_FIELD.getPreferredName(), value);
-        }
-        builder.endObject();
-        return builder;
-    }
-
-    public Map<String, Object> toMap() {
-        Map<String, Object> map = new HashMap<>();
-        map.put(LABEL_FIELD.getPreferredName(), label);
-        map.put(VALUE_FIELD.getPreferredName(), value);
-        return map;
-    }
-
-    public static SettingsConfigurationSelectOption fromXContent(XContentParser parser) throws IOException {
-        return PARSER.parse(parser, null);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(label);
-        out.writeGenericValue(value);
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        SettingsConfigurationSelectOption that = (SettingsConfigurationSelectOption) o;
-        return Objects.equals(label, that.label) && Objects.equals(value, that.value);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(label, value);
-    }
-
-    public static class Builder {
-
-        private String label;
-        private Object value;
-
-        public Builder setLabel(String label) {
-            this.label = label;
-            return this;
-        }
-
-        public Builder setValue(Object value) {
-            this.value = value;
-            return this;
-        }
-
-        public Builder setLabelAndValue(String labelAndValue) {
-            this.label = labelAndValue;
-            this.value = labelAndValue;
-            return this;
-        }
-
-        public SettingsConfigurationSelectOption build() {
-            return new SettingsConfigurationSelectOption(label, value);
-        }
-    }
-
-}
diff --git a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidation.java b/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidation.java
deleted file mode 100644
index f106442d6d4ac..0000000000000
--- a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidation.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.inference.configuration;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ObjectParser;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParseException;
-import org.elasticsearch.xcontent.XContentParser;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-
-/**
- * Represents a configuration validation entity, encapsulating a validation constraint and its corresponding type.
- * This class is used to define and handle specific validation rules or requirements within a configuration context.
- */
-public class SettingsConfigurationValidation implements Writeable, ToXContentObject {
-
-    private final Object constraint;
-    private final SettingsConfigurationValidationType type;
-
-    /**
-     * Constructs a new SettingsConfigurationValidation instance with specified constraint and type.
-     * This constructor initializes the object with a given validation constraint and its associated validation type.
-     *
-     * @param constraint The validation constraint (string, number or list), represented as generic Object type.
-     * @param type       The type of configuration validation, specified as an instance of {@link SettingsConfigurationValidationType}.
-     */
-    private SettingsConfigurationValidation(Object constraint, SettingsConfigurationValidationType type) {
-        this.constraint = constraint;
-        this.type = type;
-    }
-
-    public SettingsConfigurationValidation(StreamInput in) throws IOException {
-        this.constraint = in.readGenericValue();
-        this.type = in.readEnum(SettingsConfigurationValidationType.class);
-    }
-
-    private static final ParseField CONSTRAINT_FIELD = new ParseField("constraint");
-    private static final ParseField TYPE_FIELD = new ParseField("type");
-
-    private static final ConstructingObjectParser<SettingsConfigurationValidation, Void> PARSER = new ConstructingObjectParser<>(
-        "service_configuration_validation",
-        true,
-        args -> new SettingsConfigurationValidation.Builder().setConstraint(args[0])
-            .setType((SettingsConfigurationValidationType) args[1])
-            .build()
-    );
-
-    static {
-        PARSER.declareField(
-            constructorArg(),
-            (p, c) -> parseConstraintValue(p),
-            CONSTRAINT_FIELD,
-            ObjectParser.ValueType.VALUE_OBJECT_ARRAY
-        );
-        PARSER.declareField(
-            constructorArg(),
-            (p, c) -> SettingsConfigurationValidationType.validationType(p.text()),
-            TYPE_FIELD,
-            ObjectParser.ValueType.STRING
-        );
-    }
-
-    /**
-     * Parses the value of a constraint from the XContentParser stream.
-     * This method is designed to handle various types of constraint values as per the connector's protocol original specification.
-     * The constraints can be of type string, number, or list of values.
-     */
-    private static Object parseConstraintValue(XContentParser p) throws IOException {
-        if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
-            return p.text();
-        } else if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) {
-            return p.numberValue();
-        } else if (p.currentToken() == XContentParser.Token.START_ARRAY) {
-            return p.list();
-        }
-        throw new XContentParseException("Unsupported token [" + p.currentToken() + "]");
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        {
-            builder.field(CONSTRAINT_FIELD.getPreferredName(), constraint);
-            builder.field(TYPE_FIELD.getPreferredName(), type.toString());
-        }
-        builder.endObject();
-        return builder;
-    }
-
-    public Map<String, Object> toMap() {
-        return Map.of(CONSTRAINT_FIELD.getPreferredName(), constraint, TYPE_FIELD.getPreferredName(), type.toString());
-    }
-
-    public static SettingsConfigurationValidation fromXContent(XContentParser parser) throws IOException {
-        return PARSER.parse(parser, null);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeGenericValue(constraint);
-        out.writeEnum(type);
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        SettingsConfigurationValidation that = (SettingsConfigurationValidation) o;
-        return Objects.equals(constraint, that.constraint) && type == that.type;
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(constraint, type);
-    }
-
-    public static class Builder {
-
-        private Object constraint;
-        private SettingsConfigurationValidationType type;
-
-        public Builder setConstraint(Object constraint) {
-            this.constraint = constraint;
-            return this;
-        }
-
-        public Builder setType(SettingsConfigurationValidationType type) {
-            this.type = type;
-            return this;
-        }
-
-        public SettingsConfigurationValidation build() {
-            return new SettingsConfigurationValidation(constraint, type);
-        }
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidationType.java b/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidationType.java
deleted file mode 100644
index 6fb07d38d7db5..0000000000000
--- a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidationType.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.inference.configuration;
-
-import java.util.Locale;
-
-public enum SettingsConfigurationValidationType {
-    LESS_THAN,
-    GREATER_THAN,
-    LIST_TYPE,
-    INCLUDED_IN,
-    REGEX;
-
-    @Override
-    public String toString() {
-        return name().toLowerCase(Locale.ROOT);
-    }
-
-    public static SettingsConfigurationValidationType validationType(String type) {
-        for (SettingsConfigurationValidationType displayType : SettingsConfigurationValidationType.values()) {
-            if (displayType.name().equalsIgnoreCase(type)) {
-                return displayType;
-            }
-        }
-        throw new IllegalArgumentException("Unknown " + SettingsConfigurationValidationType.class.getSimpleName() + " [" + type + "].");
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/ingest/EnterpriseGeoIpTask.java b/server/src/main/java/org/elasticsearch/ingest/EnterpriseGeoIpTask.java
index ff6a687da9b4d..56c08f2b2fb85 100644
--- a/server/src/main/java/org/elasticsearch/ingest/EnterpriseGeoIpTask.java
+++ b/server/src/main/java/org/elasticsearch/ingest/EnterpriseGeoIpTask.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.persistent.PersistentTaskParams;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -36,7 +35,6 @@ private EnterpriseGeoIpTask() {
     }
 
     public static final String ENTERPRISE_GEOIP_DOWNLOADER = "enterprise-geoip-downloader";
-    public static final NodeFeature GEOIP_DOWNLOADER_DATABASE_CONFIGURATION = new NodeFeature("geoip.downloader.database.configuration");
 
     public static class EnterpriseGeoIpTaskParams implements PersistentTaskParams {
 
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java b/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java
deleted file mode 100644
index 77b11357d79b1..0000000000000
--- a/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.ingest;
-
-import org.elasticsearch.features.FeatureSpecification;
-import org.elasticsearch.features.NodeFeature;
-
-import java.util.Set;
-
-import static org.elasticsearch.ingest.EnterpriseGeoIpTask.GEOIP_DOWNLOADER_DATABASE_CONFIGURATION;
-
-public class IngestGeoIpFeatures implements FeatureSpecification {
-
-    public static final NodeFeature GET_DATABASE_CONFIGURATION_ACTION_MULTI_NODE = new NodeFeature(
-        "get_database_configuration_action.multi_node"
-    );
-
-    public static final NodeFeature PUT_DATABASE_CONFIGURATION_ACTION_IPINFO = new NodeFeature("put_database_configuration_action.ipinfo");
-
-    public Set<NodeFeature> getFeatures() {
-        return Set.of(
-            GEOIP_DOWNLOADER_DATABASE_CONFIGURATION,
-            GET_DATABASE_CONFIGURATION_ACTION_MULTI_NODE,
-            PUT_DATABASE_CONFIGURATION_ACTION_IPINFO
-        );
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java b/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
index 8654142016572..ca927e5af74c7 100644
--- a/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
+++ b/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
@@ -98,8 +98,8 @@ public static IngestMetadata fromXContent(XContentParser parser) throws IOExcept
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).array(PIPELINES_FIELD.getPreferredName(), pipelines.values().iterator());
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.array(PIPELINES_FIELD.getPreferredName(), pipelines.values().iterator());
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestStats.java b/server/src/main/java/org/elasticsearch/ingest/IngestStats.java
index 39df4e53423a2..c75cd3a022cb8 100644
--- a/server/src/main/java/org/elasticsearch/ingest/IngestStats.java
+++ b/server/src/main/java/org/elasticsearch/ingest/IngestStats.java
@@ -143,12 +143,10 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerP
                             return builder;
                         }
                     ),
-
-                    Iterators.<ToXContent>single((builder, params) -> builder.endArray().endObject())
+                    Iterators.single((builder, params) -> builder.endArray().endObject())
                 )
             ),
-
-            Iterators.<ToXContent>single((builder, params) -> builder.endObject().endObject())
+            Iterators.single((builder, params) -> builder.endObject().endObject())
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomFieldHighlighter.java b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomFieldHighlighter.java
index 3e59814de0585..acf186faf20b2 100644
--- a/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomFieldHighlighter.java
+++ b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomFieldHighlighter.java
@@ -34,7 +34,7 @@ class CustomFieldHighlighter extends FieldHighlighter {
     private final Locale breakIteratorLocale;
     private final int noMatchSize;
     private String fieldValue;
-    private final Integer queryMaxAnalyzedOffset;
+    private final QueryMaxAnalyzedOffset queryMaxAnalyzedOffset;
 
     CustomFieldHighlighter(
         String field,
@@ -47,7 +47,7 @@ class CustomFieldHighlighter extends FieldHighlighter {
         PassageFormatter passageFormatter,
         Comparator<Passage> passageSortComparator,
         int noMatchSize,
-        Integer queryMaxAnalyzedOffset
+        QueryMaxAnalyzedOffset queryMaxAnalyzedOffset
     ) {
         super(
             field,
@@ -113,7 +113,7 @@ protected Passage[] getSummaryPassagesNoHighlight(int maxPassages) {
     @Override
     protected Passage[] highlightOffsetsEnums(OffsetsEnum off) throws IOException {
         if (queryMaxAnalyzedOffset != null) {
-            off = new LimitedOffsetsEnum(off, queryMaxAnalyzedOffset);
+            off = new LimitedOffsetsEnum(off, queryMaxAnalyzedOffset.getNotNull());
         }
         return super.highlightOffsetsEnums(off);
     }
diff --git a/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java
index d1c7d0415ad15..59dffb73985ac 100644
--- a/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java
+++ b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java
@@ -66,7 +66,7 @@ public final class CustomUnifiedHighlighter extends UnifiedHighlighter {
     private final int noMatchSize;
     private final CustomFieldHighlighter fieldHighlighter;
     private final int maxAnalyzedOffset;
-    private final Integer queryMaxAnalyzedOffset;
+    private final QueryMaxAnalyzedOffset queryMaxAnalyzedOffset;
 
     /**
      * Creates a new instance of {@link CustomUnifiedHighlighter}
@@ -94,7 +94,7 @@ public CustomUnifiedHighlighter(
         int noMatchSize,
         int maxPassages,
         int maxAnalyzedOffset,
-        Integer queryMaxAnalyzedOffset,
+        QueryMaxAnalyzedOffset queryMaxAnalyzedOffset,
         boolean requireFieldMatch,
         boolean weightMatchesEnabled
     ) {
@@ -125,9 +125,9 @@ public Snippet[] highlightField(LeafReader reader, int docId, CheckedSupplier<St
             return null;
         }
         int fieldValueLength = fieldValue.length();
-        if (((queryMaxAnalyzedOffset == null || queryMaxAnalyzedOffset > maxAnalyzedOffset)
+        if ((queryMaxAnalyzedOffset == null || queryMaxAnalyzedOffset.getNotNull() > maxAnalyzedOffset)
             && (getOffsetSource(field) == OffsetSource.ANALYSIS)
-            && (fieldValueLength > maxAnalyzedOffset))) {
+            && (fieldValueLength > maxAnalyzedOffset)) {
             throw new IllegalArgumentException(
                 "The length ["
                     + fieldValueLength
diff --git a/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/QueryMaxAnalyzedOffset.java b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/QueryMaxAnalyzedOffset.java
new file mode 100644
index 0000000000000..e74b11d4e1a91
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/QueryMaxAnalyzedOffset.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.lucene.search.uhighlight;
+
+public class QueryMaxAnalyzedOffset {
+    private final int queryMaxAnalyzedOffset;
+
+    private QueryMaxAnalyzedOffset(final int queryMaxAnalyzedOffset) {
+        // If we have a negative value, grab value for the actual maximum from the index.
+        this.queryMaxAnalyzedOffset = queryMaxAnalyzedOffset;
+    }
+
+    public static QueryMaxAnalyzedOffset create(final Integer queryMaxAnalyzedOffset, final int indexMaxAnalyzedOffset) {
+        if (queryMaxAnalyzedOffset == null) {
+            return null;
+        }
+        return new QueryMaxAnalyzedOffset(queryMaxAnalyzedOffset < 0 ? indexMaxAnalyzedOffset : queryMaxAnalyzedOffset);
+    }
+
+    public int getNotNull() {
+        return queryMaxAnalyzedOffset;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/lucene/util/automaton/MinimizationOperations.java b/server/src/main/java/org/elasticsearch/lucene/util/automaton/MinimizationOperations.java
new file mode 100644
index 0000000000000..68d848d8af165
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/lucene/util/automaton/MinimizationOperations.java
@@ -0,0 +1,344 @@
+/*
+ * @notice
+ * Copyright (c) 2001-2009 Anders Moeller
+ * Modifications copyright (C) 2024 Elasticsearch B.V.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Sourced from: https://github.com/apache/lucene/blob/main/lucene/core/src/test/org/apache/lucene/util/automaton/MinimizationOperations.java
+ */
+
+package org.elasticsearch.lucene.util.automaton;
+
+import org.apache.lucene.internal.hppc.IntArrayList;
+import org.apache.lucene.internal.hppc.IntCursor;
+import org.apache.lucene.internal.hppc.IntHashSet;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.Operations;
+import org.apache.lucene.util.automaton.Transition;
+
+import java.util.BitSet;
+import java.util.LinkedList;
+
+/**
+ * Operations for minimizing automata.
+ * <p>
+ * Lucene 10 removed minimization, but Elasticsearch still requires it.
+ * Minimization is critical in the security codebase to reduce the heap
+ * usage of automata used for permission checks.
+ * <p>
+ * Copied of Lucene's AutomatonTestUtil
+ */
+public final class MinimizationOperations {
+
+    private MinimizationOperations() {}
+
+    /**
+     * Minimizes (and determinizes if not already deterministic) the given automaton using Hopcroft's
+     * algorithm.
+     *
+     * @param determinizeWorkLimit maximum effort to spend determinizing the automaton. Set higher to
+     *     allow more complex queries and lower to prevent memory exhaustion. Use {@link
+     *     Operations#DEFAULT_DETERMINIZE_WORK_LIMIT} as a decent default if you don't otherwise know
+     *     what to specify.
+     */
+    public static Automaton minimize(Automaton a, int determinizeWorkLimit) {
+
+        if (a.getNumStates() == 0 || (a.isAccept(0) == false && a.getNumTransitions(0) == 0)) {
+            // Fastmatch for common case
+            return new Automaton();
+        }
+        a = Operations.determinize(a, determinizeWorkLimit);
+        // a.writeDot("adet");
+        if (a.getNumTransitions(0) == 1) {
+            Transition t = new Transition();
+            a.getTransition(0, 0, t);
+            if (t.dest == 0 && t.min == Character.MIN_CODE_POINT && t.max == Character.MAX_CODE_POINT) {
+                // Accepts all strings
+                return a;
+            }
+        }
+        a = totalize(a);
+        // a.writeDot("atot");
+
+        // initialize data structures
+        final int[] sigma = a.getStartPoints();
+        final int sigmaLen = sigma.length, statesLen = a.getNumStates();
+
+        final IntArrayList[][] reverse = new IntArrayList[statesLen][sigmaLen];
+        final IntHashSet[] partition = new IntHashSet[statesLen];
+        final IntArrayList[] splitblock = new IntArrayList[statesLen];
+        final int[] block = new int[statesLen];
+        final StateList[][] active = new StateList[statesLen][sigmaLen];
+        final StateListNode[][] active2 = new StateListNode[statesLen][sigmaLen];
+        final LinkedList<IntPair> pending = new LinkedList<>();
+        final BitSet pending2 = new BitSet(sigmaLen * statesLen);
+        final BitSet split = new BitSet(statesLen), refine = new BitSet(statesLen), refine2 = new BitSet(statesLen);
+        for (int q = 0; q < statesLen; q++) {
+            splitblock[q] = new IntArrayList();
+            partition[q] = new IntHashSet();
+            for (int x = 0; x < sigmaLen; x++) {
+                active[q][x] = StateList.EMPTY;
+            }
+        }
+        // find initial partition and reverse edges
+        for (int q = 0; q < statesLen; q++) {
+            // TODO moved the following into the loop because we cannot reset transition.transitionUpto (pkg private)
+            Transition transition = new Transition();
+            final int j = a.isAccept(q) ? 0 : 1;
+            partition[j].add(q);
+            block[q] = j;
+            transition.source = q;
+            // TODO we'd need to be able to access pkg private transition.transitionUpto if we want to optimize the following
+            // transition.transitionUpto = -1;
+            for (int x = 0; x < sigmaLen; x++) {
+                final IntArrayList[] r = reverse[a.next(transition, sigma[x])];
+                if (r[x] == null) {
+                    r[x] = new IntArrayList();
+                }
+                r[x].add(q);
+            }
+        }
+        // initialize active sets
+        for (int j = 0; j <= 1; j++) {
+            for (int x = 0; x < sigmaLen; x++) {
+                for (IntCursor qCursor : partition[j]) {
+                    int q = qCursor.value;
+                    if (reverse[q][x] != null) {
+                        StateList stateList = active[j][x];
+                        if (stateList == StateList.EMPTY) {
+                            stateList = new StateList();
+                            active[j][x] = stateList;
+                        }
+                        active2[q][x] = stateList.add(q);
+                    }
+                }
+            }
+        }
+
+        // initialize pending
+        for (int x = 0; x < sigmaLen; x++) {
+            final int j = (active[0][x].size <= active[1][x].size) ? 0 : 1;
+            pending.add(new IntPair(j, x));
+            pending2.set(x * statesLen + j);
+        }
+
+        // process pending until fixed point
+        int k = 2;
+        // System.out.println("start min");
+        while (false == pending.isEmpty()) {
+            // System.out.println(" cycle pending");
+            final IntPair ip = pending.removeFirst();
+            final int p = ip.n1;
+            final int x = ip.n2;
+            // System.out.println(" pop n1=" + ip.n1 + " n2=" + ip.n2);
+            pending2.clear(x * statesLen + p);
+            // find states that need to be split off their blocks
+            for (StateListNode m = active[p][x].first; m != null; m = m.next) {
+                final IntArrayList r = reverse[m.q][x];
+                if (r != null) {
+                    for (IntCursor iCursor : r) {
+                        final int i = iCursor.value;
+                        if (false == split.get(i)) {
+                            split.set(i);
+                            final int j = block[i];
+                            splitblock[j].add(i);
+                            if (false == refine2.get(j)) {
+                                refine2.set(j);
+                                refine.set(j);
+                            }
+                        }
+                    }
+                }
+            }
+
+            // refine blocks
+            for (int j = refine.nextSetBit(0); j >= 0; j = refine.nextSetBit(j + 1)) {
+                final IntArrayList sb = splitblock[j];
+                if (sb.size() < partition[j].size()) {
+                    final IntHashSet b1 = partition[j];
+                    final IntHashSet b2 = partition[k];
+                    for (IntCursor iCursor : sb) {
+                        final int s = iCursor.value;
+                        b1.remove(s);
+                        b2.add(s);
+                        block[s] = k;
+                        for (int c = 0; c < sigmaLen; c++) {
+                            final StateListNode sn = active2[s][c];
+                            if (sn != null && sn.sl == active[j][c]) {
+                                sn.remove();
+                                StateList stateList = active[k][c];
+                                if (stateList == StateList.EMPTY) {
+                                    stateList = new StateList();
+                                    active[k][c] = stateList;
+                                }
+                                active2[s][c] = stateList.add(s);
+                            }
+                        }
+                    }
+                    // update pending
+                    for (int c = 0; c < sigmaLen; c++) {
+                        final int aj = active[j][c].size, ak = active[k][c].size, ofs = c * statesLen;
+                        if ((false == pending2.get(ofs + j)) && 0 < aj && aj <= ak) {
+                            pending2.set(ofs + j);
+                            pending.add(new IntPair(j, c));
+                        } else {
+                            pending2.set(ofs + k);
+                            pending.add(new IntPair(k, c));
+                        }
+                    }
+                    k++;
+                }
+                refine2.clear(j);
+                for (IntCursor iCursor : sb) {
+                    final int s = iCursor.value;
+                    split.clear(s);
+                }
+                sb.clear();
+            }
+            refine.clear();
+        }
+
+        Automaton result = new Automaton();
+
+        Transition t = new Transition();
+
+        // System.out.println(" k=" + k);
+
+        // make a new state for each equivalence class, set initial state
+        int[] stateMap = new int[statesLen];
+        int[] stateRep = new int[k];
+
+        result.createState();
+
+        // System.out.println("min: k=" + k);
+        for (int n = 0; n < k; n++) {
+            // System.out.println(" n=" + n);
+
+            boolean isInitial = partition[n].contains(0);
+
+            int newState;
+            if (isInitial) {
+                // System.out.println(" isInitial!");
+                newState = 0;
+            } else {
+                newState = result.createState();
+            }
+
+            // System.out.println(" newState=" + newState);
+
+            for (IntCursor qCursor : partition[n]) {
+                int q = qCursor.value;
+                stateMap[q] = newState;
+                // System.out.println(" q=" + q + " isAccept?=" + a.isAccept(q));
+                result.setAccept(newState, a.isAccept(q));
+                stateRep[newState] = q; // select representative
+            }
+        }
+
+        // build transitions and set acceptance
+        for (int n = 0; n < k; n++) {
+            int numTransitions = a.initTransition(stateRep[n], t);
+            for (int i = 0; i < numTransitions; i++) {
+                a.getNextTransition(t);
+                // System.out.println(" add trans");
+                result.addTransition(n, stateMap[t.dest], t.min, t.max);
+            }
+        }
+        result.finishState();
+        // System.out.println(result.getNumStates() + " states");
+
+        return Operations.removeDeadStates(result);
+    }
+
+    record IntPair(int n1, int n2) {}
+
+    static final class StateList {
+
+        // Empty list that should never be mutated, used as a memory saving optimization instead of null
+        // so we don't need to branch the read path in #minimize
+        static final StateList EMPTY = new StateList();
+
+        int size;
+
+        StateListNode first, last;
+
+        StateListNode add(int q) {
+            assert this != EMPTY;
+            return new StateListNode(q, this);
+        }
+    }
+
+    static final class StateListNode {
+
+        final int q;
+
+        StateListNode next, prev;
+
+        final StateList sl;
+
+        StateListNode(int q, StateList sl) {
+            this.q = q;
+            this.sl = sl;
+            if (sl.size++ == 0) sl.first = sl.last = this;
+            else {
+                sl.last.next = this;
+                prev = sl.last;
+                sl.last = this;
+            }
+        }
+
+        void remove() {
+            sl.size--;
+            if (sl.first == this) sl.first = next;
+            else prev.next = next;
+            if (sl.last == this) sl.last = prev;
+            else next.prev = prev;
+        }
+    }
+
+    static Automaton totalize(Automaton a) {
+        Automaton result = new Automaton();
+        int numStates = a.getNumStates();
+        for (int i = 0; i < numStates; i++) {
+            result.createState();
+            result.setAccept(i, a.isAccept(i));
+        }
+
+        int deadState = result.createState();
+        result.addTransition(deadState, deadState, Character.MIN_CODE_POINT, Character.MAX_CODE_POINT);
+
+        Transition t = new Transition();
+        for (int i = 0; i < numStates; i++) {
+            int maxi = Character.MIN_CODE_POINT;
+            int count = a.initTransition(i, t);
+            for (int j = 0; j < count; j++) {
+                a.getNextTransition(t);
+                result.addTransition(i, t.dest, t.min, t.max);
+                if (t.min > maxi) {
+                    result.addTransition(i, deadState, maxi, t.min - 1);
+                }
+                if (t.max + 1 > maxi) {
+                    maxi = t.max + 1;
+                }
+            }
+
+            if (maxi <= Character.MAX_CODE_POINT) {
+                result.addTransition(i, deadState, maxi, Character.MAX_CODE_POINT);
+            }
+        }
+
+        result.finishState();
+        return result;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java b/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java
index 99011d101d342..dfaf6535e4d85 100644
--- a/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java
+++ b/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java
@@ -53,7 +53,7 @@ public IndicesMetrics(MeterRegistry meterRegistry, IndicesService indicesService
     }
 
     private static List<AutoCloseable> registerAsyncMetrics(MeterRegistry registry, IndicesStatsCache cache) {
-        final int TOTAL_METRICS = 48;
+        final int TOTAL_METRICS = 52;
         List<AutoCloseable> metrics = new ArrayList<>(TOTAL_METRICS);
         for (IndexMode indexMode : IndexMode.values()) {
             String name = indexMode.getName();
@@ -156,6 +156,14 @@ private static List<AutoCloseable> registerAsyncMetrics(MeterRegistry registry,
                     diffGauge(() -> cache.getOrRefresh().get(indexMode).indexing.getIndexFailedCount())
                 )
             );
+            metrics.add(
+                registry.registerLongGauge(
+                    "es.indices." + name + ".indexing.failure.version_conflict.total",
+                    "current indexing failures due to version conflict of " + name + " indices",
+                    "unit",
+                    diffGauge(() -> cache.getOrRefresh().get(indexMode).indexing.getIndexFailedDueToVersionConflictCount())
+                )
+            );
         }
         assert metrics.size() == TOTAL_METRICS : "total number of metrics has changed";
         return metrics;
diff --git a/server/src/main/java/org/elasticsearch/monitor/metrics/NodeMetrics.java b/server/src/main/java/org/elasticsearch/monitor/metrics/NodeMetrics.java
index 94395193622e0..2fabd03970f05 100644
--- a/server/src/main/java/org/elasticsearch/monitor/metrics/NodeMetrics.java
+++ b/server/src/main/java/org/elasticsearch/monitor/metrics/NodeMetrics.java
@@ -365,6 +365,22 @@ private void registerAsyncMetrics(MeterRegistry registry) {
             )
         );
 
+        metrics.add(
+            registry.registerLongAsyncCounter(
+                "es.indexing.indexing.failed.version_conflict.total",
+                "Total number of failed indexing operations due to version conflict",
+                "operations",
+                () -> new LongWithAttributes(
+                    Optional.ofNullable(stats.getOrRefresh())
+                        .map(o -> o.getIndices())
+                        .map(o -> o.getIndexing())
+                        .map(o -> o.getTotal())
+                        .map(o -> o.getIndexFailedDueToVersionConflictCount())
+                        .orElse(0L)
+                )
+            )
+        );
+
         metrics.add(
             registry.registerLongAsyncCounter(
                 "es.indexing.deletion.docs.total",
diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
index 17e56a392daff..4693b4fcf718a 100644
--- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
+++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
@@ -42,6 +42,7 @@
 import org.elasticsearch.cluster.coordination.Coordinator;
 import org.elasticsearch.cluster.coordination.MasterHistoryService;
 import org.elasticsearch.cluster.coordination.StableMasterHealthIndicatorService;
+import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.IndexMetadataVerifier;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -83,7 +84,7 @@
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.core.UpdateForV9;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
@@ -111,6 +112,8 @@
 import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexSettingProviders;
 import org.elasticsearch.index.IndexingPressure;
+import org.elasticsearch.index.SlowLogFieldProvider;
+import org.elasticsearch.index.SlowLogFields;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.SourceFieldMetrics;
@@ -553,9 +556,10 @@ private SettingsModule validateSettings(Settings envSettings, Settings settings,
         return settingsModule;
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_FOUNDATIONS)
+    @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS)
     private static void addBwcSearchWorkerSettings(List<Setting<?>> additionalSettings) {
-        // TODO remove the below settings, they are unused and only here to enable BwC for deployments that still use them
+        // Search workers thread pool has been removed in Elasticsearch 8.16.0. These settings are deprecated and take no effect.
+        // They are here only to enable BwC for deployments that still use them
         additionalSettings.add(
             Setting.intSetting("thread_pool.search_worker.queue_size", 0, Setting.Property.NodeScope, Setting.Property.DeprecatedWarning)
         );
@@ -626,7 +630,6 @@ private ScriptService createScriptService(SettingsModule settingsModule, ThreadP
     }
 
     private DataStreamGlobalRetentionSettings createDataStreamServicesAndGlobalRetentionResolver(
-        Settings settings,
         ThreadPool threadPool,
         ClusterService clusterService,
         IndicesService indicesService,
@@ -636,6 +639,10 @@ private DataStreamGlobalRetentionSettings createDataStreamServicesAndGlobalReten
             clusterService.getClusterSettings()
         );
         modules.bindToInstance(DataStreamGlobalRetentionSettings.class, dataStreamGlobalRetentionSettings);
+        modules.bindToInstance(
+            DataStreamFailureStoreSettings.class,
+            DataStreamFailureStoreSettings.create(clusterService.getClusterSettings())
+        );
         modules.bindToInstance(
             MetadataCreateDataStreamService.class,
             new MetadataCreateDataStreamService(threadPool, clusterService, metadataCreateIndexService)
@@ -800,6 +807,31 @@ private void construct(
             new ShardSearchPhaseAPMMetrics(telemetryProvider.getMeterRegistry())
         );
 
+        List<? extends SlowLogFieldProvider> slowLogFieldProviders = pluginsService.loadServiceProviders(SlowLogFieldProvider.class);
+        // NOTE: the response of index/search slow log fields below must be calculated dynamically on every call
+        // because the responses may change dynamically at runtime
+        SlowLogFieldProvider slowLogFieldProvider = indexSettings -> {
+            final List<SlowLogFields> fields = new ArrayList<>();
+            for (var provider : slowLogFieldProviders) {
+                fields.add(provider.create(indexSettings));
+            }
+            return new SlowLogFields() {
+                @Override
+                public Map<String, String> indexFields() {
+                    return fields.stream()
+                        .flatMap(f -> f.indexFields().entrySet().stream())
+                        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+                }
+
+                @Override
+                public Map<String, String> searchFields() {
+                    return fields.stream()
+                        .flatMap(f -> f.searchFields().entrySet().stream())
+                        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+                }
+            };
+        };
+
         IndicesService indicesService = new IndicesServiceBuilder().settings(settings)
             .pluginsService(pluginsService)
             .nodeEnvironment(nodeEnvironment)
@@ -821,6 +853,7 @@ private void construct(
             .requestCacheKeyDifferentiator(searchModule.getRequestCacheKeyDifferentiator())
             .mapperMetrics(mapperMetrics)
             .searchOperationListeners(searchOperationListeners)
+            .slowLogFieldProvider(slowLogFieldProvider)
             .build();
 
         final var parameters = new IndexSettingProvider.Parameters(clusterService, indicesService::createIndexMapperServiceForValidation);
@@ -858,7 +891,6 @@ private void construct(
         );
 
         final DataStreamGlobalRetentionSettings dataStreamGlobalRetentionSettings = createDataStreamServicesAndGlobalRetentionResolver(
-            settings,
             threadPool,
             clusterService,
             indicesService,
@@ -1168,7 +1200,6 @@ private void construct(
         DataStreamAutoShardingService dataStreamAutoShardingService = new DataStreamAutoShardingService(
             settings,
             clusterService,
-            featureService,
             threadPool::absoluteTimeInMillis
         );
         dataStreamAutoShardingService.init();
@@ -1319,7 +1350,7 @@ private Module loadDiagnosticServices(
 
         var serverHealthIndicatorServices = Stream.of(
             new StableMasterHealthIndicatorService(coordinationDiagnosticsService, clusterService),
-            new RepositoryIntegrityHealthIndicatorService(clusterService, featureService),
+            new RepositoryIntegrityHealthIndicatorService(clusterService),
             new DiskHealthIndicatorService(clusterService, featureService),
             new ShardsCapacityHealthIndicatorService(clusterService, featureService),
             fileSettingsHealthIndicatorService
@@ -1344,14 +1375,7 @@ private Module loadDiagnosticServices(
             new DiskHealthTracker(nodeService, clusterService),
             new RepositoriesHealthTracker(repositoriesService)
         );
-        LocalHealthMonitor localHealthMonitor = LocalHealthMonitor.create(
-            settings,
-            clusterService,
-            threadPool,
-            client,
-            featureService,
-            healthTrackers
-        );
+        LocalHealthMonitor localHealthMonitor = LocalHealthMonitor.create(settings, clusterService, threadPool, client, healthTrackers);
         HealthInfoCache nodeHealthOverview = HealthInfoCache.create(clusterService);
 
         return b -> {
diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java
index bde2d55ef2940..55753f2827d2a 100644
--- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java
+++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java
@@ -17,11 +17,12 @@
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ObjectParser;
@@ -551,8 +552,11 @@ public static NamedDiff<Metadata.Custom> readDiffFrom(StreamInput in) throws IOE
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).field("last_allocation_id", lastAllocationId).array("tasks", tasks.values().iterator());
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.concat(
+            Iterators.single((builder, params) -> builder.field("last_allocation_id", lastAllocationId)),
+            ChunkedToXContentHelper.array("tasks", tasks.values().iterator())
+        );
     }
 
     public static Builder builder() {
diff --git a/server/src/main/java/org/elasticsearch/readiness/ReadinessService.java b/server/src/main/java/org/elasticsearch/readiness/ReadinessService.java
index 15b9eacfa2118..1a169699d4131 100644
--- a/server/src/main/java/org/elasticsearch/readiness/ReadinessService.java
+++ b/server/src/main/java/org/elasticsearch/readiness/ReadinessService.java
@@ -22,9 +22,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.reservedstate.service.FileSettingsFeatures;
 import org.elasticsearch.reservedstate.service.FileSettingsService;
 import org.elasticsearch.shutdown.PluginShutdownService;
 import org.elasticsearch.transport.BindTransportException;
@@ -280,22 +278,7 @@ private boolean isMasterElected(ClusterState clusterState) {
     // protected to allow mock service to override
     protected boolean areFileSettingsApplied(ClusterState clusterState) {
         ReservedStateMetadata fileSettingsMetadata = clusterState.metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
-        if (fileSettingsMetadata == null) {
-            // In order to block readiness on file settings being applied, we need to know that the master node has written an initial
-            // version, or a marker that file settings don't exist. When upgrading from a version that did not have file settings, the
-            // current master node may not be the first node upgraded. To be safe, we wait to consider file settings application for
-            // readiness until the whole cluster supports file settings. Note that this only applies when no reserved state metadata
-            // exists, so either we are starting up a current cluster (and the feature will be found) or we are upgrading from
-            // a version before file settings existed (before 8.4).
-            return supportsFileSettings(clusterState) == false;
-        } else {
-            return fileSettingsMetadata.version().equals(ReservedStateMetadata.NO_VERSION) == false;
-        }
-    }
-
-    @SuppressForbidden(reason = "need to check file settings support on exact cluster state")
-    private static boolean supportsFileSettings(ClusterState clusterState) {
-        return clusterState.clusterFeatures().clusterHasFeature(FileSettingsFeatures.FILE_SETTINGS_SUPPORTED);
+        return fileSettingsMetadata != null && fileSettingsMetadata.version().equals(ReservedStateMetadata.NO_VERSION) == false;
     }
 
     private void setReady(boolean ready) {
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesFeatures.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesFeatures.java
deleted file mode 100644
index 37d8b6a293dec..0000000000000
--- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesFeatures.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.repositories;
-
-import org.elasticsearch.features.FeatureSpecification;
-import org.elasticsearch.features.NodeFeature;
-
-import java.util.Set;
-
-public class RepositoriesFeatures implements FeatureSpecification {
-    public static final NodeFeature SUPPORTS_REPOSITORIES_USAGE_STATS = new NodeFeature("repositories.supports_usage_stats");
-
-    @Override
-    public Set<NodeFeature> getFeatures() {
-        return Set.of(SUPPORTS_REPOSITORIES_USAGE_STATS);
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java
index 2cd6e2b11ef7a..a4185a12d7c3b 100644
--- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java
+++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java
@@ -33,7 +33,7 @@ public record RepositoriesMetrics(
     LongHistogram httpRequestTimeInMillisHistogram
 ) {
 
-    public static RepositoriesMetrics NOOP = new RepositoriesMetrics(MeterRegistry.NOOP);
+    public static final RepositoriesMetrics NOOP = new RepositoriesMetrics(MeterRegistry.NOOP);
 
     /**
      * Is incremented for each request sent to the blob store (including retries)
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
index 1c4b7cfdab4ef..abff4d0725355 100644
--- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
+++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
@@ -955,7 +955,7 @@ public List<BiConsumer<Snapshot, IndexVersion>> getPreRestoreVersionChecks() {
         return preRestoreChecks;
     }
 
-    public static String COUNT_USAGE_STATS_NAME = "count";
+    public static final String COUNT_USAGE_STATS_NAME = "count";
 
     public RepositoryUsageStats getUsageStats() {
         if (repositories.isEmpty()) {
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java
index 713f7eacb2c16..2ebbf24d65670 100644
--- a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java
+++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java
@@ -1153,7 +1153,7 @@ private static String parseLegacySnapshotUUID(XContentParser parser) throws IOEx
      */
     public static class SnapshotDetails {
 
-        public static SnapshotDetails EMPTY = new SnapshotDetails(null, null, -1, -1, null);
+        public static final SnapshotDetails EMPTY = new SnapshotDetails(null, null, -1, -1, null);
 
         @Nullable // TODO forbid nulls here, this only applies to very old repositories
         private final SnapshotState snapshotState;
diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsFeatures.java b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsFeatures.java
deleted file mode 100644
index e5f1e7c1c2210..0000000000000
--- a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsFeatures.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.reservedstate.service;
-
-import org.elasticsearch.features.FeatureSpecification;
-import org.elasticsearch.features.NodeFeature;
-
-import java.util.Set;
-
-public class FileSettingsFeatures implements FeatureSpecification {
-
-    // Although file settings were supported starting in 8.4.0, this is really about whether file settings
-    // are used in readiness.
-    public static final NodeFeature FILE_SETTINGS_SUPPORTED = new NodeFeature("file_settings");
-
-    @Override
-    public Set<NodeFeature> getFeatures() {
-        return Set.of(FILE_SETTINGS_SUPPORTED);
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
index 4564a37dacf4a..509086b982319 100644
--- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
+++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
@@ -269,5 +269,4 @@ protected Set<String> responseParams() {
     protected Set<String> responseParams(RestApiVersion restApiVersion) {
         return responseParams();
     }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/RestCompatibleVersionHelper.java b/server/src/main/java/org/elasticsearch/rest/RestCompatibleVersionHelper.java
index 07cc73f4da2b1..31758be719a66 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestCompatibleVersionHelper.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestCompatibleVersionHelper.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.RestApiVersion;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.xcontent.MediaType;
 import org.elasticsearch.xcontent.ParsedMediaType;
 
@@ -27,7 +26,6 @@ class RestCompatibleVersionHelper {
     /**
      * @return The requested API version, or {@link Optional#empty()} if there was no explicit version in the request.
      */
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
     static Optional<RestApiVersion> getCompatibleVersion(
         @Nullable ParsedMediaType acceptHeader,
         @Nullable ParsedMediaType contentTypeHeader,
@@ -52,8 +50,7 @@ static Optional<RestApiVersion> getCompatibleVersion(
         if (hasContent) {
 
             // content-type version must be current or prior
-            // This can be uncommented once all references to RestApiVersion.V_7 are removed
-            /*if (contentTypeVersion > RestApiVersion.current().major || contentTypeVersion < RestApiVersion.minimumSupported().major) {
+            if (contentTypeVersion > RestApiVersion.current().major || contentTypeVersion < RestApiVersion.minimumSupported().major) {
                 throw new ElasticsearchStatusException(
                     "Content-Type version must be either version {} or {}, but found {}. Content-Type={}",
                     RestStatus.BAD_REQUEST,
@@ -62,7 +59,7 @@ static Optional<RestApiVersion> getCompatibleVersion(
                     contentTypeVersion,
                     contentTypeHeader
                 );
-            }*/
+            }
             // if both accept and content-type are sent, the version must match
             if (contentTypeVersion != acceptVersion) {
                 throw new ElasticsearchStatusException(
diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java
index 49fe794bbe615..49801499ea991 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestController.java
@@ -93,6 +93,7 @@ public class RestController implements HttpServerTransport.Dispatcher {
     public static final String STATUS_CODE_KEY = "es_rest_status_code";
     public static final String HANDLER_NAME_KEY = "es_rest_handler_name";
     public static final String REQUEST_METHOD_KEY = "es_rest_request_method";
+    public static final boolean ERROR_TRACE_DEFAULT = false;
 
     static {
         try (InputStream stream = RestController.class.getResourceAsStream("/config/favicon.ico")) {
@@ -638,7 +639,7 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel
     private static void validateErrorTrace(RestRequest request, RestChannel channel) {
         // error_trace cannot be used when we disable detailed errors
         // we consume the error_trace parameter first to ensure that it is always consumed
-        if (request.paramAsBoolean("error_trace", false) && channel.detailedErrorsEnabled() == false) {
+        if (request.paramAsBoolean("error_trace", ERROR_TRACE_DEFAULT) && channel.detailedErrorsEnabled() == false) {
             throw new IllegalArgumentException("error traces in responses are disabled.");
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/rest/RestFeatures.java b/server/src/main/java/org/elasticsearch/rest/RestFeatures.java
deleted file mode 100644
index e72b30526c8e3..0000000000000
--- a/server/src/main/java/org/elasticsearch/rest/RestFeatures.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.rest;
-
-import org.elasticsearch.features.FeatureSpecification;
-import org.elasticsearch.features.NodeFeature;
-import org.elasticsearch.rest.action.admin.cluster.RestNodesCapabilitiesAction;
-
-import java.util.Set;
-
-import static org.elasticsearch.search.fetch.subphase.highlight.DefaultHighlighter.UNIFIED_HIGHLIGHTER_MATCHED_FIELDS;
-
-public class RestFeatures implements FeatureSpecification {
-    @Override
-    public Set<NodeFeature> getFeatures() {
-        return Set.of(
-            RestNodesCapabilitiesAction.CAPABILITIES_ACTION,
-            RestNodesCapabilitiesAction.LOCAL_ONLY_CAPABILITIES,
-            UNIFIED_HIGHLIGHTER_MATCHED_FIELDS
-        );
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/rest/RestResponse.java b/server/src/main/java/org/elasticsearch/rest/RestResponse.java
index d043974055667..0c359e0a4a053 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestResponse.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestResponse.java
@@ -37,6 +37,7 @@
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE;
 import static org.elasticsearch.rest.RestController.ELASTIC_PRODUCT_HTTP_HEADER;
+import static org.elasticsearch.rest.RestController.ERROR_TRACE_DEFAULT;
 
 public final class RestResponse implements Releasable {
 
@@ -143,7 +144,7 @@ public RestResponse(RestChannel channel, RestStatus status, Exception e) throws
         // switched in the xcontent rendering parameters.
         // For authorization problems (RestStatus.UNAUTHORIZED) we don't want to do this since this could
         // leak information to the caller who is unauthorized to make this call
-        if (params.paramAsBoolean("error_trace", false) && status != RestStatus.UNAUTHORIZED) {
+        if (params.paramAsBoolean("error_trace", ERROR_TRACE_DEFAULT) && status != RestStatus.UNAUTHORIZED) {
             params = new ToXContent.DelegatingMapParams(singletonMap(REST_EXCEPTION_SKIP_STACK_TRACE, "false"), params);
         }
 
diff --git a/server/src/main/java/org/elasticsearch/rest/RestUtils.java b/server/src/main/java/org/elasticsearch/rest/RestUtils.java
index df51b57f0859c..10e72035cf1f5 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestUtils.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestUtils.java
@@ -9,12 +9,18 @@
 
 package org.elasticsearch.rest;
 
+import org.elasticsearch.action.support.local.TransportLocalClusterStateAction;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.logging.DeprecationCategory;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV10;
 
+import java.net.URI;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
@@ -35,6 +41,13 @@ public class RestUtils {
 
     public static final UnaryOperator<String> REST_DECODER = RestUtils::decodeComponent;
 
+    public static void decodeQueryString(URI uri, Map<String, String> params) {
+        final var rawQuery = uri.getRawQuery();
+        if (Strings.hasLength(rawQuery)) {
+            decodeQueryString(rawQuery, 0, params);
+        }
+    }
+
     public static void decodeQueryString(String s, int fromIndex, Map<String, String> params) {
         if (fromIndex < 0) {
             return;
@@ -315,4 +328,23 @@ public static TimeValue getTimeout(RestRequest restRequest) {
         assert restRequest != null;
         return restRequest.paramAsTime(REST_TIMEOUT_PARAM, null);
     }
+
+    // Remove the BWC support for the deprecated ?local parameter.
+    // NOTE: ensure each usage of this method has been deprecated for long enough to remove it.
+    @UpdateForV10(owner = UpdateForV10.Owner.DISTRIBUTED_COORDINATION)
+    public static void consumeDeprecatedLocalParameter(RestRequest request) {
+        if (request.hasParam("local") == false) {
+            return;
+        }
+        // Consume this param just for validation when in BWC mode.
+        final var local = request.paramAsBoolean("local", false);
+        if (request.getRestApiVersion() != RestApiVersion.V_8) {
+            DeprecationLogger.getLogger(TransportLocalClusterStateAction.class)
+                .critical(
+                    DeprecationCategory.API,
+                    "TransportLocalClusterStateAction-local-parameter",
+                    "the [?local] query parameter to this API has no effect, is now deprecated, and will be removed in a future version"
+                );
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java
index ca9e4abcaeec7..477cc1acb8319 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java
@@ -11,15 +11,16 @@
 
 import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsAction;
 import org.elasticsearch.action.admin.cluster.settings.RestClusterGetSettingsResponse;
-import org.elasticsearch.action.support.master.MasterNodeReadRequest;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.rest.action.RestCancellableNodeClient;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
 import java.io.IOException;
@@ -52,19 +53,14 @@ public String getName() {
         return "cluster_get_settings_action";
     }
 
-    private static void setUpRequestParams(MasterNodeReadRequest<?> clusterRequest, RestRequest request) {
-        clusterRequest.local(request.paramAsBoolean("local", clusterRequest.local()));
-    }
-
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final boolean renderDefaults = request.paramAsBoolean("include_defaults", false);
 
         ClusterGetSettingsAction.Request clusterSettingsRequest = new ClusterGetSettingsAction.Request(getMasterNodeTimeout(request));
+        RestUtils.consumeDeprecatedLocalParameter(request);
 
-        setUpRequestParams(clusterSettingsRequest, request);
-
-        return channel -> client.execute(
+        return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute(
             ClusterGetSettingsAction.INSTANCE,
             clusterSettingsRequest,
             new RestToXContentListener<RestClusterGetSettingsResponse>(channel).map(
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java
index 63bd4523f9bd1..690f3155971ca 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java
@@ -33,7 +33,8 @@ public class RestClusterStatsAction extends BaseRestHandler {
         "human-readable-total-docs-size",
         "verbose-dense-vector-mapping-stats",
         "ccs-stats",
-        "retrievers-usage-stats"
+        "retrievers-usage-stats",
+        "esql-stats"
     );
     private static final Set<String> SUPPORTED_QUERY_PARAMETERS = Set.of("include_remotes", "nodeId", REST_TIMEOUT_PARAM);
 
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterUpdateSettingsAction.java
index cd154d2e5c50a..b4641a49d6977 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterUpdateSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterUpdateSettingsAction.java
@@ -11,6 +11,7 @@
 
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -33,6 +34,9 @@ public class RestClusterUpdateSettingsAction extends BaseRestHandler {
     private static final String PERSISTENT = "persistent";
     private static final String TRANSIENT = "transient";
 
+    // TODO: Remove this and use a single cluster feature / capability for the whole failure store feature when the feature flag is removed
+    private static final String DATA_STREAM_FAILURE_STORE_CLUSTER_SETTING_CAPABILITY = "data_stream_failure_store_cluster_setting";
+
     @Override
     public List<Route> routes() {
         return List.of(new Route(PUT, "/_cluster/settings"));
@@ -73,4 +77,9 @@ protected Set<String> responseParams() {
     public boolean canTripCircuitBreaker() {
         return false;
     }
+
+    @Override
+    public Set<String> supportedCapabilities() {
+        return DataStream.isFailureStoreFeatureFlagEnabled() ? Set.of(DATA_STREAM_FAILURE_STORE_CLUSTER_SETTING_CAPABILITY) : Set.of();
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesCapabilitiesAction.java
index 897dd1b52e56f..7d660e527a814 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesCapabilitiesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesCapabilitiesAction.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.action.admin.cluster.node.capabilities.NodesCapabilitiesRequest;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -30,8 +29,6 @@
 @ServerlessScope(Scope.INTERNAL)
 public class RestNodesCapabilitiesAction extends BaseRestHandler {
 
-    public static final NodeFeature CAPABILITIES_ACTION = new NodeFeature("rest.capabilities_action");
-    public static final NodeFeature LOCAL_ONLY_CAPABILITIES = new NodeFeature("rest.local_only_capabilities");
     private static final Set<String> SUPPORTED_QUERY_PARAMETERS = Set.of(
         "timeout",
         "method",
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
index c2848781bc7a7..1ed9317c790b8 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -39,8 +38,6 @@
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestNodesStatsAction extends BaseRestHandler {
-    private final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestNodesStatsAction.class);
-    private static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in nodes stats requests is deprecated.";
 
     @Override
     public List<Route> routes() {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestUpdateDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestUpdateDesiredNodesAction.java
index ec8bb6285bdd4..da7a7d3379ee0 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestUpdateDesiredNodesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestUpdateDesiredNodesAction.java
@@ -14,16 +14,13 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.DesiredNode;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xcontent.XContentParseException;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.List;
-import java.util.function.Predicate;
 
 import static org.elasticsearch.rest.RestUtils.getAckTimeout;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
@@ -33,11 +30,6 @@ public class RestUpdateDesiredNodesAction extends BaseRestHandler {
     private final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestUpdateDesiredNodesAction.class);
     private static final String VERSION_DEPRECATION_MESSAGE =
         "[version removal] Specifying node_version in desired nodes requests is deprecated.";
-    private final Predicate<NodeFeature> clusterSupportsFeature;
-
-    public RestUpdateDesiredNodesAction(Predicate<NodeFeature> clusterSupportsFeature) {
-        this.clusterSupportsFeature = clusterSupportsFeature;
-    }
 
     @Override
     public String getName() {
@@ -67,14 +59,8 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
             );
         }
 
-        if (clusterSupportsFeature.test(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED)) {
-            if (updateDesiredNodesRequest.getNodes().stream().anyMatch(DesiredNode::hasVersion)) {
-                deprecationLogger.compatibleCritical("desired_nodes_version", VERSION_DEPRECATION_MESSAGE);
-            }
-        } else {
-            if (updateDesiredNodesRequest.getNodes().stream().anyMatch(n -> n.hasVersion() == false)) {
-                throw new XContentParseException("[node_version] field is required and must have a valid value");
-            }
+        if (updateDesiredNodesRequest.getNodes().stream().anyMatch(DesiredNode::hasVersion)) {
+            deprecationLogger.compatibleCritical("desired_nodes_version", VERSION_DEPRECATION_MESSAGE);
         }
 
         return restChannel -> client.execute(
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java
index 900a352d42f30..9083c781ae167 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java
@@ -26,5 +26,5 @@ public class CreateIndexCapabilities {
      */
     private static final String LOOKUP_INDEX_MODE_CAPABILITY = "lookup_index_mode";
 
-    public static Set<String> CAPABILITIES = Set.of(LOGSDB_INDEX_MODE_CAPABILITY, LOOKUP_INDEX_MODE_CAPABILITY);
+    public static final Set<String> CAPABILITIES = Set.of(LOGSDB_INDEX_MODE_CAPABILITY, LOOKUP_INDEX_MODE_CAPABILITY);
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCloseIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCloseIndexAction.java
index a2cf3c08196a6..21274e3e85361 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCloseIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCloseIndexAction.java
@@ -17,7 +17,8 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.core.UpdateForV9;
+import org.elasticsearch.core.RestApiVersion;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -26,6 +27,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestUtils.getAckTimeout;
@@ -35,6 +37,7 @@
 public class RestCloseIndexAction extends BaseRestHandler {
 
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestCloseIndexAction.class);
+    public static final Set<String> WAIT_FOR_ACTIVE_SHARDS_INDEX_SETTING_REMOVED = Set.of("wait-for-active-shards-index-setting-removed");
 
     @Override
     public List<Route> routes() {
@@ -47,30 +50,49 @@ public String getName() {
     }
 
     @Override
-    @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT)
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         CloseIndexRequest closeIndexRequest = new CloseIndexRequest(Strings.splitStringByCommaToArray(request.param("index")));
         closeIndexRequest.masterNodeTimeout(getMasterNodeTimeout(request));
         closeIndexRequest.ackTimeout(getAckTimeout(request));
         closeIndexRequest.indicesOptions(IndicesOptions.fromRequest(request, closeIndexRequest.indicesOptions()));
+        handleIndexSettingWaitForActiveShards(request, closeIndexRequest);
+        return channel -> client.admin().indices().close(closeIndexRequest, new RestToXContentListener<>(channel));
+    }
+
+    /**
+     * Special handling of the query param 'wait_for_active_shards' because of the deprecated param value 'index-setting'.
+     * - In 10.x this code should be simplified to simply parsing the active shard count with {@link ActiveShardCount#parseString(String)}
+     * - In 9.x we throw an error that informs the user about the parameter value removal; unless we are running in rest-compatibility mode
+     * with 8.x when we log a deprecation warning and handle the value as a noop.
+     */
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
+    private void handleIndexSettingWaitForActiveShards(final RestRequest request, final CloseIndexRequest closeIndexRequest) {
         String waitForActiveShards = request.param("wait_for_active_shards");
+        if (waitForActiveShards == null) {
+            return;
+        }
         if ("index-setting".equalsIgnoreCase(waitForActiveShards)) {
-            deprecationLogger.warn(
-                DeprecationCategory.SETTINGS,
-                "close-index-wait_for_active_shards-index-setting",
-                "?wait_for_active_shards=index-setting is now the default behaviour; the 'index-setting' value for this parameter "
-                    + "should no longer be used since it will become unsupported in version "
-                    + (Version.V_8_0_0.major + 1)
-            );
-            // TODO in v9:
-            // - throw an IllegalArgumentException here
-            // - record the removal of support for this value as a breaking change
-            // TODO in v10:
-            // - remove the IllegalArgumentException here
-        } else if (waitForActiveShards != null) {
+            if (request.getRestApiVersion() == RestApiVersion.V_8) {
+                deprecationLogger.warn(
+                    DeprecationCategory.SETTINGS,
+                    "close-index-wait_for_active_shards-index-setting",
+                    "?wait_for_active_shards=index-setting is now the default behaviour; the 'index-setting' value for this parameter "
+                        + "should no longer be used since it will become unsupported in version "
+                        + Version.V_9_0_0.major
+                );
+            } else {
+                throw new IllegalArgumentException(
+                    "The 'index-setting' value for parameter 'wait_for_active_shards' is the default behaviour and this configuration"
+                        + " value is not supported anymore. Please remove 'wait_for_active_shards=index-setting'."
+                );
+            }
+        } else {
             closeIndexRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards));
         }
-        return channel -> client.admin().indices().close(closeIndexRequest, new RestToXContentListener<>(channel));
     }
 
+    @Override
+    public Set<String> supportedCapabilities() {
+        return WAIT_FOR_ACTIVE_SHARDS_INDEX_SETTING_REMOVED;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetComponentTemplateAction.java
index 466e45ff5dfc1..390870be9de04 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetComponentTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetComponentTemplateAction.java
@@ -14,8 +14,10 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.rest.action.RestCancellableNodeClient;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
 import java.io.IOException;
@@ -26,11 +28,12 @@
 import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
-import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestGetComponentTemplateAction extends BaseRestHandler {
 
+    private static final Set<String> SUPPORTED_CAPABILITIES = Set.of("local_param_deprecated");
+
     @Override
     public List<Route> routes() {
         return List.of(
@@ -47,18 +50,23 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-
-        final GetComponentTemplateAction.Request getRequest = new GetComponentTemplateAction.Request(request.param("name"));
+        final GetComponentTemplateAction.Request getRequest = new GetComponentTemplateAction.Request(
+            RestUtils.getMasterNodeTimeout(request),
+            request.param("name")
+        );
         getRequest.includeDefaults(request.paramAsBoolean("include_defaults", false));
-        getRequest.local(request.paramAsBoolean("local", getRequest.local()));
-        getRequest.masterNodeTimeout(getMasterNodeTimeout(request));
+        RestUtils.consumeDeprecatedLocalParameter(request);
 
         final boolean implicitAll = getRequest.name() == null;
 
-        return channel -> client.execute(GetComponentTemplateAction.INSTANCE, getRequest, new RestToXContentListener<>(channel, r -> {
-            final boolean templateExists = r.getComponentTemplates().isEmpty() == false;
-            return (templateExists || implicitAll) ? OK : NOT_FOUND;
-        }));
+        return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute(
+            GetComponentTemplateAction.INSTANCE,
+            getRequest,
+            new RestToXContentListener<>(channel, r -> {
+                final boolean templateExists = r.getComponentTemplates().isEmpty() == false;
+                return (templateExists || implicitAll) ? OK : NOT_FOUND;
+            })
+        );
     }
 
     @Override
@@ -66,4 +74,8 @@ protected Set<String> responseParams() {
         return Settings.FORMAT_PARAMS;
     }
 
+    @Override
+    public Set<String> supportedCapabilities() {
+        return SUPPORTED_CAPABILITIES;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetComposableIndexTemplateAction.java
index 1a9bfe3bac9b8..78b2dc0963301 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetComposableIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetComposableIndexTemplateAction.java
@@ -14,8 +14,10 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.rest.action.RestCancellableNodeClient;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
 import java.io.IOException;
@@ -26,7 +28,6 @@
 import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
-import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestGetComposableIndexTemplateAction extends BaseRestHandler {
@@ -47,17 +48,23 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        final GetComposableIndexTemplateAction.Request getRequest = new GetComposableIndexTemplateAction.Request(request.param("name"));
-
-        getRequest.local(request.paramAsBoolean("local", getRequest.local()));
-        getRequest.masterNodeTimeout(getMasterNodeTimeout(request));
+        final GetComposableIndexTemplateAction.Request getRequest = new GetComposableIndexTemplateAction.Request(
+            RestUtils.getMasterNodeTimeout(request),
+            request.param("name")
+        );
         getRequest.includeDefaults(request.paramAsBoolean("include_defaults", false));
+        RestUtils.consumeDeprecatedLocalParameter(request);
+
         final boolean implicitAll = getRequest.name() == null;
 
-        return channel -> client.execute(GetComposableIndexTemplateAction.INSTANCE, getRequest, new RestToXContentListener<>(channel, r -> {
-            final boolean templateExists = r.indexTemplates().isEmpty() == false;
-            return (templateExists || implicitAll) ? OK : NOT_FOUND;
-        }));
+        return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute(
+            GetComposableIndexTemplateAction.INSTANCE,
+            getRequest,
+            new RestToXContentListener<>(channel, r -> {
+                final boolean templateExists = r.indexTemplates().isEmpty() == false;
+                return (templateExists || implicitAll) ? OK : NOT_FOUND;
+            })
+        );
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java
index b0b879a6d787c..4d67a33194b4b 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java
@@ -9,12 +9,15 @@
 
 package org.elasticsearch.rest.action.admin.indices;
 
+import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesAction;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
+import org.elasticsearch.rest.action.RestCancellableNodeClient;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
 import java.io.IOException;
@@ -25,12 +28,14 @@
 import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
-import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
 
 /**
  * The REST handler for get template and head template APIs.
  */
 public class RestGetIndexTemplateAction extends BaseRestHandler {
+
+    private static final Set<String> CAPABILITIES = Set.of("local_param_deprecated");
+
     @Override
     public List<Route> routes() {
         return List.of(new Route(GET, "/_template"), new Route(GET, "/_template/{name}"), new Route(HEAD, "/_template/{name}"));
@@ -45,21 +50,31 @@ public String getName() {
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final String[] names = Strings.splitStringByCommaToArray(request.param("name"));
 
-        final GetIndexTemplatesRequest getIndexTemplatesRequest = new GetIndexTemplatesRequest(names);
-
-        getIndexTemplatesRequest.local(request.paramAsBoolean("local", getIndexTemplatesRequest.local()));
-        getIndexTemplatesRequest.masterNodeTimeout(getMasterNodeTimeout(request));
+        final GetIndexTemplatesRequest getIndexTemplatesRequest = new GetIndexTemplatesRequest(
+            RestUtils.getMasterNodeTimeout(request),
+            names
+        );
+        RestUtils.consumeDeprecatedLocalParameter(request);
 
         final boolean implicitAll = getIndexTemplatesRequest.names().length == 0;
 
-        return channel -> client.admin().indices().getTemplates(getIndexTemplatesRequest, new RestToXContentListener<>(channel, r -> {
-            final boolean templateExists = r.getIndexTemplates().isEmpty() == false;
-            return (templateExists || implicitAll) ? OK : NOT_FOUND;
-        }));
+        return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute(
+            GetIndexTemplatesAction.INSTANCE,
+            getIndexTemplatesRequest,
+            new RestToXContentListener<>(channel, r -> {
+                final boolean templateExists = r.getIndexTemplates().isEmpty() == false;
+                return (templateExists || implicitAll) ? OK : NOT_FOUND;
+            })
+        );
     }
 
     @Override
     protected Set<String> responseParams() {
         return Settings.FORMAT_PARAMS;
     }
+
+    @Override
+    public Set<String> supportedCapabilities() {
+        return CAPABILITIES;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java
index 27620fa750ea9..f9e02646041a6 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.http.HttpChannel;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -51,11 +50,9 @@ public String getName() {
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
-        final GetMappingsRequest getMappingsRequest = new GetMappingsRequest();
+        final GetMappingsRequest getMappingsRequest = new GetMappingsRequest(getMasterNodeTimeout(request));
         getMappingsRequest.indices(indices);
         getMappingsRequest.indicesOptions(IndicesOptions.fromRequest(request, getMappingsRequest.indicesOptions()));
-        final TimeValue timeout = getMasterNodeTimeout(request);
-        getMappingsRequest.masterNodeTimeout(timeout);
         getMappingsRequest.local(request.paramAsBoolean("local", getMappingsRequest.local()));
         final HttpChannel httpChannel = request.getHttpChannel();
         return channel -> new RestCancellableNodeClient(client, httpChannel).admin()
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
index b5050eb8007a8..822d9c7151ecf 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
@@ -16,14 +16,12 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestCancellableNodeClient;
 import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener;
-import org.elasticsearch.rest.action.document.RestMultiTermVectorsAction;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -39,9 +37,6 @@
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestIndicesStatsAction extends BaseRestHandler {
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestMultiTermVectorsAction.class);
-    private static final String TYPES_DEPRECATION_MESSAGE = "[types removal] "
-        + "Specifying types in indices stats requests is deprecated.";
 
     @Override
     public List<Route> routes() {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java
index ee11f7f520463..f9689cf0d06ac 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -34,9 +33,6 @@
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestPutMappingAction extends BaseRestHandler {
-    public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in put mapping request is deprecated. "
-        + "Use typeless api instead";
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestPutMappingAction.class);
 
     @Override
     public List<Route> routes() {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
index 39d7b1d5851db..25cefb428c4e7 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
@@ -11,7 +11,6 @@
 
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -45,7 +44,7 @@ public String getName() {
     @Override
     public Set<String> supportedCapabilities() {
         if (DataStream.isFailureStoreFeatureFlagEnabled()) {
-            return Set.of("lazy-rollover-failure-store");
+            return Set.of("lazy-rollover-failure-store", "index-expression-selectors");
         } else {
             return Set.of();
         }
@@ -59,16 +58,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         rolloverIndexRequest.lazy(request.paramAsBoolean("lazy", false));
         rolloverIndexRequest.ackTimeout(getAckTimeout(request));
         rolloverIndexRequest.masterNodeTimeout(getMasterNodeTimeout(request));
-        if (DataStream.isFailureStoreFeatureFlagEnabled()) {
-            boolean failureStore = request.paramAsBoolean("target_failure_store", false);
-            if (failureStore) {
-                rolloverIndexRequest.setIndicesOptions(
-                    IndicesOptions.builder(rolloverIndexRequest.indicesOptions())
-                        .selectorOptions(IndicesOptions.SelectorOptions.FAILURES)
-                        .build()
-                );
-            }
-        }
         rolloverIndexRequest.getCreateIndexRequest()
             .waitForActiveShards(ActiveShardCount.parseString(request.param("wait_for_active_shards")));
         return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).admin()
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java
index 4cc010c15ffb9..1689d234387cc 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.rest.action.RestCancellableNodeClient;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
 import java.io.IOException;
@@ -41,8 +42,10 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        SimulateIndexTemplateRequest simulateIndexTemplateRequest = new SimulateIndexTemplateRequest(request.param("name"));
-        simulateIndexTemplateRequest.masterNodeTimeout(getMasterNodeTimeout(request));
+        SimulateIndexTemplateRequest simulateIndexTemplateRequest = new SimulateIndexTemplateRequest(
+            getMasterNodeTimeout(request),
+            request.param("name")
+        );
         simulateIndexTemplateRequest.includeDefaults(request.paramAsBoolean("include_defaults", false));
         if (request.hasContent()) {
             TransportPutComposableIndexTemplateAction.Request indexTemplateRequest = new TransportPutComposableIndexTemplateAction.Request(
@@ -57,7 +60,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
             simulateIndexTemplateRequest.indexTemplateRequest(indexTemplateRequest);
         }
 
-        return channel -> client.execute(
+        return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute(
             SimulateIndexTemplateAction.INSTANCE,
             simulateIndexTemplateRequest,
             new RestToXContentListener<>(channel)
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java
index cc2e3136a4163..47b68e24a9ed0 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.rest.action.RestCancellableNodeClient;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
 import java.io.IOException;
@@ -39,8 +40,10 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request();
-        simulateRequest.templateName(request.param("name"));
+        SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request(
+            getMasterNodeTimeout(request),
+            request.param("name")
+        );
         simulateRequest.includeDefaults(request.paramAsBoolean("include_defaults", false));
         if (request.hasContent()) {
             TransportPutComposableIndexTemplateAction.Request indexTemplateRequest = new TransportPutComposableIndexTemplateAction.Request(
@@ -54,8 +57,11 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
 
             simulateRequest.indexTemplateRequest(indexTemplateRequest);
         }
-        simulateRequest.masterNodeTimeout(getMasterNodeTimeout(request));
 
-        return channel -> client.execute(SimulateTemplateAction.INSTANCE, simulateRequest, new RestToXContentListener<>(channel));
+        return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute(
+            SimulateTemplateAction.INSTANCE,
+            simulateRequest,
+            new RestToXContentListener<>(channel)
+        );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java
index 8bc7e9aa76551..84f68eceb7c39 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
@@ -36,8 +35,6 @@
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestValidateQueryAction extends BaseRestHandler {
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestValidateQueryAction.class);
-    static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in validate query requests is deprecated.";
 
     @Override
     public List<Route> routes() {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index a968ea4520f40..703f1f2c18408 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -303,6 +303,15 @@ protected Table getTableWithHeader(final RestRequest request) {
             "sibling:pri;alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops"
         );
         table.addCell("pri.indexing.index_failed", "default:false;text-align:right;desc:number of failed indexing ops");
+        table.addCell(
+            "indexing.index_failed_due_to_version_conflict",
+            "sibling:pri;alias:iifvc,indexingIndexFailedDueToVersionConflict;default:false;text-align:right;"
+                + "desc:number of failed indexing ops due to version conflict"
+        );
+        table.addCell(
+            "pri.indexing.index_failed_due_to_version_conflict",
+            "default:false;text-align:right;desc:number of failed indexing ops due to version conflict"
+        );
 
         table.addCell("merges.current", "sibling:pri;alias:mc,mergesCurrent;default:false;text-align:right;desc:number of current merges");
         table.addCell("pri.merges.current", "default:false;text-align:right;desc:number of current merges");
@@ -670,6 +679,13 @@ Table buildTable(
             table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexFailedCount());
             table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexFailedCount());
 
+            table.addCell(
+                totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexFailedDueToVersionConflictCount()
+            );
+            table.addCell(
+                primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexFailedDueToVersionConflictCount()
+            );
+
             table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrent());
             table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrent());
 
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
index 39e679f2c0ad0..5560bd6dc49c8 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
@@ -230,6 +230,11 @@ protected Table getTableWithHeader(final RestRequest request) {
             "indexing.index_failed",
             "alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops"
         );
+        table.addCell(
+            "indexing.index_failed_due_to_version_conflict",
+            "alias:iifvc,indexingIndexFailedDueToVersionConflict;default:false;text-align:right;"
+                + "desc:number of failed indexing ops due to version conflict"
+        );
 
         table.addCell("merges.current", "alias:mc,mergesCurrent;default:false;text-align:right;desc:number of current merges");
         table.addCell(
@@ -466,6 +471,7 @@ Table buildTable(
             table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexTime());
             table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexCount());
             table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexFailedCount());
+            table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexFailedDueToVersionConflictCount());
 
             MergeStats mergeStats = indicesStats == null ? null : indicesStats.getMerge();
             table.addCell(mergeStats == null ? null : mergeStats.getCurrent());
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
index 8b5e21de2d741..717e6e2d8f35b 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
@@ -169,6 +169,11 @@ protected Table getTableWithHeader(final RestRequest request) {
             "indexing.index_failed",
             "alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops"
         );
+        table.addCell(
+            "indexing.index_failed_due_to_version_conflict",
+            "alias:iifvc,indexingIndexFailedDueToVersionConflict;default:false;text-align:right;"
+                + "desc:number of failed indexing ops due to version conflict"
+        );
 
         table.addCell("merges.current", "alias:mc,mergesCurrent;default:false;text-align:right;desc:number of current merges");
         table.addCell(
@@ -373,6 +378,7 @@ Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsRe
             table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexTime()));
             table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexCount()));
             table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexFailedCount()));
+            table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexFailedDueToVersionConflictCount()));
 
             table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrent));
             table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrentNumDocs));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java
index 3d355f1905555..8175524c5e2df 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestResponseListener;
@@ -30,7 +31,6 @@
 import java.util.Map;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
-import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestTemplatesAction extends AbstractCatAction {
@@ -54,17 +54,16 @@ protected void documentation(StringBuilder sb) {
     protected RestChannelConsumer doCatRequest(final RestRequest request, NodeClient client) {
         final String matchPattern = request.hasParam("name") ? request.param("name") : null;
 
+        final var masterNodeTimeout = RestUtils.getMasterNodeTimeout(request);
         final GetIndexTemplatesRequest getIndexTemplatesRequest = matchPattern == null
-            ? new GetIndexTemplatesRequest()
-            : new GetIndexTemplatesRequest(matchPattern);
-        getIndexTemplatesRequest.local(request.paramAsBoolean("local", getIndexTemplatesRequest.local()));
-        getIndexTemplatesRequest.masterNodeTimeout(getMasterNodeTimeout(request));
+            ? new GetIndexTemplatesRequest(masterNodeTimeout)
+            : new GetIndexTemplatesRequest(masterNodeTimeout, matchPattern);
+        RestUtils.consumeDeprecatedLocalParameter(request);
 
         final GetComposableIndexTemplateAction.Request getComposableTemplatesRequest = new GetComposableIndexTemplateAction.Request(
+            masterNodeTimeout,
             matchPattern
         );
-        getComposableTemplatesRequest.local(request.paramAsBoolean("local", getComposableTemplatesRequest.local()));
-        getComposableTemplatesRequest.masterNodeTimeout(getMasterNodeTimeout(request));
 
         return channel -> {
 
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
index 24fab92ced392..87b1a6b9c2fa8 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
@@ -72,6 +72,9 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
+        if (client.threadPool() != null && client.threadPool().getThreadContext() != null) {
+            client.threadPool().getThreadContext().setErrorTraceTransportHeader(request);
+        }
         final MultiSearchRequest multiSearchRequest = parseRequest(request, allowExplicitIndex, searchUsageHolder, clusterSupportsFeature);
         return channel -> {
             final RestCancellableNodeClient cancellableClient = new RestCancellableNodeClient(client, request.getHttpChannel());
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
index a9c2ff7576b05..5eda47bc32354 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestCancellableNodeClient;
 import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.StoredFieldsContext;
@@ -95,7 +94,9 @@ public Set<String> supportedCapabilities() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-
+        if (client.threadPool() != null && client.threadPool().getThreadContext() != null) {
+            client.threadPool().getThreadContext().setErrorTraceTransportHeader(request);
+        }
         SearchRequest searchRequest = new SearchRequest();
         // access the BwC param, but just drop it
         // this might be set by old clients
@@ -204,7 +205,7 @@ public static void parseSearchRequest(
 
         String scroll = request.param("scroll");
         if (scroll != null) {
-            searchRequest.scroll(new Scroll(parseTimeValue(scroll, null, "scroll")));
+            searchRequest.scroll(parseTimeValue(scroll, null, "scroll"));
         }
         searchRequest.routing(request.param("routing"));
         searchRequest.preference(request.param("preference"));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java
index ccf94f9eebffa..758538f6302ab 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.xcontent.XContentParseException;
 
 import java.io.IOException;
@@ -54,7 +53,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         searchScrollRequest.scrollId(scrollId);
         String scroll = request.param("scroll");
         if (scroll != null) {
-            searchScrollRequest.scroll(new Scroll(parseTimeValue(scroll, null, "scroll")));
+            searchScrollRequest.scroll(parseTimeValue(scroll, null, "scroll"));
         }
 
         request.withContentOrSourceParamParserOrNull(xContentParser -> {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java
index 06f8f8f3c1be6..8231046c6586f 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java
@@ -9,9 +9,6 @@
 
 package org.elasticsearch.rest.action.search;
 
-import org.elasticsearch.Build;
-import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper;
-
 import java.util.HashSet;
 import java.util.Set;
 
@@ -34,20 +31,18 @@ private SearchCapabilities() {}
     private static final String TRANSFORM_RANK_RRF_TO_RETRIEVER = "transform_rank_rrf_to_retriever";
     /** Support kql query. */
     private static final String KQL_QUERY_SUPPORTED = "kql_query";
-    /** Support rank-vectors field mapper. */
-    private static final String RANK_VECTORS_FIELD_MAPPER = "rank_vectors_field_mapper";
     /** Support propagating nested retrievers' inner_hits to top-level compound retrievers . */
     private static final String NESTED_RETRIEVER_INNER_HITS_SUPPORT = "nested_retriever_inner_hits_support";
-    /** Support rank-vectors script field access. */
-    private static final String RANK_VECTORS_SCRIPT_ACCESS = "rank_vectors_script_access";
-    /** Initial support for rank-vectors maxSim functions access. */
-    private static final String RANK_VECTORS_SCRIPT_MAX_SIM = "rank_vectors_script_max_sim_with_bugfix";
     /** Fixed the math in {@code moving_fn}'s {@code linearWeightedAvg}. */
     private static final String MOVING_FN_RIGHT_MATH = "moving_fn_right_math";
+    /** knn query where k defaults to the request size. */
+    private static final String K_DEFAULT_TO_SIZE = "k_default_to_size";
 
     private static final String RANDOM_SAMPLER_WITH_SCORED_SUBAGGS = "random_sampler_with_scored_subaggs";
     private static final String OPTIMIZED_SCALAR_QUANTIZATION_BBQ = "optimized_scalar_quantization_bbq";
-    private static final String KNN_QUANTIZED_VECTOR_RESCORE = "knn_quantized_vector_rescore";
+    private static final String KNN_QUANTIZED_VECTOR_RESCORE_OVERSAMPLE = "knn_quantized_vector_rescore_oversample";
+
+    private static final String HIGHLIGHT_MAX_ANALYZED_OFFSET_DEFAULT = "highlight_max_analyzed_offset_default";
 
     public static final Set<String> CAPABILITIES;
     static {
@@ -60,16 +55,11 @@ private SearchCapabilities() {}
         capabilities.add(NESTED_RETRIEVER_INNER_HITS_SUPPORT);
         capabilities.add(RANDOM_SAMPLER_WITH_SCORED_SUBAGGS);
         capabilities.add(OPTIMIZED_SCALAR_QUANTIZATION_BBQ);
-        capabilities.add(KNN_QUANTIZED_VECTOR_RESCORE);
+        capabilities.add(KNN_QUANTIZED_VECTOR_RESCORE_OVERSAMPLE);
         capabilities.add(MOVING_FN_RIGHT_MATH);
-        if (RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()) {
-            capabilities.add(RANK_VECTORS_FIELD_MAPPER);
-            capabilities.add(RANK_VECTORS_SCRIPT_ACCESS);
-            capabilities.add(RANK_VECTORS_SCRIPT_MAX_SIM);
-        }
-        if (Build.current().isSnapshot()) {
-            capabilities.add(KQL_QUERY_SUPPORTED);
-        }
+        capabilities.add(K_DEFAULT_TO_SIZE);
+        capabilities.add(KQL_QUERY_SUPPORTED);
+        capabilities.add(HIGHLIGHT_MAX_ANALYZED_OFFSET_DEFAULT);
         CAPABILITIES = Set.copyOf(capabilities);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/script/Metadata.java b/server/src/main/java/org/elasticsearch/script/Metadata.java
index f6212de58df84..dc5ae51e45af0 100644
--- a/server/src/main/java/org/elasticsearch/script/Metadata.java
+++ b/server/src/main/java/org/elasticsearch/script/Metadata.java
@@ -51,7 +51,7 @@ public class Metadata {
     protected static final String IF_PRIMARY_TERM = "_if_primary_term";
     protected static final String DYNAMIC_TEMPLATES = "_dynamic_templates";
 
-    public static FieldProperty<Object> ObjectField = new FieldProperty<>(Object.class);
+    public static final FieldProperty<Object> ObjectField = new FieldProperty<>(Object.class);
     public static FieldProperty<String> StringField = new FieldProperty<>(String.class);
     public static FieldProperty<Number> LongField = new FieldProperty<>(Number.class).withValidation(FieldProperty.LONGABLE_NUMBER);
 
@@ -335,7 +335,7 @@ public FieldProperty<T> withValidation(BiConsumer<String, T> extendedValidation)
             return new FieldProperty<>(type, nullable, writable, extendedValidation);
         }
 
-        public static BiConsumer<String, Number> LONGABLE_NUMBER = (k, v) -> {
+        public static final BiConsumer<String, Number> LONGABLE_NUMBER = (k, v) -> {
             long version = v.longValue();
             // did we round?
             if (v.doubleValue() == version) {
@@ -346,7 +346,7 @@ public FieldProperty<T> withValidation(BiConsumer<String, T> extendedValidation)
             );
         };
 
-        public static FieldProperty<?> ALLOW_ALL = new FieldProperty<>(null, true, true, null);
+        public static final FieldProperty<?> ALLOW_ALL = new FieldProperty<>(null, true, true, null);
 
         @SuppressWarnings("fallthrough")
         public void check(MapOperation op, String key, Object value) {
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java b/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java
index adc1f65b88732..e752f5f811a43 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java
@@ -13,20 +13,16 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
-import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.ToXContentFragment;
+import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.Map;
 import java.util.Objects;
-import java.util.function.Function;
-
-import static org.elasticsearch.script.ScriptCacheStats.Fields.SCRIPT_CACHE_STATS;
 
 // This class is deprecated in favor of ScriptStats and ScriptContextStats
-public record ScriptCacheStats(Map<String, ScriptStats> context, ScriptStats general) implements Writeable, ChunkedToXContent {
+public record ScriptCacheStats(Map<String, ScriptStats> context, ScriptStats general) implements Writeable, ToXContentFragment {
 
     public ScriptCacheStats(Map<String, ScriptStats> context) {
         this(Collections.unmodifiableMap(context), null);
@@ -68,26 +64,37 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
-        Function<ScriptStats, ToXContent> statsFields = s -> (b, p) -> b.field(ScriptStats.Fields.COMPILATIONS, s.getCompilations())
-            .field(ScriptStats.Fields.CACHE_EVICTIONS, s.getCacheEvictions())
-            .field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, s.getCompilationLimitTriggered());
-
-        return ChunkedToXContent.builder(outerParams).object(SCRIPT_CACHE_STATS, sb -> {
-            if (general != null) {
-                sb.xContentObject(Fields.SUM, statsFields.apply(general));
-            } else {
-                sb.xContentObject(Fields.SUM, statsFields.apply(sum()));
-                sb.array(
-                    Fields.CONTEXTS,
-                    context.entrySet().stream().sorted(Map.Entry.comparingByKey()).iterator(),
-                    (eb, e) -> eb.object(ebo -> {
-                        ebo.field(Fields.CONTEXT, e.getKey());
-                        ebo.append(statsFields.apply(e.getValue()));
-                    })
-                );
-            }
-        });
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(Fields.SCRIPT_CACHE_STATS);
+        builder.startObject(Fields.SUM);
+        if (general != null) {
+            builder.field(ScriptStats.Fields.COMPILATIONS, general.getCompilations());
+            builder.field(ScriptStats.Fields.CACHE_EVICTIONS, general.getCacheEvictions());
+            builder.field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, general.getCompilationLimitTriggered());
+            builder.endObject().endObject();
+            return builder;
+        }
+
+        ScriptStats sum = sum();
+        builder.field(ScriptStats.Fields.COMPILATIONS, sum.getCompilations());
+        builder.field(ScriptStats.Fields.CACHE_EVICTIONS, sum.getCacheEvictions());
+        builder.field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, sum.getCompilationLimitTriggered());
+        builder.endObject();
+
+        builder.startArray(Fields.CONTEXTS);
+        for (String name : context.keySet().stream().sorted().toList()) {
+            ScriptStats stats = context.get(name);
+            builder.startObject();
+            builder.field(Fields.CONTEXT, name);
+            builder.field(ScriptStats.Fields.COMPILATIONS, stats.getCompilations());
+            builder.field(ScriptStats.Fields.CACHE_EVICTIONS, stats.getCacheEvictions());
+            builder.field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, stats.getCompilationLimitTriggered());
+            builder.endObject();
+        }
+        builder.endArray();
+        builder.endObject();
+
+        return builder;
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptFeatures.java b/server/src/main/java/org/elasticsearch/script/ScriptFeatures.java
index 88756ddbf4ed5..cd781e42379d1 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptFeatures.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptFeatures.java
@@ -17,6 +17,6 @@
 public final class ScriptFeatures implements FeatureSpecification {
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(VectorScoreScriptUtils.HAMMING_DISTANCE_FUNCTION, ScriptTermStats.TERM_STAT_FEATURE);
+        return Set.of();
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptStats.java b/server/src/main/java/org/elasticsearch/script/ScriptStats.java
index f24052ef7e3a9..a25d9587bcb0e 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptStats.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptStats.java
@@ -10,11 +10,13 @@
 package org.elasticsearch.script;
 
 import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
@@ -25,6 +27,7 @@
 import java.util.Map;
 import java.util.Objects;
 
+import static org.elasticsearch.common.collect.Iterators.single;
 import static org.elasticsearch.script.ScriptContextStats.Fields.COMPILATIONS_HISTORY;
 import static org.elasticsearch.script.ScriptStats.Fields.CACHE_EVICTIONS;
 import static org.elasticsearch.script.ScriptStats.Fields.COMPILATIONS;
@@ -190,19 +193,24 @@ public ScriptCacheStats toScriptCacheStats() {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).object(SCRIPT_STATS, ob -> {
-            ob.field(COMPILATIONS, compilations);
-            ob.field(CACHE_EVICTIONS, cacheEvictions);
-            ob.field(COMPILATION_LIMIT_TRIGGERED, compilationLimitTriggered);
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
+        return Iterators.concat(single((builder, params) -> {
+            builder.startObject(SCRIPT_STATS)
+                .field(COMPILATIONS, compilations)
+                .field(CACHE_EVICTIONS, cacheEvictions)
+                .field(COMPILATION_LIMIT_TRIGGERED, compilationLimitTriggered);
             if (compilationsHistory != null && compilationsHistory.areTimingsEmpty() == false) {
-                ob.xContentObject(COMPILATIONS_HISTORY, compilationsHistory);
+                builder.startObject(COMPILATIONS_HISTORY);
+                compilationsHistory.toXContent(builder, params);
+                builder.endObject();
             }
             if (cacheEvictionsHistory != null && cacheEvictionsHistory.areTimingsEmpty() == false) {
-                ob.xContentObject(COMPILATIONS_HISTORY, cacheEvictionsHistory);
+                builder.startObject(COMPILATIONS_HISTORY);
+                cacheEvictionsHistory.toXContent(builder, params);
+                builder.endObject();
             }
-            ob.array(CONTEXTS, contextStats.iterator());
-        });
+            return builder;
+        }), ChunkedToXContentHelper.array(CONTEXTS, contextStats.iterator()), ChunkedToXContentHelper.endObject());
     }
 
     static final class Fields {
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java b/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java
index b27019765e33b..9c51afce3a49d 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java
@@ -16,7 +16,6 @@
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.elasticsearch.common.util.CachedSupplier;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.search.internal.ContextIndexSearcher;
 
 import java.io.IOException;
@@ -30,8 +29,6 @@
  */
 public class ScriptTermStats {
 
-    public static final NodeFeature TERM_STAT_FEATURE = new NodeFeature("script.term_stats");
-
     private final IntSupplier docIdSupplier;
     private final Term[] terms;
     private final IndexSearcher searcher;
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java b/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java
index 2e2ae4e73c6f0..79725eb9c3b87 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java
@@ -170,8 +170,8 @@ public interface Factory extends ScriptFactory {
             CombineScript newInstance(Map<String, Object> params, Map<String, Object> state);
         }
 
-        public static String[] PARAMETERS = {};
-        public static ScriptContext<Factory> CONTEXT = new ScriptContext<>("aggs_combine", Factory.class);
+        public static final String[] PARAMETERS = {};
+        public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("aggs_combine", Factory.class);
     }
 
     public abstract static class ReduceScript {
@@ -198,6 +198,6 @@ public interface Factory extends ScriptFactory {
         }
 
         public static String[] PARAMETERS = {};
-        public static ScriptContext<Factory> CONTEXT = new ScriptContext<>("aggs_reduce", Factory.class);
+        public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("aggs_reduce", Factory.class);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java b/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java
index e773bceb5ec05..bdebdcc1eecb4 100644
--- a/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java
+++ b/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java
@@ -10,7 +10,6 @@
 package org.elasticsearch.script;
 
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.script.field.vectors.DenseVector;
 import org.elasticsearch.script.field.vectors.DenseVectorDocValuesField;
@@ -21,8 +20,6 @@
 
 public class VectorScoreScriptUtils {
 
-    public static final NodeFeature HAMMING_DISTANCE_FUNCTION = new NodeFeature("script.hamming");
-
     public static class DenseVectorFunction {
         protected final ScoreScript scoreScript;
         protected final DenseVectorDocValuesField field;
diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java
index db81bb6ebe1cb..1bff1b50fb5ac 100644
--- a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java
+++ b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java
@@ -111,13 +111,13 @@ public boolean isEmpty() {
         return value == null;
     }
 
-    static class ByteVectorIterator implements VectorIterator<byte[]> {
+    public static class ByteVectorIterator implements VectorIterator<byte[]> {
         private final byte[] buffer;
         private final BytesRef vectorValues;
         private final int size;
         private int idx = 0;
 
-        ByteVectorIterator(BytesRef vectorValues, byte[] buffer, int size) {
+        public ByteVectorIterator(BytesRef vectorValues, byte[] buffer, int size) {
             assert vectorValues.length == (buffer.length * size);
             this.vectorValues = vectorValues;
             this.size = size;
diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java
index 39bc1e621113b..d47795a3b2401 100644
--- a/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java
+++ b/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java
@@ -110,14 +110,14 @@ private void decodeVectorIfNecessary() {
         }
     }
 
-    static class FloatVectorIterator implements VectorIterator<float[]> {
+    public static class FloatVectorIterator implements VectorIterator<float[]> {
         private final float[] buffer;
         private final FloatBuffer vectorValues;
         private final BytesRef vectorValueBytesRef;
         private final int size;
         private int idx = 0;
 
-        FloatVectorIterator(BytesRef vectorValues, float[] buffer, int size) {
+        public FloatVectorIterator(BytesRef vectorValues, float[] buffer, int size) {
             assert vectorValues.length == (buffer.length * Float.BYTES * size);
             this.vectorValueBytesRef = vectorValues;
             this.vectorValues = ByteBuffer.wrap(vectorValues.bytes, vectorValues.offset, vectorValues.length)
diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index b87d097413b67..47d3ed337af73 100644
--- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -73,6 +73,7 @@
 import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext;
 import org.elasticsearch.search.rank.feature.RankFeatureResult;
 import org.elasticsearch.search.rescore.RescoreContext;
+import org.elasticsearch.search.rescore.RescorePhase;
 import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
@@ -377,7 +378,7 @@ public void preProcess() {
             );
         }
         if (rescore != null) {
-            if (sort != null) {
+            if (RescorePhase.validateSort(sort) == false) {
                 throw new IllegalArgumentException("Cannot use [sort] option in conjunction with [rescore].");
             }
             int maxWindow = indexService.getIndexSettings().getMaxRescoreWindow();
diff --git a/server/src/main/java/org/elasticsearch/search/Scroll.java b/server/src/main/java/org/elasticsearch/search/Scroll.java
deleted file mode 100644
index 9c978fdea9d4c..0000000000000
--- a/server/src/main/java/org/elasticsearch/search/Scroll.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.search;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.core.TimeValue;
-
-import java.io.IOException;
-import java.util.Objects;
-
-/**
- * A scroll enables scrolling of search request. It holds a {@link #keepAlive()} time that
- * will control how long to keep the scrolling resources open.
- *
- *
- */
-public final class Scroll implements Writeable {
-
-    private final TimeValue keepAlive;
-
-    public Scroll(StreamInput in) throws IOException {
-        this.keepAlive = in.readTimeValue();
-    }
-
-    /**
-     * Constructs a new scroll of the provided keep alive.
-     */
-    public Scroll(TimeValue keepAlive) {
-        this.keepAlive = Objects.requireNonNull(keepAlive, "keepAlive must not be null");
-    }
-
-    /**
-     * How long the resources will be kept open to support the scroll request.
-     */
-    public TimeValue keepAlive() {
-        return keepAlive;
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeTimeValue(keepAlive);
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-        Scroll scroll = (Scroll) o;
-        return Objects.equals(keepAlive, scroll.keepAlive);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(keepAlive);
-    }
-
-    @Override
-    public String toString() {
-        return "Scroll{keepAlive=" + keepAlive + '}';
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/search/SearchFeatures.java b/server/src/main/java/org/elasticsearch/search/SearchFeatures.java
index beac39c2de304..98dd7f9388c1f 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchFeatures.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchFeatures.java
@@ -11,7 +11,6 @@
 
 import org.elasticsearch.features.FeatureSpecification;
 import org.elasticsearch.features.NodeFeature;
-import org.elasticsearch.search.vectors.KnnVectorQueryBuilder;
 
 import java.util.Set;
 
@@ -21,6 +20,13 @@ public final class SearchFeatures implements FeatureSpecification {
 
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(KnnVectorQueryBuilder.K_PARAM_SUPPORTED, LUCENE_10_0_0_UPGRADE);
+        return Set.of(LUCENE_10_0_0_UPGRADE);
+    }
+
+    public static final NodeFeature RETRIEVER_RESCORER_ENABLED = new NodeFeature("search.retriever.rescorer.enabled");
+
+    @Override
+    public Set<NodeFeature> getTestFeatures() {
+        return Set.of(RETRIEVER_RESCORER_ENABLED);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java
index 98f7c92d9997a..701d451ac2c14 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchHit.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java
@@ -517,6 +517,10 @@ public void addDocumentFields(Map<String, DocumentField> docFields, Map<String,
         this.metaFields.putAll(metaFields);
     }
 
+    public DocumentField removeDocumentField(String field) {
+        return documentFields.remove(field);
+    }
+
     /**
      * @return a map of metadata fields for this hit
      */
diff --git a/server/src/main/java/org/elasticsearch/search/SearchHits.java b/server/src/main/java/org/elasticsearch/search/SearchHits.java
index fe133cbac335d..b2e9970631f4a 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchHits.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchHits.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.RefCounted;
 import org.elasticsearch.core.SimpleRefCounted;
@@ -284,22 +285,25 @@ public static final class Fields {
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         assert hasReferences();
-        return ChunkedToXContent.builder(params).object(Fields.HITS, ob -> {
-            boolean totalHitAsInt = ob.params().paramAsBoolean(RestSearchAction.TOTAL_HITS_AS_INT_PARAM, false);
+        return Iterators.concat(Iterators.single((b, p) -> {
+            b.startObject(Fields.HITS);
+            boolean totalHitAsInt = params.paramAsBoolean(RestSearchAction.TOTAL_HITS_AS_INT_PARAM, false);
             if (totalHitAsInt) {
-                ob.field(Fields.TOTAL, totalHits == null ? -1 : totalHits.value());
+                long total = totalHits == null ? -1 : totalHits.value();
+                b.field(Fields.TOTAL, total);
             } else if (totalHits != null) {
-                ob.append((b, p) -> {
-                    b.startObject(Fields.TOTAL);
-                    b.field("value", totalHits.value());
-                    b.field("relation", totalHits.relation() == Relation.EQUAL_TO ? "eq" : "gte");
-                    return b.endObject();
-                });
+                b.startObject(Fields.TOTAL);
+                b.field("value", totalHits.value());
+                b.field("relation", totalHits.relation() == Relation.EQUAL_TO ? "eq" : "gte");
+                b.endObject();
             }
-
-            ob.field(Fields.MAX_SCORE, Float.isNaN(maxScore) ? null : maxScore);
-            ob.array(Fields.HITS, Iterators.forArray(hits));
-        });
+            if (Float.isNaN(maxScore)) {
+                b.nullField(Fields.MAX_SCORE);
+            } else {
+                b.field(Fields.MAX_SCORE, maxScore);
+            }
+            return b;
+        }), ChunkedToXContentHelper.array(Fields.HITS, Iterators.forArray(hits)), ChunkedToXContentHelper.endObject());
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java
index d282ba425b126..d983dd1ff78d4 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -231,6 +231,7 @@
 import org.elasticsearch.search.rescore.QueryRescorerBuilder;
 import org.elasticsearch.search.rescore.RescorerBuilder;
 import org.elasticsearch.search.retriever.KnnRetrieverBuilder;
+import org.elasticsearch.search.retriever.RescorerRetrieverBuilder;
 import org.elasticsearch.search.retriever.RetrieverBuilder;
 import org.elasticsearch.search.retriever.RetrieverParserContext;
 import org.elasticsearch.search.retriever.StandardRetrieverBuilder;
@@ -1048,8 +1049,9 @@ private void registerFetchSubPhases(List<SearchPlugin> plugins) {
         registerFetchSubPhase(new StoredFieldsPhase());
         registerFetchSubPhase(new FetchDocValuesPhase());
         registerFetchSubPhase(new ScriptFieldsPhase());
-        registerFetchSubPhase(new FetchSourcePhase());
         registerFetchSubPhase(new FetchFieldsPhase());
+        // register after fetch fields to handle metadata fields that needs to be copied in _source (e.g. _inference_fields).
+        registerFetchSubPhase(new FetchSourcePhase());
         registerFetchSubPhase(new FetchVersionPhase());
         registerFetchSubPhase(new SeqNoPrimaryTermPhase());
         registerFetchSubPhase(new MatchedQueriesPhase());
@@ -1080,6 +1082,7 @@ private void registerFetchSubPhase(FetchSubPhase subPhase) {
     private void registerRetrieverParsers(List<SearchPlugin> plugins) {
         registerRetriever(new RetrieverSpec<>(StandardRetrieverBuilder.NAME, StandardRetrieverBuilder::fromXContent));
         registerRetriever(new RetrieverSpec<>(KnnRetrieverBuilder.NAME, KnnRetrieverBuilder::fromXContent));
+        registerRetriever(new RetrieverSpec<>(RescorerRetrieverBuilder.NAME, RescorerRetrieverBuilder::fromXContent));
 
         registerFromPlugin(plugins, SearchPlugin::getRetrievers, this::registerRetriever);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index b9bd398500c71..d228d93f897f2 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -17,6 +17,8 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TopDocs;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.ResolvedIndices;
@@ -152,6 +154,7 @@
 import java.util.function.LongSupplier;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.TransportVersions.ERROR_TRACE_IN_TRANSPORT_HEADER;
 import static org.elasticsearch.core.TimeValue.timeValueHours;
 import static org.elasticsearch.core.TimeValue.timeValueMillis;
 import static org.elasticsearch.core.TimeValue.timeValueMinutes;
@@ -272,6 +275,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
 
     public static final int DEFAULT_SIZE = 10;
     public static final int DEFAULT_FROM = 0;
+    private static final StackTraceElement[] EMPTY_STACK_TRACE_ARRAY = new StackTraceElement[0];
 
     private final ThreadPool threadPool;
 
@@ -506,7 +510,41 @@ protected void doClose() {
         keepAliveReaper.cancel();
     }
 
+    /**
+     * Wraps the listener to avoid sending StackTraces back to the coordinating
+     * node if the `error_trace` header is set to {@code false}. Upon reading we
+     * default to {@code true} to maintain the same behavior as before the change,
+     * due to older nodes not being able to specify whether it needs stack traces.
+     *
+     * @param <T>            the type of the response
+     * @param listener       the action listener to be wrapped
+     * @param version        channel version of the request
+     * @param threadPool     with context where to write the new header
+     * @return the wrapped action listener
+     */
+    static <T> ActionListener<T> maybeWrapListenerForStackTrace(
+        ActionListener<T> listener,
+        TransportVersion version,
+        ThreadPool threadPool
+    ) {
+        boolean header = true;
+        if (version.onOrAfter(ERROR_TRACE_IN_TRANSPORT_HEADER) && threadPool.getThreadContext() != null) {
+            header = Boolean.parseBoolean(threadPool.getThreadContext().getHeaderOrDefault("error_trace", "false"));
+        }
+        if (header == false) {
+            return listener.delegateResponse((l, e) -> {
+                ExceptionsHelper.unwrapCausesAndSuppressed(e, err -> {
+                    err.setStackTrace(EMPTY_STACK_TRACE_ARRAY);
+                    return false;
+                });
+                l.onFailure(e);
+            });
+        }
+        return listener;
+    }
+
     public void executeDfsPhase(ShardSearchRequest request, SearchShardTask task, ActionListener<SearchPhaseResult> listener) {
+        listener = maybeWrapListenerForStackTrace(listener, request.getChannelVersion(), threadPool);
         final IndexShard shard = getShard(request);
         rewriteAndFetchShardRequest(shard, request, listener.delegateFailure((l, rewritten) -> {
             // fork the execution in the search thread pool
@@ -544,10 +582,11 @@ private void loadOrExecuteQueryPhase(final ShardSearchRequest request, final Sea
     }
 
     public void executeQueryPhase(ShardSearchRequest request, SearchShardTask task, ActionListener<SearchPhaseResult> listener) {
+        ActionListener<SearchPhaseResult> finalListener = maybeWrapListenerForStackTrace(listener, request.getChannelVersion(), threadPool);
         assert request.canReturnNullResponseIfMatchNoDocs() == false || request.numberOfShards() > 1
             : "empty responses require more than one shard";
         final IndexShard shard = getShard(request);
-        rewriteAndFetchShardRequest(shard, request, listener.delegateFailure((l, orig) -> {
+        rewriteAndFetchShardRequest(shard, request, finalListener.delegateFailure((l, orig) -> {
             // check if we can shortcut the query phase entirely.
             if (orig.canReturnNullResponseIfMatchNoDocs()) {
                 assert orig.scroll() == null;
@@ -561,7 +600,7 @@ public void executeQueryPhase(ShardSearchRequest request, SearchShardTask task,
                 );
                 CanMatchShardResponse canMatchResp = canMatch(canMatchContext, false);
                 if (canMatchResp.canMatch() == false) {
-                    listener.onResponse(QuerySearchResult.nullInstance());
+                    finalListener.onResponse(QuerySearchResult.nullInstance());
                     return;
                 }
             }
@@ -736,6 +775,7 @@ private SearchPhaseResult executeQueryPhase(ShardSearchRequest request, SearchSh
     }
 
     public void executeRankFeaturePhase(RankFeatureShardRequest request, SearchShardTask task, ActionListener<RankFeatureResult> listener) {
+        listener = maybeWrapListenerForStackTrace(listener, request.getShardSearchRequest().getChannelVersion(), threadPool);
         final ReaderContext readerContext = findReaderContext(request.contextId(), request);
         final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.getShardSearchRequest());
         final Releasable markAsUsed = readerContext.markAsUsed(getKeepAlive(shardSearchRequest));
@@ -779,8 +819,10 @@ private QueryFetchSearchResult executeFetchPhase(ReaderContext reader, SearchCon
     public void executeQueryPhase(
         InternalScrollSearchRequest request,
         SearchShardTask task,
-        ActionListener<ScrollQuerySearchResult> listener
+        ActionListener<ScrollQuerySearchResult> listener,
+        TransportVersion version
     ) {
+        listener = maybeWrapListenerForStackTrace(listener, version, threadPool);
         final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId(), request);
         final Releasable markAsUsed;
         try {
@@ -816,7 +858,13 @@ public void executeQueryPhase(
      * It is the responsibility of the caller to ensure that the ref count is correctly decremented
      * when the object is no longer needed.
      */
-    public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task, ActionListener<QuerySearchResult> listener) {
+    public void executeQueryPhase(
+        QuerySearchRequest request,
+        SearchShardTask task,
+        ActionListener<QuerySearchResult> listener,
+        TransportVersion version
+    ) {
+        listener = maybeWrapListenerForStackTrace(listener, version, threadPool);
         final ReaderContext readerContext = findReaderContext(request.contextId(), request.shardSearchRequest());
         final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.shardSearchRequest());
         final Releasable markAsUsed = readerContext.markAsUsed(getKeepAlive(shardSearchRequest));
@@ -1208,14 +1256,14 @@ private static long getKeepAlive(ShardSearchRequest request, long defaultKeepAli
         }
     }
 
-    private long getScrollKeepAlive(Scroll scroll) {
-        return getScrollKeepAlive(scroll, defaultKeepAlive, maxKeepAlive);
+    private long getScrollKeepAlive(TimeValue keepAlive) {
+        return getScrollKeepAlive(keepAlive, defaultKeepAlive, maxKeepAlive);
     }
 
-    private static long getScrollKeepAlive(Scroll scroll, long defaultKeepAlive, long maxKeepAlive) {
-        if (scroll != null && scroll.keepAlive() != null) {
-            checkKeepAliveLimit(scroll.keepAlive().millis(), maxKeepAlive);
-            return scroll.keepAlive().getMillis();
+    private static long getScrollKeepAlive(TimeValue keepAlive, long defaultKeepAlive, long maxKeepAlive) {
+        if (keepAlive != null) {
+            checkKeepAliveLimit(keepAlive.millis(), maxKeepAlive);
+            return keepAlive.getMillis();
         }
         return defaultKeepAlive;
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorsReducer.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorsReducer.java
index 6682fb2a83418..4c89877b7b1c6 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorsReducer.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorsReducer.java
@@ -12,7 +12,10 @@
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -54,7 +57,12 @@ public void accept(InternalAggregations aggregations) {
      * returns the reduced {@link InternalAggregations}.
      */
     public InternalAggregations get() {
-        return InternalAggregations.from(aggByName.values().stream().map(AggregatorReducer::get).toList());
+        final Collection<AggregatorReducer> reducers = aggByName.values();
+        final List<InternalAggregation> aggs = new ArrayList<>(reducers.size());
+        for (AggregatorReducer reducer : reducers) {
+            aggs.add(reducer.get());
+        }
+        return InternalAggregations.from(aggs);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java
index 3593eb5adf7e4..afeeaa9bd6752 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.common.io.stream.DelayableWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.search.aggregations.Aggregator.BucketComparator;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
 import org.elasticsearch.search.aggregations.bucket.terms.BucketAndOrd;
@@ -559,7 +558,6 @@ public static void writeHistogramOrder(BucketOrder order, StreamOutput out) thro
      * Contains logic for parsing a {@link BucketOrder} from a {@link XContentParser}.
      */
     public static class Parser {
-        private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(Parser.class);
 
         /**
          * Parse a {@link BucketOrder} from {@link XContent}.
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
index e86c7127ec2f4..592f7b4887598 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
@@ -59,13 +59,6 @@ public BucketsAggregator(
         docCountProvider = new DocCountProvider();
     }
 
-    /**
-     * Return an upper bound of the maximum bucket ordinal seen so far.
-     */
-    public final long maxBucketOrd() {
-        return docCounts.size();
-    }
-
     /**
      * Ensure there are at least <code>maxBucketOrd</code> buckets available.
      */
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DelayedBucketReducer.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DelayedBucketReducer.java
deleted file mode 100644
index 28b184797ce1e..0000000000000
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DelayedBucketReducer.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.search.aggregations.bucket;
-
-import org.elasticsearch.search.aggregations.AggregationReduceContext;
-import org.elasticsearch.search.aggregations.AggregatorsReducer;
-import org.elasticsearch.search.aggregations.InternalAggregations;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- *  Class for reducing a list of {@link B} to a single {@link InternalAggregations}
- *  and the number of documents in a delayable fashion.
- *
- *  This class can be reused by calling {@link #reset(B)}.
- *
- * @see BucketReducer
- */
-public final class DelayedBucketReducer<B extends MultiBucketsAggregation.Bucket> {
-
-    private final AggregationReduceContext context;
-    // changes at reset time
-    private B proto;
-    // the maximum size of this array is the number of shards to be reduced. We currently do it in a batches of 256
-    // by default. if we expect bigger batches, we might consider to use ObjectArray.
-    private final List<InternalAggregations> internalAggregations;
-    private long count = 0;
-
-    public DelayedBucketReducer(B proto, AggregationReduceContext context) {
-        this.proto = proto;
-        this.context = context;
-        this.internalAggregations = new ArrayList<>();
-    }
-
-    /**
-     * Adds a {@link B} for reduction.
-     */
-    public void accept(B bucket) {
-        count += bucket.getDocCount();
-        internalAggregations.add(bucket.getAggregations());
-    }
-
-    /**
-     * returns the bucket prototype.
-     */
-    public B getProto() {
-        return proto;
-    }
-
-    /**
-     * Reset the content of this reducer.
-     */
-    public void reset(B proto) {
-        this.proto = proto;
-        count = 0L;
-        internalAggregations.clear();
-    }
-
-    /**
-     * returns the reduced {@link InternalAggregations}.
-     */
-    public InternalAggregations getAggregations() {
-        try (
-            AggregatorsReducer aggregatorsReducer = new AggregatorsReducer(proto.getAggregations(), context, internalAggregations.size())
-        ) {
-            for (InternalAggregations agg : internalAggregations) {
-                aggregatorsReducer.accept(agg);
-            }
-            return aggregatorsReducer.get();
-        }
-    }
-
-    /**
-     * returns the number of docs
-     */
-    public long getDocCount() {
-        return count;
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java
index a8ccd1c76d031..ce6e97e822b8d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java
@@ -11,8 +11,6 @@
 
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.common.Rounding;
-import org.elasticsearch.common.logging.DeprecationCategory;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
@@ -31,7 +29,6 @@
 import java.util.Map;
 
 public final class DateHistogramAggregatorFactory extends ValuesSourceAggregatorFactory {
-    private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(DateHistogramAggregatorFactory.class);
 
     public static void registerAggregators(ValuesSourceRegistry.Builder builder) {
         builder.register(
@@ -42,49 +39,6 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) {
         );
 
         builder.register(DateHistogramAggregationBuilder.REGISTRY_KEY, CoreValuesSourceType.RANGE, DateRangeHistogramAggregator::new, true);
-
-        builder.register(
-            DateHistogramAggregationBuilder.REGISTRY_KEY,
-            CoreValuesSourceType.BOOLEAN,
-            (
-                name,
-                factories,
-                rounding,
-                order,
-                keyed,
-                minDocCount,
-                downsampledResultsOffset,
-                extendedBounds,
-                hardBounds,
-                valuesSourceConfig,
-                context,
-                parent,
-                cardinality,
-                metadata) -> {
-                DEPRECATION_LOGGER.warn(
-                    DeprecationCategory.AGGREGATIONS,
-                    "date-histogram-boolean",
-                    "Running DateHistogram aggregations on [boolean] fields is deprecated"
-                );
-                return DateHistogramAggregator.build(
-                    name,
-                    factories,
-                    rounding,
-                    order,
-                    keyed,
-                    minDocCount,
-                    downsampledResultsOffset,
-                    extendedBounds,
-                    hardBounds,
-                    valuesSourceConfig,
-                    context,
-                    parent,
-                    cardinality,
-                    metadata
-                );
-            },
-            true
-        );
     }
 
     private final DateHistogramAggregationSupplier aggregatorSupplier;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java
index 6886b0e22c20a..752ea46b4da02 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -34,9 +33,6 @@
  * - Provides a variety of helper methods to interpret the intervals as different types, depending on caller's need
  */
 public class DateIntervalWrapper implements ToXContentFragment, Writeable {
-    private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(DateHistogramAggregationBuilder.class);
-    private static final String DEPRECATION_TEXT = "[interval] on [date_histogram] is deprecated, use [fixed_interval] or "
-        + "[calendar_interval] in the future.";
     private static final ParseField FIXED_INTERVAL_FIELD = new ParseField("fixed_interval");
     private static final ParseField CALENDAR_INTERVAL_FIELD = new ParseField("calendar_interval");
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/LongKeyedMultiBucketsAggregatorReducer.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/LongKeyedMultiBucketsAggregatorReducer.java
deleted file mode 100644
index ccb4a7294e58d..0000000000000
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/LongKeyedMultiBucketsAggregatorReducer.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-package org.elasticsearch.search.aggregations.bucket.histogram;
-
-import org.elasticsearch.common.util.LongObjectPagedHashMap;
-import org.elasticsearch.core.Releasable;
-import org.elasticsearch.core.Releasables;
-import org.elasticsearch.search.aggregations.AggregationReduceContext;
-import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
-import org.elasticsearch.search.aggregations.bucket.BucketReducer;
-import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- *  Reduces aggregations where buckets are represented by a long key. It uses a {@link LongObjectPagedHashMap}
- *  to keep track of the different buckets.
- */
-abstract class LongKeyedMultiBucketsAggregatorReducer<B extends MultiBucketsAggregation.Bucket> implements Releasable {
-
-    private final AggregationReduceContext reduceContext;
-    private final int size;
-    private final long minDocCount;
-    private final LongObjectPagedHashMap<BucketReducer<B>> bucketsReducer;
-    int consumeBucketCount = 0;
-
-    LongKeyedMultiBucketsAggregatorReducer(AggregationReduceContext reduceContext, int size, long minDocCount) {
-        this.reduceContext = reduceContext;
-        this.size = size;
-        this.minDocCount = minDocCount;
-        bucketsReducer = new LongObjectPagedHashMap<>(size, reduceContext.bigArrays());
-    }
-
-    /**
-     * The bucket to reduce with its corresponding long key.
-     */
-    public final void accept(long key, B bucket) {
-        BucketReducer<B> reducer = bucketsReducer.get(key);
-        if (reducer == null) {
-            reducer = new BucketReducer<>(bucket, reduceContext, size);
-            bucketsReducer.put(key, reducer);
-        }
-        consumeBucketsAndMaybeBreak(reducer, bucket);
-        reducer.accept(bucket);
-    }
-
-    private void consumeBucketsAndMaybeBreak(BucketReducer<B> reducer, B bucket) {
-        if (reduceContext.isFinalReduce() == false || minDocCount == 0) {
-            if (reducer.getDocCount() == 0 && bucket.getDocCount() > 0) {
-                consumeBucketsAndMaybeBreak();
-            }
-        } else {
-            if (reducer.getDocCount() < minDocCount && (reducer.getDocCount() + bucket.getDocCount()) >= minDocCount) {
-                consumeBucketsAndMaybeBreak();
-            }
-        }
-    }
-
-    private void consumeBucketsAndMaybeBreak() {
-        if (consumeBucketCount++ >= InternalMultiBucketAggregation.REPORT_EMPTY_EVERY) {
-            reduceContext.consumeBucketsAndMaybeBreak(consumeBucketCount);
-            consumeBucketCount = 0;
-        }
-    }
-
-    /**
-     * Returns the reduced buckets.
-     */
-    public final List<B> get() {
-        reduceContext.consumeBucketsAndMaybeBreak(consumeBucketCount);
-        final List<B> reducedBuckets = new ArrayList<>((int) bucketsReducer.size());
-        bucketsReducer.forEach(entry -> {
-            if (reduceContext.isFinalReduce() == false || entry.value.getDocCount() >= minDocCount) {
-                reducedBuckets.add(createBucket(entry.key, entry.value.getDocCount(), entry.value.getAggregations()));
-            }
-        });
-        return reducedBuckets;
-    }
-
-    /**
-     * Builds a bucket provided the key, the number of documents and the sub-aggregations.
-     */
-    protected abstract B createBucket(long key, long docCount, InternalAggregations aggregations);
-
-    @Override
-    public final void close() {
-        bucketsReducer.forEach(r -> Releasables.close(r.value));
-        Releasables.close(bucketsReducer);
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
index 34a2ebea88440..740d53eb245e4 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
@@ -271,13 +271,6 @@ public InternalAggregation finalizeSampling(SamplingContext samplingContext) {
         );
     }
 
-    private Bucket reduceBucket(List<Bucket> buckets, AggregationReduceContext context) {
-        assert buckets.isEmpty() == false;
-        final List<InternalAggregations> aggregations = new BucketAggregationList<>(buckets);
-        final InternalAggregations aggs = InternalAggregations.reduce(aggregations, context);
-        return createBucket(aggs, buckets.get(0));
-    }
-
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         if (keyed) {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/ChiSquare.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/ChiSquare.java
index 64a589f38910d..8c856750f053f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/ChiSquare.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/ChiSquare.java
@@ -38,10 +38,7 @@ public ChiSquare(StreamInput in) throws IOException {
 
     @Override
     public boolean equals(Object other) {
-        if ((other instanceof ChiSquare) == false) {
-            return false;
-        }
-        return super.equals(other);
+        return other instanceof ChiSquare && super.equals(other);
     }
 
     @Override
@@ -80,17 +77,4 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    public static class ChiSquareBuilder extends NXYSignificanceHeuristic.NXYBuilder {
-        public ChiSquareBuilder(boolean includeNegatives, boolean backgroundIsSuperset) {
-            super(includeNegatives, backgroundIsSuperset);
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(NAME);
-            super.build(builder);
-            builder.endObject();
-            return builder;
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/GND.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/GND.java
index 2020b00c3c68c..6b00245d51468 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/GND.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/GND.java
@@ -46,10 +46,7 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public boolean equals(Object other) {
-        if ((other instanceof GND) == false) {
-            return false;
-        }
-        return super.equals(other);
+        return other instanceof GND && super.equals(other);
     }
 
     @Override
@@ -99,17 +96,4 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    public static class GNDBuilder extends NXYBuilder {
-        public GNDBuilder(boolean backgroundIsSuperset) {
-            super(true, backgroundIsSuperset);
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(NAME);
-            builder.field(BACKGROUND_IS_SUPERSET.getPreferredName(), backgroundIsSuperset);
-            builder.endObject();
-            return builder;
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/JLHScore.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/JLHScore.java
index 80e477dfab3dd..028a4fab6d574 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/JLHScore.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/JLHScore.java
@@ -90,10 +90,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public boolean equals(Object obj) {
-        if (obj == null || obj.getClass() != getClass()) {
-            return false;
-        }
-        return true;
+        return obj != null && obj.getClass() == getClass();
     }
 
     @Override
@@ -101,12 +98,4 @@ public int hashCode() {
         return getClass().hashCode();
     }
 
-    public static class JLHScoreBuilder implements SignificanceHeuristicBuilder {
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(NAME).endObject();
-            return builder;
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/MutualInformation.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/MutualInformation.java
index efdd10ca145f0..a10f4d62bfe94 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/MutualInformation.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/MutualInformation.java
@@ -40,10 +40,7 @@ public MutualInformation(StreamInput in) throws IOException {
 
     @Override
     public boolean equals(Object other) {
-        if ((other instanceof MutualInformation) == false) {
-            return false;
-        }
-        return super.equals(other);
+        return other instanceof MutualInformation && super.equals(other);
     }
 
     @Override
@@ -119,17 +116,4 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    public static class MutualInformationBuilder extends NXYBuilder {
-        public MutualInformationBuilder(boolean includeNegatives, boolean backgroundIsSuperset) {
-            super(includeNegatives, backgroundIsSuperset);
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(NAME);
-            super.build(builder);
-            builder.endObject();
-            return builder;
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/NXYSignificanceHeuristic.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/NXYSignificanceHeuristic.java
index 2e38219bae337..d17d3a85fee61 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/NXYSignificanceHeuristic.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/NXYSignificanceHeuristic.java
@@ -66,9 +66,7 @@ public boolean equals(Object obj) {
         if (obj == null) return false;
         if (getClass() != obj.getClass()) return false;
         NXYSignificanceHeuristic other = (NXYSignificanceHeuristic) obj;
-        if (backgroundIsSuperset != other.backgroundIsSuperset) return false;
-        if (includeNegatives != other.includeNegatives) return false;
-        return true;
+        return backgroundIsSuperset == other.backgroundIsSuperset && includeNegatives == other.includeNegatives;
     }
 
     @Override
@@ -160,24 +158,10 @@ protected static void declareParseFields(ConstructingObjectParser<? extends NXYS
      */
     protected static <T> Function<Object[], T> buildFromParsedArgs(BiFunction<Boolean, Boolean, T> ctor) {
         return args -> {
-            boolean includeNegatives = args[0] == null ? false : (boolean) args[0];
-            boolean backgroundIsSuperset = args[1] == null ? true : (boolean) args[1];
+            boolean includeNegatives = args[0] != null && (boolean) args[0];
+            boolean backgroundIsSuperset = args[1] == null || (boolean) args[1];
             return ctor.apply(includeNegatives, backgroundIsSuperset);
         };
     }
 
-    protected abstract static class NXYBuilder implements SignificanceHeuristicBuilder {
-        protected boolean includeNegatives = true;
-        protected boolean backgroundIsSuperset = true;
-
-        public NXYBuilder(boolean includeNegatives, boolean backgroundIsSuperset) {
-            this.includeNegatives = includeNegatives;
-            this.backgroundIsSuperset = backgroundIsSuperset;
-        }
-
-        protected void build(XContentBuilder builder) throws IOException {
-            builder.field(INCLUDE_NEGATIVES_FIELD.getPreferredName(), includeNegatives)
-                .field(BACKGROUND_IS_SUPERSET.getPreferredName(), backgroundIsSuperset);
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/PercentageScore.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/PercentageScore.java
index 5f200b8778897..133de5a066dac 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/PercentageScore.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/PercentageScore.java
@@ -56,10 +56,7 @@ public double getScore(long subsetFreq, long subsetSize, long supersetFreq, long
 
     @Override
     public boolean equals(Object obj) {
-        if (obj == null || obj.getClass() != getClass()) {
-            return false;
-        }
-        return true;
+        return obj != null && obj.getClass() == getClass();
     }
 
     @Override
@@ -67,12 +64,4 @@ public int hashCode() {
         return getClass().hashCode();
     }
 
-    public static class PercentageScoreBuilder implements SignificanceHeuristicBuilder {
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(NAME).endObject();
-            return builder;
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index 3554a6dc08b90..6d47493e4d063 100644
--- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -48,9 +48,7 @@
 import org.elasticsearch.search.retriever.RetrieverParserContext;
 import org.elasticsearch.search.searchafter.SearchAfterBuilder;
 import org.elasticsearch.search.slice.SliceBuilder;
-import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
-import org.elasticsearch.search.sort.ShardDocSortField;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
@@ -1404,9 +1402,6 @@ private SearchSourceBuilder parseXContent(
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
                 if (RETRIEVER.match(currentFieldName, parser.getDeprecationHandler())) {
-                    if (clusterSupportsFeature.test(RetrieverBuilder.RETRIEVERS_SUPPORTED) == false) {
-                        throw new ParsingException(parser.getTokenLocation(), "Unknown key for a START_OBJECT in [retriever].");
-                    }
                     retrieverBuilder = RetrieverBuilder.parseTopLevelRetrieverBuilder(
                         parser,
                         new RetrieverParserContext(searchUsage, clusterSupportsFeature)
@@ -2341,18 +2336,6 @@ public ActionRequestValidationException validate(
                 validationException = rescorer.validate(this, validationException);
             }
         }
-
-        if (pointInTimeBuilder() == null && sorts() != null) {
-            for (var sortBuilder : sorts()) {
-                if (sortBuilder instanceof FieldSortBuilder fieldSortBuilder
-                    && ShardDocSortField.NAME.equals(fieldSortBuilder.getFieldName())) {
-                    validationException = addValidationError(
-                        "[" + FieldSortBuilder.SHARD_DOC_FIELD_NAME + "] sort field cannot be used without [point in time]",
-                        validationException
-                    );
-                }
-            }
-        }
         return validationException;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java
index 0bbbff3a5d5f4..2523c62015215 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java
@@ -10,6 +10,7 @@
 package org.elasticsearch.search.fetch;
 
 import org.apache.lucene.search.Query;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.query.ParsedQuery;
@@ -87,6 +88,10 @@ private static StoredFieldsContext buildStoredFieldsContext(SearchContext in) {
         return sfc;
     }
 
+    public BitsetFilterCache bitsetFilterCache() {
+        return searchContext.bitsetFilterCache();
+    }
+
     /**
      * The name of the index that documents are being fetched from
      */
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java
index 79e51036a91be..df99a718887e1 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java
@@ -10,6 +10,7 @@
 package org.elasticsearch.search.fetch.subphase;
 
 import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.fetch.FetchContext;
 import org.elasticsearch.search.fetch.FetchSubPhase;
@@ -63,6 +64,7 @@ private void hitExecute(HitContext hitContext) {
 
                 // If this is a parent document and there are no source filters, then add the source as-is.
                 if (nestedHit == false && sourceFilter == null) {
+                    source = replaceInferenceMetadataFields(hitContext.hit(), source);
                     hitContext.hit().sourceRef(source.internalSourceRef());
                     fastPath++;
                     return;
@@ -77,10 +79,30 @@ private void hitExecute(HitContext hitContext) {
                 }
                 if (nestedHit) {
                     source = extractNested(source, hitContext.hit().getNestedIdentity());
+                } else {
+                    source = replaceInferenceMetadataFields(hitContext.hit(), source);
                 }
                 hitContext.hit().sourceRef(source.internalSourceRef());
             }
 
+            /**
+             * Transfers the {@link InferenceMetadataFieldsMapper#NAME} field from the document fields
+             * to the original _source if it has been requested.
+             */
+            private Source replaceInferenceMetadataFields(SearchHit hit, Source source) {
+                if (InferenceMetadataFieldsMapper.isEnabled(fetchContext.getSearchExecutionContext().getMappingLookup()) == false) {
+                    return source;
+                }
+
+                var field = hit.removeDocumentField(InferenceMetadataFieldsMapper.NAME);
+                if (field == null || field.getValues().isEmpty()) {
+                    return source;
+                }
+                var newSource = source.source();
+                newSource.put(InferenceMetadataFieldsMapper.NAME, field.getValues().get(0));
+                return Source.fromMap(newSource, source.sourceContentType());
+            }
+
             @Override
             public Map<String, Object> getDebugInfo() {
                 return Map.of("fast_path", fastPath);
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java
index 5220dadec7a1a..53116ff8f6ebf 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java
@@ -9,10 +9,12 @@
 package org.elasticsearch.search.fetch.subphase;
 
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.ScorerSupplier;
+import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.search.fetch.FetchContext;
@@ -52,8 +54,9 @@ public FetchSubPhaseProcessor getProcessor(FetchContext context) throws IOExcept
             );
         }
         return new FetchSubPhaseProcessor() {
+            record ScorerAndIterator(Scorer scorer, DocIdSetIterator approximation, TwoPhaseIterator twoPhase) {}
 
-            final Map<String, Scorer> matchingIterators = new HashMap<>();
+            final Map<String, ScorerAndIterator> matchingIterators = new HashMap<>();
 
             @Override
             public void setNextReader(LeafReaderContext readerContext) throws IOException {
@@ -63,7 +66,14 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException {
                     if (ss != null) {
                         Scorer scorer = ss.get(0L);
                         if (scorer != null) {
-                            matchingIterators.put(entry.getKey(), scorer);
+                            final TwoPhaseIterator twoPhase = scorer.twoPhaseIterator();
+                            final DocIdSetIterator iterator;
+                            if (twoPhase == null) {
+                                iterator = scorer.iterator();
+                            } else {
+                                iterator = twoPhase.approximation();
+                            }
+                            matchingIterators.put(entry.getKey(), new ScorerAndIterator(scorer, iterator, twoPhase));
                         }
                     }
                 }
@@ -73,13 +83,13 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException {
             public void process(HitContext hitContext) throws IOException {
                 Map<String, Float> matches = new LinkedHashMap<>();
                 int doc = hitContext.docId();
-                for (Map.Entry<String, Scorer> entry : matchingIterators.entrySet()) {
-                    Scorer scorer = entry.getValue();
-                    if (scorer.iterator().docID() < doc) {
-                        scorer.iterator().advance(doc);
+                for (Map.Entry<String, ScorerAndIterator> entry : matchingIterators.entrySet()) {
+                    ScorerAndIterator query = entry.getValue();
+                    if (query.approximation.docID() < doc) {
+                        query.approximation.advance(doc);
                     }
-                    if (scorer.iterator().docID() == doc) {
-                        matches.put(entry.getKey(), scorer.score());
+                    if (query.approximation.docID() == doc && (query.twoPhase == null || query.twoPhase.matches())) {
+                        matches.put(entry.getKey(), query.scorer.score());
                     }
                 }
                 hitContext.hit().matchedQueries(matches);
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java
index a8db0f26d2966..64b1d76f05122 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java
@@ -561,13 +561,12 @@ public Integer phraseLimit() {
     }
 
     /**
-     * Set to a non-negative value which represents the max offset used to analyze
-     * the field thus avoiding exceptions if the field exceeds this limit.
+     * "maxAnalyzedOffset" might be non-negative int, null (unknown), or a negative int (defaulting to index analyzed offset).
      */
     @SuppressWarnings("unchecked")
     public HB maxAnalyzedOffset(Integer maxAnalyzedOffset) {
-        if (maxAnalyzedOffset != null && maxAnalyzedOffset <= 0) {
-            throw new IllegalArgumentException("[" + MAX_ANALYZED_OFFSET_FIELD + "] must be a positive integer");
+        if (maxAnalyzedOffset != null && (maxAnalyzedOffset < -1 || maxAnalyzedOffset == 0)) {
+            throw new IllegalArgumentException("[" + MAX_ANALYZED_OFFSET_FIELD + "] must be a positive integer, or -1");
         }
         this.maxAnalyzedOffset = maxAnalyzedOffset;
         return (HB) this;
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java
index 75f8e5588761a..c47f815c18639 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.text.Text;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -31,6 +30,7 @@
 import org.elasticsearch.lucene.search.uhighlight.BoundedBreakIteratorScanner;
 import org.elasticsearch.lucene.search.uhighlight.CustomPassageFormatter;
 import org.elasticsearch.lucene.search.uhighlight.CustomUnifiedHighlighter;
+import org.elasticsearch.lucene.search.uhighlight.QueryMaxAnalyzedOffset;
 import org.elasticsearch.lucene.search.uhighlight.Snippet;
 import org.elasticsearch.search.fetch.FetchContext;
 import org.elasticsearch.search.fetch.FetchSubPhase;
@@ -50,8 +50,6 @@
 
 public class DefaultHighlighter implements Highlighter {
 
-    public static final NodeFeature UNIFIED_HIGHLIGHTER_MATCHED_FIELDS = new NodeFeature("unified_highlighter_matched_fields");
-
     @Override
     public boolean canHighlight(MappedFieldType fieldType) {
         return true;
@@ -121,7 +119,10 @@ CustomUnifiedHighlighter buildHighlighter(FieldHighlightContext fieldContext) {
         int maxAnalyzedOffset = indexSettings.getHighlightMaxAnalyzedOffset();
         boolean weightMatchesEnabled = indexSettings.isWeightMatchesEnabled();
         int numberOfFragments = fieldContext.field.fieldOptions().numberOfFragments();
-        Integer queryMaxAnalyzedOffset = fieldContext.field.fieldOptions().maxAnalyzedOffset();
+        QueryMaxAnalyzedOffset queryMaxAnalyzedOffset = QueryMaxAnalyzedOffset.create(
+            fieldContext.field.fieldOptions().maxAnalyzedOffset(),
+            maxAnalyzedOffset
+        );
         Analyzer analyzer = wrapAnalyzer(
             fieldContext.context.getSearchExecutionContext().getIndexAnalyzer(f -> Lucene.KEYWORD_ANALYZER),
             queryMaxAnalyzedOffset
@@ -171,7 +172,7 @@ CustomUnifiedHighlighter buildHighlighter(FieldHighlightContext fieldContext) {
             fieldContext.field.fieldOptions().noMatchSize(),
             highlighterNumberOfFragments,
             maxAnalyzedOffset,
-            fieldContext.field.fieldOptions().maxAnalyzedOffset(),
+            queryMaxAnalyzedOffset,
             fieldContext.field.fieldOptions().requireFieldMatch(),
             weightMatchesEnabled
         );
@@ -186,9 +187,9 @@ protected PassageFormatter getPassageFormatter(SearchHighlightContext.Field fiel
         );
     }
 
-    protected Analyzer wrapAnalyzer(Analyzer analyzer, Integer maxAnalyzedOffset) {
+    protected Analyzer wrapAnalyzer(Analyzer analyzer, QueryMaxAnalyzedOffset maxAnalyzedOffset) {
         if (maxAnalyzedOffset != null) {
-            analyzer = new LimitTokenOffsetAnalyzer(analyzer, maxAnalyzedOffset);
+            analyzer = new LimitTokenOffsetAnalyzer(analyzer, maxAnalyzedOffset.getNotNull());
         }
         return analyzer;
     }
@@ -227,7 +228,7 @@ protected static BreakIterator getBreakIterator(SearchHighlightContext.Field fie
         }
     }
 
-    protected static String convertFieldValue(MappedFieldType type, Object value) {
+    public static String convertFieldValue(MappedFieldType type, Object value) {
         if (value instanceof BytesRef) {
             return type.valueForDisplay(value).toString();
         } else {
@@ -235,7 +236,7 @@ protected static String convertFieldValue(MappedFieldType type, Object value) {
         }
     }
 
-    protected static String mergeFieldValues(List<Object> fieldValues, char valuesSeparator) {
+    public static String mergeFieldValues(List<Object> fieldValues, char valuesSeparator) {
         // postings highlighter accepts all values in a single string, as offsets etc. need to match with content
         // loaded from stored fields, we merge all values using a proper separator
         String rawValue = Strings.collectionToDelimitedString(fieldValues, String.valueOf(valuesSeparator));
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
index 3dd3dd1b42c8c..e1c09e925c1b4 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.lucene.search.uhighlight.QueryMaxAnalyzedOffset;
 import org.elasticsearch.search.fetch.FetchContext;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 
@@ -107,7 +108,10 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc
         ArrayList<OrderedTextFragment> fragsList = new ArrayList<>();
         List<Object> textsToHighlight;
         final int maxAnalyzedOffset = context.getSearchExecutionContext().getIndexSettings().getHighlightMaxAnalyzedOffset();
-        Integer queryMaxAnalyzedOffset = fieldContext.field.fieldOptions().maxAnalyzedOffset();
+        QueryMaxAnalyzedOffset queryMaxAnalyzedOffset = QueryMaxAnalyzedOffset.create(
+            fieldContext.field.fieldOptions().maxAnalyzedOffset(),
+            maxAnalyzedOffset
+        );
         Analyzer analyzer = wrapAnalyzer(
             context.getSearchExecutionContext().getIndexAnalyzer(f -> Lucene.KEYWORD_ANALYZER),
             queryMaxAnalyzedOffset
@@ -119,7 +123,8 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc
         for (Object textToHighlight : textsToHighlight) {
             String text = convertFieldValue(fieldType, textToHighlight);
             int textLength = text.length();
-            if ((queryMaxAnalyzedOffset == null || queryMaxAnalyzedOffset > maxAnalyzedOffset) && (textLength > maxAnalyzedOffset)) {
+            if ((queryMaxAnalyzedOffset == null || queryMaxAnalyzedOffset.getNotNull() > maxAnalyzedOffset)
+                && (textLength > maxAnalyzedOffset)) {
                 throw new IllegalArgumentException(
                     "The length ["
                         + textLength
@@ -241,9 +246,9 @@ private static int findGoodEndForNoHighlightExcerpt(int noMatchSize, Analyzer an
         }
     }
 
-    private static Analyzer wrapAnalyzer(Analyzer analyzer, Integer maxAnalyzedOffset) {
+    private static Analyzer wrapAnalyzer(Analyzer analyzer, QueryMaxAnalyzedOffset maxAnalyzedOffset) {
         if (maxAnalyzedOffset != null) {
-            return new LimitTokenOffsetAnalyzer(analyzer, maxAnalyzedOffset);
+            return new LimitTokenOffsetAnalyzer(analyzer, maxAnalyzedOffset.getNotNull());
         }
         return analyzer;
     }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java
index 523ec965d271e..76b553f568ee8 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.action.search.SearchShardTask;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.search.Scroll;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportRequest;
@@ -25,7 +25,7 @@ public class InternalScrollSearchRequest extends TransportRequest {
 
     private ShardSearchContextId contextId;
 
-    private Scroll scroll;
+    private TimeValue scroll;
 
     public InternalScrollSearchRequest() {}
 
@@ -37,21 +37,21 @@ public InternalScrollSearchRequest(SearchScrollRequest request, ShardSearchConte
     public InternalScrollSearchRequest(StreamInput in) throws IOException {
         super(in);
         contextId = new ShardSearchContextId(in);
-        scroll = in.readOptionalWriteable(Scroll::new);
+        scroll = in.readOptionalTimeValue();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         contextId.writeTo(out);
-        out.writeOptionalWriteable(scroll);
+        out.writeOptionalTimeValue(scroll);
     }
 
     public ShardSearchContextId contextId() {
         return contextId;
     }
 
-    public Scroll scroll() {
+    public TimeValue scroll() {
         return scroll;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java b/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
index 98886f21987f7..d6f97b9ab781b 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
@@ -11,12 +11,12 @@
 
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TotalHits;
-import org.elasticsearch.search.Scroll;
+import org.elasticsearch.core.TimeValue;
 
 /** Wrapper around information that needs to stay around when scrolling. */
 public final class ScrollContext {
     public TotalHits totalHits = null;
     public float maxScore = Float.NaN;
     public ScoreDoc lastEmittedDoc;
-    public Scroll scroll;
+    public TimeValue scroll;
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index 49e41559702ea..14e2007befa55 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -385,7 +385,7 @@ public String toString() {
         }
         if (scrollContext() != null) {
             if (scrollContext().scroll != null) {
-                result.append("scroll=[").append(scrollContext().scroll.keepAlive()).append("]");
+                result.append("scroll=[").append(scrollContext().scroll).append("]");
             } else {
                 result.append("scroll=[null]");
             }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
index fcfe6fbb70030..5e4ffbdba9ad2 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
@@ -42,7 +42,6 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.AliasFilterParsingException;
 import org.elasticsearch.indices.InvalidAliasNameException;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.SearchSortValuesAndFormats;
 import org.elasticsearch.search.builder.PointInTimeBuilder;
@@ -77,7 +76,7 @@ public class ShardSearchRequest extends TransportRequest implements IndicesReque
     private final long waitForCheckpoint;
     private final TimeValue waitForCheckpointsTimeout;
     private final SearchType searchType;
-    private final Scroll scroll;
+    private final TimeValue scroll;
     private final float indexBoost;
     private Boolean requestCache;
     private final long nowInMillis;
@@ -221,7 +220,7 @@ public ShardSearchRequest(
         AliasFilter aliasFilter,
         float indexBoost,
         boolean allowPartialSearchResults,
-        Scroll scroll,
+        TimeValue scroll,
         long nowInMillis,
         @Nullable String clusterAlias,
         ShardSearchContextId readerId,
@@ -283,7 +282,7 @@ public ShardSearchRequest(StreamInput in) throws IOException {
         searchType = SearchType.fromId(in.readByte());
         shardRequestIndex = in.readVInt();
         numberOfShards = in.readVInt();
-        scroll = in.readOptionalWriteable(Scroll::new);
+        scroll = in.readOptionalTimeValue();
         source = in.readOptionalWriteable(SearchSourceBuilder::new);
         if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0) && in.getTransportVersion().before(TransportVersions.V_8_9_X)) {
             // to deserialize between the 8.8 and 8.500.020 version we need to translate
@@ -358,7 +357,7 @@ protected final void innerWriteTo(StreamOutput out, boolean asKey) throws IOExce
             out.writeVInt(shardRequestIndex);
             out.writeVInt(numberOfShards);
         }
-        out.writeOptionalWriteable(scroll);
+        out.writeOptionalTimeValue(scroll);
         out.writeOptionalWriteable(source);
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0) && out.getTransportVersion().before(TransportVersions.V_8_9_X)) {
             // to serialize between the 8.8 and 8.500.020 version we need to translate
@@ -482,7 +481,7 @@ public boolean allowPartialSearchResults() {
         return allowPartialSearchResults;
     }
 
-    public Scroll scroll() {
+    public TimeValue scroll() {
         return scroll;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java
index 9eb0170af5efb..d899a390d8be7 100644
--- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java
+++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java
@@ -102,14 +102,6 @@ private SearchLookup(SearchLookup searchLookup, Set<String> fieldChain) {
         this.fieldLookupProvider = searchLookup.fieldLookupProvider;
     }
 
-    private SearchLookup(SearchLookup searchLookup, SourceProvider sourceProvider, Set<String> fieldChain) {
-        this.fieldChain = Collections.unmodifiableSet(fieldChain);
-        this.sourceProvider = sourceProvider;
-        this.fieldTypeLookup = searchLookup.fieldTypeLookup;
-        this.fieldDataLookup = searchLookup.fieldDataLookup;
-        this.fieldLookupProvider = searchLookup.fieldLookupProvider;
-    }
-
     /**
      * Creates a copy of the current {@link SearchLookup} that looks fields up in the same way, but also tracks field references
      * in order to detect cycles and prevent resolving fields that depend on more than {@link #MAX_FIELD_CHAIN_DEPTH} other fields.
@@ -153,7 +145,4 @@ public Source getSource(LeafReaderContext ctx, int doc) throws IOException {
         return sourceProvider.getSource(ctx, doc);
     }
 
-    public SearchLookup swapSourceProvider(SourceProvider sourceProvider) {
-        return new SearchLookup(this, sourceProvider, fieldChain);
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhaseCollectorManager.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhaseCollectorManager.java
index cbc04dd460ff5..3d793a164f40a 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QueryPhaseCollectorManager.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhaseCollectorManager.java
@@ -58,6 +58,7 @@
 import org.elasticsearch.search.profile.query.CollectorResult;
 import org.elasticsearch.search.profile.query.InternalProfileCollector;
 import org.elasticsearch.search.rescore.RescoreContext;
+import org.elasticsearch.search.rescore.RescorePhase;
 import org.elasticsearch.search.sort.SortAndFormats;
 
 import java.io.IOException;
@@ -238,7 +239,7 @@ static CollectorManager<Collector, QueryPhaseResult> createQueryPhaseCollectorMa
             int numDocs = Math.min(searchContext.from() + searchContext.size(), totalNumDocs);
             final boolean rescore = searchContext.rescore().isEmpty() == false;
             if (rescore) {
-                assert searchContext.sort() == null;
+                assert RescorePhase.validateSort(searchContext.sort());
                 for (RescoreContext rescoreContext : searchContext.rescore()) {
                     numDocs = Math.max(numDocs, rescoreContext.getWindowSize());
                 }
diff --git a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardPhase.java b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardPhase.java
index e64bbe3c39d79..4374c06da365d 100644
--- a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardPhase.java
@@ -83,7 +83,7 @@ public static void processFetch(SearchContext searchContext) {
             // FetchSearchResult#shardResult()
             SearchHits hits = fetchSearchResult.hits();
             RankFeatureShardResult featureRankShardResult = (RankFeatureShardResult) rankFeaturePhaseRankShardContext
-                .buildRankFeatureShardResult(hits, searchContext.shardTarget().getShardId().id());
+                .buildRankFeatureShardResult(hits, searchContext.request().shardRequestIndex());
             // save the result in the search context
             // need to add profiling info as well available from fetch
             if (featureRankShardResult != null) {
diff --git a/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java b/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
index 7e3646e7689cc..c23df9cdfa441 100644
--- a/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
+++ b/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
@@ -13,6 +13,7 @@
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TopFieldDocs;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.search.SearchShardTask;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
@@ -22,9 +23,12 @@
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.query.QueryPhase;
 import org.elasticsearch.search.query.SearchTimeoutException;
+import org.elasticsearch.search.sort.ShardDocSortField;
+import org.elasticsearch.search.sort.SortAndFormats;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -39,15 +43,27 @@ public static void execute(SearchContext context) {
         if (context.size() == 0 || context.rescore() == null || context.rescore().isEmpty()) {
             return;
         }
-
+        if (validateSort(context.sort()) == false) {
+            throw new IllegalStateException("Cannot use [sort] option in conjunction with [rescore], missing a validate?");
+        }
         TopDocs topDocs = context.queryResult().topDocs().topDocs;
         if (topDocs.scoreDocs.length == 0) {
             return;
         }
+        // Populate FieldDoc#score using the primary sort field (_score) to ensure compatibility with top docs rescoring
+        Arrays.stream(topDocs.scoreDocs).forEach(t -> {
+            if (t instanceof FieldDoc fieldDoc) {
+                fieldDoc.score = (float) fieldDoc.fields[0];
+            }
+        });
         TopFieldGroups topGroups = null;
+        TopFieldDocs topFields = null;
         if (topDocs instanceof TopFieldGroups topFieldGroups) {
-            assert context.collapse() != null;
+            assert context.collapse() != null && validateSortFields(topFieldGroups.fields);
             topGroups = topFieldGroups;
+        } else if (topDocs instanceof TopFieldDocs topFieldDocs) {
+            assert validateSortFields(topFieldDocs.fields);
+            topFields = topFieldDocs;
         }
         try {
             Runnable cancellationCheck = getCancellationChecks(context);
@@ -56,17 +72,18 @@ public static void execute(SearchContext context) {
                 topDocs = ctx.rescorer().rescore(topDocs, context.searcher(), ctx);
                 // It is the responsibility of the rescorer to sort the resulted top docs,
                 // here we only assert that this condition is met.
-                assert context.sort() == null && topDocsSortedByScore(topDocs) : "topdocs should be sorted after rescore";
+                assert topDocsSortedByScore(topDocs) : "topdocs should be sorted after rescore";
                 ctx.setCancellationChecker(null);
             }
+            /**
+             * Since rescorers are building top docs with score only, we must reconstruct the {@link TopFieldGroups}
+             * or {@link TopFieldDocs} using their original version before rescoring.
+             */
             if (topGroups != null) {
                 assert context.collapse() != null;
-                /**
-                 * Since rescorers don't preserve collapsing, we must reconstruct the group and field
-                 * values from the originalTopGroups to create a new {@link TopFieldGroups} from the
-                 * rescored top documents.
-                 */
-                topDocs = rewriteTopGroups(topGroups, topDocs);
+                topDocs = rewriteTopFieldGroups(topGroups, topDocs);
+            } else if (topFields != null) {
+                topDocs = rewriteTopFieldDocs(topFields, topDocs);
             }
             context.queryResult()
                 .topDocs(new TopDocsAndMaxScore(topDocs, topDocs.scoreDocs[0].score), context.queryResult().sortValueFormats());
@@ -81,29 +98,84 @@ public static void execute(SearchContext context) {
         }
     }
 
-    private static TopFieldGroups rewriteTopGroups(TopFieldGroups originalTopGroups, TopDocs rescoredTopDocs) {
-        assert originalTopGroups.fields.length == 1 && SortField.FIELD_SCORE.equals(originalTopGroups.fields[0])
-            : "rescore must always sort by score descending";
+    /**
+     * Returns whether the provided {@link SortAndFormats} can be used to rescore
+     * top documents.
+     */
+    public static boolean validateSort(SortAndFormats sortAndFormats) {
+        if (sortAndFormats == null) {
+            return true;
+        }
+        return validateSortFields(sortAndFormats.sort.getSort());
+    }
+
+    private static boolean validateSortFields(SortField[] fields) {
+        if (fields[0].equals(SortField.FIELD_SCORE) == false) {
+            return false;
+        }
+        if (fields.length == 1) {
+            return true;
+        }
+
+        // The ShardDocSortField can be used as a tiebreaker because it maintains
+        // the natural document ID order within the shard.
+        if (fields[1] instanceof ShardDocSortField == false || fields[1].getReverse()) {
+            return false;
+        }
+        return true;
+    }
+
+    private static TopFieldDocs rewriteTopFieldDocs(TopFieldDocs originalTopFieldDocs, TopDocs rescoredTopDocs) {
+        Map<Integer, FieldDoc> docIdToFieldDoc = Maps.newMapWithExpectedSize(originalTopFieldDocs.scoreDocs.length);
+        for (int i = 0; i < originalTopFieldDocs.scoreDocs.length; i++) {
+            docIdToFieldDoc.put(originalTopFieldDocs.scoreDocs[i].doc, (FieldDoc) originalTopFieldDocs.scoreDocs[i]);
+        }
+        var newScoreDocs = new FieldDoc[rescoredTopDocs.scoreDocs.length];
+        int pos = 0;
+        for (var doc : rescoredTopDocs.scoreDocs) {
+            newScoreDocs[pos] = docIdToFieldDoc.get(doc.doc);
+            newScoreDocs[pos].score = doc.score;
+            newScoreDocs[pos].fields[0] = newScoreDocs[pos].score;
+            pos++;
+        }
+        return new TopFieldDocs(originalTopFieldDocs.totalHits, newScoreDocs, originalTopFieldDocs.fields);
+    }
+
+    private static TopFieldGroups rewriteTopFieldGroups(TopFieldGroups originalTopGroups, TopDocs rescoredTopDocs) {
+        var newFieldDocs = rewriteFieldDocs((FieldDoc[]) originalTopGroups.scoreDocs, rescoredTopDocs.scoreDocs);
+
         Map<Integer, Object> docIdToGroupValue = Maps.newMapWithExpectedSize(originalTopGroups.scoreDocs.length);
         for (int i = 0; i < originalTopGroups.scoreDocs.length; i++) {
             docIdToGroupValue.put(originalTopGroups.scoreDocs[i].doc, originalTopGroups.groupValues[i]);
         }
-        var newScoreDocs = new FieldDoc[rescoredTopDocs.scoreDocs.length];
         var newGroupValues = new Object[originalTopGroups.groupValues.length];
         int pos = 0;
         for (var doc : rescoredTopDocs.scoreDocs) {
-            newScoreDocs[pos] = new FieldDoc(doc.doc, doc.score, new Object[] { doc.score });
             newGroupValues[pos++] = docIdToGroupValue.get(doc.doc);
         }
         return new TopFieldGroups(
             originalTopGroups.field,
             originalTopGroups.totalHits,
-            newScoreDocs,
+            newFieldDocs,
             originalTopGroups.fields,
             newGroupValues
         );
     }
 
+    private static FieldDoc[] rewriteFieldDocs(FieldDoc[] originalTopDocs, ScoreDoc[] rescoredTopDocs) {
+        Map<Integer, FieldDoc> docIdToFieldDoc = Maps.newMapWithExpectedSize(rescoredTopDocs.length);
+        Arrays.stream(originalTopDocs).forEach(d -> docIdToFieldDoc.put(d.doc, d));
+        var newDocs = new FieldDoc[rescoredTopDocs.length];
+        int pos = 0;
+        for (var doc : rescoredTopDocs) {
+            newDocs[pos] = docIdToFieldDoc.get(doc.doc);
+            newDocs[pos].score = doc.score;
+            newDocs[pos].fields[0] = doc.score;
+            pos++;
+        }
+        return newDocs;
+    }
+
     /**
      * Returns true if the provided docs are sorted by score.
      */
diff --git a/server/src/main/java/org/elasticsearch/search/rescore/RescorerBuilder.java b/server/src/main/java/org/elasticsearch/search/rescore/RescorerBuilder.java
index f624961515389..38a319321207f 100644
--- a/server/src/main/java/org/elasticsearch/search/rescore/RescorerBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/rescore/RescorerBuilder.java
@@ -39,7 +39,7 @@ public abstract class RescorerBuilder<RB extends RescorerBuilder<RB>>
 
     protected Integer windowSize;
 
-    private static final ParseField WINDOW_SIZE_FIELD = new ParseField("window_size");
+    public static final ParseField WINDOW_SIZE_FIELD = new ParseField("window_size");
 
     /**
      * Construct an empty RescoreBuilder.
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java
index 2ab6395db73b5..298340e5c579e 100644
--- a/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java
@@ -32,10 +32,12 @@
 import org.elasticsearch.search.sort.ScoreSortBuilder;
 import org.elasticsearch.search.sort.ShardDocSortField;
 import org.elasticsearch.search.sort.SortBuilder;
+import org.elasticsearch.xcontent.ParseField;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
@@ -49,6 +51,8 @@ public abstract class CompoundRetrieverBuilder<T extends CompoundRetrieverBuilde
 
     public static final NodeFeature INNER_RETRIEVERS_FILTER_SUPPORT = new NodeFeature("inner_retrievers_filter_support");
 
+    public static final ParseField RANK_WINDOW_SIZE_FIELD = new ParseField("rank_window_size");
+
     public record RetrieverSource(RetrieverBuilder retriever, SearchSourceBuilder source) {}
 
     protected final int rankWindowSize;
@@ -81,6 +85,14 @@ public final boolean isCompound() {
         return true;
     }
 
+    /**
+     * Retrieves the {@link ParseField} used to configure the {@link CompoundRetrieverBuilder#rankWindowSize}
+     * at the REST layer.
+     */
+    public ParseField getRankWindowSizeField() {
+        return RANK_WINDOW_SIZE_FIELD;
+    }
+
     @Override
     public final RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException {
         if (ctx.getPointInTimeBuilder() == null) {
@@ -209,14 +221,14 @@ public ActionRequestValidationException validate(
         validationException = super.validate(source, validationException, isScroll, allowPartialSearchResults);
         if (source.size() > rankWindowSize) {
             validationException = addValidationError(
-                "["
-                    + this.getName()
-                    + "] requires [rank_window_size: "
-                    + rankWindowSize
-                    + "]"
-                    + " be greater than or equal to [size: "
-                    + source.size()
-                    + "]",
+                String.format(
+                    Locale.ROOT,
+                    "[%s] requires [%s: %d] be greater than or equal to [size: %d]",
+                    getName(),
+                    getRankWindowSizeField().getPreferredName(),
+                    rankWindowSize,
+                    source.size()
+                ),
                 validationException
             );
         }
@@ -231,6 +243,21 @@ public ActionRequestValidationException validate(
         }
         for (RetrieverSource innerRetriever : innerRetrievers) {
             validationException = innerRetriever.retriever().validate(source, validationException, isScroll, allowPartialSearchResults);
+            if (innerRetriever.retriever() instanceof CompoundRetrieverBuilder<?> compoundChild) {
+                if (rankWindowSize > compoundChild.rankWindowSize) {
+                    String errorMessage = String.format(
+                        Locale.ROOT,
+                        "[%s] requires [%s: %d] to be smaller than or equal to its sub retriever's %s [%s: %d]",
+                        this.getName(),
+                        getRankWindowSizeField().getPreferredName(),
+                        rankWindowSize,
+                        compoundChild.getName(),
+                        compoundChild.getRankWindowSizeField(),
+                        compoundChild.rankWindowSize
+                    );
+                    validationException = addValidationError(errorMessage, validationException);
+                }
+            }
         }
         return validationException;
     }
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java
index b29546ded75cd..737d2aa397c34 100644
--- a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java
@@ -10,8 +10,6 @@
 package org.elasticsearch.search.retriever;
 
 import org.apache.lucene.util.SetOnce;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryRewriteContext;
@@ -46,7 +44,6 @@
 public final class KnnRetrieverBuilder extends RetrieverBuilder {
 
     public static final String NAME = "knn";
-    public static final NodeFeature KNN_RETRIEVER_SUPPORTED = new NodeFeature("knn_retriever_supported");
 
     public static final ParseField FIELD_FIELD = new ParseField("field");
     public static final ParseField K_FIELD = new ParseField("k");
@@ -103,9 +100,6 @@ public final class KnnRetrieverBuilder extends RetrieverBuilder {
     }
 
     public static KnnRetrieverBuilder fromXContent(XContentParser parser, RetrieverParserContext context) throws IOException {
-        if (context.clusterSupportsFeature(KNN_RETRIEVER_SUPPORTED) == false) {
-            throw new ParsingException(parser.getTokenLocation(), "unknown retriever [" + NAME + "]");
-        }
         return PARSER.apply(parser, context);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/RescorerRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/RescorerRetrieverBuilder.java
new file mode 100644
index 0000000000000..09688b5b9b001
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/retriever/RescorerRetrieverBuilder.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.search.retriever;
+
+import org.apache.lucene.search.ScoreDoc;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.rank.RankDoc;
+import org.elasticsearch.search.rescore.RescorerBuilder;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.search.builder.SearchSourceBuilder.RESCORE_FIELD;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+
+/**
+ * A {@link CompoundRetrieverBuilder} that re-scores only the results produced by its child retriever.
+ */
+public final class RescorerRetrieverBuilder extends CompoundRetrieverBuilder<RescorerRetrieverBuilder> {
+
+    public static final String NAME = "rescorer";
+    public static final ParseField RETRIEVER_FIELD = new ParseField("retriever");
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<RescorerRetrieverBuilder, RetrieverParserContext> PARSER = new ConstructingObjectParser<>(
+        NAME,
+        args -> new RescorerRetrieverBuilder((RetrieverBuilder) args[0], (List<RescorerBuilder<?>>) args[1])
+    );
+
+    static {
+        PARSER.declareNamedObject(constructorArg(), (parser, context, n) -> {
+            RetrieverBuilder innerRetriever = parser.namedObject(RetrieverBuilder.class, n, context);
+            context.trackRetrieverUsage(innerRetriever.getName());
+            return innerRetriever;
+        }, RETRIEVER_FIELD);
+        PARSER.declareField(constructorArg(), (parser, context) -> {
+            if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
+                List<RescorerBuilder<?>> rescorers = new ArrayList<>();
+                while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                    rescorers.add(RescorerBuilder.parseFromXContent(parser, name -> context.trackRescorerUsage(name)));
+                }
+                return rescorers;
+            } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+                return List.of(RescorerBuilder.parseFromXContent(parser, name -> context.trackRescorerUsage(name)));
+            } else {
+                throw new IllegalArgumentException(
+                    "Unknown format for [rescorer.rescore], expects an object or an array of objects, got: " + parser.currentToken()
+                );
+            }
+        }, RESCORE_FIELD, ObjectParser.ValueType.OBJECT_ARRAY);
+        RetrieverBuilder.declareBaseParserFields(NAME, PARSER);
+    }
+
+    public static RescorerRetrieverBuilder fromXContent(XContentParser parser, RetrieverParserContext context) throws IOException {
+        try {
+            return PARSER.apply(parser, context);
+        } catch (Exception e) {
+            throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e);
+        }
+    }
+
+    private final List<RescorerBuilder<?>> rescorers;
+
+    public RescorerRetrieverBuilder(RetrieverBuilder retriever, List<RescorerBuilder<?>> rescorers) {
+        super(List.of(new RetrieverSource(retriever, null)), extractMinWindowSize(rescorers));
+        if (rescorers.isEmpty()) {
+            throw new IllegalArgumentException("Missing rescore definition");
+        }
+        this.rescorers = rescorers;
+    }
+
+    private RescorerRetrieverBuilder(RetrieverSource retriever, List<RescorerBuilder<?>> rescorers) {
+        super(List.of(retriever), extractMinWindowSize(rescorers));
+        this.rescorers = rescorers;
+    }
+
+    /**
+     * The minimum window size is used as the {@link CompoundRetrieverBuilder#rankWindowSize},
+     * the final number of top documents to return in this retriever.
+     */
+    private static int extractMinWindowSize(List<RescorerBuilder<?>> rescorers) {
+        int windowSize = Integer.MAX_VALUE;
+        for (var rescore : rescorers) {
+            windowSize = Math.min(rescore.windowSize() == null ? RescorerBuilder.DEFAULT_WINDOW_SIZE : rescore.windowSize(), windowSize);
+        }
+        return windowSize;
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public ParseField getRankWindowSizeField() {
+        return RescorerBuilder.WINDOW_SIZE_FIELD;
+    }
+
+    @Override
+    protected SearchSourceBuilder finalizeSourceBuilder(SearchSourceBuilder source) {
+        /**
+         * The re-scorer is passed downstream because this query operates only on
+         * the top documents retrieved by the child retriever.
+         *
+         * - If the sub-retriever is a {@link CompoundRetrieverBuilder}, only the top
+         *   documents are re-scored since they are already determined at this stage.
+         * - For other retrievers that do not require a rewrite, the re-scorer's window
+         *   size is applied per shard. As a result, more documents are re-scored
+         *   compared to the final top documents produced by these retrievers in isolation.
+         */
+        for (var rescorer : rescorers) {
+            source.addRescorer(rescorer);
+        }
+        return source;
+    }
+
+    @Override
+    public void doToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(RETRIEVER_FIELD.getPreferredName(), innerRetrievers.getFirst().retriever());
+        builder.startArray(RESCORE_FIELD.getPreferredName());
+        for (RescorerBuilder<?> rescorer : rescorers) {
+            rescorer.toXContent(builder, params);
+        }
+        builder.endArray();
+    }
+
+    @Override
+    protected RescorerRetrieverBuilder clone(List<RetrieverSource> newChildRetrievers, List<QueryBuilder> newPreFilterQueryBuilders) {
+        var newInstance = new RescorerRetrieverBuilder(newChildRetrievers.get(0), rescorers);
+        newInstance.preFilterQueryBuilders = newPreFilterQueryBuilders;
+        return newInstance;
+    }
+
+    @Override
+    protected RankDoc[] combineInnerRetrieverResults(List<ScoreDoc[]> rankResults) {
+        assert rankResults.size() == 1;
+        ScoreDoc[] scoreDocs = rankResults.getFirst();
+        RankDoc[] rankDocs = new RankDoc[scoreDocs.length];
+        for (int i = 0; i < scoreDocs.length; i++) {
+            ScoreDoc scoreDoc = scoreDocs[i];
+            rankDocs[i] = new RankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex);
+            rankDocs[i].rank = i + 1;
+        }
+        return rankDocs;
+    }
+
+    @Override
+    public boolean doEquals(Object o) {
+        RescorerRetrieverBuilder that = (RescorerRetrieverBuilder) o;
+        return super.doEquals(o) && Objects.equals(rescorers, that.rescorers);
+    }
+
+    @Override
+    public int doHashCode() {
+        return Objects.hash(super.doHashCode(), rescorers);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java
index d52c354cad69e..ce852a44c28ec 100644
--- a/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.SuggestingErrorOnUnknown;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryRewriteContext;
@@ -50,8 +49,6 @@
  */
 public abstract class RetrieverBuilder implements Rewriteable<RetrieverBuilder>, ToXContent {
 
-    public static final NodeFeature RETRIEVERS_SUPPORTED = new NodeFeature("retrievers_supported");
-
     public static final ParseField PRE_FILTER_FIELD = new ParseField("filter");
 
     public static final ParseField MIN_SCORE_FIELD = new ParseField("min_score");
@@ -63,7 +60,7 @@ protected static void declareBaseParserFields(
         AbstractObjectParser<? extends RetrieverBuilder, RetrieverParserContext> parser
     ) {
         parser.declareObjectArray(
-            (r, v) -> r.preFilterQueryBuilders = v,
+            (r, v) -> r.preFilterQueryBuilders = new ArrayList<>(v),
             (p, c) -> AbstractQueryBuilder.parseTopLevelQuery(p, c::trackQueryUsage),
             PRE_FILTER_FIELD
         );
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/RetrieversFeatures.java b/server/src/main/java/org/elasticsearch/search/retriever/RetrieversFeatures.java
index 74a8b30c8e7dc..bfd6f572a9e65 100644
--- a/server/src/main/java/org/elasticsearch/search/retriever/RetrieversFeatures.java
+++ b/server/src/main/java/org/elasticsearch/search/retriever/RetrieversFeatures.java
@@ -22,10 +22,6 @@ public class RetrieversFeatures implements FeatureSpecification {
 
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(
-            RetrieverBuilder.RETRIEVERS_SUPPORTED,
-            StandardRetrieverBuilder.STANDARD_RETRIEVER_SUPPORTED,
-            KnnRetrieverBuilder.KNN_RETRIEVER_SUPPORTED
-        );
+        return Set.of();
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java
index 4e875a97fdfc4..3ca74dc133d47 100644
--- a/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java
@@ -9,8 +9,6 @@
 
 package org.elasticsearch.search.retriever;
 
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -39,7 +37,6 @@
 public final class StandardRetrieverBuilder extends RetrieverBuilder implements ToXContent {
 
     public static final String NAME = "standard";
-    public static final NodeFeature STANDARD_RETRIEVER_SUPPORTED = new NodeFeature("standard_retriever_supported");
 
     public static final ParseField QUERY_FIELD = new ParseField("query");
     public static final ParseField SEARCH_AFTER_FIELD = new ParseField("search_after");
@@ -81,9 +78,6 @@ public final class StandardRetrieverBuilder extends RetrieverBuilder implements
     }
 
     public static StandardRetrieverBuilder fromXContent(XContentParser parser, RetrieverParserContext context) throws IOException {
-        if (context.clusterSupportsFeature(STANDARD_RETRIEVER_SUPPORTED) == false) {
-            throw new ParsingException(parser.getTokenLocation(), "unknown retriever [" + NAME + "]");
-        }
         return PARSER.apply(parser, context);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java
index b18ce2dff65cb..9b9718efcf523 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java
@@ -465,7 +465,7 @@ public KnnVectorQueryBuilder toQueryBuilder() {
         if (queryVectorBuilder != null) {
             throw new IllegalArgumentException("missing rewrite");
         }
-        return new KnnVectorQueryBuilder(field, queryVector, null, numCands, rescoreVectorBuilder, similarity).boost(boost)
+        return new KnnVectorQueryBuilder(field, queryVector, numCands, numCands, rescoreVectorBuilder, similarity).boost(boost)
             .queryName(queryName)
             .addFilterQueries(filterQueries);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java
index 81b00f1329591..12573d5ad496e 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java
@@ -256,7 +256,7 @@ public KnnVectorQueryBuilder toQueryBuilder() {
             if (numCands > NUM_CANDS_LIMIT) {
                 throw new IllegalArgumentException("[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot exceed [" + NUM_CANDS_LIMIT + "]");
             }
-            return new KnnVectorQueryBuilder(field, queryVector, null, numCands, null, null);
+            return new KnnVectorQueryBuilder(field, queryVector, numCands, numCands, null, null);
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java
index 88f6312fa7e6f..565fd7325a5ac 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NestedObjectMapper;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
@@ -56,8 +55,6 @@
  * {@link org.apache.lucene.search.KnnByteVectorQuery}.
  */
 public class KnnVectorQueryBuilder extends AbstractQueryBuilder<KnnVectorQueryBuilder> {
-    public static final NodeFeature K_PARAM_SUPPORTED = new NodeFeature("search.vectors.k_param_supported");
-
     public static final String NAME = "knn";
     private static final int NUM_CANDS_LIMIT = 10_000;
     private static final float NUM_CANDS_MULTIPLICATIVE_FACTOR = 1.5f;
@@ -487,23 +484,19 @@ protected QueryBuilder doRewrite(QueryRewriteContext ctx) throws IOException {
         return this;
     }
 
-    @Override
-    protected QueryBuilder doIndexMetadataRewrite(QueryRewriteContext context) throws IOException {
-        return super.doIndexMetadataRewrite(context);
-    }
-
     @Override
     protected Query doToQuery(SearchExecutionContext context) throws IOException {
         MappedFieldType fieldType = context.getFieldType(fieldName);
-        int requestSize;
-        if (k != null) {
-            requestSize = k;
+        int k;
+        if (this.k != null) {
+            k = this.k;
         } else {
-            requestSize = context.requestSize() == null || context.requestSize() < 0 ? DEFAULT_SIZE : context.requestSize();
+            k = context.requestSize() == null || context.requestSize() < 0 ? DEFAULT_SIZE : context.requestSize();
+            if (numCands != null) {
+                k = Math.min(k, numCands);
+            }
         }
-        int adjustedNumCands = numCands == null
-            ? Math.round(Math.min(NUM_CANDS_MULTIPLICATIVE_FACTOR * requestSize, NUM_CANDS_LIMIT))
-            : numCands;
+        int adjustedNumCands = numCands == null ? Math.round(Math.min(NUM_CANDS_MULTIPLICATIVE_FACTOR * k, NUM_CANDS_LIMIT)) : numCands;
         if (fieldType == null) {
             return new MatchNoDocsQuery();
         }
@@ -526,10 +519,10 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException {
 
         DenseVectorFieldType vectorFieldType = (DenseVectorFieldType) fieldType;
         String parentPath = context.nestedLookup().getNestedParent(fieldName);
-        Float numCandidatesFactor = rescoreVectorBuilder() == null ? null : rescoreVectorBuilder.numCandidatesFactor();
+        Float oversample = rescoreVectorBuilder() == null ? null : rescoreVectorBuilder.oversample();
 
+        BitSetProducer parentBitSet = null;
         if (parentPath != null) {
-            final BitSetProducer parentBitSet;
             final Query parentFilter;
             NestedObjectMapper originalObjectMapper = context.nestedScope().getObjectMapper();
             if (originalObjectMapper != null) {
@@ -558,17 +551,9 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException {
                 // Now join the filterQuery & parentFilter to provide the matching blocks of children
                 filterQuery = new ToChildBlockJoinQuery(filterQuery, parentBitSet);
             }
-            return vectorFieldType.createKnnQuery(
-                queryVector,
-                k,
-                adjustedNumCands,
-                numCandidatesFactor,
-                filterQuery,
-                vectorSimilarity,
-                parentBitSet
-            );
         }
-        return vectorFieldType.createKnnQuery(queryVector, k, adjustedNumCands, numCandidatesFactor, filterQuery, vectorSimilarity, null);
+
+        return vectorFieldType.createKnnQuery(queryVector, k, adjustedNumCands, oversample, filterQuery, vectorSimilarity, parentBitSet);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/RescoreKnnVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/RescoreKnnVectorQuery.java
index a9c606b1f8618..79ede6873ad1f 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/RescoreKnnVectorQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/RescoreKnnVectorQuery.java
@@ -32,16 +32,15 @@ public class RescoreKnnVectorQuery extends Query implements QueryProfilerProvide
     private final String fieldName;
     private final float[] floatTarget;
     private final VectorSimilarityFunction vectorSimilarityFunction;
-    private final Integer k;
+    private final int k;
     private final Query innerQuery;
-
-    private QueryProfilerProvider vectorProfiling;
+    private long vectorOperations = 0;
 
     public RescoreKnnVectorQuery(
         String fieldName,
         float[] floatTarget,
         VectorSimilarityFunction vectorSimilarityFunction,
-        Integer k,
+        int k,
         Query innerQuery
     ) {
         this.fieldName = fieldName;
@@ -54,19 +53,12 @@ public RescoreKnnVectorQuery(
     @Override
     public Query rewrite(IndexSearcher searcher) throws IOException {
         DoubleValuesSource valueSource = new VectorSimilarityFloatValueSource(fieldName, floatTarget, vectorSimilarityFunction);
-        // Vector similarity VectorSimilarityFloatValueSource keep track of the compared vectors - we need that in case we don't need
-        // to calculate top k and return directly the query to understand how many comparisons were done
-        vectorProfiling = (QueryProfilerProvider) valueSource;
         FunctionScoreQuery functionScoreQuery = new FunctionScoreQuery(innerQuery, valueSource);
         Query query = searcher.rewrite(functionScoreQuery);
 
-        if (k == null) {
-            // No need to calculate top k - let the request size limit the results.
-            return query;
-        }
-
         // Retrieve top k documents from the rescored query
         TopDocs topDocs = searcher.search(query, k);
+        vectorOperations = topDocs.totalHits.value();
         ScoreDoc[] scoreDocs = topDocs.scoreDocs;
         int[] docIds = new int[scoreDocs.length];
         float[] scores = new float[scoreDocs.length];
@@ -82,7 +74,7 @@ public Query innerQuery() {
         return innerQuery;
     }
 
-    public Integer k() {
+    public int k() {
         return k;
     }
 
@@ -92,10 +84,7 @@ public void profile(QueryProfiler queryProfiler) {
             queryProfilerProvider.profile(queryProfiler);
         }
 
-        if (vectorProfiling == null) {
-            throw new IllegalStateException("Query should have been rewritten");
-        }
-        vectorProfiling.profile(queryProfiler);
+        queryProfiler.addVectorOpsCount(vectorOperations);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java
index 4604d4f0ea325..0e110a57d1e14 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java
@@ -23,7 +23,7 @@
 
 public class RescoreVectorBuilder implements Writeable, ToXContentObject {
 
-    public static final ParseField NUM_CANDIDATES_FACTOR_FIELD = new ParseField("num_candidates_factor");
+    public static final ParseField OVERSAMPLE_FIELD = new ParseField("oversample");
     public static final float MIN_OVERSAMPLE = 1.0F;
     private static final ConstructingObjectParser<RescoreVectorBuilder, Void> PARSER = new ConstructingObjectParser<>(
         "rescore_vector",
@@ -31,33 +31,33 @@ public class RescoreVectorBuilder implements Writeable, ToXContentObject {
     );
 
     static {
-        PARSER.declareFloat(ConstructingObjectParser.constructorArg(), NUM_CANDIDATES_FACTOR_FIELD);
+        PARSER.declareFloat(ConstructingObjectParser.constructorArg(), OVERSAMPLE_FIELD);
     }
 
     // Oversample is required as of now as it is the only field in the rescore vector
-    private final float numCandidatesFactor;
+    private final float oversample;
 
     public RescoreVectorBuilder(float numCandidatesFactor) {
-        Objects.requireNonNull(numCandidatesFactor, "[" + NUM_CANDIDATES_FACTOR_FIELD.getPreferredName() + "] must be set");
+        Objects.requireNonNull(numCandidatesFactor, "[" + OVERSAMPLE_FIELD.getPreferredName() + "] must be set");
         if (numCandidatesFactor < MIN_OVERSAMPLE) {
-            throw new IllegalArgumentException("[" + NUM_CANDIDATES_FACTOR_FIELD.getPreferredName() + "] must be >= " + MIN_OVERSAMPLE);
+            throw new IllegalArgumentException("[" + OVERSAMPLE_FIELD.getPreferredName() + "] must be >= " + MIN_OVERSAMPLE);
         }
-        this.numCandidatesFactor = numCandidatesFactor;
+        this.oversample = numCandidatesFactor;
     }
 
     public RescoreVectorBuilder(StreamInput in) throws IOException {
-        this.numCandidatesFactor = in.readFloat();
+        this.oversample = in.readFloat();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeFloat(numCandidatesFactor);
+        out.writeFloat(oversample);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(NUM_CANDIDATES_FACTOR_FIELD.getPreferredName(), numCandidatesFactor);
+        builder.field(OVERSAMPLE_FIELD.getPreferredName(), oversample);
         builder.endObject();
         return builder;
     }
@@ -71,15 +71,15 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         RescoreVectorBuilder that = (RescoreVectorBuilder) o;
-        return Objects.equals(numCandidatesFactor, that.numCandidatesFactor);
+        return Objects.equals(oversample, that.oversample);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hashCode(numCandidatesFactor);
+        return Objects.hashCode(oversample);
     }
 
-    public float numCandidatesFactor() {
-        return numCandidatesFactor;
+    public float oversample() {
+        return oversample;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorService.java
index 4be1ee9ddc513..a6a7b458c216d 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorService.java
@@ -13,9 +13,7 @@
 import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.health.Diagnosis;
-import org.elasticsearch.health.HealthFeatures;
 import org.elasticsearch.health.HealthIndicatorDetails;
 import org.elasticsearch.health.HealthIndicatorImpact;
 import org.elasticsearch.health.HealthIndicatorResult;
@@ -100,11 +98,9 @@ public class RepositoryIntegrityHealthIndicatorService implements HealthIndicato
     );
 
     private final ClusterService clusterService;
-    private final FeatureService featureService;
 
-    public RepositoryIntegrityHealthIndicatorService(ClusterService clusterService, FeatureService featureService) {
+    public RepositoryIntegrityHealthIndicatorService(ClusterService clusterService) {
         this.clusterService = clusterService;
-        this.featureService = featureService;
     }
 
     @Override
@@ -175,15 +171,8 @@ private RepositoryHealthAnalyzer(
                 || invalidRepositories.isEmpty() == false) {
                 healthStatus = YELLOW;
             } else if (repositoriesHealthByNode.isEmpty()) {
-                clusterHasFeature = featureService.clusterHasFeature(
-                    clusterState,
-                    HealthFeatures.SUPPORTS_EXTENDED_REPOSITORY_INDICATOR
-                ) == false;
-                if (clusterHasFeature) {
-                    healthStatus = GREEN;
-                } else {
-                    healthStatus = UNKNOWN;
-                }
+                clusterHasFeature = false;
+                healthStatus = UNKNOWN;
             } else {
                 healthStatus = GREEN;
             }
diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index ddb1e3d384fbe..debe3d6e6bd92 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -11,7 +11,6 @@
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -71,7 +70,6 @@
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.Mapping;
-import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.shard.IndexLongFieldRange;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
@@ -159,7 +157,7 @@ public final class RestoreService implements ClusterStateApplier {
         SETTING_CREATION_DATE,
         SETTING_HISTORY_UUID,
         IndexSettings.MODE.getKey(),
-        SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(),
+        IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(),
         IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(),
         IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(),
         IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(),
@@ -362,6 +360,7 @@ private void startRestore(
             metadataBuilder = Metadata.builder();
         }
 
+        // TODO: https://github.com/elastic/elasticsearch/issues/119545 - This does not yet support selectors
         final String[] indicesInRequest = request.indices();
         List<String> requestIndices = new ArrayList<>(indicesInRequest.length);
         if (indicesInRequest.length == 0) {
@@ -1348,6 +1347,7 @@ public ClusterState execute(ClusterState currentState) {
             final Map<ShardId, ShardRestoreStatus> shards = new HashMap<>();
 
             final IndexVersion minIndexCompatibilityVersion = currentState.getNodes().getMinSupportedIndexVersion();
+            final IndexVersion minReadOnlyIndexCompatibilityVersion = currentState.getNodes().getMinReadOnlySupportedIndexVersion();
             final String localNodeId = clusterService.state().nodes().getLocalNodeId();
             for (Map.Entry<String, IndexId> indexEntry : indicesToRestore.entrySet()) {
                 final IndexId index = indexEntry.getValue();
@@ -1360,12 +1360,16 @@ public ClusterState execute(ClusterState currentState) {
                     request.indexSettings(),
                     request.ignoreIndexSettings()
                 );
-                if (snapshotIndexMetadata.getCompatibilityVersion().before(minIndexCompatibilityVersion)) {
+                if (snapshotIndexMetadata.getCompatibilityVersion().isLegacyIndexVersion()) {
                     // adapt index metadata so that it can be understood by current version
                     snapshotIndexMetadata = convertLegacyIndex(snapshotIndexMetadata, currentState, indicesService);
                 }
                 try {
-                    snapshotIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(snapshotIndexMetadata, minIndexCompatibilityVersion);
+                    snapshotIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(
+                        snapshotIndexMetadata,
+                        minIndexCompatibilityVersion,
+                        minReadOnlyIndexCompatibilityVersion
+                    );
                 } catch (Exception ex) {
                     throw new SnapshotRestoreException(snapshot, "cannot restore index [" + index + "] because it cannot be upgraded", ex);
                 }
@@ -1393,11 +1397,7 @@ public ClusterState execute(ClusterState currentState) {
                         currentState.metadata()
                     );
 
-                    final IndexMetadata.Builder indexMdBuilder = restoreToCreateNewIndex(
-                        snapshotIndexMetadata,
-                        renamedIndexName,
-                        currentState.getMinTransportVersion()
-                    );
+                    final IndexMetadata.Builder indexMdBuilder = restoreToCreateNewIndex(snapshotIndexMetadata, renamedIndexName);
                     if (request.includeAliases() == false
                         && snapshotIndexMetadata.getAliases().isEmpty() == false
                         && isSystemIndex(snapshotIndexMetadata) == false) {
@@ -1415,11 +1415,7 @@ && isSystemIndex(snapshotIndexMetadata) == false) {
                 } else {
                     // Index exists and it's closed - open it in metadata and start recovery
                     validateExistingClosedIndex(currentIndexMetadata, snapshotIndexMetadata, renamedIndexName, partial);
-                    final IndexMetadata.Builder indexMdBuilder = restoreOverClosedIndex(
-                        snapshotIndexMetadata,
-                        currentIndexMetadata,
-                        currentState.getMinTransportVersion()
-                    );
+                    final IndexMetadata.Builder indexMdBuilder = restoreOverClosedIndex(snapshotIndexMetadata, currentIndexMetadata);
 
                     if (request.includeAliases() == false && isSystemIndex(snapshotIndexMetadata) == false) {
                         // Remove all snapshot aliases
@@ -1794,11 +1790,7 @@ private static IndexMetadata convertLegacyIndex(
         return convertedIndexMetadataBuilder.build();
     }
 
-    private static IndexMetadata.Builder restoreToCreateNewIndex(
-        IndexMetadata snapshotIndexMetadata,
-        String renamedIndexName,
-        TransportVersion minClusterTransportVersion
-    ) {
+    private static IndexMetadata.Builder restoreToCreateNewIndex(IndexMetadata snapshotIndexMetadata, String renamedIndexName) {
         return IndexMetadata.builder(snapshotIndexMetadata)
             .state(IndexMetadata.State.OPEN)
             .index(renamedIndexName)
@@ -1806,14 +1798,10 @@ private static IndexMetadata.Builder restoreToCreateNewIndex(
                 Settings.builder().put(snapshotIndexMetadata.getSettings()).put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
             )
             .timestampRange(IndexLongFieldRange.NO_SHARDS)
-            .eventIngestedRange(IndexLongFieldRange.NO_SHARDS, minClusterTransportVersion);
+            .eventIngestedRange(IndexLongFieldRange.NO_SHARDS);
     }
 
-    private static IndexMetadata.Builder restoreOverClosedIndex(
-        IndexMetadata snapshotIndexMetadata,
-        IndexMetadata currentIndexMetadata,
-        TransportVersion minTransportVersion
-    ) {
+    private static IndexMetadata.Builder restoreOverClosedIndex(IndexMetadata snapshotIndexMetadata, IndexMetadata currentIndexMetadata) {
         final IndexMetadata.Builder indexMdBuilder = IndexMetadata.builder(snapshotIndexMetadata)
             .state(IndexMetadata.State.OPEN)
             .version(Math.max(snapshotIndexMetadata.getVersion(), 1 + currentIndexMetadata.getVersion()))
@@ -1822,7 +1810,7 @@ private static IndexMetadata.Builder restoreOverClosedIndex(
             .settingsVersion(Math.max(snapshotIndexMetadata.getSettingsVersion(), 1 + currentIndexMetadata.getSettingsVersion()))
             .aliasesVersion(Math.max(snapshotIndexMetadata.getAliasesVersion(), 1 + currentIndexMetadata.getAliasesVersion()))
             .timestampRange(IndexLongFieldRange.NO_SHARDS)
-            .eventIngestedRange(IndexLongFieldRange.NO_SHARDS, minTransportVersion)
+            .eventIngestedRange(IndexLongFieldRange.NO_SHARDS)
             .index(currentIndexMetadata.getIndex().getName())
             .settings(
                 Settings.builder()
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index 8d526f3e114e1..7ee706acbceda 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -786,15 +786,7 @@ private static Metadata metadataForSnapshot(SnapshotsInProgress.Entry snapshot,
                 assert snapshot.partial()
                     : "Data stream [" + dataStreamName + "] was deleted during a snapshot but snapshot was not partial.";
             } else {
-                boolean missingIndex = false;
-                for (Index index : dataStream.getIndices()) {
-                    final String indexName = index.getName();
-                    if (builder.get(indexName) == null || indicesInSnapshot.contains(indexName) == false) {
-                        missingIndex = true;
-                        break;
-                    }
-                }
-                final DataStream reconciled = missingIndex ? dataStream.snapshot(indicesInSnapshot) : dataStream;
+                final DataStream reconciled = dataStream.snapshot(indicesInSnapshot, builder);
                 if (reconciled != null) {
                     dataStreams.put(dataStreamName, reconciled);
                 }
@@ -4145,6 +4137,9 @@ private SnapshotsInProgress createSnapshot(
                 request.partial(),
                 indexIds,
                 CollectionUtils.concatLists(
+                    // It's ok to just get the data stream names here because we have already resolved every concrete index that will be
+                    // in the snapshot, and thus already resolved any selectors that might be present. We now only care about which data
+                    // streams we're packing up in the resulting snapshot, not what their contents are.
                     indexNameExpressionResolver.dataStreamNames(currentState, request.indicesOptions(), request.indices()),
                     systemDataStreamNames
                 ),
diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index f55e3740aaa8f..37a3ec586d104 100644
--- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.ReportingService;
 import org.elasticsearch.telemetry.metric.Instrument;
@@ -120,13 +119,7 @@ public static class Names {
     public static final String THREAD_POOL_METRIC_NAME_REJECTED = ".threads.rejected.total";
 
     public enum ThreadPoolType {
-        @Deprecated(forRemoval = true)
-        @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // no longer used, remove in v9
-        DIRECT("direct"),
         FIXED("fixed"),
-        @Deprecated(forRemoval = true)
-        @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // no longer used, remove in v9
-        FIXED_AUTO_QUEUE_SIZE("fixed_auto_queue_size"),
         SCALING("scaling");
 
         private final String type;
diff --git a/server/src/main/java/org/elasticsearch/transport/ConnectTransportException.java b/server/src/main/java/org/elasticsearch/transport/ConnectTransportException.java
index 648d27c885843..302175cc4f5a0 100644
--- a/server/src/main/java/org/elasticsearch/transport/ConnectTransportException.java
+++ b/server/src/main/java/org/elasticsearch/transport/ConnectTransportException.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 
@@ -41,6 +42,18 @@ public ConnectTransportException(StreamInput in) throws IOException {
         }
     }
 
+    /**
+     * The ES REST API is a gateway to a single or multiple clusters. If there is an error connecting to other servers, then we should
+     * return a 502 BAD_GATEWAY status code instead of the parent class' 500 INTERNAL_SERVER_ERROR. Clients tend to retry on a 502 but not
+     * on a 500, and retrying may help on a connection error.
+     *
+     * @return a {@link RestStatus#BAD_GATEWAY} code
+     */
+    @Override
+    public final RestStatus status() {
+        return RestStatus.BAD_GATEWAY;
+    }
+
     @Override
     protected void writeTo(StreamOutput out, Writer<Throwable> nestedExceptionsWriter) throws IOException {
         super.writeTo(out, nestedExceptionsWriter);
diff --git a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
index d5047a61e4606..eb2eab75d3fe3 100644
--- a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
+++ b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
@@ -303,6 +303,7 @@ public void onFailure(Exception e) {
                     new VersionInformation(
                         Version.CURRENT.minimumCompatibilityVersion(),
                         IndexVersions.MINIMUM_COMPATIBLE,
+                        IndexVersions.MINIMUM_READONLY_COMPATIBLE,
                         IndexVersion.current()
                     )
                 );
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
index ccb00181798db..8399f5dd72f7d 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
@@ -10,11 +10,14 @@
 package org.elasticsearch.transport;
 
 import org.elasticsearch.cluster.metadata.ClusterNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SelectorResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.node.Node;
 
 import java.util.ArrayList;
@@ -64,8 +67,11 @@ public static boolean isRemoteIndexName(String indexExpression) {
             // Thus, whatever it is, this is definitely not a remote index.
             return false;
         }
+        int idx = indexExpression.indexOf(RemoteClusterService.REMOTE_CLUSTER_INDEX_SEPARATOR);
+        // Check to make sure the remote cluster separator ':' isn't actually a selector separator '::'
+        boolean isSelector = indexExpression.startsWith(SelectorResolver.SELECTOR_SEPARATOR, idx);
         // Note remote index name also can not start with ':'
-        return indexExpression.indexOf(RemoteClusterService.REMOTE_CLUSTER_INDEX_SEPARATOR) > 0;
+        return idx > 0 && isSelector == false;
     }
 
     /**
@@ -99,7 +105,8 @@ public static String[] splitIndexName(String indexExpression) {
         if (i == 0) {
             throw new IllegalArgumentException("index name [" + indexExpression + "] is invalid because the remote part is empty");
         }
-        if (i < 0) {
+        if (i < 0 || indexExpression.startsWith(SelectorResolver.SELECTOR_SEPARATOR, i)) {
+            // Either no colon present, or the colon was a part of a selector separator (::)
             return new String[] { null, indexExpression };
         } else {
             return new String[] { indexExpression.substring(0, i), indexExpression.substring(i + 1) };
@@ -144,6 +151,9 @@ protected Map<String, List<String>> groupClusterIndices(Set<String> remoteCluste
                     isNegative ? remoteClusterName.substring(1) : remoteClusterName
                 );
                 if (isNegative) {
+                    Tuple<String, String> indexAndSelector = IndexNameExpressionResolver.splitSelectorExpression(indexName);
+                    indexName = indexAndSelector.v1();
+                    String selectorString = indexAndSelector.v2();
                     if (indexName.equals("*") == false) {
                         throw new IllegalArgumentException(
                             Strings.format(
@@ -152,6 +162,14 @@ protected Map<String, List<String>> groupClusterIndices(Set<String> remoteCluste
                             )
                         );
                     }
+                    if (selectorString != null && selectorString.equals("*") == false) {
+                        throw new IllegalArgumentException(
+                            Strings.format(
+                                "To exclude a cluster you must specify the '::*' selector or leave it off, but found: [%s]",
+                                selectorString
+                            )
+                        );
+                    }
                     clustersToRemove.addAll(clusters);
                 } else {
                     for (String clusterName : clusters) {
diff --git a/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java
index 2c198caf22354..854072c49e354 100644
--- a/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java
+++ b/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java
@@ -505,6 +505,7 @@ private static DiscoveryNode resolveSeedNode(String clusterAlias, String address
         var seedVersion = new VersionInformation(
             Version.CURRENT.minimumCompatibilityVersion(),
             IndexVersions.MINIMUM_COMPATIBLE,
+            IndexVersions.MINIMUM_READONLY_COMPATIBLE,
             IndexVersion.current()
         );
         if (proxyAddress == null || proxyAddress.isEmpty()) {
diff --git a/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java b/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java
index a43b1e7440bda..ec9b30e71e62c 100644
--- a/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java
+++ b/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java
@@ -41,9 +41,14 @@ final class TransportHandshaker {
      * ignores the body of the request. After the handshake, the OutboundHandler uses the min(local,remote) protocol version for all later
      * messages.
      *
-     * This version supports two handshake protocols, v6080099 and v7170099, which respectively have the same message structure as the
-     * transport protocols of v6.8.0 and v7.17.0. This node only sends v7170099 requests, but it can send a valid response to any v6080099
-     * requests that it receives.
+     * This version supports three handshake protocols, v6080099, v7170099 and v8800000, which respectively have the same message structure
+     * as the transport protocols of v6.8.0, v7.17.0, and v8.18.0. This node only sends v7170099 requests, but it can send a valid response
+     * to any v6080099 or v8800000 requests that it receives.
+     *
+     * Note that these are not really TransportVersion constants as used elsewhere in ES, they're independent things that just happen to be
+     * stored in the same location in the message header and which roughly match the same ID numbering scheme. Older versions of ES did
+     * rely on them matching the real transport protocol (which itself matched the release version numbers), but these days that's no longer
+     * true.
      *
      * Here are some example messages, broken down to show their structure:
      *
@@ -79,7 +84,7 @@ final class TransportHandshaker {
      *    c3 f9 eb 03                   -- max acceptable protocol version (vInt: 00000011 11101011 11111001 11000011 == 8060099)
      *
      *
-     * ## v7170099 Request:
+     * ## v7170099 and v8800000 Requests:
      *
      * 45 53                            -- 'ES' marker
      * 00 00 00 31                      -- total message length
@@ -98,7 +103,7 @@ final class TransportHandshaker {
      *    04                            -- payload length
      *       c3 f9 eb 03                -- max acceptable protocol version (vInt: 00000011 11101011 11111001 11000011 == 8060099)
      *
-     * ## v7170099 Response:
+     * ## v7170099 and v8800000 Responses:
      *
      * 45 53                            -- 'ES' marker
      * 00 00 00 17                      -- total message length
@@ -118,7 +123,12 @@ final class TransportHandshaker {
 
     static final TransportVersion EARLIEST_HANDSHAKE_VERSION = TransportVersion.fromId(6080099);
     static final TransportVersion REQUEST_HANDSHAKE_VERSION = TransportVersions.MINIMUM_COMPATIBLE;
-    static final Set<TransportVersion> ALLOWED_HANDSHAKE_VERSIONS = Set.of(EARLIEST_HANDSHAKE_VERSION, REQUEST_HANDSHAKE_VERSION);
+    static final TransportVersion V9_HANDSHAKE_VERSION = TransportVersion.fromId(8_800_00_0);
+    static final Set<TransportVersion> ALLOWED_HANDSHAKE_VERSIONS = Set.of(
+        EARLIEST_HANDSHAKE_VERSION,
+        REQUEST_HANDSHAKE_VERSION,
+        V9_HANDSHAKE_VERSION
+    );
 
     static final String HANDSHAKE_ACTION_NAME = "internal:tcp/handshake";
     private final ConcurrentMap<Long, HandshakeResponseHandler> pendingHandshakes = new ConcurrentHashMap<>();
diff --git a/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java b/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java
index 1aeab06146834..38d6f338073de 100644
--- a/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java
+++ b/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
 
@@ -57,8 +57,8 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).xContentObjectFields(RESULTS_FIELD.getPreferredName(), featureStatuses);
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.xContentObjectFields(RESULTS_FIELD.getPreferredName(), featureStatuses);
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/usage/UsageService.java b/server/src/main/java/org/elasticsearch/usage/UsageService.java
index dd4895eb4bdc2..5b4fa0f27bf48 100644
--- a/server/src/main/java/org/elasticsearch/usage/UsageService.java
+++ b/server/src/main/java/org/elasticsearch/usage/UsageService.java
@@ -26,11 +26,13 @@ public class UsageService {
     private final Map<String, BaseRestHandler> handlers;
     private final SearchUsageHolder searchUsageHolder;
     private final CCSUsageTelemetry ccsUsageHolder;
+    private final CCSUsageTelemetry esqlUsageHolder;
 
     public UsageService() {
         this.handlers = new HashMap<>();
         this.searchUsageHolder = new SearchUsageHolder();
         this.ccsUsageHolder = new CCSUsageTelemetry();
+        this.esqlUsageHolder = new CCSUsageTelemetry(false);
     }
 
     /**
@@ -89,4 +91,8 @@ public SearchUsageHolder getSearchUsageHolder() {
     public CCSUsageTelemetry getCcsUsageHolder() {
         return ccsUsageHolder;
     }
+
+    public CCSUsageTelemetry getEsqlUsageHolder() {
+        return esqlUsageHolder;
+    }
 }
diff --git a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
index 12965152f260c..a4f606b54827b 100644
--- a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
+++ b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
@@ -7,20 +7,13 @@
  # License v3.0 only", or the "Server Side Public License, v 1".
 #
 
-org.elasticsearch.action.admin.indices.stats.IndicesStatsFeatures
 org.elasticsearch.action.bulk.BulkFeatures
 org.elasticsearch.features.FeatureInfrastructureFeatures
-org.elasticsearch.health.HealthFeatures
-org.elasticsearch.cluster.metadata.MetadataFeatures
-org.elasticsearch.rest.RestFeatures
-org.elasticsearch.repositories.RepositoriesFeatures
-org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures
 org.elasticsearch.rest.action.admin.cluster.ClusterRerouteFeatures
+org.elasticsearch.index.IndexFeatures
 org.elasticsearch.index.mapper.MapperFeatures
-org.elasticsearch.ingest.IngestGeoIpFeatures
 org.elasticsearch.search.SearchFeatures
 org.elasticsearch.search.retriever.RetrieversFeatures
 org.elasticsearch.script.ScriptFeatures
-org.elasticsearch.reservedstate.service.FileSettingsFeatures
 org.elasticsearch.cluster.routing.RoutingFeatures
 org.elasticsearch.action.admin.cluster.stats.ClusterStatsFeatures
diff --git a/server/src/main/resources/org/elasticsearch/TransportVersions.csv b/server/src/main/resources/org/elasticsearch/TransportVersions.csv
index 08db0822dfef5..2016f59b58a3e 100644
--- a/server/src/main/resources/org/elasticsearch/TransportVersions.csv
+++ b/server/src/main/resources/org/elasticsearch/TransportVersions.csv
@@ -135,4 +135,5 @@
 8.15.5,8702003
 8.16.0,8772001
 8.16.1,8772004
+8.16.2,8772004
 8.17.0,8797002
diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
index 040a7a6205f9c..462fab651c211 100644
--- a/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+++ b/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
@@ -88,6 +88,7 @@ grant codeBase "${codebase.elasticsearch}" {
 // this is the test-framework, but the jar is horribly named
 grant codeBase "${codebase.framework}" {
   permission java.lang.RuntimePermission "setSecurityManager";
+  permission java.lang.RuntimePermission "createClassLoader";
 };
 
 grant codeBase "${codebase.elasticsearch-rest-client}" {
@@ -129,4 +130,5 @@ grant {
   permission java.nio.file.LinkPermission "symbolic";
   // needed for keystore tests
   permission java.lang.RuntimePermission "accessUserInformation";
+  permission java.lang.RuntimePermission "getClassLoader";
 };
diff --git a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv
index afe696f31d323..3bfeeded6494c 100644
--- a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv
+++ b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv
@@ -135,4 +135,5 @@
 8.15.5,8512000
 8.16.0,8518000
 8.16.1,8518000
+8.16.2,8518000
 8.17.0,8521000
diff --git a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index 2abe4157583cd..31f54f9a16359 100644
--- a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -409,6 +409,7 @@ public void testConnectTransportException() throws IOException {
         ex = serialize(new ConnectTransportException(node, "msg", "action", new NullPointerException()));
         assertEquals("[][" + transportAddress + "][action] msg", ex.getMessage());
         assertThat(ex.getCause(), instanceOf(NullPointerException.class));
+        assertEquals(RestStatus.BAD_GATEWAY, ex.status());
     }
 
     public void testSearchPhaseExecutionException() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java b/server/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java
index 1db1b62facc9d..bc1c1e527683f 100644
--- a/server/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java
+++ b/server/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java
@@ -9,6 +9,7 @@
 
 package org.elasticsearch.action;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -42,7 +43,12 @@ public void testOriginalIndicesSerialization() throws IOException {
             OriginalIndices originalIndices2 = OriginalIndices.readOriginalIndices(in);
 
             assertThat(originalIndices2.indices(), equalTo(originalIndices.indices()));
-            assertThat(originalIndices2.indicesOptions(), equalTo(originalIndices.indicesOptions()));
+            // indices options are not equivalent when sent to an older version and re-read due
+            // to the addition of selector settings. Allow selectors is always true when read
+            // from a version prior to its addition, since true is the default value.
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) || originalIndices.indicesOptions().allowSelectors()) {
+                assertThat(originalIndices2.indicesOptions(), equalTo(originalIndices.indicesOptions()));
+            }
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java
index f3d8f8860ba83..d9bf3e0e99c81 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsTests;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ClusterServiceUtils;
@@ -35,8 +34,6 @@
 
 import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.not;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
@@ -48,7 +45,6 @@ public class TransportGetAllocationStatsActionTests extends ESTestCase {
     private ClusterService clusterService;
     private TransportService transportService;
     private AllocationStatsService allocationStatsService;
-    private FeatureService featureService;
 
     private TransportGetAllocationStatsAction action;
 
@@ -67,15 +63,13 @@ public void setUp() throws Exception {
             Set.of()
         );
         allocationStatsService = mock(AllocationStatsService.class);
-        featureService = mock(FeatureService.class);
         action = new TransportGetAllocationStatsAction(
             transportService,
             clusterService,
             threadPool,
             new ActionFilters(Set.of()),
             null,
-            allocationStatsService,
-            featureService
+            allocationStatsService
         );
     }
 
@@ -99,8 +93,6 @@ public void testReturnsOnlyRequestedStats() throws Exception {
         );
 
         when(allocationStatsService.stats()).thenReturn(Map.of(randomIdentifier(), NodeAllocationStatsTests.randomNodeAllocationStats()));
-        when(featureService.clusterHasFeature(any(ClusterState.class), eq(AllocationStatsFeatures.INCLUDE_DISK_THRESHOLD_SETTINGS)))
-            .thenReturn(true);
 
         var future = new PlainActionFuture<TransportGetAllocationStatsAction.Response>();
         action.masterOperation(mock(Task.class), request, ClusterState.EMPTY_STATE, future);
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java
index 8a51963097dae..9fef4c4ed328f 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.indices.SystemIndexDescriptorUtils;
 import org.elasticsearch.indices.SystemIndices;
@@ -31,7 +31,8 @@
 public class TransportGetFeatureUpgradeStatusActionTests extends ESTestCase {
 
     public static String TEST_SYSTEM_INDEX_PATTERN = ".test*";
-    private static final IndexVersion TEST_OLD_VERSION = IndexVersion.fromId(6000099);
+    // Version just before MINIMUM_COMPATIBLE in order to check that UpgradeStatus.MIGRATION_NEEDED is set correctly
+    private static final IndexVersion TEST_OLD_VERSION = IndexVersion.fromId(IndexVersions.MINIMUM_COMPATIBLE.id() - 1);
     private static final ClusterState CLUSTER_STATE = getClusterState();
     private static final SystemIndices.Feature FEATURE = getFeature();
 
@@ -85,8 +86,6 @@ private static ClusterState getClusterState() {
             .numberOfReplicas(0)
             .build();
 
-        @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
-        // Once we start testing 9.x, we should update this test to use a 7.x "version created"
         IndexMetadata indexMetadata2 = IndexMetadata.builder(".test-index-2")
             .settings(Settings.builder().put("index.version.created", TEST_OLD_VERSION).build())
             .numberOfShards(1)
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
index a7058e5d6cd8c..0e80b70c4a0fd 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -497,7 +497,7 @@ private static int expectedChunks(NodeStats nodeStats, ToXContent.Params params)
             + chunkIfPresent(nodeStats.getDiscoveryStats()) // <br/>
             + assertExpectedChunks(nodeStats.getIngestStats(), NodeStatsTests::expectedChunks, params) // <br/>
             + chunkIfPresent(nodeStats.getAdaptiveSelectionStats()) // <br/>
-            + assertExpectedChunks(nodeStats.getScriptCacheStats(), NodeStatsTests::expectedChunks, params);
+            + chunkIfPresent(nodeStats.getScriptCacheStats());
     }
 
     private static int chunkIfPresent(ToXContent xcontent) {
@@ -511,17 +511,8 @@ private static <T extends ChunkedToXContent> int assertExpectedChunks(T obj, ToI
         return chunks;
     }
 
-    private static int expectedChunks(ScriptCacheStats scriptCacheStats) {
-        var chunks = 3; // start, end, SUM
-        if (scriptCacheStats.general() == null) {
-            chunks += 2 + scriptCacheStats.context().size() * 4;
-        }
-
-        return chunks;
-    }
-
     private static int expectedChunks(ScriptStats scriptStats) {
-        return 7 + (scriptStats.compilationsHistory() != null && scriptStats.compilationsHistory().areTimingsEmpty() == false ? 1 : 0)
+        return 4 + (scriptStats.compilationsHistory() != null && scriptStats.compilationsHistory().areTimingsEmpty() == false ? 1 : 0)
             + (scriptStats.cacheEvictionsHistory() != null && scriptStats.cacheEvictionsHistory().areTimingsEmpty() == false ? 1 : 0)
             + scriptStats.contextStats().size();
     }
@@ -555,7 +546,7 @@ private static int expectedChunks(TransportStats transportStats) {
     }
 
     private static int expectedChunks(NodeIndicesStats nodeIndicesStats, NodeStatsLevel level) {
-        return switch (level) {
+        return nodeIndicesStats == null ? 0 : switch (level) {
             case NODE -> 2;
             case INDICES -> 5; // only one index
             case SHARDS -> 9; // only one shard
@@ -596,6 +587,7 @@ private static CommonStats createShardLevelCommonStats() {
             ++iota,
             ++iota,
             ++iota,
+            ++iota,
             false,
             ++iota,
             ++iota,
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java
index b59cc13a20ff2..a73e9635a53b3 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java
@@ -9,7 +9,6 @@
 
 package org.elasticsearch.action.admin.cluster.reroute;
 
-import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
@@ -312,15 +311,11 @@ private void assertXContent(
             fail(e);
         }
 
-        int[] expectedChunks = new int[] { 3 };
-        if (Objects.equals(params.param("metric"), "none") == false) {
-            expectedChunks[0] += 2 + ClusterStateTests.expectedChunkCount(params, response.getState());
-        }
-        if (params.paramAsBoolean("explain", false)) {
-            expectedChunks[0]++;
-        }
+        final var expectedChunks = Objects.equals(params.param("metric"), "none")
+            ? 2
+            : 4 + ClusterStateTests.expectedChunkCount(params, response.getState());
 
-        AbstractChunkedSerializingTestCase.assertChunkCount(response, params, o -> expectedChunks[0]);
+        AbstractChunkedSerializingTestCase.assertChunkCount(response, params, o -> expectedChunks);
         assertCriticalWarnings(criticalDeprecationWarnings);
     }
 
@@ -351,7 +346,7 @@ private static ClusterState createClusterState() {
                                     .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                                     .build()
                             )
-                            .eventIngestedRange(IndexLongFieldRange.UNKNOWN, TransportVersion.current())
+                            .eventIngestedRange(IndexLongFieldRange.UNKNOWN)
                             .build(),
                         false
                     )
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsSerializationTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsSerializationTests.java
deleted file mode 100644
index 5954de0586985..0000000000000
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsSerializationTests.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.action.admin.cluster.settings;
-
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.AbstractWireSerializingTestCase;
-
-public class ClusterGetSettingsSerializationTests extends AbstractWireSerializingTestCase<ClusterGetSettingsAction.Response> {
-    @Override
-    protected Writeable.Reader<ClusterGetSettingsAction.Response> instanceReader() {
-        return ClusterGetSettingsAction.Response::new;
-    }
-
-    @Override
-    protected ClusterGetSettingsAction.Response createTestInstance() {
-        final Settings persistentSettings = Settings.builder()
-            .put("persistent.foo.filtered", "bar")
-            .put("persistent.foo.non_filtered", "baz")
-            .build();
-
-        final Settings transientSettings = Settings.builder()
-            .put("transient.foo.filtered", "bar")
-            .put("transient.foo.non_filtered", "baz")
-            .build();
-
-        final Settings allSettings = Settings.builder().put(persistentSettings).put(transientSettings).build();
-
-        return new ClusterGetSettingsAction.Response(persistentSettings, transientSettings, allSettings);
-    }
-
-    @Override
-    protected ClusterGetSettingsAction.Response mutateInstance(ClusterGetSettingsAction.Response instance) {
-        final Settings otherSettings = Settings.builder().put("random.setting", randomAlphaOfLength(randomIntBetween(1, 12))).build();
-        return switch (between(0, 2)) {
-            case 0 -> new ClusterGetSettingsAction.Response(otherSettings, instance.transientSettings(), instance.settings());
-            case 1 -> new ClusterGetSettingsAction.Response(instance.persistentSettings(), otherSettings, instance.settings());
-            case 2 -> new ClusterGetSettingsAction.Response(instance.persistentSettings(), instance.transientSettings(), otherSettings);
-            default -> throw new IllegalStateException("Unexpected switch value");
-        };
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java
index 47f3f71bcc11b..58341b7d7fb9a 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java
@@ -12,17 +12,18 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockUtils;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.util.List;
+import java.util.Map;
 
 import static org.mockito.Mockito.mock;
 
@@ -54,10 +55,8 @@ public void testTransportFilters() throws Exception {
         TransportClusterGetSettingsAction action = new TransportClusterGetSettingsAction(
             transportService,
             mock(ClusterService.class),
-            threadPool,
             filter,
-            mock(ActionFilters.class),
-            mock(IndexNameExpressionResolver.class)
+            mock(ActionFilters.class)
         );
 
         final Settings persistentSettings = Settings.builder()
@@ -74,7 +73,8 @@ public void testTransportFilters() throws Exception {
         final ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).metadata(metadata).build();
 
         final PlainActionFuture<ClusterGetSettingsAction.Response> future = new PlainActionFuture<>();
-        action.masterOperation(null, null, clusterState, future);
+        final var task = new CancellableTask(1, "test", ClusterGetSettingsAction.NAME, "", null, Map.of());
+        action.localClusterStateOperation(task, null, clusterState, future);
         assertTrue(future.isDone());
 
         final ClusterGetSettingsAction.Response response = future.get();
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java
index a72630c327ea2..6444caf08f831 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java
@@ -352,4 +352,20 @@ public void testRanges() throws IOException {
         assertThat(value2Read.count(), equalTo(count1 + count2));
         assertThat(value2Read.max(), equalTo(max1));
     }
+
+    public void testUseMRTFalse() {
+        CCSTelemetrySnapshot empty = new CCSTelemetrySnapshot();
+        // Ignore MRT data
+        empty.setUseMRT(false);
+
+        var randomWithMRT = randomValueOtherThanMany(
+            v -> v.getTookMrtTrue().count() == 0 || v.getTookMrtFalse().count() == 0,
+            this::randomCCSTelemetrySnapshot
+        );
+
+        empty.add(randomWithMRT);
+        assertThat(empty.getTook().count(), equalTo(randomWithMRT.getTook().count()));
+        assertThat(empty.getTookMrtFalse().count(), equalTo(0L));
+        assertThat(empty.getTookMrtTrue().count(), equalTo(0L));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetryTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetryTests.java
index c4a2fdee1111e..5eb2224ec5f8e 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetryTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetryTests.java
@@ -340,4 +340,23 @@ public void testConcurrentUpdates() throws InterruptedException {
         CCSTelemetrySnapshot expectedSnapshot = ccsUsageHolder.getCCSTelemetrySnapshot();
         assertThat(snapshot, equalTo(expectedSnapshot));
     }
+
+    public void testUseMRTFalse() {
+        // Ignore MRT counters if instructed.
+        CCSUsageTelemetry ccsUsageHolder = new CCSUsageTelemetry(false);
+
+        CCSUsage.Builder builder = new CCSUsage.Builder();
+        builder.took(10L).setRemotesCount(1).setClient("kibana");
+        builder.setFeature(MRT_FEATURE);
+        ccsUsageHolder.updateUsage(builder.build());
+
+        builder = new CCSUsage.Builder();
+        builder.took(11L).setRemotesCount(1).setClient("kibana");
+        ccsUsageHolder.updateUsage(builder.build());
+
+        CCSTelemetrySnapshot snapshot = ccsUsageHolder.getCCSTelemetrySnapshot();
+        assertThat(snapshot.getTook().count(), equalTo(2L));
+        assertThat(snapshot.getTookMrtFalse().count(), equalTo(0L));
+        assertThat(snapshot.getTookMrtTrue().count(), equalTo(0L));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java
index 96954458c18c4..0617053769da0 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable.Reader;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
-import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
@@ -586,7 +586,7 @@ public void testSourceModes() {
         int numDisabledIndices = randomIntBetween(1, 5);
         for (int i = 0; i < numSyntheticIndices; i++) {
             IndexMetadata.Builder indexMetadata = new IndexMetadata.Builder("foo-synthetic-" + i).settings(
-                indexSettings(IndexVersion.current(), 4, 1).put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic")
+                indexSettings(IndexVersion.current(), 4, 1).put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic")
             );
             builder.put(indexMetadata);
         }
@@ -594,7 +594,7 @@ public void testSourceModes() {
             IndexMetadata.Builder indexMetadata;
             if (randomBoolean()) {
                 indexMetadata = new IndexMetadata.Builder("foo-stored-" + i).settings(
-                    indexSettings(IndexVersion.current(), 4, 1).put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "stored")
+                    indexSettings(IndexVersion.current(), 4, 1).put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "stored")
                 );
             } else {
                 indexMetadata = new IndexMetadata.Builder("foo-stored-" + i).settings(indexSettings(IndexVersion.current(), 4, 1));
@@ -603,7 +603,7 @@ public void testSourceModes() {
         }
         for (int i = 0; i < numDisabledIndices; i++) {
             IndexMetadata.Builder indexMetadata = new IndexMetadata.Builder("foo-disabled-" + i).settings(
-                indexSettings(IndexVersion.current(), 4, 1).put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "disabled")
+                indexSettings(IndexVersion.current(), 4, 1).put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "disabled")
             );
             builder.put(indexMetadata);
         }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java
index 9bf4ad7c3cb64..fbd6e0916eefe 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java
@@ -130,6 +130,7 @@ public void testCreation() {
             new ShardStats[] { shardStats },
             new SearchUsageStats(),
             RepositoryUsageStats.EMPTY,
+            null,
             null
         );
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java
index ec5f0b489d6ef..31c22422ec538 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java
@@ -44,7 +44,7 @@ public void testPostProcess() {
         metadata.put(IndexMetadata.builder("c").settings(settings(IndexVersion.current())).numberOfShards(1).numberOfReplicas(0));
         ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).metadata(metadata).build();
 
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         Map<String, List<AliasMetadata>> aliases = Map.of("b", Collections.singletonList(new AliasMetadata.Builder("y").build()));
         Map<String, List<AliasMetadata>> result = TransportGetAliasesAction.postProcess(
             request,
@@ -60,7 +60,7 @@ public void testPostProcess() {
         assertThat(result.get("b").size(), equalTo(1));
         assertThat(result.get("c").size(), equalTo(0));
 
-        request = new GetAliasesRequest();
+        request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.replaceAliases("y", "z");
         aliases = Map.of("b", Collections.singletonList(new AliasMetadata.Builder("y").build()));
         result = TransportGetAliasesAction.postProcess(
@@ -77,7 +77,7 @@ public void testPostProcess() {
         assertThat(result.get("b").size(), equalTo(1));
         assertThat(result.get("c").size(), equalTo(0));
 
-        request = new GetAliasesRequest("y", "z");
+        request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "y", "z");
         aliases = Map.of("b", Collections.singletonList(new AliasMetadata.Builder("y").build()));
         result = TransportGetAliasesAction.postProcess(
             request,
@@ -95,7 +95,7 @@ public void testPostProcess() {
     public void testDeprecationWarningEmittedForTotalWildcard() {
         ClusterState state = systemIndexTestClusterState();
 
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         Map<String, List<AliasMetadata>> aliases = Map.of(
             ".b",
             Collections.singletonList(new AliasMetadata.Builder(".y").build()),
@@ -130,7 +130,7 @@ public void testDeprecationWarningEmittedForTotalWildcard() {
     public void testDeprecationWarningEmittedWhenSystemIndexIsRequested() {
         ClusterState state = systemIndexTestClusterState();
 
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.indices(".b");
         Map<String, List<AliasMetadata>> aliases = Map.of(".b", Collections.singletonList(new AliasMetadata.Builder(".y").build()));
         final String[] concreteIndices = { ".b" };
@@ -159,7 +159,7 @@ public void testDeprecationWarningEmittedWhenSystemIndexIsRequested() {
     public void testDeprecationWarningEmittedWhenSystemIndexIsRequestedByAlias() {
         ClusterState state = systemIndexTestClusterState();
 
-        GetAliasesRequest request = new GetAliasesRequest(".y");
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, ".y");
         Map<String, List<AliasMetadata>> aliases = Map.of(".b", Collections.singletonList(new AliasMetadata.Builder(".y").build()));
         final String[] concreteIndices = { "a", ".b", "c" };
         assertEquals(state.metadata().findAliases(request.aliases(), concreteIndices), aliases);
@@ -187,7 +187,7 @@ public void testDeprecationWarningEmittedWhenSystemIndexIsRequestedByAlias() {
     public void testDeprecationWarningNotEmittedWhenSystemAccessAllowed() {
         ClusterState state = systemIndexTestClusterState();
 
-        GetAliasesRequest request = new GetAliasesRequest(".y");
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, ".y");
         Map<String, List<AliasMetadata>> aliases = Map.of(".b", Collections.singletonList(new AliasMetadata.Builder(".y").build()));
         final String[] concreteIndices = { "a", ".b", "c" };
         assertEquals(state.metadata().findAliases(request.aliases(), concreteIndices), aliases);
@@ -210,7 +210,7 @@ public void testDeprecationWarningNotEmittedWhenSystemAccessAllowed() {
     public void testDeprecationWarningNotEmittedWhenOnlyNonsystemIndexRequested() {
         ClusterState state = systemIndexTestClusterState();
 
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.indices("c");
         Map<String, List<AliasMetadata>> aliases = Map.of("c", Collections.singletonList(new AliasMetadata.Builder("d").build()));
         final String[] concreteIndices = { "c" };
@@ -239,7 +239,7 @@ public void testPostProcessDataStreamAliases() {
         clusterState = ClusterState.builder(clusterState).metadata(builder).build();
 
         // return all all data streams with aliases
-        var getAliasesRequest = new GetAliasesRequest();
+        var getAliasesRequest = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         var result = TransportGetAliasesAction.postProcess(resolver, getAliasesRequest, clusterState);
         assertThat(result.keySet(), containsInAnyOrder("logs-foo", "logs-bar"));
         assertThat(result.get("logs-foo"), contains(new DataStreamAlias("logs", List.of("logs-bar", "logs-foo"), null, null)));
@@ -252,20 +252,20 @@ public void testPostProcessDataStreamAliases() {
         );
 
         // filter by alias name
-        getAliasesRequest = new GetAliasesRequest("secret");
+        getAliasesRequest = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "secret");
         result = TransportGetAliasesAction.postProcess(resolver, getAliasesRequest, clusterState);
         assertThat(result.keySet(), containsInAnyOrder("logs-bar"));
         assertThat(result.get("logs-bar"), contains(new DataStreamAlias("secret", List.of("logs-bar"), null, null)));
 
         // filter by data stream:
-        getAliasesRequest = new GetAliasesRequest().indices("logs-foo");
+        getAliasesRequest = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).indices("logs-foo");
         result = TransportGetAliasesAction.postProcess(resolver, getAliasesRequest, clusterState);
         assertThat(result.keySet(), containsInAnyOrder("logs-foo"));
         assertThat(result.get("logs-foo"), contains(new DataStreamAlias("logs", List.of("logs-bar", "logs-foo"), null, null)));
     }
 
     public void testNetNewSystemIndicesDontErrorWhenNotRequested() {
-        GetAliasesRequest aliasesRequest = new GetAliasesRequest();
+        GetAliasesRequest aliasesRequest = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         // `.b` will be the "net new" system index this test case
         ClusterState clusterState = systemIndexTestClusterState();
         String[] concreteIndices;
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/TransportAnalyzeIndexDiskUsageActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/TransportAnalyzeIndexDiskUsageActionTests.java
index b6a26d8df2fa1..f06ad47d306ef 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/TransportAnalyzeIndexDiskUsageActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/TransportAnalyzeIndexDiskUsageActionTests.java
@@ -132,7 +132,6 @@ public void testSimpleLimitRequests() throws Exception {
             assertBusy(() -> assertThat(transportService.getRequestsSentPerNode(), equalTo(expectedRequestCounts)));
             pendingRequests.addAll(transportService.getCapturedRequests(true));
         }
-        assertBusy(future::isDone);
         AnalyzeIndexDiskUsageResponse response = future.actionGet();
         assertThat(response.getTotalShards(), equalTo(numberOfShards));
         assertThat(response.getFailedShards(), equalTo(0));
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestTests.java
index 3bbc03a333438..35a7e4d655f57 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestTests.java
@@ -82,6 +82,5 @@ public void testIndicesOptions() {
         );
         assertThat(getIndexRequest.indicesOptions().wildcardOptions(), equalTo(IndicesOptions.strictExpandOpen().wildcardOptions()));
         assertThat(getIndexRequest.indicesOptions().gatekeeperOptions(), equalTo(IndicesOptions.strictExpandOpen().gatekeeperOptions()));
-        assertThat(getIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.ALL_APPLICABLE));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java
index 37957df46a796..958dab50e4a02 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedDataStream;
 import org.elasticsearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedIndex;
 import org.elasticsearch.action.admin.indices.resolve.ResolveIndexAction.TransportAction;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -240,7 +241,10 @@ public void testResolveHiddenProperlyWithDateMath() {
         assertThat(
             resolvedIndices,
             contains(
-                oneOf(new ResolvedExpression("logs-pgsql-prod-" + todaySuffix), new ResolvedExpression("logs-pgsql-prod-" + tomorrowSuffix))
+                oneOf(
+                    new ResolvedExpression("logs-pgsql-prod-" + todaySuffix, IndexComponentSelector.DATA),
+                    new ResolvedExpression("logs-pgsql-prod-" + tomorrowSuffix, IndexComponentSelector.DATA)
+                )
             )
         );
     }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
index 1a30fae1ebc00..42984dc9da828 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.AliasAction;
@@ -747,15 +746,15 @@ public void testValidation() throws Exception {
         final String defaultRolloverIndexName;
         final boolean useDataStream = randomBoolean();
         final Metadata.Builder builder = Metadata.builder();
-        var defaultSelectorOptions = IndicesOptions.SelectorOptions.DEFAULT;
+        boolean isFailureStoreRollover = false;
         if (useDataStream) {
             DataStream dataStream = DataStreamTestHelper.randomInstance()
                 // ensure no replicate data stream
                 .promoteDataStream();
             rolloverTarget = dataStream.getName();
-            if (dataStream.isFailureStoreEnabled() && randomBoolean()) {
-                defaultSelectorOptions = IndicesOptions.SelectorOptions.FAILURES;
+            if (dataStream.isFailureStoreExplicitlyEnabled() && randomBoolean()) {
                 sourceIndexName = dataStream.getFailureStoreWriteIndex().getName();
+                isFailureStoreRollover = true;
                 defaultRolloverIndexName = DataStream.getDefaultFailureStoreName(
                     dataStream.getName(),
                     dataStream.getGeneration() + 1,
@@ -815,7 +814,7 @@ public void testValidation() throws Exception {
             true,
             null,
             null,
-            IndicesOptions.SelectorOptions.FAILURES.equals(defaultSelectorOptions)
+            isFailureStoreRollover
         );
 
         newIndexName = newIndexName == null ? defaultRolloverIndexName : newIndexName;
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java
index f0190790ba001..ee95f7ffb5b9a 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java
@@ -11,7 +11,6 @@
 
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -174,9 +173,6 @@ public void testSerialize() throws Exception {
                 .build()
         );
         originalRequest.lazy(randomBoolean());
-        originalRequest.setIndicesOptions(
-            IndicesOptions.builder(originalRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE).build()
-        );
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             originalRequest.writeTo(out);
@@ -186,7 +182,6 @@ public void testSerialize() throws Exception {
                 assertThat(cloneRequest.getNewIndexName(), equalTo(originalRequest.getNewIndexName()));
                 assertThat(cloneRequest.getRolloverTarget(), equalTo(originalRequest.getRolloverTarget()));
                 assertThat(cloneRequest.isLazy(), equalTo(originalRequest.isLazy()));
-                assertThat(cloneRequest.indicesOptions().selectorOptions(), equalTo(originalRequest.indicesOptions().selectorOptions()));
                 for (Map.Entry<String, Condition<?>> entry : cloneRequest.getConditions().getConditions().entrySet()) {
                     Condition<?> condition = originalRequest.getConditions().getConditions().get(entry.getKey());
                     // here we compare the string representation as there is some information loss when serializing
@@ -253,12 +248,7 @@ public void testValidation() {
         }
 
         {
-            RolloverRequest rolloverRequest = new RolloverRequest("alias-index", "new-index-name");
-            rolloverRequest.setIndicesOptions(
-                IndicesOptions.builder(rolloverRequest.indicesOptions())
-                    .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
-                    .build()
-            );
+            RolloverRequest rolloverRequest = new RolloverRequest("alias-index::*", "new-index-name");
             ActionRequestValidationException validationException = rolloverRequest.validate();
             assertNotNull(validationException);
             assertEquals(1, validationException.validationErrors().size());
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
index f04180bde30f2..657a03066ada9 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -43,7 +43,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.cache.query.QueryCacheStats;
@@ -121,7 +120,6 @@ public class TransportRolloverActionTests extends ESTestCase {
     final DataStreamAutoShardingService dataStreamAutoShardingService = new DataStreamAutoShardingService(
         Settings.EMPTY,
         mockClusterService,
-        new FeatureService(List.of()),
         System::currentTimeMillis
     );
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java
index 144db693b8ab3..e33f97e7cb580 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java
@@ -68,7 +68,7 @@ public void testChunking() {
                 0,
                 Collections.emptyList()
             ),
-            response -> 11 * response.getIndices().size() + 5
+            response -> 11 * response.getIndices().size() + 4
         );
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java
index a34e36e81dd83..7b707eb4c31ba 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java
@@ -47,7 +47,7 @@ public void testToXContentChunkPerIndex() {
 
         AbstractChunkedSerializingTestCase.assertChunkCount(
             new FieldUsageStatsResponse(indices, indices, 0, List.of(), perIndex),
-            ignored -> 3 * indices + 3
+            ignored -> 3 * indices + 2
         );
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java
index bce4d20a06fbe..2e915083476b8 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java
@@ -138,12 +138,12 @@ public void testChunkedEncodingPerIndex() {
         AbstractChunkedSerializingTestCase.assertChunkCount(
             indicesStatsResponse,
             new ToXContent.MapParams(Map.of("level", "cluster")),
-            ignored1 -> 4
+            ignored1 -> 3
         );
         AbstractChunkedSerializingTestCase.assertChunkCount(
             indicesStatsResponse,
             new ToXContent.MapParams(Map.of("level", "indices")),
-            ignored -> 5 + 2 * shards
+            ignored -> 4 + 2 * shards
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java
index 994f71c82e6e1..6d72ff9558105 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java
@@ -19,16 +19,12 @@
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.Map;
 
 import static org.elasticsearch.cluster.metadata.ComponentTemplateTests.randomAliases;
@@ -37,30 +33,7 @@
 import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
 import static org.hamcrest.Matchers.containsString;
 
-public class GetComponentTemplateResponseTests extends AbstractWireSerializingTestCase<GetComponentTemplateAction.Response> {
-    @Override
-    protected Writeable.Reader<GetComponentTemplateAction.Response> instanceReader() {
-        return GetComponentTemplateAction.Response::new;
-    }
-
-    @Override
-    protected GetComponentTemplateAction.Response createTestInstance() {
-        return new GetComponentTemplateAction.Response(
-            randomBoolean() ? Map.of() : randomTemplates(),
-            RolloverConfigurationTests.randomRolloverConditions()
-        );
-    }
-
-    @Override
-    protected GetComponentTemplateAction.Response mutateInstance(GetComponentTemplateAction.Response instance) {
-        var templates = instance.getComponentTemplates();
-        var rolloverConditions = instance.getRolloverConfiguration();
-        switch (randomInt(1)) {
-            case 0 -> templates = templates == null ? randomTemplates() : null;
-            case 1 -> rolloverConditions = randomValueOtherThan(rolloverConditions, RolloverConfigurationTests::randomRolloverConditions);
-        }
-        return new GetComponentTemplateAction.Response(templates, rolloverConditions);
-    }
+public class GetComponentTemplateResponseTests extends ESTestCase {
 
     public void testXContentSerializationWithRolloverAndEffectiveRetention() throws IOException {
         Settings settings = null;
@@ -102,17 +75,4 @@ public void testXContentSerializationWithRolloverAndEffectiveRetention() throws
             }
         }
     }
-
-    @Override
-    protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(IndicesModule.getNamedWriteables());
-    }
-
-    private static Map<String, ComponentTemplate> randomTemplates() {
-        Map<String, ComponentTemplate> templates = new HashMap<>();
-        for (int i = 0; i < randomIntBetween(1, 4); i++) {
-            templates.put(randomAlphaOfLength(4), ComponentTemplateTests.randomInstance());
-        }
-        return templates;
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateRequestTests.java
deleted file mode 100644
index 08e40e6a548ac..0000000000000
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateRequestTests.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.action.admin.indices.template.get;
-
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.test.AbstractWireSerializingTestCase;
-
-public class GetComposableIndexTemplateRequestTests extends AbstractWireSerializingTestCase<GetComposableIndexTemplateAction.Request> {
-    @Override
-    protected Writeable.Reader<GetComposableIndexTemplateAction.Request> instanceReader() {
-        return GetComposableIndexTemplateAction.Request::new;
-    }
-
-    @Override
-    protected GetComposableIndexTemplateAction.Request createTestInstance() {
-        return new GetComposableIndexTemplateAction.Request(randomBoolean() ? null : randomAlphaOfLength(4));
-    }
-
-    @Override
-    protected GetComposableIndexTemplateAction.Request mutateInstance(GetComposableIndexTemplateAction.Request instance) {
-        return randomValueOtherThan(instance, this::createTestInstance);
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateResponseTests.java
deleted file mode 100644
index ead01c977c8e4..0000000000000
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateResponseTests.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.action.admin.indices.template.get;
-
-import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration;
-import org.elasticsearch.action.admin.indices.rollover.RolloverConfigurationTests;
-import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
-import org.elasticsearch.cluster.metadata.ComposableIndexTemplateTests;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.test.AbstractWireSerializingTestCase;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class GetComposableIndexTemplateResponseTests extends AbstractWireSerializingTestCase<GetComposableIndexTemplateAction.Response> {
-    @Override
-    protected Writeable.Reader<GetComposableIndexTemplateAction.Response> instanceReader() {
-        return GetComposableIndexTemplateAction.Response::new;
-    }
-
-    @Override
-    protected GetComposableIndexTemplateAction.Response createTestInstance() {
-        RolloverConfiguration rolloverConfiguration = randomBoolean() ? null : RolloverConfigurationTests.randomRolloverConditions();
-        if (randomBoolean()) {
-            return new GetComposableIndexTemplateAction.Response(Map.of(), rolloverConfiguration);
-        }
-        Map<String, ComposableIndexTemplate> templates = new HashMap<>();
-        for (int i = 0; i < randomIntBetween(1, 4); i++) {
-            templates.put(randomAlphaOfLength(4), ComposableIndexTemplateTests.randomInstance());
-        }
-        return new GetComposableIndexTemplateAction.Response(templates, rolloverConfiguration);
-    }
-
-    @Override
-    protected GetComposableIndexTemplateAction.Response mutateInstance(GetComposableIndexTemplateAction.Response instance) {
-        var rolloverConfiguration = instance.getRolloverConfiguration();
-        var templates = instance.indexTemplates();
-        switch (randomInt(1)) {
-            case 0 -> rolloverConfiguration = randomBoolean() || rolloverConfiguration == null
-                ? randomValueOtherThan(rolloverConfiguration, RolloverConfigurationTests::randomRolloverConditions)
-                : null;
-            case 1 -> {
-                var updatedTemplates = new HashMap<String, ComposableIndexTemplate>();
-                for (String name : templates.keySet()) {
-                    if (randomBoolean()) {
-                        updatedTemplates.put(name, templates.get(name));
-                    }
-                }
-                updatedTemplates.put(randomAlphaOfLength(4), ComposableIndexTemplateTests.randomInstance());
-                templates = updatedTemplates;
-            }
-        }
-        return new GetComposableIndexTemplateAction.Response(templates, rolloverConfiguration);
-    }
-
-    @Override
-    protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(IndicesModule.getNamedWriteables());
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponseTests.java
deleted file mode 100644
index d6ab47f448f8b..0000000000000
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponseTests.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.action.admin.indices.template.get;
-
-import org.elasticsearch.cluster.metadata.AliasMetadata;
-import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.AbstractWireSerializingTestCase;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.stream.IntStream;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class GetIndexTemplatesResponseTests extends AbstractWireSerializingTestCase<GetIndexTemplatesResponse> {
-
-    @Override
-    protected GetIndexTemplatesResponse createTestInstance() {
-        List<IndexTemplateMetadata> templates = new ArrayList<>();
-        int numTemplates = between(0, 10);
-        for (int t = 0; t < numTemplates; t++) {
-            IndexTemplateMetadata.Builder templateBuilder = IndexTemplateMetadata.builder("template-" + t);
-            templateBuilder.patterns(IntStream.range(0, between(1, 5)).mapToObj(i -> "pattern-" + i).toList());
-            int numAlias = between(0, 5);
-            for (int i = 0; i < numAlias; i++) {
-                templateBuilder.putAlias(AliasMetadata.builder(randomAlphaOfLengthBetween(1, 10)));
-            }
-            if (randomBoolean()) {
-                templateBuilder.settings(Settings.builder().put("index.setting-1", randomLong()));
-            }
-            if (randomBoolean()) {
-                templateBuilder.order(randomInt());
-            }
-            if (randomBoolean()) {
-                templateBuilder.version(between(0, 100));
-            }
-            if (randomBoolean()) {
-                try {
-                    templateBuilder.putMapping("doc", "{\"properties\":{\"type\":\"text\"}}");
-                } catch (IOException ex) {
-                    throw new UncheckedIOException(ex);
-                }
-            }
-            templates.add(templateBuilder.build());
-        }
-        return new GetIndexTemplatesResponse(templates);
-    }
-
-    @Override
-    protected GetIndexTemplatesResponse mutateInstance(GetIndexTemplatesResponse instance) {
-        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
-    }
-
-    @Override
-    protected Writeable.Reader<GetIndexTemplatesResponse> instanceReader() {
-        return GetIndexTemplatesResponse::new;
-    }
-
-    @Override
-    protected void assertEqualInstances(GetIndexTemplatesResponse expectedInstance, GetIndexTemplatesResponse newInstance) {
-        assertNotSame(newInstance, expectedInstance);
-        assertThat(new HashSet<>(newInstance.getIndexTemplates()), equalTo(new HashSet<>(expectedInstance.getIndexTemplates())));
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java
index 02a4c134ccd15..1d861b340543d 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java
@@ -12,45 +12,21 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
-import org.elasticsearch.cluster.metadata.ComposableIndexTemplateTests;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Template;
-import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.test.ESTestCase;
 
 import java.util.List;
 
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class SimulateIndexTemplateRequestTests extends AbstractWireSerializingTestCase<SimulateIndexTemplateRequest> {
-
-    @Override
-    protected Writeable.Reader<SimulateIndexTemplateRequest> instanceReader() {
-        return SimulateIndexTemplateRequest::new;
-    }
-
-    @Override
-    protected SimulateIndexTemplateRequest createTestInstance() {
-        SimulateIndexTemplateRequest req = new SimulateIndexTemplateRequest(randomAlphaOfLength(10));
-        TransportPutComposableIndexTemplateAction.Request newTemplateRequest = new TransportPutComposableIndexTemplateAction.Request(
-            randomAlphaOfLength(4)
-        );
-        newTemplateRequest.indexTemplate(ComposableIndexTemplateTests.randomInstance());
-        req.indexTemplateRequest(newTemplateRequest);
-        req.includeDefaults(randomBoolean());
-        return req;
-    }
-
-    @Override
-    protected SimulateIndexTemplateRequest mutateInstance(SimulateIndexTemplateRequest instance) {
-        return randomValueOtherThan(instance, this::createTestInstance);
-    }
+public class SimulateIndexTemplateRequestTests extends ESTestCase {
 
     public void testIndexNameCannotBeNullOrEmpty() {
-        expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest((String) null));
-        expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest(""));
+        expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest(TEST_REQUEST_TIMEOUT, null));
+        expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest(TEST_REQUEST_TIMEOUT, ""));
     }
 
     public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() {
@@ -60,7 +36,7 @@ public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() {
         TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test");
         request.indexTemplate(globalTemplate);
 
-        SimulateIndexTemplateRequest simulateRequest = new SimulateIndexTemplateRequest("testing");
+        SimulateIndexTemplateRequest simulateRequest = new SimulateIndexTemplateRequest(TEST_REQUEST_TIMEOUT, "testing");
         simulateRequest.indexTemplateRequest(request);
 
         ActionRequestValidationException validationException = simulateRequest.validate();
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java
index 14ebf260d3bf9..6163566a5e593 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java
@@ -12,49 +12,17 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
-import org.elasticsearch.cluster.metadata.ComposableIndexTemplateTests;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Template;
-import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.test.ESTestCase;
 
 import java.util.List;
 
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class SimulateTemplateRequestTests extends AbstractWireSerializingTestCase<SimulateTemplateAction.Request> {
-
-    @Override
-    protected Writeable.Reader<SimulateTemplateAction.Request> instanceReader() {
-        return SimulateTemplateAction.Request::new;
-    }
-
-    @Override
-    protected SimulateTemplateAction.Request createTestInstance() {
-        SimulateTemplateAction.Request req = new SimulateTemplateAction.Request(randomAlphaOfLength(10));
-        TransportPutComposableIndexTemplateAction.Request newTemplateRequest = new TransportPutComposableIndexTemplateAction.Request(
-            randomAlphaOfLength(4)
-        );
-        newTemplateRequest.indexTemplate(ComposableIndexTemplateTests.randomInstance());
-        req.indexTemplateRequest(newTemplateRequest);
-        req.includeDefaults(randomBoolean());
-        return req;
-    }
-
-    @Override
-    protected SimulateTemplateAction.Request mutateInstance(SimulateTemplateAction.Request instance) {
-        return randomValueOtherThan(instance, this::createTestInstance);
-    }
-
-    public void testIndexNameCannotBeNullOrEmpty() {
-        expectThrows(IllegalArgumentException.class, () -> new SimulateTemplateAction.Request((String) null));
-        expectThrows(
-            IllegalArgumentException.class,
-            () -> new SimulateTemplateAction.Request((TransportPutComposableIndexTemplateAction.Request) null)
-        );
-    }
+public class SimulateTemplateRequestTests extends ESTestCase {
 
     public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() {
         Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null);
@@ -63,7 +31,7 @@ public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() {
         TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test");
         request.indexTemplate(globalTemplate);
 
-        SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request("testing");
+        SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request(TEST_REQUEST_TIMEOUT, "testing");
         simulateRequest.indexTemplateRequest(request);
 
         ActionRequestValidationException validationException = simulateRequest.validate();
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java
index 9a12b05d1cfd8..9360ce1719634 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.cluster.coordination.NoMasterBlockService;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
 import org.elasticsearch.cluster.metadata.DataStreamOptions;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -40,6 +41,7 @@
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -89,6 +91,7 @@ public class BulkOperationTests extends ESTestCase {
     private final String dataStreamName = "my_data_stream";
     private final String fsDataStreamName = "my_failure_store_data_stream";
     private final String fsRolloverDataStreamName = "my_failure_store_to_be_rolled_over_data_stream";
+    private final String fsBySettingsDataStreamName = "my_failure_store_enabled_by_setting_data_stream";
 
     private final IndexMetadata indexMetadata = IndexMetadata.builder(indexName)
         .settings(
@@ -120,6 +123,12 @@ public class BulkOperationTests extends ESTestCase {
     private final IndexMetadata ds3FailureStore2 = DataStreamTestHelper.createFailureStore(fsRolloverDataStreamName, 2, millis)
         .numberOfShards(1)
         .build();
+    private final IndexMetadata ds4BackingIndex1 = DataStreamTestHelper.createBackingIndex(fsBySettingsDataStreamName, 1, millis)
+        .numberOfShards(2)
+        .build();
+    private final IndexMetadata ds4FailureStore1 = DataStreamTestHelper.createFailureStore(fsBySettingsDataStreamName, 1, millis)
+        .numberOfShards(1)
+        .build();
 
     private final DataStream dataStream1 = DataStreamTestHelper.newInstance(
         dataStreamName,
@@ -137,6 +146,11 @@ public class BulkOperationTests extends ESTestCase {
             DataStream.DataStreamIndices.failureIndicesBuilder(List.of(ds3FailureStore1.getIndex())).setRolloverOnWrite(true).build()
         )
         .build();
+    private final DataStream dataStream4 = DataStream.builder(fsBySettingsDataStreamName, List.of(ds4BackingIndex1.getIndex()))
+        .setGeneration(1)
+        .setDataStreamOptions(DataStreamOptions.EMPTY)
+        .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(List.of(ds4FailureStore1.getIndex())).build())
+        .build();
 
     private final ClusterState DEFAULT_STATE = ClusterState.builder(ClusterName.DEFAULT)
         .metadata(
@@ -172,11 +186,24 @@ public class BulkOperationTests extends ESTestCase {
                         ds3BackingIndex1.getIndex().getName(),
                         ds3BackingIndex1,
                         ds3FailureStore1.getIndex().getName(),
-                        ds3FailureStore1
+                        ds3FailureStore1,
+                        ds4BackingIndex1.getIndex().getName(),
+                        ds4BackingIndex1,
+                        ds4FailureStore1.getIndex().getName(),
+                        ds4FailureStore1
                     )
                 )
                 .dataStreams(
-                    Map.of(dataStreamName, dataStream1, fsDataStreamName, dataStream2, fsRolloverDataStreamName, dataStream3),
+                    Map.of(
+                        dataStreamName,
+                        dataStream1,
+                        fsDataStreamName,
+                        dataStream2,
+                        fsRolloverDataStreamName,
+                        dataStream3,
+                        fsBySettingsDataStreamName,
+                        dataStream4
+                    ),
                     Map.of()
                 )
                 .build()
@@ -409,6 +436,58 @@ public void testFailingDocumentRedirectsToFailureStore() throws Exception {
         assertThat(failedItem.getFailureStoreStatus(), equalTo(IndexDocFailureStoreStatus.USED));
     }
 
+    public void testFailingDocumentRedirectsToFailureStoreWhenEnabledByClusterSetting() {
+        Assume.assumeTrue(DataStream.isFailureStoreFeatureFlagEnabled());
+
+        BulkRequest bulkRequest = new BulkRequest();
+        bulkRequest.add(
+            new IndexRequest(fsBySettingsDataStreamName).id("1").source(Map.of("key", "val")).opType(DocWriteRequest.OpType.CREATE)
+        );
+
+        NodeClient client = getNodeClient(
+            thatFailsDocuments(
+                Map.of(new IndexAndId(ds4BackingIndex1.getIndex().getName(), "1"), () -> new MapperException("mapping go boom"))
+            )
+        );
+        DataStreamFailureStoreSettings dataStreamFailureStoreSettings = DataStreamFailureStoreSettings.create(
+            ClusterSettings.createBuiltInClusterSettings(
+                Settings.builder()
+                    .put(DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), fsBySettingsDataStreamName)
+                    .build()
+            )
+        );
+
+        // Without the cluster setting, this bulk request should fail:
+        BulkResponse bulkItemResponsesWithoutClusterSetting = safeAwait(l -> newBulkOperation(client, bulkRequest, l).run());
+        assertThat(bulkItemResponsesWithoutClusterSetting.hasFailures(), is(true));
+        BulkItemResponse failedItem = Arrays.stream(bulkItemResponsesWithoutClusterSetting.getItems())
+            .filter(BulkItemResponse::isFailed)
+            .findFirst()
+            .orElseThrow(() -> new AssertionError("Could not find redirected item"));
+        assertThat(failedItem.getFailure().getCause(), is(instanceOf(MapperException.class)));
+        assertThat(failedItem.getFailure().getCause().getMessage(), is(equalTo("mapping go boom")));
+
+        // With a cluster setting to enable the failure store for this data stream, the same request should be redirected:
+        BulkResponse bulkItemResponsesUsingClusterSetting = safeAwait(
+            l -> newBulkOperation(
+                DEFAULT_STATE,
+                client,
+                bulkRequest,
+                new AtomicArray<>(bulkRequest.numberOfActions()),
+                mockObserver(DEFAULT_STATE),
+                l,
+                new FailureStoreDocumentConverter(),
+                dataStreamFailureStoreSettings
+            ).run()
+        );
+        assertThat(bulkItemResponsesUsingClusterSetting.hasFailures(), is(false));
+        BulkItemResponse redirectedItem = Arrays.stream(bulkItemResponsesUsingClusterSetting.getItems())
+            .filter(item -> item.getIndex().equals(ds4FailureStore1.getIndex().getName()))
+            .findFirst()
+            .orElseThrow(() -> new AssertionError("Could not find redirected item"));
+        assertThat(redirectedItem.getFailureStoreStatus(), equalTo(IndexDocFailureStoreStatus.USED));
+    }
+
     /**
      * A bulk operation to a data stream with a failure store enabled may still partially fail if the redirected documents experience
      * a shard-level failure while writing to the failure store indices.
@@ -957,6 +1036,7 @@ private static BulkItemResponse requestToFailedResponse(BulkItemRequest itemRequ
 
     /**
      * Create a client that redirects expected actions to the provided function and fails if an unexpected operation happens.
+     *
      * @param onShardAction Called when TransportShardBulkAction is executed.
      * @return A node client for the test.
      */
@@ -966,6 +1046,7 @@ private NodeClient getNodeClient(BiConsumer<BulkShardRequest, ActionListener<Bul
 
     /**
      * Create a client that redirects expected actions to the provided function and fails if an unexpected operation happens.
+     *
      * @param onShardAction Called when TransportShardBulkAction is executed.
      * @return A node client for the test.
      */
@@ -1074,6 +1155,28 @@ private BulkOperation newBulkOperation(
         ClusterStateObserver observer,
         ActionListener<BulkResponse> listener,
         FailureStoreDocumentConverter failureStoreDocumentConverter
+    ) {
+        return newBulkOperation(
+            state,
+            client,
+            request,
+            existingResponses,
+            observer,
+            listener,
+            failureStoreDocumentConverter,
+            DataStreamFailureStoreSettings.create(ClusterSettings.createBuiltInClusterSettings())
+        );
+    }
+
+    private BulkOperation newBulkOperation(
+        ClusterState state,
+        NodeClient client,
+        BulkRequest request,
+        AtomicArray<BulkItemResponse> existingResponses,
+        ClusterStateObserver observer,
+        ActionListener<BulkResponse> listener,
+        FailureStoreDocumentConverter failureStoreDocumentConverter,
+        DataStreamFailureStoreSettings dataStreamFailureStoreSettings
     ) {
         // Time provision
         long timeZero = TimeUnit.MILLISECONDS.toNanos(randomMillisUpToYear9999() - TimeUnit.DAYS.toMillis(1));
@@ -1105,7 +1208,8 @@ private BulkOperation newBulkOperation(
             listener,
             observer,
             failureStoreDocumentConverter,
-            FailureStoreMetrics.NOOP
+            FailureStoreMetrics.NOOP,
+            dataStreamFailureStoreSettings
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
index 96b62056b6dc4..c6f923ce7cc03 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
@@ -36,6 +37,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.TriConsumer;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.core.Nullable;
@@ -151,13 +153,13 @@ class TestTransportBulkAction extends TransportBulkAction {
                 transportService,
                 TransportBulkActionIngestTests.this.clusterService,
                 ingestService,
-                mockFeatureService,
                 new NodeClient(Settings.EMPTY, TransportBulkActionIngestTests.this.threadPool),
                 new ActionFilters(Collections.emptySet()),
                 TestIndexNameExpressionResolver.newInstance(),
                 new IndexingPressure(SETTINGS),
                 EmptySystemIndices.INSTANCE,
-                FailureStoreMetrics.NOOP
+                FailureStoreMetrics.NOOP,
+                DataStreamFailureStoreSettings.create(ClusterSettings.createBuiltInClusterSettings())
             );
         }
 
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java
index 6bc08995b932e..616488d3472ff 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java
@@ -22,16 +22,20 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.ActionTestUtils;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
+import org.elasticsearch.cluster.metadata.DataStreamOptions;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexAbstraction.ConcreteIndex;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -39,6 +43,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.features.FeatureService;
@@ -84,6 +89,8 @@
 
 public class TransportBulkActionTests extends ESTestCase {
 
+    private final ClusterSettings clusterSettings = ClusterSettings.createBuiltInClusterSettings();
+
     /** Services needed by bulk action */
     private TransportService transportService;
     private ClusterService clusterService;
@@ -106,13 +113,13 @@ class TestTransportBulkAction extends TransportBulkAction {
                 transportService,
                 TransportBulkActionTests.this.clusterService,
                 null,
-                mockFeatureService,
                 new NodeClient(Settings.EMPTY, TransportBulkActionTests.this.threadPool),
                 new ActionFilters(Collections.emptySet()),
                 new Resolver(),
                 new IndexingPressure(Settings.EMPTY),
                 EmptySystemIndices.INSTANCE,
-                FailureStoreMetrics.NOOP
+                FailureStoreMetrics.NOOP,
+                DataStreamFailureStoreSettings.create(clusterSettings)
             );
         }
 
@@ -131,9 +138,11 @@ void createIndex(CreateIndexRequest createIndexRequest, ActionListener<CreateInd
 
         @Override
         void rollOver(RolloverRequest rolloverRequest, ActionListener<RolloverResponse> listener) {
-            if (failDataStreamRolloverException != null && rolloverRequest.targetsFailureStore() == false) {
+            String selectorString = IndexNameExpressionResolver.splitSelectorExpression(rolloverRequest.getRolloverTarget()).v2();
+            boolean isFailureStoreRollover = IndexComponentSelector.FAILURES.getKey().equals(selectorString);
+            if (failDataStreamRolloverException != null && isFailureStoreRollover == false) {
                 listener.onFailure(failDataStreamRolloverException);
-            } else if (failFailureStoreRolloverException != null && rolloverRequest.targetsFailureStore()) {
+            } else if (failFailureStoreRolloverException != null && isFailureStoreRollover) {
                 listener.onFailure(failFailureStoreRolloverException);
             } else {
                 listener.onResponse(
@@ -392,36 +401,52 @@ public void testRejectionAfterCreateIndexIsPropagated() {
     public void testResolveFailureStoreFromMetadata() throws Exception {
         assumeThat(DataStream.isFailureStoreFeatureFlagEnabled(), is(true));
 
-        String dataStreamWithFailureStore = "test-data-stream-failure-enabled";
-        String dataStreamWithoutFailureStore = "test-data-stream-failure-disabled";
+        String dataStreamWithFailureStoreEnabled = "test-data-stream-failure-enabled";
+        String dataStreamWithFailureStoreDefault = "test-data-stream-failure-default";
+        String dataStreamWithFailureStoreDisabled = "test-data-stream-failure-disabled";
         long testTime = randomMillisUpToYear9999();
 
-        IndexMetadata backingIndex1 = DataStreamTestHelper.createFirstBackingIndex(dataStreamWithFailureStore, testTime).build();
-        IndexMetadata backingIndex2 = DataStreamTestHelper.createFirstBackingIndex(dataStreamWithoutFailureStore, testTime).build();
-        IndexMetadata failureStoreIndex1 = DataStreamTestHelper.createFirstFailureStore(dataStreamWithFailureStore, testTime).build();
+        IndexMetadata backingIndex1 = DataStreamTestHelper.createFirstBackingIndex(dataStreamWithFailureStoreEnabled, testTime).build();
+        IndexMetadata backingIndex2 = DataStreamTestHelper.createFirstBackingIndex(dataStreamWithFailureStoreDefault, testTime).build();
+        IndexMetadata backingIndex3 = DataStreamTestHelper.createFirstBackingIndex(dataStreamWithFailureStoreDisabled, testTime).build();
+        IndexMetadata failureStoreIndex1 = DataStreamTestHelper.createFirstFailureStore(dataStreamWithFailureStoreEnabled, testTime)
+            .build();
 
         Metadata metadata = Metadata.builder()
             .dataStreams(
                 Map.of(
-                    dataStreamWithFailureStore,
+                    dataStreamWithFailureStoreEnabled,
                     DataStreamTestHelper.newInstance(
-                        dataStreamWithFailureStore,
+                        dataStreamWithFailureStoreEnabled,
                         List.of(backingIndex1.getIndex()),
                         1L,
                         Map.of(),
                         false,
                         null,
-                        List.of(failureStoreIndex1.getIndex())
+                        List.of(),
+                        DataStreamOptions.FAILURE_STORE_ENABLED
                     ),
-                    dataStreamWithoutFailureStore,
+                    dataStreamWithFailureStoreDefault,
                     DataStreamTestHelper.newInstance(
-                        dataStreamWithoutFailureStore,
+                        dataStreamWithFailureStoreDefault,
                         List.of(backingIndex2.getIndex()),
                         1L,
                         Map.of(),
                         false,
                         null,
-                        List.of()
+                        List.of(),
+                        DataStreamOptions.EMPTY
+                    ),
+                    dataStreamWithFailureStoreDisabled,
+                    DataStreamTestHelper.newInstance(
+                        dataStreamWithFailureStoreDisabled,
+                        List.of(backingIndex3.getIndex()),
+                        1L,
+                        Map.of(),
+                        false,
+                        null,
+                        List.of(),
+                        DataStreamOptions.FAILURE_STORE_DISABLED
                     )
                 ),
                 Map.of()
@@ -432,6 +457,8 @@ public void testResolveFailureStoreFromMetadata() throws Exception {
                     backingIndex1,
                     backingIndex2.getIndex().getName(),
                     backingIndex2,
+                    backingIndex3.getIndex().getName(),
+                    backingIndex3,
                     failureStoreIndex1.getIndex().getName(),
                     failureStoreIndex1
                 )
@@ -439,38 +466,56 @@ public void testResolveFailureStoreFromMetadata() throws Exception {
             .build();
 
         // Data stream with failure store should store failures
-        assertThat(TransportBulkAction.resolveFailureInternal(dataStreamWithFailureStore, metadata, testTime), is(true));
-        // Data stream without failure store should not
-        assertThat(TransportBulkAction.resolveFailureInternal(dataStreamWithoutFailureStore, metadata, testTime), is(false));
+        assertThat(bulkAction.resolveFailureInternal(dataStreamWithFailureStoreEnabled, metadata, testTime), is(true));
+        // Data stream with the default failure store options should not...
+        assertThat(bulkAction.resolveFailureInternal(dataStreamWithFailureStoreDefault, metadata, testTime), is(false));
+        // ...unless we change the cluster setting to enable it that way.
+        clusterSettings.applySettings(
+            Settings.builder()
+                .put(DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), dataStreamWithFailureStoreDefault)
+                .build()
+        );
+        assertThat(bulkAction.resolveFailureInternal(dataStreamWithFailureStoreDefault, metadata, testTime), is(true));
+        // Data stream with failure store explicitly disabled should not store failures even if it matches the cluster setting
+        clusterSettings.applySettings(
+            Settings.builder()
+                .put(DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), dataStreamWithFailureStoreDisabled)
+                .build()
+        );
+        assertThat(bulkAction.resolveFailureInternal(dataStreamWithFailureStoreDisabled, metadata, testTime), is(false));
         // An index should not be considered for failure storage
-        assertThat(TransportBulkAction.resolveFailureInternal(backingIndex1.getIndex().getName(), metadata, testTime), is(nullValue()));
+        assertThat(bulkAction.resolveFailureInternal(backingIndex1.getIndex().getName(), metadata, testTime), is(nullValue()));
         // even if that index is itself a failure store
-        assertThat(
-            TransportBulkAction.resolveFailureInternal(failureStoreIndex1.getIndex().getName(), metadata, testTime),
-            is(nullValue())
-        );
+        assertThat(bulkAction.resolveFailureInternal(failureStoreIndex1.getIndex().getName(), metadata, testTime), is(nullValue()));
     }
 
     public void testResolveFailureStoreFromTemplate() throws Exception {
         assumeThat(DataStream.isFailureStoreFeatureFlagEnabled(), is(true));
 
-        String dsTemplateWithFailureStore = "test-data-stream-failure-enabled";
-        String dsTemplateWithoutFailureStore = "test-data-stream-failure-disabled";
+        String dsTemplateWithFailureStoreEnabled = "test-data-stream-failure-enabled";
+        String dsTemplateWithFailureStoreDefault = "test-data-stream-failure-default";
+        String dsTemplateWithFailureStoreDisabled = "test-data-stream-failure-disabled";
         String indexTemplate = "test-index";
         long testTime = randomMillisUpToYear9999();
 
         Metadata metadata = Metadata.builder()
             .indexTemplates(
                 Map.of(
-                    dsTemplateWithFailureStore,
+                    dsTemplateWithFailureStoreEnabled,
                     ComposableIndexTemplate.builder()
-                        .indexPatterns(List.of(dsTemplateWithFailureStore + "-*"))
+                        .indexPatterns(List.of(dsTemplateWithFailureStoreEnabled + "-*"))
                         .template(Template.builder().dataStreamOptions(DataStreamTestHelper.createDataStreamOptionsTemplate(true)))
                         .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
                         .build(),
-                    dsTemplateWithoutFailureStore,
+                    dsTemplateWithFailureStoreDefault,
+                    ComposableIndexTemplate.builder()
+                        .indexPatterns(List.of(dsTemplateWithFailureStoreDefault + "-*"))
+                        .template(Template.builder().dataStreamOptions(DataStreamTestHelper.createDataStreamOptionsTemplate(null)))
+                        .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
+                        .build(),
+                    dsTemplateWithFailureStoreDisabled,
                     ComposableIndexTemplate.builder()
-                        .indexPatterns(List.of(dsTemplateWithoutFailureStore + "-*"))
+                        .indexPatterns(List.of(dsTemplateWithFailureStoreDisabled + "-*"))
                         .template(Template.builder().dataStreamOptions(DataStreamTestHelper.createDataStreamOptionsTemplate(false)))
                         .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
                         .build(),
@@ -481,11 +526,36 @@ public void testResolveFailureStoreFromTemplate() throws Exception {
             .build();
 
         // Data stream with failure store should store failures
-        assertThat(TransportBulkAction.resolveFailureInternal(dsTemplateWithFailureStore + "-1", metadata, testTime), is(true));
-        // Data stream without failure store should not
-        assertThat(TransportBulkAction.resolveFailureInternal(dsTemplateWithoutFailureStore + "-1", metadata, testTime), is(false));
+        assertThat(bulkAction.resolveFailureInternal(dsTemplateWithFailureStoreEnabled + "-1", metadata, testTime), is(true));
+        // Same if date math is used
+        assertThat(bulkAction.resolveFailureInternal("<" + dsTemplateWithFailureStoreEnabled + "-{now}>", metadata, testTime), is(true));
+        // Data stream with the default failure store options should not...
+        assertThat(bulkAction.resolveFailureInternal(dsTemplateWithFailureStoreDefault + "-1", metadata, testTime), is(false));
+        assertThat(bulkAction.resolveFailureInternal("<" + dsTemplateWithFailureStoreDefault + "-{now}>", metadata, testTime), is(false));
+        // ...unless we change the cluster setting to enable it that way.
+        clusterSettings.applySettings(
+            Settings.builder()
+                .put(
+                    DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(),
+                    dsTemplateWithFailureStoreDefault + "*"
+                )
+                .build()
+        );
+        assertThat(bulkAction.resolveFailureInternal(dsTemplateWithFailureStoreDefault + "-1", metadata, testTime), is(true));
+        assertThat(bulkAction.resolveFailureInternal("<" + dsTemplateWithFailureStoreDefault + "-{now}>", metadata, testTime), is(true));
+        // Data stream with failure store explicitly disabled should not store failures even if it matches the cluster setting
+        clusterSettings.applySettings(
+            Settings.builder()
+                .put(
+                    DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(),
+                    dsTemplateWithFailureStoreDisabled + "*"
+                )
+                .build()
+        );
+        assertThat(bulkAction.resolveFailureInternal(dsTemplateWithFailureStoreDisabled + "-1", metadata, testTime), is(false));
+        assertThat(bulkAction.resolveFailureInternal("<" + dsTemplateWithFailureStoreDisabled + "-{now}>", metadata, testTime), is(false));
         // An index template should not be considered for failure storage
-        assertThat(TransportBulkAction.resolveFailureInternal(indexTemplate + "-1", metadata, testTime), is(nullValue()));
+        assertThat(bulkAction.resolveFailureInternal(indexTemplate + "-1", metadata, testTime), is(nullValue()));
     }
 
     /**
@@ -558,38 +628,6 @@ public void testFailuresDuringPrerequisiteActions() throws InterruptedException
         assertNull(bulkRequest.requests.get(2));
     }
 
-    public void testFailureStoreFromTemplateResolution() {
-        Metadata metadata = Metadata.builder()
-            .indexTemplates(
-                Map.of(
-                    "my-index-template",
-                    ComposableIndexTemplate.builder().indexPatterns(List.of("my-index*")).build(),
-                    "my-enabled-fs-template",
-                    ComposableIndexTemplate.builder()
-                        .indexPatterns(List.of("my-enabled*"))
-                        .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
-                        .template(Template.builder().dataStreamOptions(DataStreamTestHelper.createDataStreamOptionsTemplate(true)))
-                        .build(),
-                    "my-disabled-fs-template",
-                    ComposableIndexTemplate.builder()
-                        .indexPatterns(List.of("my-disabled*"))
-                        .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
-                        .template(Template.builder().dataStreamOptions(DataStreamTestHelper.createDataStreamOptionsTemplate(false)))
-                        .build(),
-                    "my-no-fs-template",
-                    ComposableIndexTemplate.builder()
-                        .indexPatterns(List.of("my-no*"))
-                        .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
-                        .build()
-                )
-            )
-            .build();
-        assertThat(TransportBulkAction.resolveFailureStoreFromTemplate("my-index", metadata), nullValue());
-        assertThat(TransportBulkAction.resolveFailureStoreFromTemplate("my-enabled-fs", metadata), equalTo(true));
-        assertThat(TransportBulkAction.resolveFailureStoreFromTemplate("my-disabled-fs", metadata), equalTo(false));
-        assertThat(TransportBulkAction.resolveFailureStoreFromTemplate("my-no-fs", metadata), equalTo(false));
-    }
-
     private BulkRequest buildBulkRequest(List<String> indices) {
         BulkRequest request = new BulkRequest();
         for (String index : indices) {
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
index b3d3ebe5e1357..544a373675ee5 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
@@ -21,11 +21,13 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -245,14 +247,14 @@ static class TestTransportBulkAction extends TransportBulkAction {
                 transportService,
                 clusterService,
                 null,
-                null,
                 client,
                 actionFilters,
                 indexNameExpressionResolver,
                 new IndexingPressure(Settings.EMPTY),
                 EmptySystemIndices.INSTANCE,
                 relativeTimeProvider,
-                FailureStoreMetrics.NOOP
+                FailureStoreMetrics.NOOP,
+                DataStreamFailureStoreSettings.create(ClusterSettings.createBuiltInClusterSettings())
             );
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java
index b389e33993b9b..f25f477655482 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java
@@ -92,8 +92,16 @@ public class TransportShardBulkActionTests extends IndexShardTestCase {
     private final ShardId shardId = new ShardId("index", "_na_", 0);
     private final Settings idxSettings = indexSettings(IndexVersion.current(), 1, 0).build();
 
-    private IndexMetadata indexMetadata() throws IOException {
-        return IndexMetadata.builder("index").putMapping("""
+    private IndexMetadata indexMetadata(String mapping) {
+        IndexMetadata.Builder builder = IndexMetadata.builder("index").settings(idxSettings).primaryTerm(0, 1);
+        if (mapping != null) {
+            builder.putMapping(mapping);
+        }
+        return builder.build();
+    }
+
+    private IndexMetadata indexMetadata() {
+        return indexMetadata("""
             {
               "properties": {
                 "foo": {
@@ -106,7 +114,7 @@ private IndexMetadata indexMetadata() throws IOException {
                   }
                 }
               }
-            }""").settings(idxSettings).primaryTerm(0, 1).build();
+            }""");
     }
 
     public void testExecuteBulkIndexRequest() throws Exception {
@@ -502,7 +510,7 @@ public void testNoopUpdateRequest() throws Exception {
         IndexShard shard = mockShard(null, null);
 
         UpdateHelper updateHelper = mock(UpdateHelper.class);
-        when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
+        when(updateHelper.prepare(any(), eq(shard), any(), any())).thenReturn(
             new UpdateHelper.Result(
                 noopUpdateResponse,
                 DocWriteResponse.Result.NOOP,
@@ -557,7 +565,7 @@ public void testUpdateRequestWithFailure() throws Exception {
         );
 
         UpdateHelper updateHelper = mock(UpdateHelper.class);
-        when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
+        when(updateHelper.prepare(any(), eq(shard), any(), any())).thenReturn(
             new UpdateHelper.Result(
                 updateResponse,
                 randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED,
@@ -618,7 +626,7 @@ public void testUpdateRequestWithConflictFailure() throws Exception {
         );
 
         UpdateHelper updateHelper = mock(UpdateHelper.class);
-        when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
+        when(updateHelper.prepare(any(), eq(shard), any(), any())).thenReturn(
             new UpdateHelper.Result(
                 updateResponse,
                 randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED,
@@ -681,7 +689,7 @@ public void testUpdateRequestWithSuccess() throws Exception {
         );
 
         UpdateHelper updateHelper = mock(UpdateHelper.class);
-        when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
+        when(updateHelper.prepare(any(), eq(shard), any(), any())).thenReturn(
             new UpdateHelper.Result(
                 updateResponse,
                 created ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED,
@@ -738,7 +746,7 @@ public void testUpdateWithDelete() throws Exception {
         when(shard.applyDeleteOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyLong())).thenReturn(deleteResult);
 
         UpdateHelper updateHelper = mock(UpdateHelper.class);
-        when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
+        when(updateHelper.prepare(any(), eq(shard), any(), any())).thenReturn(
             new UpdateHelper.Result(
                 updateResponse,
                 DocWriteResponse.Result.DELETED,
@@ -784,7 +792,7 @@ public void testFailureDuringUpdateProcessing() throws Exception {
 
         UpdateHelper updateHelper = mock(UpdateHelper.class);
         final ElasticsearchException err = new ElasticsearchException("oops");
-        when(updateHelper.prepare(any(), eq(shard), any())).thenThrow(err);
+        when(updateHelper.prepare(any(), eq(shard), any(), any())).thenThrow(err);
         BulkItemRequest[] items = new BulkItemRequest[] { primaryRequest };
         BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
 
@@ -916,7 +924,7 @@ public void testRetries() throws Exception {
         });
 
         UpdateHelper updateHelper = mock(UpdateHelper.class);
-        when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
+        when(updateHelper.prepare(any(), eq(shard), any(), any())).thenReturn(
             new UpdateHelper.Result(
                 updateResponse,
                 randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED,
@@ -1129,7 +1137,7 @@ public void testNoopMappingUpdateInfiniteLoopPrevention() throws Exception {
         );
 
         UpdateHelper updateHelper = mock(UpdateHelper.class);
-        when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
+        when(updateHelper.prepare(any(), eq(shard), any(), any())).thenReturn(
             new UpdateHelper.Result(
                 new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"),
                 randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED,
@@ -1195,7 +1203,7 @@ public void testNoopMappingUpdateSuccessOnRetry() throws Exception {
         );
 
         UpdateHelper updateHelper = mock(UpdateHelper.class);
-        when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
+        when(updateHelper.prepare(any(), eq(shard), any(), any())).thenReturn(
             new UpdateHelper.Result(
                 new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"),
                 randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED,
@@ -1235,6 +1243,9 @@ private IndexShard mockShard(IndexSettings indexSettings, MapperService mapperSe
 
         if (indexSettings != null) {
             when(shard.indexSettings()).thenReturn(indexSettings);
+        } else {
+            IndexSettings defaultIndexSettings = new IndexSettings(indexMetadata(null), Settings.EMPTY);
+            when(shard.indexSettings()).thenReturn(defaultIndexSettings);
         }
 
         if (mapperService != null) {
diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/DataStreamsActionUtilTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/DataStreamsActionUtilTests.java
index a4e45b0ee6f7c..445a5ad067b79 100644
--- a/server/src/test/java/org/elasticsearch/action/datastreams/DataStreamsActionUtilTests.java
+++ b/server/src/test/java/org/elasticsearch/action/datastreams/DataStreamsActionUtilTests.java
@@ -9,6 +9,7 @@
 
 package org.elasticsearch.action.datastreams;
 
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -17,6 +18,7 @@
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
@@ -46,6 +48,8 @@ public void testDataStreamsResolveConcreteIndexNames() {
         var dataStreamIndex2 = new Index(".ds-bar2", IndexMetadata.INDEX_UUID_NA_VALUE);
         var dataStreamIndex3 = new Index(".ds-foo2", IndexMetadata.INDEX_UUID_NA_VALUE);
         var dataStreamIndex4 = new Index(".ds-baz1", IndexMetadata.INDEX_UUID_NA_VALUE);
+        var dataStreamFailureIndex1 = new Index(".fs-foo1", IndexMetadata.INDEX_UUID_NA_VALUE);
+        var dataStreamFailureIndex2 = new Index(".fs-bar2", IndexMetadata.INDEX_UUID_NA_VALUE);
 
         ClusterState clusterState = ClusterState.builder(new ClusterName("test-cluster"))
             .metadata(
@@ -54,8 +58,14 @@ public void testDataStreamsResolveConcreteIndexNames() {
                         DataStreamMetadata.TYPE,
                         new DataStreamMetadata(
                             ImmutableOpenMap.<String, DataStream>builder()
-                                .fPut("fooDs", DataStreamTestHelper.newInstance("fooDs", List.of(dataStreamIndex1)))
-                                .fPut("barDs", DataStreamTestHelper.newInstance("barDs", List.of(dataStreamIndex2)))
+                                .fPut(
+                                    "fooDs",
+                                    DataStreamTestHelper.newInstance("fooDs", List.of(dataStreamIndex1), List.of(dataStreamFailureIndex1))
+                                )
+                                .fPut(
+                                    "barDs",
+                                    DataStreamTestHelper.newInstance("barDs", List.of(dataStreamIndex2), List.of(dataStreamFailureIndex2))
+                                )
                                 .fPut("foo2Ds", DataStreamTestHelper.newInstance("foo2Ds", List.of(dataStreamIndex3)))
                                 .fPut("bazDs", DataStreamTestHelper.newInstance("bazDs", List.of(dataStreamIndex4)))
                                 .build(),
@@ -69,7 +79,9 @@ public void testDataStreamsResolveConcreteIndexNames() {
                             dataStreamIndex1,
                             dataStreamIndex2,
                             dataStreamIndex3,
-                            dataStreamIndex4
+                            dataStreamIndex4,
+                            dataStreamFailureIndex1,
+                            dataStreamFailureIndex2
                         )
                     )
                     .build()
@@ -78,16 +90,39 @@ public void testDataStreamsResolveConcreteIndexNames() {
 
         var query = new String[] { "foo*", "baz*" };
         var indexNameExpressionResolver = mock(IndexNameExpressionResolver.class);
-        when(indexNameExpressionResolver.dataStreamNames(any(), any(), eq(query))).thenReturn(List.of("fooDs", "foo2Ds", "bazDs"));
+
+        when(indexNameExpressionResolver.dataStreams(any(), any(), eq(query))).thenReturn(
+            List.of(new ResolvedExpression("fooDs"), new ResolvedExpression("foo2Ds"), new ResolvedExpression("bazDs"))
+        );
 
         var resolved = DataStreamsActionUtil.resolveConcreteIndexNames(
             indexNameExpressionResolver,
             clusterState,
             query,
             IndicesOptions.builder().wildcardOptions(IndicesOptions.WildcardOptions.builder().includeHidden(true)).build()
-        ).toList();
+        );
 
         assertThat(resolved, containsInAnyOrder(".ds-foo1", ".ds-foo2", ".ds-baz1"));
+
+        when(indexNameExpressionResolver.dataStreams(any(), any(), eq(query))).thenReturn(
+            List.of(
+                new ResolvedExpression("fooDs", IndexComponentSelector.DATA),
+                new ResolvedExpression("fooDs", IndexComponentSelector.FAILURES),
+                new ResolvedExpression("foo2Ds", IndexComponentSelector.DATA),
+                new ResolvedExpression("foo2Ds", IndexComponentSelector.FAILURES),
+                new ResolvedExpression("bazDs", IndexComponentSelector.DATA),
+                new ResolvedExpression("bazDs", IndexComponentSelector.FAILURES)
+            )
+        );
+
+        resolved = DataStreamsActionUtil.resolveConcreteIndexNames(
+            indexNameExpressionResolver,
+            clusterState,
+            query,
+            IndicesOptions.builder().wildcardOptions(IndicesOptions.WildcardOptions.builder().includeHidden(true)).build()
+        );
+
+        assertThat(resolved, containsInAnyOrder(".ds-foo1", ".fs-foo1", ".ds-foo2", ".ds-baz1"));
     }
 
     private Map<String, IndexMetadata> createLocalOnlyIndicesMetadata(Index... indices) {
diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java
index b0b3531f54b48..feb00728c858e 100644
--- a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java
@@ -85,6 +85,7 @@ private static GetDataStreamAction.Response.DataStreamInfo newDataStreamInfo(boo
         DataStream dataStream = newDataStreamInstance(isSystem, retention);
         return new GetDataStreamAction.Response.DataStreamInfo(
             dataStream,
+            randomBoolean(),
             randomFrom(ClusterHealthStatus.values()),
             null,
             null,
diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingServiceTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingServiceTests.java
index 2c6e273bb6e23..14ecf85b3aa7e 100644
--- a/server/src/test/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingServiceTests.java
@@ -19,15 +19,14 @@
 import org.elasticsearch.cluster.metadata.IndexMetadataStats;
 import org.elasticsearch.cluster.metadata.IndexWriteLoad;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.features.FeatureService;
-import org.elasticsearch.features.FeatureSpecification;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexVersion;
@@ -79,12 +78,6 @@ public void setupService() {
         service = new DataStreamAutoShardingService(
             Settings.builder().put(DataStreamAutoShardingService.DATA_STREAMS_AUTO_SHARDING_ENABLED, true).build(),
             clusterService,
-            new FeatureService(List.of(new FeatureSpecification() {
-                @Override
-                public Set<NodeFeature> getFeatures() {
-                    return Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE);
-                }
-            })),
             () -> now
         );
         dataStreamName = randomAlphaOfLengthBetween(10, 100);
@@ -110,14 +103,7 @@ public void testCalculateValidations() {
         );
         builder.put(dataStream);
         ClusterState state = ClusterState.builder(ClusterName.DEFAULT)
-            .nodeFeatures(
-                Map.of(
-                    "n1",
-                    Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()),
-                    "n2",
-                    Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id())
-                )
-            )
+            .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2")))
             .metadata(builder)
             .build();
 
@@ -126,12 +112,6 @@ public void testCalculateValidations() {
             DataStreamAutoShardingService disabledAutoshardingService = new DataStreamAutoShardingService(
                 Settings.EMPTY,
                 clusterService,
-                new FeatureService(List.of(new FeatureSpecification() {
-                    @Override
-                    public Set<NodeFeature> getFeatures() {
-                        return Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE);
-                    }
-                })),
                 System::currentTimeMillis
             );
 
@@ -139,40 +119,6 @@ public Set<NodeFeature> getFeatures() {
             assertThat(autoShardingResult, is(NOT_APPLICABLE_RESULT));
         }
 
-        {
-            // cluster doesn't have feature
-            ClusterState stateNoFeature = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder()).build();
-
-            DataStreamAutoShardingService noFeatureService = new DataStreamAutoShardingService(
-                Settings.builder().put(DataStreamAutoShardingService.DATA_STREAMS_AUTO_SHARDING_ENABLED, true).build(),
-                clusterService,
-                new FeatureService(List.of()),
-                () -> now
-            );
-
-            AutoShardingResult autoShardingResult = noFeatureService.calculate(stateNoFeature, dataStream, 2.0);
-            assertThat(autoShardingResult, is(NOT_APPLICABLE_RESULT));
-        }
-
-        {
-            // patterns are configured to exclude the current data stream
-            DataStreamAutoShardingService noFeatureService = new DataStreamAutoShardingService(
-                Settings.builder()
-                    .put(DataStreamAutoShardingService.DATA_STREAMS_AUTO_SHARDING_ENABLED, true)
-                    .putList(
-                        DataStreamAutoShardingService.DATA_STREAMS_AUTO_SHARDING_EXCLUDES_SETTING.getKey(),
-                        List.of("foo", dataStreamName + "*")
-                    )
-                    .build(),
-                clusterService,
-                new FeatureService(List.of()),
-                () -> now
-            );
-
-            AutoShardingResult autoShardingResult = noFeatureService.calculate(state, dataStream, 2.0);
-            assertThat(autoShardingResult, is(NOT_APPLICABLE_RESULT));
-        }
-
         {
             // null write load passed
             AutoShardingResult autoShardingResult = service.calculate(state, dataStream, null);
@@ -199,14 +145,7 @@ public void testCalculateIncreaseShardingRecommendations() {
             DataStream dataStream = dataStreamSupplier.apply(null);
             builder.put(dataStream);
             ClusterState state = ClusterState.builder(ClusterName.DEFAULT)
-                .nodeFeatures(
-                    Map.of(
-                        "n1",
-                        Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()),
-                        "n2",
-                        Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id())
-                    )
-                )
+                .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2")))
                 .metadata(builder)
                 .build();
 
@@ -237,14 +176,7 @@ public void testCalculateIncreaseShardingRecommendations() {
             );
             builder.put(dataStream);
             ClusterState state = ClusterState.builder(ClusterName.DEFAULT)
-                .nodeFeatures(
-                    Map.of(
-                        "n1",
-                        Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()),
-                        "n2",
-                        Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id())
-                    )
-                )
+                .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2")))
                 .metadata(builder)
                 .build();
 
@@ -275,14 +207,7 @@ public void testCalculateIncreaseShardingRecommendations() {
             );
             builder.put(dataStream);
             ClusterState state = ClusterState.builder(ClusterName.DEFAULT)
-                .nodeFeatures(
-                    Map.of(
-                        "n1",
-                        Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()),
-                        "n2",
-                        Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id())
-                    )
-                )
+                .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2")))
                 .metadata(builder)
                 .build();
 
@@ -313,14 +238,7 @@ public void testCalculateDecreaseShardingRecommendations() {
             DataStream dataStream = dataStreamSupplier.apply(null);
             builder.put(dataStream);
             ClusterState state = ClusterState.builder(ClusterName.DEFAULT)
-                .nodeFeatures(
-                    Map.of(
-                        "n1",
-                        Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()),
-                        "n2",
-                        Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id())
-                    )
-                )
+                .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2")))
                 .metadata(builder)
                 .build();
 
@@ -353,14 +271,7 @@ public void testCalculateDecreaseShardingRecommendations() {
             DataStream dataStream = dataStreamSupplier.apply(null);
             builder.put(dataStream);
             ClusterState state = ClusterState.builder(ClusterName.DEFAULT)
-                .nodeFeatures(
-                    Map.of(
-                        "n1",
-                        Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()),
-                        "n2",
-                        Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id())
-                    )
-                )
+                .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2")))
                 .metadata(builder)
                 .build();
 
@@ -401,14 +312,7 @@ public void testCalculateDecreaseShardingRecommendations() {
             );
             builder.put(dataStream);
             ClusterState state = ClusterState.builder(ClusterName.DEFAULT)
-                .nodeFeatures(
-                    Map.of(
-                        "n1",
-                        Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()),
-                        "n2",
-                        Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id())
-                    )
-                )
+                .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2")))
                 .metadata(builder)
                 .build();
 
@@ -447,14 +351,7 @@ public void testCalculateDecreaseShardingRecommendations() {
             );
             builder.put(dataStream);
             ClusterState state = ClusterState.builder(ClusterName.DEFAULT)
-                .nodeFeatures(
-                    Map.of(
-                        "n1",
-                        Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()),
-                        "n2",
-                        Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id())
-                    )
-                )
+                .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2")))
                 .metadata(builder)
                 .build();
 
@@ -487,14 +384,7 @@ public void testCalculateDecreaseShardingRecommendations() {
             DataStream dataStream = dataStreamSupplier.apply(null);
             builder.put(dataStream);
             ClusterState state = ClusterState.builder(ClusterName.DEFAULT)
-                .nodeFeatures(
-                    Map.of(
-                        "n1",
-                        Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()),
-                        "n2",
-                        Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id())
-                    )
-                )
+                .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2")))
                 .metadata(builder)
                 .build();
 
diff --git a/server/src/test/java/org/elasticsearch/action/get/TransportMultiGetActionTests.java b/server/src/test/java/org/elasticsearch/action/get/TransportMultiGetActionTests.java
index 667d6686767fa..dad392970b6a1 100644
--- a/server/src/test/java/org/elasticsearch/action/get/TransportMultiGetActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/get/TransportMultiGetActionTests.java
@@ -145,15 +145,9 @@ public static void beforeClass() throws Exception {
         when(
             operationRouting.getShards(eq(clusterState), eq(index1.getName()), anyString(), nullable(String.class), nullable(String.class))
         ).thenReturn(index1ShardIterator);
-        when(operationRouting.shardId(eq(clusterState), eq(index1.getName()), nullable(String.class), nullable(String.class))).thenReturn(
-            new ShardId(index1, randomInt())
-        );
         when(
             operationRouting.getShards(eq(clusterState), eq(index2.getName()), anyString(), nullable(String.class), nullable(String.class))
         ).thenReturn(index2ShardIterator);
-        when(operationRouting.shardId(eq(clusterState), eq(index2.getName()), nullable(String.class), nullable(String.class))).thenReturn(
-            new ShardId(index2, randomInt())
-        );
 
         clusterService = mock(ClusterService.class);
         when(clusterService.localNode()).thenReturn(transportService.getLocalNode());
diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateIndexResponseTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateIndexResponseTests.java
index 4b226bc41b09a..f3ec4fc1ac1c3 100644
--- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateIndexResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateIndexResponseTests.java
@@ -48,6 +48,7 @@ public void testToXContent() throws IOException {
             sourceBytes,
             XContentType.JSON,
             pipelines,
+            List.of(),
             null
         );
 
@@ -79,6 +80,7 @@ public void testToXContent() throws IOException {
             sourceBytes,
             XContentType.JSON,
             pipelines,
+            List.of(),
             new ElasticsearchException("Some failure")
         );
 
@@ -103,6 +105,39 @@ public void testToXContent() throws IOException {
             ),
             Strings.toString(indexResponseWithException)
         );
+
+        SimulateIndexResponse indexResponseWithIgnoredFields = new SimulateIndexResponse(
+            id,
+            index,
+            version,
+            sourceBytes,
+            XContentType.JSON,
+            pipelines,
+            List.of("abc", "def"),
+            null
+        );
+
+        assertEquals(
+            XContentHelper.stripWhitespace(
+                Strings.format(
+                    """
+                        {
+                          "_id": "%s",
+                          "_index": "%s",
+                          "_version": %d,
+                          "_source": %s,
+                          "executed_pipelines": [%s],
+                          "ignored_fields": [{"field": "abc"}, {"field": "def"}]
+                        }""",
+                    id,
+                    index,
+                    version,
+                    source,
+                    pipelines.stream().map(pipeline -> "\"" + pipeline + "\"").collect(Collectors.joining(","))
+                )
+            ),
+            Strings.toString(indexResponseWithIgnoredFields)
+        );
     }
 
     public void testSerialization() throws IOException {
@@ -135,6 +170,7 @@ private static SimulateIndexResponse randomIndexResponse() {
             sourceBytes,
             xContentType,
             pipelines,
+            randomList(0, 20, () -> randomAlphaOfLength(15)),
             randomBoolean() ? null : new ElasticsearchException("failed")
         );
     }
diff --git a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
index 71bf2a47cfa47..47ff4ca6f0600 100644
--- a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
@@ -207,12 +207,7 @@ public void testOnPhaseFailure() {
         List<Tuple<String, String>> nodeLookups = new ArrayList<>();
         ArraySearchPhaseResults<SearchPhaseResult> phaseResults = phaseResults(requestIds, nodeLookups, 0);
         AbstractSearchAsyncAction<SearchPhaseResult> action = createAction(searchRequest, phaseResults, listener, false, new AtomicLong());
-        action.onPhaseFailure(new SearchPhase("test") {
-            @Override
-            public void run() {
-
-            }
-        }, "message", null);
+        action.onPhaseFailure("test", "message", null);
         assertThat(exception.get(), instanceOf(SearchPhaseExecutionException.class));
         SearchPhaseExecutionException searchPhaseExecutionException = (SearchPhaseExecutionException) exception.get();
         assertEquals("message", searchPhaseExecutionException.getMessage());
diff --git a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
index 69872b5e4b546..1460270c48293 100644
--- a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
@@ -10,7 +10,6 @@
 package org.elasticsearch.action.search;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.search.CanMatchNodeResponse.ResponseOrFailure;
@@ -1462,7 +1461,7 @@ private void addIndexMinMaxTimestamps(Index index, String fieldName, long minTim
                 indexMetadataBuilder.timestampRange(timestampRange);
                 fields.put(index, new DateFieldRangeInfo(new DateFieldMapper.DateFieldType(fieldName), null, null, null));
             } else if (fieldName.equals(IndexMetadata.EVENT_INGESTED_FIELD_NAME)) {
-                indexMetadataBuilder.eventIngestedRange(timestampRange, TransportVersion.current());
+                indexMetadataBuilder.eventIngestedRange(timestampRange);
                 fields.put(index, new DateFieldRangeInfo(null, null, new DateFieldMapper.DateFieldType(fieldName), null));
             }
 
@@ -1502,7 +1501,7 @@ private void addIndexMinMaxForTimestampAndEventIngested(
                 .numberOfShards(1)
                 .numberOfReplicas(0)
                 .timestampRange(tsTimestampRange)
-                .eventIngestedRange(eventIngestedTimestampRange, TransportVersion.current());
+                .eventIngestedRange(eventIngestedTimestampRange);
 
             Metadata.Builder metadataBuilder = Metadata.builder(clusterState.metadata()).put(indexMetadataBuilder);
             clusterState = ClusterState.builder(clusterState).metadata(metadataBuilder).build();
diff --git a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
index 193855a4c835f..bf62973b9b052 100644
--- a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
@@ -141,7 +141,7 @@ public void sendExecuteQuery(
         ) {
             DfsQueryPhase phase = new DfsQueryPhase(results.asList(), null, null, consumer, (response) -> new SearchPhase("test") {
                 @Override
-                public void run() throws IOException {
+                protected void run() {
                     responseRef.set(((QueryPhaseResultConsumer) response).results);
                 }
             }, mockSearchPhaseContext);
@@ -227,7 +227,7 @@ public void sendExecuteQuery(
         ) {
             DfsQueryPhase phase = new DfsQueryPhase(results.asList(), null, null, consumer, (response) -> new SearchPhase("test") {
                 @Override
-                public void run() throws IOException {
+                protected void run() {
                     responseRef.set(((QueryPhaseResultConsumer) response).results);
                 }
             }, mockSearchPhaseContext);
@@ -315,7 +315,7 @@ public void sendExecuteQuery(
         ) {
             DfsQueryPhase phase = new DfsQueryPhase(results.asList(), null, null, consumer, (response) -> new SearchPhase("test") {
                 @Override
-                public void run() throws IOException {
+                protected void run() {
                     responseRef.set(((QueryPhaseResultConsumer) response).results);
                 }
             }, mockSearchPhaseContext);
diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
index 23184be02f9c3..65fdec96c92f0 100644
--- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
@@ -96,11 +96,10 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
 
                         List<MultiSearchResponse.Item> mSearchResponses = new ArrayList<>(numInnerHits);
                         for (int innerHitNum = 0; innerHitNum < numInnerHits; innerHitNum++) {
-                            var sections = new SearchResponseSections(collapsedHits.get(innerHitNum), null, null, false, null, null, 1);
-                            try {
+                            try (
+                                var sections = new SearchResponseSections(collapsedHits.get(innerHitNum), null, null, false, null, null, 1)
+                            ) {
                                 mockSearchPhaseContext.sendSearchResponse(sections, null);
-                            } finally {
-                                sections.decRef();
                             }
                             mSearchResponses.add(new MultiSearchResponse.Item(mockSearchPhaseContext.searchResponse.get(), null));
                             // transferring ownership to the multi-search response so no need to release here
@@ -120,12 +119,9 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
                 try {
                     ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, hits, () -> new SearchPhase("test") {
                         @Override
-                        public void run() {
-                            var sections = new SearchResponseSections(hits, null, null, false, null, null, 1);
-                            try {
+                        protected void run() {
+                            try (var sections = new SearchResponseSections(hits, null, null, false, null, null, 1)) {
                                 mockSearchPhaseContext.sendSearchResponse(sections, null);
-                            } finally {
-                                sections.decRef();
                             }
                         }
                     });
@@ -214,12 +210,9 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
         try {
             ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, hits, () -> new SearchPhase("test") {
                 @Override
-                public void run() {
-                    var sections = new SearchResponseSections(hits, null, null, false, null, null, 1);
-                    try {
+                protected void run() {
+                    try (var sections = new SearchResponseSections(hits, null, null, false, null, null, 1)) {
                         mockSearchPhaseContext.sendSearchResponse(sections, null);
-                    } finally {
-                        sections.decRef();
                     }
                 }
             });
@@ -253,12 +246,9 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
             try {
                 ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, hits, () -> new SearchPhase("test") {
                     @Override
-                    public void run() {
-                        var sections = new SearchResponseSections(hits, null, null, false, null, null, 1);
-                        try {
+                    protected void run() {
+                        try (var sections = new SearchResponseSections(hits, null, null, false, null, null, 1)) {
                             mockSearchPhaseContext.sendSearchResponse(sections, null);
-                        } finally {
-                            sections.decRef();
                         }
                     }
                 });
@@ -296,7 +286,7 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
             SearchHits hits = SearchHits.empty(new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0f);
             ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, hits, () -> new SearchPhase("test") {
                 @Override
-                public void run() {
+                protected void run() {
                     mockSearchPhaseContext.sendSearchResponse(new SearchResponseSections(hits, null, null, false, null, null, 1), null);
                 }
             });
@@ -346,7 +336,7 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
             try {
                 ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, hits, () -> new SearchPhase("test") {
                     @Override
-                    public void run() {
+                    protected void run() {
                         mockSearchPhaseContext.sendSearchResponse(new SearchResponseSections(hits, null, null, false, null, null, 1), null);
                     }
                 });
@@ -395,7 +385,7 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
             try {
                 ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, hits, () -> new SearchPhase("test") {
                     @Override
-                    public void run() {
+                    protected void run() {
                         mockSearchPhaseContext.sendSearchResponse(
                             new SearchResponseSections(hits, null, null, false, null, null, 1),
                             new AtomicArray<>(0)
diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchLookupFieldsPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchLookupFieldsPhaseTests.java
index 1d2daf0cd660e..5c508dce61fc3 100644
--- a/server/src/test/java/org/elasticsearch/action/search/FetchLookupFieldsPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/FetchLookupFieldsPhaseTests.java
@@ -47,12 +47,10 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
                 searchHits[i] = SearchHitTests.createTestItem(randomBoolean(), randomBoolean());
             }
             SearchHits hits = new SearchHits(searchHits, new TotalHits(numHits, TotalHits.Relation.EQUAL_TO), 1.0f);
-            var sections = new SearchResponseSections(hits, null, null, false, null, null, 1);
-            try {
+            try (var sections = new SearchResponseSections(hits, null, null, false, null, null, 1)) {
                 FetchLookupFieldsPhase phase = new FetchLookupFieldsPhase(searchPhaseContext, sections, null);
                 phase.run();
             } finally {
-                sections.decRef();
                 hits.decRef();
             }
             searchPhaseContext.assertNoFailure();
@@ -189,12 +187,10 @@ void sendExecuteMultiSearch(
                 new TotalHits(2, TotalHits.Relation.EQUAL_TO),
                 1.0f
             );
-            var sections = new SearchResponseSections(searchHits, null, null, false, null, null, 1);
-            try {
+            try (var sections = new SearchResponseSections(searchHits, null, null, false, null, null, 1)) {
                 FetchLookupFieldsPhase phase = new FetchLookupFieldsPhase(searchPhaseContext, sections, null);
                 phase.run();
             } finally {
-                sections.decRef();
                 searchHits.decRef();
             }
             assertTrue(requestSent.get());
diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
index dda20dfb37e9d..fd60621c7e400 100644
--- a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
@@ -488,7 +488,7 @@ public void sendExecuteFetch(
                 reducedQueryPhase,
                 (searchResponse, scrollId) -> new SearchPhase("test") {
                     @Override
-                    public void run() {
+                    protected void run() {
                         mockSearchPhaseContext.sendSearchResponse(searchResponse, null);
                         latch.countDown();
                     }
@@ -764,7 +764,7 @@ private static BiFunction<SearchResponseSections, AtomicArray<SearchPhaseResult>
     ) {
         return (searchResponse, scrollId) -> new SearchPhase("test") {
             @Override
-            public void run() {
+            protected void run() {
                 mockSearchPhaseContext.sendSearchResponse(searchResponse, null);
             }
         };
diff --git a/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java b/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java
index 353188af8be3c..568186e0cae5b 100644
--- a/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java
@@ -418,7 +418,7 @@ public void testKnnSearchAction() throws IOException {
         float[] queryVector = randomVector();
         assertResponse(
             client().prepareSearch("index1", "index2")
-                .setQuery(new KnnVectorQueryBuilder("vector", queryVector, null, 5, null, null))
+                .setQuery(new KnnVectorQueryBuilder("vector", queryVector, 5, 5, null, null))
                 .setSize(2),
             response -> {
                 // The total hits is num_cands * num_shards, since the query gathers num_cands hits from each shard
diff --git a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
index cf65d756811ad..e8e12300c23e3 100644
--- a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
+++ b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
@@ -113,7 +113,7 @@ public void sendSearchResponse(SearchResponseSections internalSearchResponse, At
     }
 
     @Override
-    public void onPhaseFailure(SearchPhase phase, String msg, Throwable cause) {
+    public void onPhaseFailure(String phase, String msg, Throwable cause) {
         phaseFailure.set(cause);
     }
 
@@ -135,12 +135,12 @@ public SearchTransportService getSearchTransport() {
     }
 
     @Override
-    public void executeNextPhase(SearchPhase currentPhase, Supplier<SearchPhase> nextPhaseSupplier) {
+    public void executeNextPhase(String currentPhase, Supplier<SearchPhase> nextPhaseSupplier) {
         var nextPhase = nextPhaseSupplier.get();
         try {
             nextPhase.run();
         } catch (Exception e) {
-            onPhaseFailure(nextPhase, "phase failed", e);
+            onPhaseFailure(nextPhase.getName(), "phase failed", e);
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
index 9f81b999c9d98..008230c5e3fe3 100644
--- a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.search.RestMultiSearchAction;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESTestCase;
@@ -578,7 +577,7 @@ private static MultiSearchRequest createMultiSearchRequest() {
             }
 
             // scroll is not supported in the current msearch api, so unset it:
-            searchRequest.scroll((Scroll) null);
+            searchRequest.scroll(null);
 
             // only expand_wildcards, ignore_unavailable and allow_no_indices can be specified from msearch api, so unset other options:
             IndicesOptions randomlyGenerated = searchRequest.indicesOptions();
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
index 2361beb7ad036..7e9e6f623cab0 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
@@ -140,7 +140,7 @@ protected void executePhaseOnShard(
             protected SearchPhase getNextPhase() {
                 return new SearchPhase("test") {
                     @Override
-                    public void run() {
+                    protected void run() {
                         assertTrue(searchPhaseDidRun.compareAndSet(false, true));
                         latch.countDown();
                     }
@@ -255,7 +255,7 @@ protected void executePhaseOnShard(
                 protected SearchPhase getNextPhase() {
                     return new SearchPhase("test") {
                         @Override
-                        public void run() {
+                        protected void run() {
                             assertTrue(searchPhaseDidRun.compareAndSet(false, true));
                             latch.countDown();
                         }
@@ -363,7 +363,7 @@ protected void executePhaseOnShard(
                 protected SearchPhase getNextPhase() {
                     return new SearchPhase("test") {
                         @Override
-                        public void run() {
+                        protected void run() {
                             for (int i = 0; i < results.getNumShards(); i++) {
                                 TestSearchPhaseResult result = results.getAtomicArray().get(i);
                                 assertEquals(result.node.getId(), result.getSearchShardTarget().getNodeId());
@@ -497,7 +497,7 @@ protected void executePhaseOnShard(
                 protected SearchPhase getNextPhase() {
                     return new SearchPhase("test") {
                         @Override
-                        public void run() {
+                        protected void run() {
                             throw new RuntimeException("boom");
                         }
                     };
@@ -608,7 +608,7 @@ protected void executePhaseOnShard(
                 protected SearchPhase getNextPhase() {
                     return new SearchPhase("test") {
                         @Override
-                        public void run() {
+                        protected void run() {
                             assertTrue(searchPhaseDidRun.compareAndSet(false, true));
                             latch.countDown();
                         }
@@ -688,7 +688,7 @@ protected void executePhaseOnShard(
             protected SearchPhase getNextPhase() {
                 return new SearchPhase("test") {
                     @Override
-                    public void run() {
+                    protected void run() {
                         assertTrue(searchPhaseDidRun.compareAndSet(false, true));
                         latch.countDown();
                     }
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
index 9a507977c0123..bf81486087361 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
@@ -292,8 +292,7 @@ public void testMerge() {
                     reducedQueryPhase.suggest(),
                     profile
                 );
-                final SearchResponseSections mergedResponse = SearchPhaseController.merge(false, reducedQueryPhase, fetchResults);
-                try {
+                try (SearchResponseSections mergedResponse = SearchPhaseController.merge(false, reducedQueryPhase, fetchResults)) {
                     if (trackTotalHits == SearchContext.TRACK_TOTAL_HITS_DISABLED) {
                         assertNull(mergedResponse.hits.getTotalHits());
                     } else {
@@ -346,7 +345,6 @@ public void testMerge() {
                         assertThat(mergedResponse.profile(), is(anEmptyMap()));
                     }
                 } finally {
-                    mergedResponse.decRef();
                     fetchResults.asList().forEach(TransportMessage::decRef);
                 }
             } finally {
@@ -410,8 +408,7 @@ protected boolean lessThan(RankDoc a, RankDoc b) {
                     reducedQueryPhase.suggest(),
                     false
                 );
-                SearchResponseSections mergedResponse = SearchPhaseController.merge(false, reducedQueryPhase, fetchResults);
-                try {
+                try (SearchResponseSections mergedResponse = SearchPhaseController.merge(false, reducedQueryPhase, fetchResults)) {
                     if (trackTotalHits == SearchContext.TRACK_TOTAL_HITS_DISABLED) {
                         assertNull(mergedResponse.hits.getTotalHits());
                     } else {
@@ -427,7 +424,6 @@ protected boolean lessThan(RankDoc a, RankDoc b) {
                     assertThat(mergedResponse.hits().getHits().length, equalTo(reducedQueryPhase.sortedTopDocs().scoreDocs().length));
                     assertThat(mergedResponse.profile(), is(anEmptyMap()));
                 } finally {
-                    mergedResponse.decRef();
                     fetchResults.asList().forEach(TransportMessage::decRef);
                 }
             } finally {
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseExecutionExceptionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseExecutionExceptionTests.java
index 50ce97a2b73af..3e483b1f2cae3 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseExecutionExceptionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseExecutionExceptionTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.NodeDisconnectedException;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContent;
 import org.elasticsearch.xcontent.XContentParser;
@@ -31,6 +32,7 @@
 
 import static org.hamcrest.CoreMatchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
 
 public class SearchPhaseExecutionExceptionTests extends ESTestCase {
 
@@ -168,4 +170,57 @@ public void testPhaseFailureWithSearchShardFailure() {
 
         assertEquals(actual.status(), RestStatus.BAD_REQUEST);
     }
+
+    public void testOnlyWithCodesThatDoNotRequirePrecedence() {
+        int pickedIndex = randomIntBetween(0, 1);
+
+        // Pick one of these exceptions randomly.
+        var searchExceptions = new ElasticsearchException[] {
+            new ElasticsearchException("simulated"),
+            new NodeDisconnectedException(null, "unused message", "unused action", null) };
+
+        // Status codes that map to searchExceptions.
+        var expectedStatusCodes = new RestStatus[] { RestStatus.INTERNAL_SERVER_ERROR, RestStatus.BAD_GATEWAY };
+
+        ShardSearchFailure shardFailure1 = new ShardSearchFailure(
+            searchExceptions[pickedIndex],
+            new SearchShardTarget("nodeID", new ShardId("someIndex", "someUUID", 1), null)
+        );
+
+        ShardSearchFailure shardFailure2 = new ShardSearchFailure(
+            searchExceptions[pickedIndex],
+            new SearchShardTarget("nodeID", new ShardId("someIndex", "someUUID", 2), null)
+        );
+
+        SearchPhaseExecutionException ex = new SearchPhaseExecutionException(
+            "search",
+            "all shards failed",
+            new ShardSearchFailure[] { shardFailure1, shardFailure2 }
+        );
+
+        assertThat(ex.status(), is(expectedStatusCodes[pickedIndex]));
+    }
+
+    public void testWithRetriableCodesThatTakePrecedence() {
+        // Maps to a 500.
+        ShardSearchFailure shardFailure1 = new ShardSearchFailure(
+            new ElasticsearchException("simulated"),
+            new SearchShardTarget("nodeID", new ShardId("someIndex", "someUUID", 1), null)
+        );
+
+        // Maps to a 502.
+        ShardSearchFailure shardFailure2 = new ShardSearchFailure(
+            new NodeDisconnectedException(null, "unused message", "unused action", null),
+            new SearchShardTarget("nodeID", new ShardId("someIndex", "someUUID", 2), null)
+        );
+
+        SearchPhaseExecutionException ex = new SearchPhaseExecutionException(
+            "search",
+            "all shards failed",
+            new ShardSearchFailure[] { shardFailure1, shardFailure2 }
+        );
+
+        // The 502 takes precedence over 500.
+        assertThat(ex.status(), is(RestStatus.BAD_GATEWAY));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
index 6357155793fdf..f005f862720ff 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
@@ -207,7 +207,7 @@ public void sendExecuteQuery(
                 protected SearchPhase getNextPhase() {
                     return new SearchPhase("test") {
                         @Override
-                        public void run() {
+                        protected void run() {
                             latch.countDown();
                         }
                     };
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
index 152e5a9e6eac6..08f717dacf47d 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.search.AbstractSearchTestCase;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.PointInTimeBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.builder.SubSearchSourceBuilder;
@@ -219,16 +218,13 @@ public void testIllegalArguments() {
 
         e = expectThrows(NullPointerException.class, () -> searchRequest.source(null));
         assertEquals("source must not be null", e.getMessage());
-
-        e = expectThrows(NullPointerException.class, () -> searchRequest.scroll((TimeValue) null));
-        assertEquals("keepAlive must not be null", e.getMessage());
     }
 
     public void testValidate() throws IOException {
         {
             // if scroll isn't set, validate should never add errors
             SearchRequest searchRequest = createSearchRequest().source(new SearchSourceBuilder());
-            searchRequest.scroll((Scroll) null);
+            searchRequest.scroll(null);
             ActionRequestValidationException validationErrors = searchRequest.validate();
             assertNull(validationErrors);
         }
@@ -294,7 +290,7 @@ public void testValidate() throws IOException {
                 new SearchSourceBuilder().retriever(new TestCompoundRetrieverBuilder(randomIntBetween(1, 10)))
             );
             searchRequest.allowPartialSearchResults(true);
-            searchRequest.scroll((Scroll) null);
+            searchRequest.scroll(null);
             ActionRequestValidationException validationErrors = searchRequest.validate();
             assertNotNull(validationErrors);
             assertEquals(1, validationErrors.validationErrors().size());
@@ -348,7 +344,7 @@ protected int doHashCode() {
                 }
             }));
             searchRequest.allowPartialSearchResults(true);
-            searchRequest.scroll((Scroll) null);
+            searchRequest.scroll(null);
             ActionRequestValidationException validationErrors = searchRequest.validate();
             assertNull(validationErrors);
         }
@@ -388,14 +384,14 @@ public QueryBuilder topDocsQuery() {
                     return null;
                 }
             }));
-            searchRequest.scroll((Scroll) null);
+            searchRequest.scroll(null);
             ActionRequestValidationException validationErrors = searchRequest.validate();
             assertNull(validationErrors);
         }
         {
             // search_after and `from` isn't valid
             SearchRequest searchRequest = createSearchRequest().source(new SearchSourceBuilder());
-            searchRequest.scroll((Scroll) null);
+            searchRequest.scroll(null);
             searchRequest.source().searchAfter(new String[] { "value" });
             searchRequest.source().from(10);
             ActionRequestValidationException validationErrors = searchRequest.validate();
@@ -406,7 +402,7 @@ public QueryBuilder topDocsQuery() {
         {
             // slice without scroll or pit
             SearchRequest searchRequest = createSearchRequest().source(new SearchSourceBuilder());
-            searchRequest.scroll((Scroll) null);
+            searchRequest.scroll(null);
             searchRequest.source().pointInTimeBuilder(null);
             searchRequest.source().slice(new SliceBuilder(1, 10));
             ActionRequestValidationException validationErrors = searchRequest.validate();
@@ -417,7 +413,7 @@ public QueryBuilder topDocsQuery() {
         {
             // stored fields disabled with _source requested
             SearchRequest searchRequest = createSearchRequest().source(new SearchSourceBuilder());
-            searchRequest.scroll((Scroll) null);
+            searchRequest.scroll(null);
             searchRequest.source().storedField("_none_");
             searchRequest.source().fetchSource(true);
             ActionRequestValidationException validationErrors = searchRequest.validate();
@@ -428,7 +424,7 @@ public QueryBuilder topDocsQuery() {
         {
             // stored fields disabled with fetch fields requested
             SearchRequest searchRequest = createSearchRequest().source(new SearchSourceBuilder());
-            searchRequest.scroll((Scroll) null);
+            searchRequest.scroll(null);
             searchRequest.source().storedField("_none_");
             searchRequest.source().fetchSource(false);
             searchRequest.source().fetchField("field");
@@ -643,9 +639,7 @@ private SearchRequest mutate(SearchRequest searchRequest) {
         mutators.add(() -> mutation.routing(randomValueOtherThan(searchRequest.routing(), () -> randomAlphaOfLengthBetween(3, 10))));
         mutators.add(() -> mutation.requestCache((randomValueOtherThan(searchRequest.requestCache(), ESTestCase::randomBoolean))));
         mutators.add(
-            () -> mutation.scroll(
-                randomValueOtherThan(searchRequest.scroll(), () -> new Scroll(new TimeValue(randomNonNegativeLong() % 100000)))
-            )
+            () -> mutation.scroll(randomValueOtherThan(searchRequest.scroll(), () -> new TimeValue(randomNonNegativeLong() % 100000)))
         );
         mutators.add(
             () -> mutation.searchType(
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
index c32255a4d4ed9..1952e9c26c570 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
@@ -17,14 +17,12 @@
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
 import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.Transport;
 
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -54,7 +52,7 @@ public void testSendRequestsToNodes() throws InterruptedException {
 
         AtomicArray<SearchAsyncActionTests.TestSearchPhaseResult> results = new AtomicArray<>(scrollId.getContext().length);
         SearchScrollRequest request = new SearchScrollRequest();
-        request.scroll(new Scroll(TimeValue.timeValueMinutes(1)));
+        request.scroll(TimeValue.timeValueMinutes(1));
         CountDownLatch latch = new CountDownLatch(1);
         AtomicInteger movedCounter = new AtomicInteger(0);
         SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult> action = new SearchScrollAsyncAction<
@@ -87,7 +85,7 @@ protected SearchPhase moveToNextPhase(BiFunction<String, String, DiscoveryNode>
                 assertEquals(1, movedCounter.incrementAndGet());
                 return new SearchPhase("test") {
                     @Override
-                    public void run() throws IOException {
+                    protected void run() {
                         latch.countDown();
                     }
                 };
@@ -128,7 +126,7 @@ public void testFailNextPhase() throws InterruptedException {
 
         AtomicArray<SearchAsyncActionTests.TestSearchPhaseResult> results = new AtomicArray<>(scrollId.getContext().length);
         SearchScrollRequest request = new SearchScrollRequest();
-        request.scroll(new Scroll(TimeValue.timeValueMinutes(1)));
+        request.scroll(TimeValue.timeValueMinutes(1));
         CountDownLatch latch = new CountDownLatch(1);
         AtomicInteger movedCounter = new AtomicInteger(0);
         ActionListener<SearchResponse> listener = new ActionListener<SearchResponse>() {
@@ -184,7 +182,7 @@ protected SearchPhase moveToNextPhase(BiFunction<String, String, DiscoveryNode>
                 assertEquals(1, movedCounter.incrementAndGet());
                 return new SearchPhase("TEST_PHASE") {
                     @Override
-                    public void run() throws IOException {
+                    protected void run() {
                         throw new IllegalArgumentException("BOOM");
                     }
                 };
@@ -224,7 +222,7 @@ public void testNodeNotAvailable() throws InterruptedException {
 
         AtomicArray<SearchAsyncActionTests.TestSearchPhaseResult> results = new AtomicArray<>(scrollId.getContext().length);
         SearchScrollRequest request = new SearchScrollRequest();
-        request.scroll(new Scroll(TimeValue.timeValueMinutes(1)));
+        request.scroll(TimeValue.timeValueMinutes(1));
         CountDownLatch latch = new CountDownLatch(1);
         AtomicInteger movedCounter = new AtomicInteger(0);
         SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult> action = new SearchScrollAsyncAction<
@@ -262,7 +260,7 @@ protected SearchPhase moveToNextPhase(BiFunction<String, String, DiscoveryNode>
                 assertEquals(1, movedCounter.incrementAndGet());
                 return new SearchPhase("test") {
                     @Override
-                    public void run() throws IOException {
+                    protected void run() {
                         latch.countDown();
                     }
                 };
@@ -309,7 +307,7 @@ public void testShardFailures() throws InterruptedException {
 
         AtomicArray<SearchAsyncActionTests.TestSearchPhaseResult> results = new AtomicArray<>(scrollId.getContext().length);
         SearchScrollRequest request = new SearchScrollRequest();
-        request.scroll(new Scroll(TimeValue.timeValueMinutes(1)));
+        request.scroll(TimeValue.timeValueMinutes(1));
         CountDownLatch latch = new CountDownLatch(1);
         AtomicInteger movedCounter = new AtomicInteger(0);
         SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult> action = new SearchScrollAsyncAction<
@@ -344,7 +342,7 @@ protected SearchPhase moveToNextPhase(BiFunction<String, String, DiscoveryNode>
                 assertEquals(1, movedCounter.incrementAndGet());
                 return new SearchPhase("test") {
                     @Override
-                    public void run() throws IOException {
+                    protected void run() {
                         latch.countDown();
                     }
                 };
@@ -390,7 +388,7 @@ public void testAllShardsFailed() throws InterruptedException {
 
         AtomicArray<SearchAsyncActionTests.TestSearchPhaseResult> results = new AtomicArray<>(scrollId.getContext().length);
         SearchScrollRequest request = new SearchScrollRequest();
-        request.scroll(new Scroll(TimeValue.timeValueMinutes(1)));
+        request.scroll(TimeValue.timeValueMinutes(1));
         CountDownLatch latch = new CountDownLatch(1);
         ActionListener<SearchResponse> listener = new ActionListener<SearchResponse>() {
             @Override
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
index 67410ddde0746..b0f1c16982ec8 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
@@ -78,7 +78,7 @@ public void testFromXContent() throws Exception {
             searchScrollRequest.fromXContent(parser);
         }
         assertEquals("SCROLL_ID", searchScrollRequest.scrollId());
-        assertEquals(TimeValue.parseTimeValue("1m", null, "scroll"), searchScrollRequest.scroll().keepAlive());
+        assertEquals(TimeValue.parseTimeValue("1m", null, "scroll"), searchScrollRequest.scroll());
     }
 
     public void testFromXContentWithUnknownParamThrowsException() throws Exception {
@@ -138,7 +138,7 @@ private static SearchScrollRequest mutate(SearchScrollRequest original) {
         if (randomBoolean()) {
             return copy.scrollId(original.scrollId() + "xyz");
         } else {
-            return copy.scroll(new TimeValue(original.scroll().keepAlive().getMillis() + 1));
+            return copy.scroll(new TimeValue(original.scroll().getMillis() + 1));
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
index 0707f1356516a..450bb1e08940e 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
@@ -66,7 +66,6 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.search.SearchResponseMetrics;
 import org.elasticsearch.search.DummyQueryBuilder;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.SearchShardTarget;
@@ -1348,7 +1347,7 @@ public void testShouldMinimizeRoundtrips() throws Exception {
             SearchRequestTests searchRequestTests = new SearchRequestTests();
             searchRequestTests.setUp();
             SearchRequest searchRequest = searchRequestTests.createSearchRequest();
-            searchRequest.scroll((Scroll) null);
+            searchRequest.scroll(null);
             searchRequest.searchType(SearchType.QUERY_THEN_FETCH);
             SearchSourceBuilder source = searchRequest.source();
             if (source != null) {
diff --git a/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java b/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
index de7b43ad091fa..e33185d64e686 100644
--- a/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
@@ -56,7 +56,6 @@ public void testSerialization() throws Exception {
                         .allowAliasToMultipleIndices(randomBoolean())
                         .allowClosedIndices(randomBoolean())
                 )
-                .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
                 .build();
 
             BytesStreamOutput output = new BytesStreamOutput();
@@ -342,9 +341,8 @@ public void testToXContent() throws IOException {
             randomBoolean()
         );
         GatekeeperOptions gatekeeperOptions = new GatekeeperOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
-        IndicesOptions.SelectorOptions selectorOptions = new IndicesOptions.SelectorOptions(randomFrom(IndexComponentSelector.values()));
 
-        IndicesOptions indicesOptions = new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions, selectorOptions);
+        IndicesOptions indicesOptions = new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions);
 
         XContentType type = randomFrom(XContentType.values());
         BytesReference xContentBytes = toXContentBytes(indicesOptions, type);
@@ -359,15 +357,6 @@ public void testToXContent() throws IOException {
         assertThat(map.get("ignore_unavailable"), equalTo(concreteTargetOptions.allowUnavailableTargets()));
         assertThat(map.get("allow_no_indices"), equalTo(wildcardOptions.allowEmptyExpressions()));
         assertThat(map.get("ignore_throttled"), equalTo(gatekeeperOptions.ignoreThrottled()));
-        String displayValue;
-        if (IndicesOptions.SelectorOptions.ALL_APPLICABLE.equals(selectorOptions)) {
-            displayValue = "include";
-        } else if (IndicesOptions.SelectorOptions.DATA.equals(selectorOptions)) {
-            displayValue = "exclude";
-        } else {
-            displayValue = "only";
-        }
-        assertThat(map.get("failure_store"), equalTo(displayValue));
     }
 
     public void testFromXContent() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponseTests.java b/server/src/test/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponseTests.java
index 03a555fac6024..81db5bf8b9553 100644
--- a/server/src/test/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponseTests.java
@@ -36,7 +36,7 @@ protected TestNodeResponse(DiscoveryNode node) {
         final var fullResponse = new BaseNodesXContentResponse<>(ClusterName.DEFAULT, List.of(new TestNodeResponse(node)), List.of()) {
             @Override
             protected Iterator<? extends ToXContent> xContentChunks(ToXContent.Params outerParams) {
-                return ChunkedToXContentHelper.singleChunk((b, p) -> b.startObject("content").endObject());
+                return ChunkedToXContentHelper.chunk((b, p) -> b.startObject("content").endObject());
             }
 
             @Override
diff --git a/server/src/test/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsActionTests.java b/server/src/test/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsActionTests.java
index df2b716ab691c..4655d2e47bac5 100644
--- a/server/src/test/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsActionTests.java
@@ -146,15 +146,9 @@ public static void beforeClass() throws Exception {
         when(
             operationRouting.getShards(eq(clusterState), eq(index1.getName()), anyString(), nullable(String.class), nullable(String.class))
         ).thenReturn(index1ShardIterator);
-        when(operationRouting.shardId(eq(clusterState), eq(index1.getName()), nullable(String.class), nullable(String.class))).thenReturn(
-            new ShardId(index1, randomInt())
-        );
         when(
             operationRouting.getShards(eq(clusterState), eq(index2.getName()), anyString(), nullable(String.class), nullable(String.class))
         ).thenReturn(index2ShardIterator);
-        when(operationRouting.shardId(eq(clusterState), eq(index2.getName()), nullable(String.class), nullable(String.class))).thenReturn(
-            new ShardId(index2, randomInt())
-        );
 
         clusterService = mock(ClusterService.class);
         when(clusterService.localNode()).thenReturn(transportService.getLocalNode());
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java
index 668aea70c23f2..5c1aed9457e81 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java
@@ -1115,7 +1115,7 @@ private ClusterState buildClusterState() throws IOException {
             .putRolloverInfo(new RolloverInfo("rolloveAlias", new ArrayList<>(), 1L))
             .stats(new IndexMetadataStats(IndexWriteLoad.builder(1).build(), 120, 1))
             .indexWriteLoadForecast(8.0)
-            .eventIngestedRange(IndexLongFieldRange.UNKNOWN, TransportVersions.V_8_0_0)
+            .eventIngestedRange(IndexLongFieldRange.UNKNOWN)
             .build();
 
         return ClusterState.builder(ClusterName.DEFAULT)
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java
index 27775270a83eb..270315f23a53c 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java
@@ -33,11 +33,13 @@
 import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.features.FeatureSpecification;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
+import org.elasticsearch.snapshots.SearchableSnapshotsSettings;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLog;
@@ -46,15 +48,18 @@
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.cluster.metadata.DesiredNodesTestCase.assertDesiredNodesStatusIsCorrect;
 import static org.elasticsearch.cluster.metadata.DesiredNodesTestCase.randomDesiredNode;
+import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
 import static org.elasticsearch.test.VersionUtils.maxCompatibleVersion;
 import static org.elasticsearch.test.VersionUtils.randomCompatibleVersion;
 import static org.elasticsearch.test.VersionUtils.randomVersion;
@@ -86,12 +91,18 @@ public void testPreventJoinClusterWithNewerIndices() {
             .build();
         metaBuilder.put(indexMetadata, false);
         Metadata metadata = metaBuilder.build();
-        NodeJoinExecutor.ensureIndexCompatibility(IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current(), metadata);
+        NodeJoinExecutor.ensureIndexCompatibility(
+            IndexVersions.MINIMUM_COMPATIBLE,
+            IndexVersions.MINIMUM_READONLY_COMPATIBLE,
+            IndexVersion.current(),
+            metadata
+        );
 
         expectThrows(
             IllegalStateException.class,
             () -> NodeJoinExecutor.ensureIndexCompatibility(
                 IndexVersions.MINIMUM_COMPATIBLE,
+                IndexVersions.MINIMUM_READONLY_COMPATIBLE,
                 IndexVersionUtils.getPreviousVersion(IndexVersion.current()),
                 metadata
             )
@@ -110,10 +121,136 @@ public void testPreventJoinClusterWithUnsupportedIndices() {
         Metadata metadata = metaBuilder.build();
         expectThrows(
             IllegalStateException.class,
-            () -> NodeJoinExecutor.ensureIndexCompatibility(IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current(), metadata)
+            () -> NodeJoinExecutor.ensureIndexCompatibility(
+                IndexVersions.MINIMUM_COMPATIBLE,
+                IndexVersions.MINIMUM_READONLY_COMPATIBLE,
+                IndexVersion.current(),
+                metadata
+            )
         );
     }
 
+    public void testJoinClusterWithReadOnlyCompatibleIndices() {
+        {
+            var indexMetadata = IndexMetadata.builder("searchable-snapshot")
+                .settings(
+                    Settings.builder()
+                        .put(INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE)
+                        .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersions.MINIMUM_READONLY_COMPATIBLE)
+                        .put(IndexMetadata.SETTING_BLOCKS_WRITE, true)
+                )
+                .numberOfShards(1)
+                .numberOfReplicas(1)
+                .build();
+
+            NodeJoinExecutor.ensureIndexCompatibility(
+                IndexVersions.MINIMUM_COMPATIBLE,
+                IndexVersions.MINIMUM_READONLY_COMPATIBLE,
+                IndexVersion.current(),
+                Metadata.builder().put(indexMetadata, false).build()
+            );
+        }
+        {
+            var indexMetadata = IndexMetadata.builder("searchable-snapshot-no-write-block")
+                .settings(
+                    Settings.builder()
+                        .put(INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE)
+                        .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersions.MINIMUM_READONLY_COMPATIBLE)
+                )
+                .numberOfShards(1)
+                .numberOfReplicas(1)
+                .build();
+
+            expectThrows(
+                IllegalStateException.class,
+                () -> NodeJoinExecutor.ensureIndexCompatibility(
+                    IndexVersions.MINIMUM_COMPATIBLE,
+                    IndexVersions.MINIMUM_READONLY_COMPATIBLE,
+                    IndexVersion.current(),
+                    Metadata.builder().put(indexMetadata, false).build()
+                )
+            );
+        }
+        {
+            var indexMetadata = IndexMetadata.builder("archive")
+                .settings(
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_VERSION_CREATED, Version.fromId(randomFrom(5000099, 6000099)))
+                        .put(IndexMetadata.SETTING_VERSION_COMPATIBILITY, IndexVersions.MINIMUM_READONLY_COMPATIBLE)
+                        .put(IndexMetadata.SETTING_BLOCKS_WRITE, true)
+                )
+                .numberOfShards(1)
+                .numberOfReplicas(1)
+                .build();
+
+            NodeJoinExecutor.ensureIndexCompatibility(
+                IndexVersions.MINIMUM_COMPATIBLE,
+                IndexVersions.MINIMUM_READONLY_COMPATIBLE,
+                IndexVersion.current(),
+                Metadata.builder().put(indexMetadata, false).build()
+            );
+        }
+        {
+            var indexMetadata = IndexMetadata.builder("archive-no-write-block")
+                .settings(
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_VERSION_CREATED, Version.fromId(randomFrom(5000099, 6000099)))
+                        .put(IndexMetadata.SETTING_VERSION_COMPATIBILITY, IndexVersions.MINIMUM_READONLY_COMPATIBLE)
+                )
+                .numberOfShards(1)
+                .numberOfReplicas(1)
+                .build();
+
+            expectThrows(
+                IllegalStateException.class,
+                () -> NodeJoinExecutor.ensureIndexCompatibility(
+                    IndexVersions.MINIMUM_COMPATIBLE,
+                    IndexVersions.MINIMUM_READONLY_COMPATIBLE,
+                    IndexVersion.current(),
+                    Metadata.builder().put(indexMetadata, false).build()
+                )
+            );
+        }
+        {
+            var indexMetadata = IndexMetadata.builder("legacy")
+                .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.fromId(randomFrom(5000099, 6000099))))
+                .numberOfShards(1)
+                .numberOfReplicas(1)
+                .build();
+
+            expectThrows(
+                IllegalStateException.class,
+                () -> NodeJoinExecutor.ensureIndexCompatibility(
+                    IndexVersions.MINIMUM_COMPATIBLE,
+                    IndexVersions.MINIMUM_READONLY_COMPATIBLE,
+                    IndexVersion.current(),
+                    Metadata.builder().put(indexMetadata, false).build()
+                )
+            );
+        }
+        {
+            var indexMetadata = IndexMetadata.builder("read-only-compatible-but-unsupported")
+                .settings(
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersions.MINIMUM_READONLY_COMPATIBLE)
+                        .put(IndexMetadata.SETTING_BLOCKS_WRITE, true)
+                )
+                .numberOfShards(1)
+                .numberOfReplicas(1)
+                .build();
+
+            expectThrows(
+                IllegalStateException.class,
+                () -> NodeJoinExecutor.ensureIndexCompatibility(
+                    IndexVersions.MINIMUM_COMPATIBLE,
+                    IndexVersions.MINIMUM_READONLY_COMPATIBLE,
+                    IndexVersion.current(),
+                    Metadata.builder().put(indexMetadata, false).build()
+                )
+            );
+        }
+    }
+
     public void testPreventJoinClusterWithUnsupportedNodeVersions() {
         DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
         final Version version = randomCompatibleVersion(random(), Version.CURRENT);
@@ -227,6 +364,227 @@ public Set<NodeFeature> getFeatures() {
         );
     }
 
+    @SuppressForbidden(reason = "we need to actually check what is in cluster state")
+    private static Map<String, Set<String>> getRecordedNodeFeatures(ClusterState state) {
+        return state.clusterFeatures().nodeFeatures();
+    }
+
+    private static Version nextMajor() {
+        return Version.fromId((Version.CURRENT.major + 1) * 1_000_000 + 99);
+    }
+
+    public void testCanJoinClusterWithAssumedFeatures() throws Exception {
+        AllocationService allocationService = createAllocationService();
+        RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
+        FeatureService featureService = new FeatureService(List.of(new FeatureSpecification() {
+            @Override
+            public Set<NodeFeature> getFeatures() {
+                return Set.of(new NodeFeature("f1"), new NodeFeature("af1", true), new NodeFeature("af2", true));
+            }
+        }));
+
+        NodeJoinExecutor executor = new NodeJoinExecutor(allocationService, rerouteService, featureService);
+
+        DiscoveryNode masterNode = DiscoveryNodeUtils.create(UUIDs.base64UUID());
+        DiscoveryNode otherNode = DiscoveryNodeUtils.create(UUIDs.base64UUID());
+        Map<String, Set<String>> features = new HashMap<>();
+        features.put(masterNode.getId(), Set.of("f1", "af1", "af2"));
+        features.put(otherNode.getId(), Set.of("f1", "af1", "af2"));
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .nodes(DiscoveryNodes.builder().add(masterNode).localNodeId(masterNode.getId()).masterNodeId(masterNode.getId()).add(otherNode))
+            .nodeFeatures(features)
+            .build();
+
+        // it is valid for major+1 versions to join clusters assumed features still present
+        // this can happen in the process of marking, then removing, assumed features
+        // they should still be recorded appropriately
+        DiscoveryNode newNode = DiscoveryNodeUtils.builder(UUIDs.base64UUID())
+            .version(nextMajor(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current())
+            .build();
+        clusterState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
+            clusterState,
+            executor,
+            List.of(
+                JoinTask.singleNode(
+                    newNode,
+                    CompatibilityVersionsUtils.staticCurrent(),
+                    Set.of("f1", "af2"),
+                    TEST_REASON,
+                    NO_FAILURE_LISTENER,
+                    0L
+                )
+            )
+        );
+        features.put(newNode.getId(), Set.of("f1", "af2"));
+
+        // extra final check that the recorded cluster features are as they should be
+        assertThat(getRecordedNodeFeatures(clusterState), equalTo(features));
+    }
+
+    public void testJoinClusterWithAssumedFeaturesDoesntAllowNonAssumed() throws Exception {
+        AllocationService allocationService = createAllocationService();
+        RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
+        FeatureService featureService = new FeatureService(List.of(new FeatureSpecification() {
+            @Override
+            public Set<NodeFeature> getFeatures() {
+                return Set.of(new NodeFeature("f1"), new NodeFeature("af1", true));
+            }
+        }));
+
+        NodeJoinExecutor executor = new NodeJoinExecutor(allocationService, rerouteService, featureService);
+
+        DiscoveryNode masterNode = DiscoveryNodeUtils.create(UUIDs.base64UUID());
+        DiscoveryNode otherNode = DiscoveryNodeUtils.create(UUIDs.base64UUID());
+        Map<String, Set<String>> features = new HashMap<>();
+        features.put(masterNode.getId(), Set.of("f1", "af1"));
+        features.put(otherNode.getId(), Set.of("f1", "af1"));
+
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .nodes(DiscoveryNodes.builder().add(masterNode).localNodeId(masterNode.getId()).masterNodeId(masterNode.getId()).add(otherNode))
+            .nodeFeatures(features)
+            .build();
+
+        DiscoveryNode newNodeNextMajor = DiscoveryNodeUtils.builder(UUIDs.base64UUID())
+            .version(nextMajor(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current())
+            .build();
+        clusterState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
+            clusterState,
+            executor,
+            List.of(
+                JoinTask.singleNode(
+                    newNodeNextMajor,
+                    CompatibilityVersionsUtils.staticCurrent(),
+                    Set.of("f1"),
+                    TEST_REASON,
+                    NO_FAILURE_LISTENER,
+                    0L
+                )
+            )
+        );
+        features.put(newNodeNextMajor.getId(), Set.of("f1"));
+
+        // even though a next major has joined without af1, this doesnt allow the current major to join with af1 missing features
+        DiscoveryNode newNodeCurMajor = DiscoveryNodeUtils.create(UUIDs.base64UUID());
+        AtomicReference<Exception> ex = new AtomicReference<>();
+        clusterState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
+            clusterState,
+            executor,
+            List.of(
+                JoinTask.singleNode(
+                    newNodeCurMajor,
+                    CompatibilityVersionsUtils.staticCurrent(),
+                    Set.of("f1"),
+                    TEST_REASON,
+                    ActionTestUtils.assertNoSuccessListener(ex::set),
+                    0L
+                )
+            )
+        );
+        assertThat(ex.get().getMessage(), containsString("missing required features [af1]"));
+
+        // a next major can't join missing non-assumed features
+        DiscoveryNode newNodeNextMajorMissing = DiscoveryNodeUtils.builder(UUIDs.base64UUID())
+            .version(nextMajor(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current())
+            .build();
+        ex.set(null);
+        clusterState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
+            clusterState,
+            executor,
+            List.of(
+                JoinTask.singleNode(
+                    newNodeNextMajorMissing,
+                    CompatibilityVersionsUtils.staticCurrent(),
+                    Set.of(),
+                    TEST_REASON,
+                    ActionTestUtils.assertNoSuccessListener(ex::set),
+                    0L
+                )
+            )
+        );
+        assertThat(ex.get().getMessage(), containsString("missing required features [f1]"));
+
+        // extra final check that the recorded cluster features are as they should be, and newNodeNextMajor hasn't gained af1
+        assertThat(getRecordedNodeFeatures(clusterState), equalTo(features));
+    }
+
+    /*
+     * Same as above but the current major missing features is processed in the same execution
+     */
+    public void testJoinClusterWithAssumedFeaturesDoesntAllowNonAssumedSameExecute() throws Exception {
+        AllocationService allocationService = createAllocationService();
+        RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
+        FeatureService featureService = new FeatureService(List.of(new FeatureSpecification() {
+            @Override
+            public Set<NodeFeature> getFeatures() {
+                return Set.of(new NodeFeature("f1"), new NodeFeature("af1", true));
+            }
+        }));
+
+        NodeJoinExecutor executor = new NodeJoinExecutor(allocationService, rerouteService, featureService);
+
+        DiscoveryNode masterNode = DiscoveryNodeUtils.create(UUIDs.base64UUID());
+        DiscoveryNode otherNode = DiscoveryNodeUtils.create(UUIDs.base64UUID());
+        Map<String, Set<String>> features = new HashMap<>();
+        features.put(masterNode.getId(), Set.of("f1", "af1"));
+        features.put(otherNode.getId(), Set.of("f1", "af1"));
+
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .nodes(DiscoveryNodes.builder().add(masterNode).localNodeId(masterNode.getId()).masterNodeId(masterNode.getId()).add(otherNode))
+            .nodeFeatures(features)
+            .build();
+
+        DiscoveryNode newNodeNextMajor = DiscoveryNodeUtils.builder(UUIDs.base64UUID())
+            .version(nextMajor(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current())
+            .build();
+        DiscoveryNode newNodeCurMajor = DiscoveryNodeUtils.create(UUIDs.base64UUID());
+        DiscoveryNode newNodeNextMajorMissing = DiscoveryNodeUtils.builder(UUIDs.base64UUID())
+            .version(nextMajor(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current())
+            .build();
+        // even though a next major could join, this doesnt allow the current major to join with missing features
+        // nor a next major missing non-assumed features
+        AtomicReference<Exception> thisMajorEx = new AtomicReference<>();
+        AtomicReference<Exception> nextMajorEx = new AtomicReference<>();
+        List<JoinTask> tasks = List.of(
+            JoinTask.singleNode(
+                newNodeNextMajor,
+                CompatibilityVersionsUtils.staticCurrent(),
+                Set.of("f1"),
+                TEST_REASON,
+                NO_FAILURE_LISTENER,
+                0L
+            ),
+            JoinTask.singleNode(
+                newNodeCurMajor,
+                CompatibilityVersionsUtils.staticCurrent(),
+                Set.of("f1"),
+                TEST_REASON,
+                ActionTestUtils.assertNoSuccessListener(thisMajorEx::set),
+                0L
+            ),
+            JoinTask.singleNode(
+                newNodeNextMajorMissing,
+                CompatibilityVersionsUtils.staticCurrent(),
+                Set.of(),
+                TEST_REASON,
+                ActionTestUtils.assertNoSuccessListener(nextMajorEx::set),
+                0L
+            )
+        );
+        if (randomBoolean()) {
+            // sometimes combine them together into a single task for completeness
+            tasks = List.of(new JoinTask(tasks.stream().flatMap(t -> t.nodeJoinTasks().stream()).toList(), false, 0L, null));
+        }
+
+        clusterState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(clusterState, executor, tasks);
+        features.put(newNodeNextMajor.getId(), Set.of("f1"));
+
+        assertThat(thisMajorEx.get().getMessage(), containsString("missing required features [af1]"));
+        assertThat(nextMajorEx.get().getMessage(), containsString("missing required features [f1]"));
+
+        // extra check that the recorded cluster features are as they should be, and newNodeNextMajor hasn't gained af1
+        assertThat(getRecordedNodeFeatures(clusterState), equalTo(features));
+    }
+
     public void testSuccess() {
         Settings.builder().build();
         Metadata.Builder metaBuilder = Metadata.builder();
@@ -243,7 +601,13 @@ public void testSuccess() {
             .build();
         metaBuilder.put(indexMetadata, false);
         Metadata metadata = metaBuilder.build();
-        NodeJoinExecutor.ensureIndexCompatibility(IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current(), metadata);
+        NodeJoinExecutor.ensureIndexCompatibility(
+            // randomCompatibleVersionSettings() can set a version as low as MINIMUM_READONLY_COMPATIBLE
+            IndexVersions.MINIMUM_READONLY_COMPATIBLE,
+            IndexVersions.MINIMUM_READONLY_COMPATIBLE,
+            IndexVersion.current(),
+            metadata
+        );
     }
 
     public static Settings.Builder randomCompatibleVersionSettings() {
@@ -921,8 +1285,8 @@ public void testSetsNodeFeaturesWhenRejoining() throws Exception {
             .nodeFeatures(Map.of(masterNode.getId(), Set.of("f1", "f2"), rejoinNode.getId(), Set.of()))
             .build();
 
-        assertThat(clusterState.clusterFeatures().clusterHasFeature(new NodeFeature("f1")), is(false));
-        assertThat(clusterState.clusterFeatures().clusterHasFeature(new NodeFeature("f2")), is(false));
+        assertThat(clusterState.clusterFeatures().clusterHasFeature(clusterState.nodes(), new NodeFeature("f1")), is(false));
+        assertThat(clusterState.clusterFeatures().clusterHasFeature(clusterState.nodes(), new NodeFeature("f2")), is(false));
 
         final var resultingState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
             clusterState,
@@ -939,8 +1303,8 @@ public void testSetsNodeFeaturesWhenRejoining() throws Exception {
             )
         );
 
-        assertThat(resultingState.clusterFeatures().clusterHasFeature(new NodeFeature("f1")), is(true));
-        assertThat(resultingState.clusterFeatures().clusterHasFeature(new NodeFeature("f2")), is(true));
+        assertThat(resultingState.clusterFeatures().clusterHasFeature(resultingState.nodes(), new NodeFeature("f1")), is(true));
+        assertThat(resultingState.clusterFeatures().clusterHasFeature(resultingState.nodes(), new NodeFeature("f2")), is(true));
     }
 
     private DesiredNodeWithStatus createActualizedDesiredNode() {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreSettingsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreSettingsTests.java
new file mode 100644
index 0000000000000..709d026cae034
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreSettingsTests.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.stream.Stream;
+
+import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiAlphanumOfLengthBetween;
+import static org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING;
+import static org.hamcrest.Matchers.is;
+
+public class DataStreamFailureStoreSettingsTests extends ESTestCase {
+
+    public void testFailureStoreEnabledForDataStreamName_defaultSettings() {
+        DataStreamFailureStoreSettings dataStreamFailureStoreSettings = DataStreamFailureStoreSettings.create(
+            ClusterSettings.createBuiltInClusterSettings()
+        );
+
+        // The default should return false for any input.
+        // The following will include some illegal names, but it's still valid to test how the method treats them.
+        Stream.generate(() -> randomAsciiAlphanumOfLengthBetween(random(), 1, 20))
+            .limit(100)
+            .forEach(name -> assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName(name), is(false)));
+        Stream.generate(() -> randomUnicodeOfLengthBetween(1, 20))
+            .limit(100)
+            .forEach(name -> assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName(name), is(false)));
+    }
+
+    public void testFailureStoreEnabledForDataStreamName_exactMatches() {
+        DataStreamFailureStoreSettings dataStreamFailureStoreSettings = DataStreamFailureStoreSettings.create(
+            ClusterSettings.createBuiltInClusterSettings(
+                // Match exactly 'foo' and 'bar' — whitespace should be stripped:
+                Settings.builder().put(DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), "  foo  , bar  ").build()
+            )
+        );
+
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName("foo"), is(true));
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName("bar"), is(true));
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName("food"), is(false));
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName("tbar"), is(false));
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName(".foo"), is(false));
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName("barf"), is(false));
+    }
+
+    public void testFailureStoreEnabledForDataStreamName_wildcardMatches() {
+        DataStreamFailureStoreSettings dataStreamFailureStoreSettings = DataStreamFailureStoreSettings.create(
+            ClusterSettings.createBuiltInClusterSettings(
+                Settings.builder().put(DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), "  foo*  , *bar  ,  a*z  ").build()
+            )
+        );
+
+        // These tests aren't exhaustive as the library used is tested thoroughly, but they provide a basic check of the correct usage:
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName("foo"), is(true));
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName("bar"), is(true));
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName("food"), is(true));
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName("tbar"), is(true));
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName("az"), is(true));
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName("a123z"), is(true));
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName(".foo"), is(false));
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName("barf"), is(false));
+    }
+
+    public void testFailureStoreEnabledForDataStreamName_respondsToSettingsChange() {
+        ClusterSettings clusterSettings = ClusterSettings.createBuiltInClusterSettings(
+            Settings.builder().put(DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), "foo").build()
+        );
+        DataStreamFailureStoreSettings dataStreamFailureStoreSettings = DataStreamFailureStoreSettings.create(clusterSettings);
+
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName("foo"), is(true));
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName("bar"), is(false));
+
+        clusterSettings.applySettings(Settings.builder().put(DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), "bar").build());
+
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName("foo"), is(false));
+        assertThat(dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName("bar"), is(true));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
index 7108a4fd4f19e..f7f299683c3fc 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
@@ -27,6 +28,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
@@ -45,8 +47,10 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName;
 import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance;
@@ -64,6 +68,8 @@
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class DataStreamTests extends AbstractXContentSerializingTestCase<DataStream> {
 
@@ -866,23 +872,39 @@ public void testReplaceFailureIndexThrowsExceptionIfReplacingWriteIndex() {
     }
 
     public void testSnapshot() {
-        var preSnapshotDataStream = DataStreamTestHelper.randomInstance();
-        var indicesToRemove = randomSubsetOf(preSnapshotDataStream.getIndices());
-        if (indicesToRemove.size() == preSnapshotDataStream.getIndices().size()) {
+        var preSnapshotDataStream = DataStreamTestHelper.randomInstance(true);
+
+        // Mutate backing indices
+        var backingIndicesToRemove = randomSubsetOf(preSnapshotDataStream.getIndices());
+        if (backingIndicesToRemove.size() == preSnapshotDataStream.getIndices().size()) {
             // never remove them all
-            indicesToRemove.remove(0);
+            backingIndicesToRemove.remove(0);
         }
-        var indicesToAdd = randomIndexInstances();
-        var postSnapshotIndices = new ArrayList<>(preSnapshotDataStream.getIndices());
-        postSnapshotIndices.removeAll(indicesToRemove);
-        postSnapshotIndices.addAll(indicesToAdd);
+        var backingIndicesToAdd = randomIndexInstances();
+        var postSnapshotBackingIndices = new ArrayList<>(preSnapshotDataStream.getIndices());
+        postSnapshotBackingIndices.removeAll(backingIndicesToRemove);
+        postSnapshotBackingIndices.addAll(backingIndicesToAdd);
+
+        // Mutate failure indices
+        var failureIndicesToRemove = randomSubsetOf(preSnapshotDataStream.getFailureIndices().getIndices());
+        var failureIndicesToAdd = randomIndexInstances();
+        var postSnapshotFailureIndices = new ArrayList<>(preSnapshotDataStream.getFailureIndices().getIndices());
+        postSnapshotFailureIndices.removeAll(failureIndicesToRemove);
+        postSnapshotFailureIndices.addAll(failureIndicesToAdd);
 
         var replicated = preSnapshotDataStream.isReplicated() && randomBoolean();
         var postSnapshotDataStream = preSnapshotDataStream.copy()
             .setBackingIndices(
                 preSnapshotDataStream.getBackingIndices()
                     .copy()
-                    .setIndices(postSnapshotIndices)
+                    .setIndices(postSnapshotBackingIndices)
+                    .setRolloverOnWrite(replicated == false && preSnapshotDataStream.rolloverOnWrite())
+                    .build()
+            )
+            .setFailureIndices(
+                preSnapshotDataStream.getFailureIndices()
+                    .copy()
+                    .setIndices(postSnapshotFailureIndices)
                     .setRolloverOnWrite(replicated == false && preSnapshotDataStream.rolloverOnWrite())
                     .build()
             )
@@ -891,9 +913,10 @@ public void testSnapshot() {
             .setReplicated(replicated)
             .build();
 
-        var reconciledDataStream = postSnapshotDataStream.snapshot(
-            preSnapshotDataStream.getIndices().stream().map(Index::getName).toList()
-        );
+        Set<String> indicesInSnapshot = new HashSet<>();
+        preSnapshotDataStream.getIndices().forEach(index -> indicesInSnapshot.add(index.getName()));
+        preSnapshotDataStream.getFailureIndices().getIndices().forEach(index -> indicesInSnapshot.add(index.getName()));
+        var reconciledDataStream = postSnapshotDataStream.snapshot(indicesInSnapshot, Metadata.builder());
 
         assertThat(reconciledDataStream.getName(), equalTo(postSnapshotDataStream.getName()));
         assertThat(reconciledDataStream.getGeneration(), equalTo(postSnapshotDataStream.getGeneration()));
@@ -907,9 +930,19 @@ public void testSnapshot() {
         }
         assertThat(reconciledDataStream.isHidden(), equalTo(postSnapshotDataStream.isHidden()));
         assertThat(reconciledDataStream.isReplicated(), equalTo(postSnapshotDataStream.isReplicated()));
-        assertThat(reconciledDataStream.getIndices(), everyItem(not(in(indicesToRemove))));
-        assertThat(reconciledDataStream.getIndices(), everyItem(not(in(indicesToAdd))));
-        assertThat(reconciledDataStream.getIndices().size(), equalTo(preSnapshotDataStream.getIndices().size() - indicesToRemove.size()));
+        assertThat(reconciledDataStream.getIndices(), everyItem(not(in(backingIndicesToRemove))));
+        assertThat(reconciledDataStream.getIndices(), everyItem(not(in(backingIndicesToAdd))));
+        assertThat(
+            reconciledDataStream.getIndices().size(),
+            equalTo(preSnapshotDataStream.getIndices().size() - backingIndicesToRemove.size())
+        );
+        var reconciledFailureIndices = reconciledDataStream.getFailureIndices().getIndices();
+        assertThat(reconciledFailureIndices, everyItem(not(in(failureIndicesToRemove))));
+        assertThat(reconciledFailureIndices, everyItem(not(in(failureIndicesToAdd))));
+        assertThat(
+            reconciledFailureIndices.size(),
+            equalTo(preSnapshotDataStream.getFailureIndices().getIndices().size() - failureIndicesToRemove.size())
+        );
     }
 
     public void testSnapshotWithAllBackingIndicesRemoved() {
@@ -920,7 +953,12 @@ public void testSnapshotWithAllBackingIndicesRemoved() {
             .setBackingIndices(preSnapshotDataStream.getBackingIndices().copy().setIndices(indicesToAdd).build())
             .build();
 
-        assertNull(postSnapshotDataStream.snapshot(preSnapshotDataStream.getIndices().stream().map(Index::getName).toList()));
+        assertNull(
+            postSnapshotDataStream.snapshot(
+                preSnapshotDataStream.getIndices().stream().map(Index::getName).collect(Collectors.toSet()),
+                Metadata.builder()
+            )
+        );
     }
 
     public void testSelectTimeSeriesWriteIndex() {
@@ -2246,6 +2284,157 @@ public void testInternalDataStream() {
         assertThat(dotPrefixedDataStream.isInternal(), is(true));
     }
 
+    public void testIsFailureStoreExplicitlyEnabled() {
+        DataStream dataStreamNoFailureStoreOptions = createTestInstance().copy().setDataStreamOptions(DataStreamOptions.EMPTY).build();
+        DataStream dataStreamFailureStoreDisabled = createTestInstance().copy()
+            .setDataStreamOptions(DataStreamOptions.FAILURE_STORE_DISABLED)
+            .build();
+        DataStream dataStreamFailureStoreEnabled = createTestInstance().copy()
+            .setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED)
+            .build();
+        assertThat(dataStreamNoFailureStoreOptions.isFailureStoreExplicitlyEnabled(), is(false));
+        assertThat(dataStreamFailureStoreDisabled.isFailureStoreExplicitlyEnabled(), is(false));
+        assertThat(dataStreamFailureStoreEnabled.isFailureStoreExplicitlyEnabled(), is(true));
+    }
+
+    public void testIsFailureStoreEffectivelyEnabled_instanceMethod() {
+        DataStream dataStreamNoFailureStoreOptions = createTestInstance().copy()
+            .setName("my-data-stream-no-failure-store-options")
+            .setDataStreamOptions(DataStreamOptions.EMPTY)
+            .build();
+        DataStream dataStreamFailureStoreExplicitlyDisabled = createTestInstance().copy()
+            .setName("my-data-stream-failure-store-explicitly-disabled")
+            .setDataStreamOptions(DataStreamOptions.FAILURE_STORE_DISABLED)
+            .build();
+        DataStream dataStreamFailureStoreExplicitlyEnabled = createTestInstance().copy()
+            .setName("my-data-stream-failure-store-explicitly-enabled")
+            .setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED)
+            .build();
+        DataStream dotPrefixDataStreamNoFailureStoreOptions = createTestInstance().copy()
+            .setName(".my-data-stream-no-failure-store-options")
+            .setDataStreamOptions(DataStreamOptions.EMPTY)
+            .build();
+        DataStream systemDataStreamNoFailureStoreOptions = createTestInstance().copy()
+            .setName("my-data-stream-system-no-failure-store-options")
+            .setDataStreamOptions(DataStreamOptions.EMPTY)
+            .setSystem(true)
+            .setHidden(true) // system indices must be hidden
+            .build();
+        DataStreamFailureStoreSettings matchingSettings = DataStreamFailureStoreSettings.create(
+            ClusterSettings.createBuiltInClusterSettings(
+                Settings.builder()
+                    .put(
+                        DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(),
+                        String.join(",", "my-data-stream-*", ".my-data-stream-*")
+                    )
+                    .build()
+            )
+        );
+        DataStreamFailureStoreSettings nonMatchingSettings = DataStreamFailureStoreSettings.create(
+            ClusterSettings.createBuiltInClusterSettings(
+                Settings.builder()
+                    .put(DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), "not-my-data-stream-*")
+                    .build()
+            )
+        );
+        assertThat(dataStreamNoFailureStoreOptions.isFailureStoreEffectivelyEnabled(matchingSettings), is(true));
+        assertThat(dataStreamNoFailureStoreOptions.isFailureStoreEffectivelyEnabled(nonMatchingSettings), is(false));
+        assertThat(dataStreamFailureStoreExplicitlyDisabled.isFailureStoreEffectivelyEnabled(matchingSettings), is(false));
+        assertThat(dataStreamFailureStoreExplicitlyDisabled.isFailureStoreEffectivelyEnabled(nonMatchingSettings), is(false));
+        assertThat(dataStreamFailureStoreExplicitlyEnabled.isFailureStoreEffectivelyEnabled(matchingSettings), is(true));
+        assertThat(dataStreamFailureStoreExplicitlyEnabled.isFailureStoreEffectivelyEnabled(nonMatchingSettings), is(true));
+        assertThat(dotPrefixDataStreamNoFailureStoreOptions.isFailureStoreEffectivelyEnabled(matchingSettings), is(false));
+        assertThat(dotPrefixDataStreamNoFailureStoreOptions.isFailureStoreEffectivelyEnabled(nonMatchingSettings), is(false));
+        assertThat(systemDataStreamNoFailureStoreOptions.isFailureStoreEffectivelyEnabled(matchingSettings), is(false));
+        assertThat(systemDataStreamNoFailureStoreOptions.isFailureStoreEffectivelyEnabled(nonMatchingSettings), is(false));
+    }
+
+    public void testIsFailureStoreEffectivelyEnabled_staticHelperMethod() {
+        String regularDataStreamName = "my-data-stream";
+        String dotPrefixedDataStreamName = ".my-dot-prefixed-data-stream";
+        String systemDataStreamName = "my-system-data-stream-name";
+        DataStreamFailureStoreSettings matchingSettings = DataStreamFailureStoreSettings.create(
+            ClusterSettings.createBuiltInClusterSettings(
+                Settings.builder()
+                    .put(
+                        DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(),
+                        String.join(",", regularDataStreamName, dotPrefixedDataStreamName, systemDataStreamName)
+                    )
+                    .build()
+            )
+        );
+        DataStreamFailureStoreSettings nonMatchingSettings = DataStreamFailureStoreSettings.create(
+            ClusterSettings.createBuiltInClusterSettings(
+                Settings.builder()
+                    .put(DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), "not-my-data-stream")
+                    .build()
+            )
+        );
+        // At time of writing, SystemDataStreamDescriptor does not allow us to declare system data streams which aren't also dot-prefixed.
+        // But we code defensively to do the system data stream and dot-prefix tests independently, as implied in the requirements.
+        // We use a mock SystemIndices instance for testing, so that we can make it treat a non-dot-prefixed name as a system data stream.
+        SystemIndices systemIndices = mock(SystemIndices.class);
+        when(systemIndices.isSystemDataStream(systemDataStreamName)).thenReturn(true);
+
+        assertThat(
+            DataStream.isFailureStoreEffectivelyEnabled(DataStreamOptions.EMPTY, matchingSettings, regularDataStreamName, systemIndices),
+            is(true)
+        );
+        assertThat(
+            DataStream.isFailureStoreEffectivelyEnabled(DataStreamOptions.EMPTY, nonMatchingSettings, regularDataStreamName, systemIndices),
+            is(false)
+        );
+        assertThat(
+            DataStream.isFailureStoreEffectivelyEnabled(
+                DataStreamOptions.EMPTY,
+                matchingSettings,
+                dotPrefixedDataStreamName,
+                systemIndices
+            ),
+            is(false)
+        );
+        assertThat(
+            DataStream.isFailureStoreEffectivelyEnabled(DataStreamOptions.EMPTY, matchingSettings, systemDataStreamName, systemIndices),
+            is(false)
+        );
+        assertThat(
+            DataStream.isFailureStoreEffectivelyEnabled(
+                DataStreamOptions.FAILURE_STORE_DISABLED,
+                matchingSettings,
+                regularDataStreamName,
+                systemIndices
+            ),
+            is(false)
+        );
+        assertThat(
+            DataStream.isFailureStoreEffectivelyEnabled(
+                DataStreamOptions.FAILURE_STORE_DISABLED,
+                nonMatchingSettings,
+                regularDataStreamName,
+                systemIndices
+            ),
+            is(false)
+        );
+        assertThat(
+            DataStream.isFailureStoreEffectivelyEnabled(
+                DataStreamOptions.FAILURE_STORE_ENABLED,
+                matchingSettings,
+                regularDataStreamName,
+                systemIndices
+            ),
+            is(true)
+        );
+        assertThat(
+            DataStream.isFailureStoreEffectivelyEnabled(
+                DataStreamOptions.FAILURE_STORE_ENABLED,
+                nonMatchingSettings,
+                regularDataStreamName,
+                systemIndices
+            ),
+            is(true)
+        );
+    }
+
     private record DataStreamMetadata(Long creationTimeInMillis, Long rolloverTimeInMillis, Long originationTimeInMillis) {
         public static DataStreamMetadata dataStreamMetadata(Long creationTimeInMillis, Long rolloverTimeInMillis) {
             return new DataStreamMetadata(creationTimeInMillis, rolloverTimeInMillis, null);
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/HumanReadableIndexSettingsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/HumanReadableIndexSettingsTests.java
index 476ade8576586..8752e68112bff 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/HumanReadableIndexSettingsTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/HumanReadableIndexSettingsTests.java
@@ -22,7 +22,7 @@
 
 public class HumanReadableIndexSettingsTests extends ESTestCase {
     public void testHumanReadableSettings() {
-        IndexVersion versionCreated = randomVersion(random());
+        IndexVersion versionCreated = randomVersion();
         long created = System.currentTimeMillis();
         Settings testSettings = Settings.builder()
             .put(IndexMetadata.SETTING_VERSION_CREATED, versionCreated)
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java
new file mode 100644
index 0000000000000..5ab5ed1c23e4f
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java
@@ -0,0 +1,241 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.indices.EmptySystemIndices;
+import org.elasticsearch.indices.InvalidIndexNameException;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.either;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class IndexAbstractionResolverTests extends ESTestCase {
+
+    private IndexNameExpressionResolver indexNameExpressionResolver;
+    private IndexAbstractionResolver indexAbstractionResolver;
+    private Metadata metadata;
+    private String dateTimeIndexToday;
+    private String dateTimeIndexTomorrow;
+
+    // Only used when resolving wildcard expressions
+    private final Supplier<Set<String>> defaultMask = () -> Set.of("index1", "index2", "data-stream1");
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        indexNameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY), EmptySystemIndices.INSTANCE);
+        indexAbstractionResolver = new IndexAbstractionResolver(indexNameExpressionResolver);
+
+        // Try to resist failing at midnight on the first/last day of the month. Time generally moves forward, so make a timestamp for
+        // the next day and if they're different, add both to the cluster state. Check for either in date math tests.
+        long timeMillis = System.currentTimeMillis();
+        long timeTomorrow = timeMillis + TimeUnit.DAYS.toMillis(1);
+        dateTimeIndexToday = IndexNameExpressionResolver.resolveDateMathExpression("<datetime-{now/M}>", timeMillis);
+        dateTimeIndexTomorrow = IndexNameExpressionResolver.resolveDateMathExpression("<datetime-{now/M}>", timeTomorrow);
+
+        metadata = DataStreamTestHelper.getClusterStateWithDataStreams(
+            List.of(new Tuple<>("data-stream1", 2), new Tuple<>("data-stream2", 2)),
+            List.of("index1", "index2", "index3", dateTimeIndexToday, dateTimeIndexTomorrow),
+            randomMillisUpToYear9999(),
+            Settings.EMPTY,
+            0,
+            false,
+            true
+        ).metadata();
+    }
+
+    public void testResolveIndexAbstractions() {
+        // == Single Concrete Index ==
+
+        // No selectors allowed, none given
+        assertThat(resolveAbstractionsSelectorNotAllowed(List.of("index1")), contains("index1"));
+        // No selectors allowed, valid selector given
+        expectThrows(IllegalArgumentException.class, () -> resolveAbstractionsSelectorNotAllowed(List.of("index1::data")));
+        // Selectors allowed, valid selector given, data selector stripped off in result since it is the default
+        assertThat(resolveAbstractionsSelectorAllowed(List.of("index1::data")), contains("index1"));
+        // Selectors allowed, wildcard selector provided, data selector stripped off in result since it is the default
+        // ** only returns ::data since expression is an index
+        assertThat(resolveAbstractionsSelectorAllowed(List.of("index1::*")), contains("index1"));
+        // Selectors allowed, invalid selector given
+        expectThrows(InvalidIndexNameException.class, () -> resolveAbstractionsSelectorAllowed(List.of("index1::custom")));
+
+        // == Single Date Math Expressions ==
+
+        // No selectors allowed, none given
+        assertThat(
+            resolveAbstractionsSelectorNotAllowed(List.of("<datetime-{now/M}>")),
+            contains(either(equalTo(dateTimeIndexToday)).or(equalTo(dateTimeIndexTomorrow)))
+        );
+        // No selectors allowed, valid selector given
+        expectThrows(IllegalArgumentException.class, () -> resolveAbstractionsSelectorNotAllowed(List.of("<datetime-{now/M}>::data")));
+        // Selectors allowed, none given
+        assertThat(
+            resolveAbstractionsSelectorAllowed(List.of("<datetime-{now/M}>")),
+            contains(either(equalTo(dateTimeIndexToday)).or(equalTo(dateTimeIndexTomorrow)))
+        );
+        // Selectors allowed, valid selector provided, data selector stripped off in result since it is the default
+        assertThat(
+            resolveAbstractionsSelectorAllowed(List.of("<datetime-{now/M}>::data")),
+            contains(either(equalTo(dateTimeIndexToday)).or(equalTo(dateTimeIndexTomorrow)))
+        );
+        // Selectors allowed, wildcard selector provided, data selector stripped off in result since it is the default
+        // ** only returns ::data since expression is an index
+        assertThat(
+            resolveAbstractionsSelectorAllowed(List.of("<datetime-{now/M}>::data")),
+            contains(either(equalTo(dateTimeIndexToday)).or(equalTo(dateTimeIndexTomorrow)))
+        );
+        // Selectors allowed, invalid selector given
+        expectThrows(InvalidIndexNameException.class, () -> resolveAbstractionsSelectorAllowed(List.of("<datetime-{now/M}>::custom")));
+
+        // == Single Patterned Index ==
+
+        // No selectors allowed, none given
+        assertThat(resolveAbstractionsSelectorNotAllowed(List.of("index*")), containsInAnyOrder("index1", "index2"));
+        // No selectors allowed, valid selector given
+        expectThrows(IllegalArgumentException.class, () -> resolveAbstractionsSelectorNotAllowed(List.of("index*::data")));
+        // Selectors allowed, valid selector given, data selector stripped off in result since it is the default
+        assertThat(resolveAbstractionsSelectorAllowed(List.of("index*::data")), containsInAnyOrder("index1", "index2"));
+        // Selectors allowed, wildcard selector provided, data selector stripped off in result since it is the default
+        // ** only returns ::data since expression is an index
+        assertThat(resolveAbstractionsSelectorAllowed(List.of("index*::*")), containsInAnyOrder("index1", "index2"));
+        // Selectors allowed, invalid selector given
+        expectThrows(InvalidIndexNameException.class, () -> resolveAbstractionsSelectorAllowed(List.of("index*::custom")));
+
+        // == Single Data Stream ==
+
+        // No selectors allowed, none given
+        assertThat(resolveAbstractionsSelectorNotAllowed(List.of("data-stream1")), contains("data-stream1"));
+        // No selectors allowed, valid selector given
+        expectThrows(IllegalArgumentException.class, () -> resolveAbstractionsSelectorNotAllowed(List.of("data-stream1::data")));
+        // Selectors allowed, valid selector given
+        assertThat(resolveAbstractionsSelectorAllowed(List.of("data-stream1::failures")), contains("data-stream1::failures"));
+        // Selectors allowed, wildcard selector provided
+        // ** returns both ::data and ::failures since expression is a data stream
+        // ** data selector stripped off in result since it is the default
+        assertThat(
+            resolveAbstractionsSelectorAllowed(List.of("data-stream1::*")),
+            containsInAnyOrder("data-stream1", "data-stream1::failures")
+        );
+        // Selectors allowed, invalid selector given
+        expectThrows(InvalidIndexNameException.class, () -> resolveAbstractionsSelectorAllowed(List.of("data-stream1::custom")));
+
+        // == Patterned Data Stream ==
+
+        // No selectors allowed, none given
+        assertThat(resolveAbstractionsSelectorNotAllowed(List.of("data-stream*")), contains("data-stream1"));
+        // No selectors allowed, valid selector given
+        expectThrows(IllegalArgumentException.class, () -> resolveAbstractionsSelectorNotAllowed(List.of("data-stream*::data")));
+        // Selectors allowed, valid selector given
+        assertThat(resolveAbstractionsSelectorAllowed(List.of("data-stream*::failures")), contains("data-stream1::failures"));
+        // Selectors allowed, wildcard selector provided
+        // ** returns both ::data and ::failures since expression is a data stream
+        assertThat(
+            resolveAbstractionsSelectorAllowed(List.of("data-stream*::*")),
+            containsInAnyOrder("data-stream1", "data-stream1::failures")
+        );
+        // Selectors allowed, invalid selector given
+        expectThrows(InvalidIndexNameException.class, () -> resolveAbstractionsSelectorAllowed(List.of("data-stream*::custom")));
+
+        // == Match All * Wildcard ==
+
+        // No selectors allowed, none given
+        assertThat(resolveAbstractionsSelectorNotAllowed(List.of("*")), containsInAnyOrder("index1", "index2", "data-stream1"));
+        // No selectors allowed, valid selector given
+        expectThrows(IllegalArgumentException.class, () -> resolveAbstractionsSelectorNotAllowed(List.of("*::data")));
+        // Selectors allowed, valid selector given
+        // ::data selector returns all values with data component
+        assertThat(resolveAbstractionsSelectorAllowed(List.of("*::data")), containsInAnyOrder("index1", "index2", "data-stream1"));
+        // Selectors allowed, valid selector given
+        // ::failures selector returns only data streams, which can have failure components
+        assertThat(resolveAbstractionsSelectorAllowed(List.of("*::failures")), contains("data-stream1::failures"));
+        // Selectors allowed, wildcard selector provided
+        // ** returns both ::data and ::failures for applicable abstractions
+        assertThat(
+            resolveAbstractionsSelectorAllowed(List.of("*::*")),
+            containsInAnyOrder("index1", "index2", "data-stream1", "data-stream1::failures")
+        );
+        // Selectors allowed, invalid selector given
+        expectThrows(InvalidIndexNameException.class, () -> resolveAbstractionsSelectorAllowed(List.of("*::custom")));
+
+        // == Wildcards with Exclusions ==
+
+        // No selectors allowed, none given
+        assertThat(resolveAbstractionsSelectorNotAllowed(List.of("*", "-index*")), containsInAnyOrder("data-stream1"));
+        // No selectors allowed, valid selector given
+        expectThrows(IllegalArgumentException.class, () -> resolveAbstractionsSelectorNotAllowed(List.of("*", "-*::data")));
+        // Selectors allowed, wildcard selector provided
+        // ** returns both ::data and ::failures for applicable abstractions
+        // ** limits the returned values based on selectors
+        assertThat(resolveAbstractionsSelectorAllowed(List.of("*::*", "-*::data")), contains("data-stream1::failures"));
+        // Selectors allowed, wildcard selector provided
+        // ** limits the returned values based on selectors
+        assertThat(
+            resolveAbstractionsSelectorAllowed(List.of("*::*", "-*::failures")),
+            containsInAnyOrder("index1", "index2", "data-stream1")
+        );
+        // Selectors allowed, none given, default to both selectors
+        // ** limits the returned values based on selectors
+        assertThat(
+            resolveAbstractionsSelectorAllowed(List.of("*", "-*::failures")),
+            containsInAnyOrder("index1", "index2", "data-stream1")
+        );
+        // Selectors allowed, invalid selector given
+        expectThrows(InvalidIndexNameException.class, () -> resolveAbstractionsSelectorAllowed(List.of("*", "-*::custom")));
+    }
+
+    public void testIsIndexVisible() {
+        assertThat(isIndexVisible("index1", null), is(true));
+        assertThat(isIndexVisible("index1", "*"), is(true));
+        assertThat(isIndexVisible("index1", "data"), is(true));
+        assertThat(isIndexVisible("index1", "failures"), is(false)); // *
+        // * Indices don't have failure components so the failure component is not visible
+
+        assertThat(isIndexVisible("data-stream1", null), is(true));
+        assertThat(isIndexVisible("data-stream1", "*"), is(true));
+        assertThat(isIndexVisible("data-stream1", "data"), is(true));
+        assertThat(isIndexVisible("data-stream1", "failures"), is(true));
+    }
+
+    private boolean isIndexVisible(String index, String selector) {
+        return IndexAbstractionResolver.isIndexVisible(
+            "*",
+            selector,
+            index,
+            IndicesOptions.strictExpandOpen(),
+            metadata,
+            indexNameExpressionResolver,
+            true
+        );
+    }
+
+    private List<String> resolveAbstractionsSelectorNotAllowed(List<String> expressions) {
+        return resolveAbstractions(expressions, IndicesOptions.strictExpandHiddenNoSelectors(), defaultMask);
+    }
+
+    private List<String> resolveAbstractionsSelectorAllowed(List<String> expressions) {
+        return resolveAbstractions(expressions, IndicesOptions.strictExpandOpen(), defaultMask);
+    }
+
+    private List<String> resolveAbstractions(List<String> expressions, IndicesOptions indicesOptions, Supplier<Set<String>> mask) {
+        return indexAbstractionResolver.resolveIndexAbstractions(expressions, indicesOptions, metadata, mask, (idx) -> true, true);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsTests.java
index 2885673ba5539..1709e2d4d372f 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsTests.java
@@ -113,7 +113,21 @@ private ShardStats createShardStats(
         commonStats.getIndexing()
             .getTotal()
             .add(
-                new IndexingStats.Stats(0, 0, 0, 0, 0, 0, 0, 0, false, 0, totalIndexingTimeSinceShardStartedInNanos, totalActiveTimeInNanos)
+                new IndexingStats.Stats(
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    false,
+                    0,
+                    totalIndexingTimeSinceShardStartedInNanos,
+                    totalActiveTimeInNanos
+                )
             );
         return new ShardStats(shardRouting, commonStats, null, null, null, null, null, false, false, 0);
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java
index 3b122864aa472..417ae89da0a69 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java
@@ -18,11 +18,13 @@
 import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.snapshots.SearchableSnapshotsSettings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.index.IndexVersionUtils;
 
 import java.util.Collections;
 
+import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
 import static org.hamcrest.Matchers.equalTo;
 
 public class IndexMetadataVerifierTests extends ESTestCase {
@@ -97,7 +99,8 @@ public void testCustomSimilarity() {
                 .put("index.similarity.my_similarity.after_effect", "l")
                 .build()
         );
-        service.verifyIndexMetadata(src, IndexVersions.MINIMUM_COMPATIBLE);
+        // The random IndexMetadata.SETTING_VERSION_CREATED in IndexMetadata can be as low as MINIMUM_READONLY_COMPATIBLE
+        service.verifyIndexMetadata(src, IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE);
     }
 
     public void testIncompatibleVersion() {
@@ -110,7 +113,7 @@ public void testIncompatibleVersion() {
         );
         String message = expectThrows(
             IllegalStateException.class,
-            () -> service.verifyIndexMetadata(metadata, IndexVersions.MINIMUM_COMPATIBLE)
+            () -> service.verifyIndexMetadata(metadata, IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE)
         ).getMessage();
         assertThat(
             message,
@@ -132,7 +135,80 @@ public void testIncompatibleVersion() {
 
         indexCreated = IndexVersionUtils.randomVersionBetween(random(), minCompat, IndexVersion.current());
         IndexMetadata goodMeta = newIndexMeta("foo", Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, indexCreated).build());
-        service.verifyIndexMetadata(goodMeta, IndexVersions.MINIMUM_COMPATIBLE);
+        service.verifyIndexMetadata(goodMeta, IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE);
+    }
+
+    public void testReadOnlyVersionCompatibility() {
+        var service = getIndexMetadataVerifier();
+        var indexCreated = IndexVersions.MINIMUM_READONLY_COMPATIBLE;
+        {
+            var idxMetadata = newIndexMeta(
+                "not-searchable-snapshot",
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean())
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, indexCreated)
+                    .build()
+            );
+            String message = expectThrows(
+                IllegalStateException.class,
+                () -> service.verifyIndexMetadata(idxMetadata, IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE)
+            ).getMessage();
+            assertThat(
+                message,
+                equalTo(
+                    "The index [not-searchable-snapshot/"
+                        + idxMetadata.getIndexUUID()
+                        + "] has current compatibility version ["
+                        + indexCreated.toReleaseVersion()
+                        + "] "
+                        + "but the minimum compatible version is ["
+                        + IndexVersions.MINIMUM_COMPATIBLE.toReleaseVersion()
+                        + "]. It should be re-indexed in Elasticsearch "
+                        + (Version.CURRENT.major - 1)
+                        + ".x before upgrading to "
+                        + Build.current().version()
+                        + "."
+                )
+            );
+        }
+        {
+            var idxMetadata = newIndexMeta(
+                "not-read-only",
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, indexCreated)
+                    .put(INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE)
+                    .build()
+            );
+            String message = expectThrows(
+                IllegalStateException.class,
+                () -> service.verifyIndexMetadata(idxMetadata, IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE)
+            ).getMessage();
+            assertThat(
+                message,
+                equalTo(
+                    "The index [not-read-only/"
+                        + idxMetadata.getIndexUUID()
+                        + "] created in version ["
+                        + indexCreated
+                        + "] with current compatibility version ["
+                        + indexCreated.toReleaseVersion()
+                        + "] must be marked as read-only using the setting [index.blocks.write] set to [true] before upgrading to "
+                        + Build.current().version()
+                        + "."
+                )
+            );
+        }
+        {
+            var idxMetadata = newIndexMeta(
+                "good",
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_BLOCKS_WRITE, true)
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, indexCreated)
+                    .put(INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE)
+                    .build()
+            );
+            service.verifyIndexMetadata(idxMetadata, IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE);
+        }
     }
 
     private IndexMetadataVerifier getIndexMetadataVerifier() {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
index c2000ede8113b..f5e4c3d8f2d09 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
@@ -32,7 +32,6 @@
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
-import org.elasticsearch.indices.FailureIndexNotSupportedException;
 import org.elasticsearch.indices.IndexClosedException;
 import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.indices.SystemIndexDescriptor;
@@ -58,6 +57,8 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.action.support.IndexComponentSelector.DATA;
+import static org.elasticsearch.action.support.IndexComponentSelector.FAILURES;
 import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo;
 import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createBackingIndex;
 import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createFailureStore;
@@ -1587,16 +1588,31 @@ public void testResolveExpressions() {
             .put(indexBuilder("test-1").state(State.OPEN).putAlias(AliasMetadata.builder("alias-1")));
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
 
-        assertEquals(resolvedExpressionsSet("alias-0", "alias-1"), indexNameExpressionResolver.resolveExpressions(state, "alias-*"));
         assertEquals(
-            resolvedExpressionsSet("test-0", "alias-0", "alias-1"),
+            Set.of(new ResolvedExpression("alias-0", DATA), new ResolvedExpression("alias-1", DATA)),
+            indexNameExpressionResolver.resolveExpressions(state, "alias-*")
+        );
+        assertEquals(
+            Set.of(
+                new ResolvedExpression("test-0", DATA),
+                new ResolvedExpression("alias-0", DATA),
+                new ResolvedExpression("alias-1", DATA)
+            ),
             indexNameExpressionResolver.resolveExpressions(state, "test-0", "alias-*")
         );
         assertEquals(
-            resolvedExpressionsSet("test-0", "test-1", "alias-0", "alias-1"),
+            Set.of(
+                new ResolvedExpression("test-0", DATA),
+                new ResolvedExpression("test-1", DATA),
+                new ResolvedExpression("alias-0", DATA),
+                new ResolvedExpression("alias-1", DATA)
+            ),
             indexNameExpressionResolver.resolveExpressions(state, "test-*", "alias-*")
         );
-        assertEquals(resolvedExpressionsSet("test-1", "alias-1"), indexNameExpressionResolver.resolveExpressions(state, "*-1"));
+        assertEquals(
+            Set.of(new ResolvedExpression("test-1", DATA), new ResolvedExpression("alias-1", DATA)),
+            indexNameExpressionResolver.resolveExpressions(state, "*-1")
+        );
     }
 
     public void testFilteringAliases() {
@@ -2314,8 +2330,7 @@ public void testIgnoreThrottled() {
                 new IndicesOptions(
                     IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS,
                     IndicesOptions.WildcardOptions.DEFAULT,
-                    IndicesOptions.GatekeeperOptions.builder().ignoreThrottled(true).build(),
-                    IndicesOptions.SelectorOptions.DEFAULT
+                    IndicesOptions.GatekeeperOptions.builder().ignoreThrottled(true).build()
                 ),
                 "ind*",
                 "test-index"
@@ -2750,12 +2765,10 @@ public void testDataStreamsWithFailureStore() {
             assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 2, epochMillis)));
         }
 
-        // Test include failure store with an exact data stream name
+        // Test explicit include failure store with an exact data stream name
         {
-            IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN)
-                .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
-                .build();
-            Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream");
+            IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
+            Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream::*");
             assertThat(result.length, equalTo(4));
             assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 1, epochMillis)));
             assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 2, epochMillis)));
@@ -2763,43 +2776,22 @@ public void testDataStreamsWithFailureStore() {
             assertThat(result[3].getName(), equalTo(DataStream.getDefaultFailureStoreName(dataStreamName, 2, epochMillis)));
         }
 
-        // Test include failure store while we do not allow failure indices and ignore unavailable
-        // We expect that they will be skipped
-        {
-            IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN)
-                .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
-                .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowFailureIndices(false).build())
-                .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
-                .build();
-            Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream");
-            assertThat(result.length, equalTo(2));
-            assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 1, epochMillis)));
-            assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 2, epochMillis)));
-        }
-
-        // Test include failure store while we do not allow failure indices
-        // We expect an error
+        // Test explicit include failure store while not allowing selectors
+        // We expect an error because selectors are disabled and one was provided
         {
             IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN)
-                .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
-                .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowFailureIndices(false).build())
+                .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowSelectors(false).build())
                 .build();
-            FailureIndexNotSupportedException failureIndexNotSupportedException = expectThrows(
-                FailureIndexNotSupportedException.class,
-                () -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream")
-            );
-            assertThat(
-                failureIndexNotSupportedException.getIndex().getName(),
-                equalTo(DataStream.getDefaultFailureStoreName(dataStreamName, 1, epochMillis))
+            expectThrows(
+                IllegalArgumentException.class,
+                () -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream::*")
             );
         }
 
-        // Test only failure store with an exact data stream name
+        // Test explicitly selecting only failure store with an exact data stream name
         {
-            IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN)
-                .selectorOptions(IndicesOptions.SelectorOptions.FAILURES)
-                .build();
-            Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream");
+            IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
+            Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream::failures");
             assertThat(result.length, equalTo(2));
             assertThat(result[0].getName(), equalTo(DataStream.getDefaultFailureStoreName(dataStreamName, 1, epochMillis)));
             assertThat(result[1].getName(), equalTo(DataStream.getDefaultFailureStoreName(dataStreamName, 2, epochMillis)));
@@ -2821,43 +2813,6 @@ public void testDataStreamsWithFailureStore() {
             );
         }
 
-        // Test include failure store without any expressions
-        {
-            IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN)
-                .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
-                .build();
-            Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true);
-            assertThat(result.length, equalTo(5));
-            List<String> indexNames = Arrays.stream(result).map(Index::getName).toList();
-            assertThat(
-                indexNames,
-                containsInAnyOrder(
-                    DataStream.getDefaultBackingIndexName(dataStreamName, 2, epochMillis),
-                    DataStream.getDefaultBackingIndexName(dataStreamName, 1, epochMillis),
-                    DataStream.getDefaultFailureStoreName(dataStreamName, 2, epochMillis),
-                    DataStream.getDefaultFailureStoreName(dataStreamName, 1, epochMillis),
-                    otherIndex.getIndex().getName()
-                )
-            );
-        }
-
-        // Test only failure store without any expressions
-        {
-            IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN)
-                .selectorOptions(IndicesOptions.SelectorOptions.FAILURES)
-                .build();
-            Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true);
-            assertThat(result.length, equalTo(2));
-            List<String> indexNames = Arrays.stream(result).map(Index::getName).toList();
-            assertThat(
-                indexNames,
-                containsInAnyOrder(
-                    DataStream.getDefaultFailureStoreName(dataStreamName, 2, epochMillis),
-                    DataStream.getDefaultFailureStoreName(dataStreamName, 1, epochMillis)
-                )
-            );
-        }
-
         // Test default with wildcard expression
         {
             IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
@@ -2874,12 +2829,10 @@ public void testDataStreamsWithFailureStore() {
             );
         }
 
-        // Test include failure store with wildcard expression
+        // Test explicit include failure store with wildcard expression
         {
-            IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN)
-                .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
-                .build();
-            Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-*");
+            IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
+            Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-*::*");
             assertThat(result.length, equalTo(5));
             List<String> indexNames = Arrays.stream(result).map(Index::getName).toList();
             assertThat(
@@ -2894,12 +2847,10 @@ public void testDataStreamsWithFailureStore() {
             );
         }
 
-        // Test only failure store with wildcard expression
+        // Test explicit only failure store with wildcard expression
         {
-            IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN)
-                .selectorOptions(IndicesOptions.SelectorOptions.FAILURES)
-                .build();
-            Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-*");
+            IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
+            Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-*::failures");
             assertThat(result.length, equalTo(2));
             List<String> indexNames = Arrays.stream(result).map(Index::getName).toList();
             assertThat(
@@ -3229,29 +3180,63 @@ public void testDataStreamsNames() {
             )
             .build();
 
+        List<ResolvedExpression> streams = indexNameExpressionResolver.dataStreams(state, IndicesOptions.lenientExpand(), "log*");
         List<String> names = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.lenientExpand(), "log*");
+        assertEquals(Collections.singletonList(new ResolvedExpression(dataStream1, DATA)), streams);
         assertEquals(Collections.singletonList(dataStream1), names);
 
+        streams = indexNameExpressionResolver.dataStreams(state, IndicesOptions.lenientExpand(), dataStream1);
         names = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.lenientExpand(), dataStream1);
+        assertEquals(Collections.singletonList(new ResolvedExpression(dataStream1, DATA)), streams);
         assertEquals(Collections.singletonList(dataStream1), names);
 
+        streams = indexNameExpressionResolver.dataStreams(state, IndicesOptions.lenientExpand(), "other*");
         names = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.lenientExpand(), "other*");
+        assertEquals(Collections.singletonList(new ResolvedExpression(dataStream2, DATA)), streams);
         assertEquals(Collections.singletonList(dataStream2), names);
 
+        streams = indexNameExpressionResolver.dataStreams(state, IndicesOptions.lenientExpand(), "*foobar");
         names = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.lenientExpand(), "*foobar");
+        assertThat(streams, containsInAnyOrder(new ResolvedExpression(dataStream1, DATA), new ResolvedExpression(dataStream2, DATA)));
         assertThat(names, containsInAnyOrder(dataStream1, dataStream2));
 
+        streams = indexNameExpressionResolver.dataStreams(state, IndicesOptions.lenientExpand(), "notmatched");
         names = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.lenientExpand(), "notmatched");
+        assertThat(streams, empty());
         assertThat(names, empty());
 
+        streams = indexNameExpressionResolver.dataStreams(state, IndicesOptions.lenientExpand(), index3.getIndex().getName());
         names = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.lenientExpand(), index3.getIndex().getName());
+        assertThat(streams, empty());
         assertThat(names, empty());
 
+        streams = indexNameExpressionResolver.dataStreams(state, IndicesOptions.lenientExpand(), "*", "-logs-foobar");
         names = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.lenientExpand(), "*", "-logs-foobar");
+        assertThat(streams, containsInAnyOrder(new ResolvedExpression(dataStream2, DATA)));
         assertThat(names, containsInAnyOrder(dataStream2));
 
+        streams = indexNameExpressionResolver.dataStreams(state, IndicesOptions.lenientExpand(), "*", "-*");
         names = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.lenientExpand(), "*", "-*");
+        assertThat(streams, empty());
         assertThat(names, empty());
+
+        streams = indexNameExpressionResolver.dataStreams(state, IndicesOptions.strictExpandOpenAndForbidClosed(), "*foobar");
+        names = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.strictExpandOpenAndForbidClosed(), "*foobar");
+        assertThat(streams, containsInAnyOrder(new ResolvedExpression(dataStream1, DATA), new ResolvedExpression(dataStream2, DATA)));
+        assertThat(names, containsInAnyOrder(dataStream1, dataStream2));
+
+        streams = indexNameExpressionResolver.dataStreams(state, IndicesOptions.lenientExpand(), "*foobar::*");
+        names = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.lenientExpand(), "*foobar::*");
+        assertThat(
+            streams,
+            containsInAnyOrder(
+                new ResolvedExpression(dataStream1, DATA),
+                new ResolvedExpression(dataStream1, FAILURES),
+                new ResolvedExpression(dataStream2, DATA),
+                new ResolvedExpression(dataStream2, FAILURES)
+            )
+        );
+        assertThat(names, containsInAnyOrder(dataStream1, dataStream2));
     }
 
     public void testDateMathMixedArray() {
@@ -3285,10 +3270,10 @@ public void testDateMathMixedArray() {
         assertThat(
             result,
             contains(
-                new ResolvedExpression("name1"),
-                new ResolvedExpression(dataMathIndex1),
-                new ResolvedExpression("name2"),
-                new ResolvedExpression(dateMathIndex2)
+                new ResolvedExpression("name1", DATA),
+                new ResolvedExpression(dataMathIndex1, DATA),
+                new ResolvedExpression("name2", DATA),
+                new ResolvedExpression(dateMathIndex2, DATA)
             )
         );
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java
index c0e397c9fb9c9..3ed74392f746e 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java
@@ -13,7 +13,6 @@
 import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceAlreadyExistsException;
-import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.Alias;
@@ -43,7 +42,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexModule;
@@ -369,11 +367,7 @@ public void testValidateSplitIndex() {
     }
 
     public void testPrepareResizeIndexSettings() {
-        final List<IndexVersion> versions = Stream.of(IndexVersionUtils.randomVersion(random()), IndexVersionUtils.randomVersion(random()))
-            .sorted()
-            .toList();
-        final IndexVersion version = versions.get(0);
-        final IndexVersion upgraded = versions.get(1);
+        final IndexVersion version = IndexVersionUtils.randomWriteVersion();
         final Settings.Builder indexSettingsBuilder = Settings.builder()
             .put("index.version.created", version)
             .put("index.similarity.default.type", "BM25")
@@ -567,6 +561,9 @@ public void testValidateIndexName() throws Exception {
             validateIndexName(checkerService, "..", "must not be '.' or '..'");
 
             validateIndexName(checkerService, "foo:bar", "must not contain ':'");
+
+            validateIndexName(checkerService, "", "must not be empty");
+            validateIndexName(checkerService, null, "must not be empty");
         }));
     }
 
@@ -1263,16 +1260,7 @@ public void testBuildIndexMetadata() {
 
         Settings indexSettings = indexSettings(IndexVersion.current(), 1, 0).build();
         List<AliasMetadata> aliases = List.of(AliasMetadata.builder("alias1").build());
-        IndexMetadata indexMetadata = buildIndexMetadata(
-            "test",
-            aliases,
-            () -> null,
-            indexSettings,
-            4,
-            sourceIndexMetadata,
-            false,
-            TransportVersion.current()
-        );
+        IndexMetadata indexMetadata = buildIndexMetadata("test", aliases, () -> null, indexSettings, 4, sourceIndexMetadata, false);
 
         assertThat(indexMetadata.getAliases().size(), is(1));
         assertThat(indexMetadata.getAliases().keySet().iterator().next(), is("alias1"));
@@ -1281,35 +1269,6 @@ public void testBuildIndexMetadata() {
         assertThat(indexMetadata.getEventIngestedRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
     }
 
-    public void testBuildIndexMetadataWithTransportVersionBeforeEventIngestedRangeAdded() {
-        IndexMetadata sourceIndexMetadata = IndexMetadata.builder("parent")
-            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build())
-            .numberOfShards(1)
-            .numberOfReplicas(0)
-            .primaryTerm(0, 3L)
-            .build();
-
-        Settings indexSettings = indexSettings(IndexVersion.current(), 1, 0).build();
-        List<AliasMetadata> aliases = List.of(AliasMetadata.builder("alias1").build());
-        IndexMetadata indexMetadata = buildIndexMetadata(
-            "test",
-            aliases,
-            () -> null,
-            indexSettings,
-            4,
-            sourceIndexMetadata,
-            false,
-            TransportVersions.V_8_0_0
-        );
-
-        assertThat(indexMetadata.getAliases().size(), is(1));
-        assertThat(indexMetadata.getAliases().keySet().iterator().next(), is("alias1"));
-        assertThat("The source index primary term must be used", indexMetadata.primaryTerm(0), is(3L));
-        assertThat(indexMetadata.getTimestampRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
-        // on versions before event.ingested was added to cluster state, it should default to UNKNOWN, not NO_SHARDS
-        assertThat(indexMetadata.getEventIngestedRange(), equalTo(IndexLongFieldRange.UNKNOWN));
-    }
-
     public void testGetIndexNumberOfRoutingShardsWithNullSourceIndex() {
         Settings indexSettings = Settings.builder()
             .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
@@ -1500,8 +1459,6 @@ public void testRejectTranslogRetentionSettings() {
         );
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT)
-    @AwaitsFix(bugUrl = "looks like a test that's not applicable to 9.0 after version bump")
     public void testDeprecateTranslogRetentionSettings() {
         request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
         final Settings.Builder settings = Settings.builder();
@@ -1589,7 +1546,7 @@ public void testClusterStateCreateIndexWithClusterBlockTransformer() {
                     .numberOfShards(1)
                     .numberOfReplicas(nbReplicas)
                     .build()
-                    .withTimestampRanges(IndexLongFieldRange.UNKNOWN, IndexLongFieldRange.UNKNOWN, minTransportVersion),
+                    .withTimestampRanges(IndexLongFieldRange.UNKNOWN, IndexLongFieldRange.UNKNOWN),
                 null,
                 MetadataCreateIndexService.createClusterBlocksTransformerForIndexCreation(settings),
                 TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java
index 0354b6f0bcea8..3ada92dbe7ae5 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java
@@ -97,7 +97,7 @@ public void testDeleteSnapshotting() {
                 Map.of(),
                 null,
                 SnapshotInfoTestUtils.randomUserMetadata(),
-                IndexVersionUtils.randomVersion(random())
+                IndexVersionUtils.randomVersion()
             )
         );
         ClusterState state = ClusterState.builder(clusterState(index)).putCustom(SnapshotsInProgress.TYPE, snaps).build();
@@ -153,7 +153,7 @@ public void testDeleteIndexWithAnAlias() {
         String alias = randomAlphaOfLength(5);
 
         IndexMetadata idxMetadata = IndexMetadata.builder(index)
-            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion(random())))
+            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion()))
             .putAlias(AliasMetadata.builder(alias).writeIndex(true).build())
             .numberOfShards(1)
             .numberOfReplicas(1)
@@ -348,7 +348,7 @@ public void testDeleteCurrentWriteFailureIndexForDataStream() {
 
     private ClusterState clusterState(String index) {
         IndexMetadata indexMetadata = IndexMetadata.builder(index)
-            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion(random())))
+            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion()))
             .numberOfShards(1)
             .numberOfReplicas(1)
             .build();
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java
index 4f2c84d76b5a4..22ddb5cc2ba35 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java
@@ -744,7 +744,7 @@ private ClusterState applyHiddenAliasMix(ClusterState before, Boolean isHidden1,
 
     private ClusterState createIndex(ClusterState state, String index) {
         IndexMetadata indexMetadata = IndexMetadata.builder(index)
-            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion(random())))
+            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomWriteVersion()))
             .numberOfShards(1)
             .numberOfReplicas(1)
             .build();
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceBatchingTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceBatchingTests.java
index 272fad595971a..e07fc038949ce 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceBatchingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceBatchingTests.java
@@ -80,8 +80,7 @@ public void testBatchOpenIndices() throws Exception {
         block1.run(); // release block
 
         // assert that the requests were acknowledged
-        assertAcked(future1.get());
-        assertAcked(future2.get());
+        assertAcked(future1, future2);
 
         // and assert that all the indices are open
         for (String index : List.of("test-1", "test-2", "test-3")) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java
index db5a98a4878ca..bd11e636d51c1 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java
@@ -377,7 +377,7 @@ private static ClusterState addSnapshotIndex(final String index, final int numSh
             shardsBuilder,
             null,
             SnapshotInfoTestUtils.randomUserMetadata(),
-            IndexVersionUtils.randomVersion(random())
+            IndexVersionUtils.randomVersion()
         );
         return ClusterState.builder(newState).putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY.withAddedEntry(entry)).build();
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java
index ba1f9f01f49d2..8f9c1826e97de 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
@@ -27,13 +28,12 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Predicates;
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -71,6 +71,7 @@
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
@@ -122,16 +123,16 @@ public void testFindAliases() {
             .build();
 
         {
-            GetAliasesRequest request = new GetAliasesRequest();
+            GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
             Map<String, List<AliasMetadata>> aliases = metadata.findAliases(request.aliases(), Strings.EMPTY_ARRAY);
             assertThat(aliases, anEmptyMap());
         }
         {
             final GetAliasesRequest request;
             if (randomBoolean()) {
-                request = new GetAliasesRequest();
+                request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
             } else {
-                request = new GetAliasesRequest(randomFrom("alias1", "alias2"));
+                request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, randomFrom("alias1", "alias2"));
                 // replacing with empty aliases behaves as if aliases were unspecified at request building
                 request.replaceAliases(Strings.EMPTY_ARRAY);
             }
@@ -141,7 +142,7 @@ public void testFindAliases() {
             assertThat(aliasMetadataList, transformedItemsMatch(AliasMetadata::alias, contains("alias1", "alias2")));
         }
         {
-            GetAliasesRequest request = new GetAliasesRequest("alias*");
+            GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "alias*");
             Map<String, List<AliasMetadata>> aliases = metadata.findAliases(request.aliases(), new String[] { "index", "index2" });
             assertThat(aliases, aMapWithSize(2));
             List<AliasMetadata> indexAliasMetadataList = aliases.get("index");
@@ -150,7 +151,7 @@ public void testFindAliases() {
             assertThat(index2AliasMetadataList, transformedItemsMatch(AliasMetadata::alias, contains("alias2", "alias3")));
         }
         {
-            GetAliasesRequest request = new GetAliasesRequest("alias1");
+            GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "alias1");
             Map<String, List<AliasMetadata>> aliases = metadata.findAliases(request.aliases(), new String[] { "index" });
             assertThat(aliases, aMapWithSize(1));
             List<AliasMetadata> aliasMetadataList = aliases.get("index");
@@ -183,19 +184,19 @@ public void testFindDataStreamAliases() {
         Metadata metadata = builder.build();
 
         {
-            GetAliasesRequest request = new GetAliasesRequest();
+            GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
             Map<String, List<DataStreamAlias>> aliases = metadata.findDataStreamAliases(request.aliases(), Strings.EMPTY_ARRAY);
             assertThat(aliases, anEmptyMap());
         }
 
         {
-            GetAliasesRequest request = new GetAliasesRequest().aliases("alias1");
+            GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("alias1");
             Map<String, List<DataStreamAlias>> aliases = metadata.findDataStreamAliases(request.aliases(), new String[] { "index" });
             assertThat(aliases, anEmptyMap());
         }
 
         {
-            GetAliasesRequest request = new GetAliasesRequest().aliases("alias1");
+            GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("alias1");
             Map<String, List<DataStreamAlias>> aliases = metadata.findDataStreamAliases(
                 request.aliases(),
                 new String[] { "index", "d1", "d2" }
@@ -206,7 +207,7 @@ public void testFindDataStreamAliases() {
         }
 
         {
-            GetAliasesRequest request = new GetAliasesRequest().aliases("ali*");
+            GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("ali*");
             Map<String, List<DataStreamAlias>> aliases = metadata.findDataStreamAliases(request.aliases(), new String[] { "index", "d2" });
             assertEquals(1, aliases.size());
             List<DataStreamAlias> found = aliases.get("d2");
@@ -215,7 +216,7 @@ public void testFindDataStreamAliases() {
 
         // test exclusion
         {
-            GetAliasesRequest request = new GetAliasesRequest().aliases("*");
+            GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("*");
             Map<String, List<DataStreamAlias>> aliases = metadata.findDataStreamAliases(
                 request.aliases(),
                 new String[] { "index", "d1", "d2", "d3", "d4" }
@@ -276,7 +277,7 @@ public void testFindAliasWithExclusion() {
                     .putAlias(AliasMetadata.builder("alias3").build())
             )
             .build();
-        GetAliasesRequest request = new GetAliasesRequest().aliases("*", "-alias1");
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("*", "-alias1");
         Map<String, List<AliasMetadata>> aliases = metadata.findAliases(request.aliases(), new String[] { "index", "index2" });
         assertThat(aliases.get("index"), transformedItemsMatch(AliasMetadata::alias, contains("alias2")));
         assertThat(aliases.get("index2"), transformedItemsMatch(AliasMetadata::alias, contains("alias3")));
@@ -311,7 +312,7 @@ public void testFindAliasWithExclusionAndOverride() {
                     .putAlias(AliasMetadata.builder("bb").build())
             )
             .build();
-        GetAliasesRequest request = new GetAliasesRequest().aliases("a*", "-*b", "b*");
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("a*", "-*b", "b*");
         List<AliasMetadata> aliases = metadata.findAliases(request.aliases(), new String[] { "index" }).get("index");
         assertThat(aliases, transformedItemsMatch(AliasMetadata::alias, contains("aa", "bb")));
     }
@@ -1985,8 +1986,6 @@ public void testHiddenAliasValidation() {
         }
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT)
-    @AwaitsFix(bugUrl = "this test needs to be updated or removed after the version 9.0 bump")
     public void testSystemAliasValidationMixedVersionSystemAndRegularFails() {
         final IndexVersion random7xVersion = IndexVersionUtils.randomVersionBetween(
             random(),
@@ -2037,8 +2036,6 @@ public void testSystemAliasValidationNewSystemAndRegularFails() {
         );
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT)
-    @AwaitsFix(bugUrl = "this test needs to be updated or removed after the version 9.0 bump")
     public void testSystemAliasOldSystemAndNewRegular() {
         final IndexVersion random7xVersion = IndexVersionUtils.randomVersionBetween(
             random(),
@@ -2052,8 +2049,6 @@ public void testSystemAliasOldSystemAndNewRegular() {
         metadataWithIndices(oldVersionSystem, regularIndex);
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT)
-    @AwaitsFix(bugUrl = "this test needs to be updated or removed after the version 9.0 bump")
     public void testSystemIndexValidationAllRegular() {
         final IndexVersion random7xVersion = IndexVersionUtils.randomVersionBetween(
             random(),
@@ -2068,8 +2063,6 @@ public void testSystemIndexValidationAllRegular() {
         metadataWithIndices(currentVersionSystem, currentVersionSystem2, oldVersionSystem);
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT)
-    @AwaitsFix(bugUrl = "this test needs to be updated or removed after the version 9.0 bump")
     public void testSystemAliasValidationAllSystemSomeOld() {
         final IndexVersion random7xVersion = IndexVersionUtils.randomVersionBetween(
             random(),
@@ -2307,32 +2300,30 @@ public static int expectedChunkCount(ToXContent.Params params, Metadata metadata
             chunkCount += 2;
 
             if (custom instanceof ComponentTemplateMetadata componentTemplateMetadata) {
-                chunkCount += checkChunkSize(custom, params, 2 + componentTemplateMetadata.componentTemplates().size());
+                chunkCount += 2 + componentTemplateMetadata.componentTemplates().size();
             } else if (custom instanceof ComposableIndexTemplateMetadata composableIndexTemplateMetadata) {
-                chunkCount += checkChunkSize(custom, params, 2 + composableIndexTemplateMetadata.indexTemplates().size());
+                chunkCount += 2 + composableIndexTemplateMetadata.indexTemplates().size();
             } else if (custom instanceof DataStreamMetadata dataStreamMetadata) {
-                chunkCount += checkChunkSize(
-                    custom,
-                    params,
-                    4 + dataStreamMetadata.dataStreams().size() + dataStreamMetadata.getDataStreamAliases().size()
-                );
+                chunkCount += 4 + dataStreamMetadata.dataStreams().size() + dataStreamMetadata.getDataStreamAliases().size();
             } else if (custom instanceof DesiredNodesMetadata) {
-                chunkCount += checkChunkSize(custom, params, 1);
+                chunkCount += 1;
             } else if (custom instanceof FeatureMigrationResults featureMigrationResults) {
-                chunkCount += checkChunkSize(custom, params, 2 + featureMigrationResults.getFeatureStatuses().size());
+                chunkCount += 2 + featureMigrationResults.getFeatureStatuses().size();
             } else if (custom instanceof IndexGraveyard indexGraveyard) {
-                chunkCount += checkChunkSize(custom, params, 2 + indexGraveyard.getTombstones().size());
+                chunkCount += 2 + indexGraveyard.getTombstones().size();
             } else if (custom instanceof IngestMetadata ingestMetadata) {
-                chunkCount += checkChunkSize(custom, params, 2 + ingestMetadata.getPipelines().size());
+                chunkCount += 2 + ingestMetadata.getPipelines().size();
             } else if (custom instanceof NodesShutdownMetadata nodesShutdownMetadata) {
-                chunkCount += checkChunkSize(custom, params, 2 + nodesShutdownMetadata.getAll().size());
+                chunkCount += 2 + nodesShutdownMetadata.getAll().size();
             } else if (custom instanceof PersistentTasksCustomMetadata persistentTasksCustomMetadata) {
-                chunkCount += checkChunkSize(custom, params, 3 + persistentTasksCustomMetadata.tasks().size());
+                chunkCount += 3 + persistentTasksCustomMetadata.tasks().size();
             } else if (custom instanceof RepositoriesMetadata repositoriesMetadata) {
-                chunkCount += checkChunkSize(custom, params, repositoriesMetadata.repositories().size());
+                chunkCount += repositoriesMetadata.repositories().size();
             } else {
                 // could be anything, we have to just try it
-                chunkCount += count(custom.toXContentChunked(params));
+                chunkCount += Iterables.size(
+                    (Iterable<ToXContent>) (() -> Iterators.map(custom.toXContentChunked(params), Function.identity()))
+                );
             }
         }
 
@@ -2344,21 +2335,6 @@ public static int expectedChunkCount(ToXContent.Params params, Metadata metadata
         return Math.toIntExact(chunkCount);
     }
 
-    private static long count(Iterator<?> it) {
-        long count = 0;
-        while (it.hasNext()) {
-            count++;
-            it.next();
-        }
-        return count;
-    }
-
-    private static long checkChunkSize(ChunkedToXContent custom, ToXContent.Params params, long expectedSize) {
-        long actualSize = count(custom.toXContentChunked(params));
-        assertThat(actualSize, equalTo(expectedSize));
-        return actualSize;
-    }
-
     /**
      * With this test we ensure that we consider whether a new field added to Metadata should be checked
      * in Metadata.isGlobalStateEquals. We force the instance fields to be either in the checked list
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java
index f3c38efb09ce9..8a801d1ece816 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java
@@ -78,6 +78,7 @@ public void testIsNodeShuttingDown() {
                     "this_node",
                     SingleNodeShutdownMetadata.builder()
                         .setNodeId("this_node")
+                        .setNodeEphemeralId("this_node")
                         .setReason("shutdown for a unit test")
                         .setType(type)
                         .setStartedAtMillis(randomNonNegativeLong())
@@ -106,6 +107,7 @@ public void testIsNodeShuttingDown() {
     public void testSigtermIsRemoveInOlderVersions() throws IOException {
         SingleNodeShutdownMetadata metadata = SingleNodeShutdownMetadata.builder()
             .setNodeId("myid")
+            .setNodeEphemeralId("myid")
             .setType(SingleNodeShutdownMetadata.Type.SIGTERM)
             .setReason("myReason")
             .setStartedAtMillis(0L)
@@ -127,6 +129,7 @@ public void testIsNodeMarkedForRemoval() {
         SingleNodeShutdownMetadata.Type type;
         SingleNodeShutdownMetadata.Builder builder = SingleNodeShutdownMetadata.builder()
             .setNodeId("thenode")
+            .setNodeEphemeralId("thenode")
             .setReason("myReason")
             .setStartedAtMillis(0L);
         switch (type = randomFrom(SingleNodeShutdownMetadata.Type.values())) {
@@ -182,6 +185,7 @@ private SingleNodeShutdownMetadata randomNodeShutdownInfo() {
         final SingleNodeShutdownMetadata.Type type = randomFrom(SingleNodeShutdownMetadata.Type.values());
         final SingleNodeShutdownMetadata.Builder builder = SingleNodeShutdownMetadata.builder()
             .setNodeId(randomAlphaOfLength(5))
+            .setNodeEphemeralId(randomAlphaOfLength(5))
             .setType(type)
             .setReason(randomAlphaOfLength(5))
             .setStartedAtMillis(randomNonNegativeLong());
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/SelectorResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/SelectorResolverTests.java
new file mode 100644
index 0000000000000..2bf34dcfd2a34
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/SelectorResolverTests.java
@@ -0,0 +1,183 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.action.support.IndexComponentSelector;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SelectorResolver;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.indices.InvalidIndexNameException;
+import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.elasticsearch.action.support.IndexComponentSelector.ALL_APPLICABLE;
+import static org.elasticsearch.action.support.IndexComponentSelector.DATA;
+import static org.elasticsearch.action.support.IndexComponentSelector.FAILURES;
+import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.Context;
+import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+
+public class SelectorResolverTests extends ESTestCase {
+
+    public void testResolveExpression() {
+        // === Parsing and defaults
+        // Allow selectors TRUE
+        Context selectorsAllowed = getContext(getOptionsForSelectors());
+
+        assertThat(resolve(selectorsAllowed, "testXXX"), equalTo(new ResolvedExpression("testXXX", DATA)));
+        assertThat(resolve(selectorsAllowed, "testXXX::data"), equalTo(new ResolvedExpression("testXXX", DATA)));
+        assertThat(resolve(selectorsAllowed, "testXXX::failures"), equalTo(new ResolvedExpression("testXXX", FAILURES)));
+        assertThat(resolve(selectorsAllowed, "testXXX::*"), equalTo(new ResolvedExpression("testXXX", ALL_APPLICABLE)));
+
+        // Disallow selectors (example: creating, modifying, or deleting indices/data streams/aliases).
+        // Accepts standard expressions but throws when selectors are specified.
+        Context noSelectors = getContext(getDisabledSelectorOptions());
+
+        assertThat(resolve(noSelectors, "testXXX"), equalTo(new ResolvedExpression("testXXX")));
+        expectThrows(IllegalArgumentException.class, () -> resolve(noSelectors, "testXXX::data"));
+        expectThrows(IllegalArgumentException.class, () -> resolve(noSelectors, "testXXX::failures"));
+        expectThrows(IllegalArgumentException.class, () -> resolve(noSelectors, "testXXX::*"));
+
+        // === Errors
+        // Only recognized components can be selected
+        expectThrows(InvalidIndexNameException.class, () -> resolve(selectorsAllowed, "testXXX::custom"));
+        // Spelling is important
+        expectThrows(InvalidIndexNameException.class, () -> resolve(selectorsAllowed, "testXXX::failres"));
+        // Only the match all wildcard is supported
+        expectThrows(InvalidIndexNameException.class, () -> resolve(selectorsAllowed, "testXXX::d*ta"));
+        // Only one selector separator is allowed per expression
+        expectThrows(InvalidIndexNameException.class, () -> resolve(selectorsAllowed, "test::XXX::data"));
+
+        // === Wildcards, Date Math, and edge cases
+        // Wildcards are left as-is (handled in wildcard resolver)
+        assertThat(resolve(selectorsAllowed, "*"), equalTo(new ResolvedExpression("*", DATA)));
+        // Exclusions are left as-is (if no wildcards are present they are not treated as exclusions)
+        assertThat(resolve(selectorsAllowed, "-testXXX"), equalTo(new ResolvedExpression("-testXXX", DATA)));
+        // Exclusion syntax with selectors will have the selectors parsed
+        assertThat(resolve(selectorsAllowed, "-testXXX::failures"), equalTo(new ResolvedExpression("-testXXX", FAILURES)));
+        // Date math is left unprocessed (handled later in date math resolver)
+        assertThat(resolve(selectorsAllowed, "<test-{now/d}>"), equalTo(new ResolvedExpression("<test-{now/d}>", DATA)));
+        // Providing a selector requires adding after the date math brackets
+        assertThat(resolve(selectorsAllowed, "<test-{now/d}>::failures"), equalTo(new ResolvedExpression("<test-{now/d}>", FAILURES)));
+        // Selectors inside of date math expressions will trip an exception because they do not match an existing component name exactly
+        expectThrows(InvalidIndexNameException.class, () -> resolve(selectorsAllowed, "<test-{now/d}::failures>"));
+
+        // === Corner Cases
+        // Empty index name is not necessarily disallowed, but will be filtered out in the next steps of resolution
+        assertThat(resolve(selectorsAllowed, "::data"), equalTo(new ResolvedExpression("", DATA)));
+        // Remote cluster syntax is respected, even if code higher up the call stack is likely to already have handled it already
+        assertThat(resolve(selectorsAllowed, "cluster:index::data"), equalTo(new ResolvedExpression("cluster:index", DATA)));
+        // CCS with an empty index name is not necessarily disallowed, though other code in the resolution logic will likely throw
+        assertThat(resolve(selectorsAllowed, "cluster:::data"), equalTo(new ResolvedExpression("cluster:", DATA)));
+        // Same for empty cluster and index names
+        assertThat(resolve(selectorsAllowed, ":::data"), equalTo(new ResolvedExpression(":", DATA)));
+        // Any more prefix colon characters will trigger the multiple separators error logic
+        expectThrows(InvalidIndexNameException.class, () -> resolve(selectorsAllowed, "::::data"));
+        // Suffix case is not supported because there is no component named with the empty string
+        expectThrows(InvalidIndexNameException.class, () -> resolve(selectorsAllowed, "index::"));
+    }
+
+    public void testResolveMatchAllToSelectors() {
+        Context selectorsAllowed = getContext(getOptionsForSelectors());
+        assertThat(resolveMatchAllToSelector(selectorsAllowed, "*"), is(DATA));
+        assertThat(resolveMatchAllToSelector(selectorsAllowed, "*::data"), is(DATA));
+        assertThat(resolveMatchAllToSelector(selectorsAllowed, "*::failures"), is(FAILURES));
+        assertThat(resolveMatchAllToSelector(selectorsAllowed, "_all"), is(DATA));
+        assertThat(resolveMatchAllToSelector(selectorsAllowed, "_all::data"), is(DATA));
+        assertThat(resolveMatchAllToSelector(selectorsAllowed, "_all::failures"), is(FAILURES));
+
+        Context selectorsDisallowed = getContext(getDisabledSelectorOptions());
+        assertThat(resolveMatchAllToSelector(selectorsDisallowed, "*"), is(nullValue()));
+        expectThrows(IllegalArgumentException.class, () -> resolveMatchAllToSelector(selectorsDisallowed, "*::data"));
+        expectThrows(IllegalArgumentException.class, () -> resolveMatchAllToSelector(selectorsDisallowed, "*::failures"));
+        assertThat(resolveMatchAllToSelector(selectorsDisallowed, "_all"), is(nullValue()));
+        expectThrows(IllegalArgumentException.class, () -> resolveMatchAllToSelector(selectorsDisallowed, "_all::data"));
+        expectThrows(IllegalArgumentException.class, () -> resolveMatchAllToSelector(selectorsDisallowed, "_all::failures"));
+    }
+
+    public void testCombineExpressionWithSelector() {
+        expectThrows(NullPointerException.class, () -> IndexNameExpressionResolver.combineSelectorExpression(null, null));
+        expectThrows(NullPointerException.class, () -> IndexNameExpressionResolver.combineSelectorExpression(null, ""));
+        expectThrows(NullPointerException.class, () -> IndexNameExpressionResolver.combineSelectorExpression(null, "a"));
+        expectThrows(NullPointerException.class, () -> IndexNameExpressionResolver.combineSelectorExpression(null, "*"));
+        assertThat(IndexNameExpressionResolver.combineSelectorExpression("", null), is(equalTo("")));
+        assertThat(IndexNameExpressionResolver.combineSelectorExpression("", ""), is(equalTo("::")));
+        assertThat(IndexNameExpressionResolver.combineSelectorExpression("a", null), is(equalTo("a")));
+        assertThat(IndexNameExpressionResolver.combineSelectorExpression("a", ""), is(equalTo("a::")));
+        assertThat(IndexNameExpressionResolver.combineSelectorExpression("a", "b"), is(equalTo("a::b")));
+        assertThat(IndexNameExpressionResolver.combineSelectorExpression("a", "*"), is(equalTo("a::*")));
+        assertThat(IndexNameExpressionResolver.combineSelectorExpression("*", "b"), is(equalTo("*::b")));
+        assertThat(IndexNameExpressionResolver.combineSelectorExpression("*", "*"), is(equalTo("*::*")));
+    }
+
+    public void testHasSelectorSuffix() {
+        assertThat(IndexNameExpressionResolver.hasSelectorSuffix(null), is(false));
+        assertThat(IndexNameExpressionResolver.hasSelectorSuffix(""), is(false));
+        assertThat(IndexNameExpressionResolver.hasSelectorSuffix("abcdefg"), is(false));
+        assertThat(IndexNameExpressionResolver.hasSelectorSuffix("*"), is(false));
+        assertThat(IndexNameExpressionResolver.hasSelectorSuffix("cluster:index"), is(false));
+        assertThat(IndexNameExpressionResolver.hasSelectorSuffix("index::data"), is(true));
+        assertThat(IndexNameExpressionResolver.hasSelectorSuffix("index::failures"), is(true));
+        assertThat(IndexNameExpressionResolver.hasSelectorSuffix("index::any"), is(true));
+        assertThat(IndexNameExpressionResolver.hasSelectorSuffix("index::*"), is(true));
+        assertThat(IndexNameExpressionResolver.hasSelectorSuffix("index::::::::::toomany"), is(true));
+        assertThat(IndexNameExpressionResolver.hasSelectorSuffix("cluster:index::data"), is(true));
+        assertThat(IndexNameExpressionResolver.hasSelectorSuffix("*:*::*"), is(true));
+        assertThat(IndexNameExpressionResolver.hasSelectorSuffix("index::value::value"), is(true));
+    }
+
+    public void testSplitSelectorExpression() {
+        expectThrows(NullPointerException.class, () -> IndexNameExpressionResolver.splitSelectorExpression(null));
+        assertThat(IndexNameExpressionResolver.splitSelectorExpression(""), is(equalTo(new Tuple<>("", null))));
+        assertThat(IndexNameExpressionResolver.splitSelectorExpression("a"), is(equalTo(new Tuple<>("a", null))));
+        assertThat(IndexNameExpressionResolver.splitSelectorExpression("*"), is(equalTo(new Tuple<>("*", null))));
+        assertThat(IndexNameExpressionResolver.splitSelectorExpression("index"), is(equalTo(new Tuple<>("index", null))));
+        assertThat(IndexNameExpressionResolver.splitSelectorExpression("cluster:index"), is(equalTo(new Tuple<>("cluster:index", null))));
+        assertThat(IndexNameExpressionResolver.splitSelectorExpression("*:index"), is(equalTo(new Tuple<>("*:index", null))));
+        assertThat(IndexNameExpressionResolver.splitSelectorExpression("cluster:*"), is(equalTo(new Tuple<>("cluster:*", null))));
+        assertThat(IndexNameExpressionResolver.splitSelectorExpression("*:*"), is(equalTo(new Tuple<>("*:*", null))));
+        assertThat(IndexNameExpressionResolver.splitSelectorExpression("*:*:*"), is(equalTo(new Tuple<>("*:*:*", null))));
+
+        assertThat(IndexNameExpressionResolver.splitSelectorExpression("a::data"), is(equalTo(new Tuple<>("a", "data"))));
+        assertThat(IndexNameExpressionResolver.splitSelectorExpression("a::failures"), is(equalTo(new Tuple<>("a", "failures"))));
+        assertThat(IndexNameExpressionResolver.splitSelectorExpression("a::*"), is(equalTo(new Tuple<>("a", "*"))));
+        expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("a::random"));
+        expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("a::d*ta"));
+        expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("a::*ailures"));
+        expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("a::"));
+        expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("a::**"));
+        expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("index::data::*"));
+        assertThat(IndexNameExpressionResolver.splitSelectorExpression("::*"), is(equalTo(new Tuple<>("", "*"))));
+    }
+
+    private static IndicesOptions getOptionsForSelectors() {
+        return IndicesOptions.builder().gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowSelectors(true)).build();
+    }
+
+    private static IndicesOptions getDisabledSelectorOptions() {
+        return IndicesOptions.builder().gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowSelectors(false)).build();
+    }
+
+    private static Context getContext(IndicesOptions indicesOptions) {
+        return new Context(mock(ClusterState.class), indicesOptions, SystemIndices.SystemIndexAccessLevel.NONE);
+    }
+
+    private static ResolvedExpression resolve(Context context, String expression) {
+        return SelectorResolver.parseExpression(expression, context.getOptions());
+    }
+
+    private static IndexComponentSelector resolveMatchAllToSelector(Context context, String matchAll) {
+        return SelectorResolver.parseMatchAllToSelector(context, matchAll);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java
index fd90e1210ce85..9d9a5ebd37218 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java
@@ -25,7 +25,11 @@
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.action.support.IndexComponentSelector.ALL_APPLICABLE;
+import static org.elasticsearch.action.support.IndexComponentSelector.DATA;
+import static org.elasticsearch.action.support.IndexComponentSelector.FAILURES;
 import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createBackingIndex;
+import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createFailureStore;
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.empty;
@@ -50,19 +54,19 @@ public void testConvertWildcardsJustIndicesTests() {
             SystemIndexAccessLevel.NONE
         );
         assertThat(
-            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "ku*")),
+            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "ku*", ALL_APPLICABLE)),
             equalTo(resolvedExpressionsSet("kuku"))
         );
         assertThat(
-            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*")),
+            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*", ALL_APPLICABLE)),
             equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY"))
         );
         assertThat(
-            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*")),
+            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", ALL_APPLICABLE)),
             equalTo(resolvedExpressionsSet("testXXX", "testXYY"))
         );
         assertThat(
-            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*")),
+            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*", ALL_APPLICABLE)),
             equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY", "kuku"))
         );
     }
@@ -83,7 +87,7 @@ public void testConvertWildcardsOpenClosedIndicesTests() {
             SystemIndexAccessLevel.NONE
         );
         assertThat(
-            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*")),
+            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", ALL_APPLICABLE)),
             equalTo(resolvedExpressionsSet("testXXX", "testXXY", "testXYY"))
         );
         context = new IndexNameExpressionResolver.Context(
@@ -92,7 +96,7 @@ public void testConvertWildcardsOpenClosedIndicesTests() {
             SystemIndexAccessLevel.NONE
         );
         assertThat(
-            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*")),
+            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", ALL_APPLICABLE)),
             equalTo(resolvedExpressionsSet("testXYY"))
         );
         context = new IndexNameExpressionResolver.Context(
@@ -101,7 +105,7 @@ public void testConvertWildcardsOpenClosedIndicesTests() {
             SystemIndexAccessLevel.NONE
         );
         assertThat(
-            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*")),
+            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", ALL_APPLICABLE)),
             equalTo(resolvedExpressionsSet("testXXX", "testXXY"))
         );
     }
@@ -124,27 +128,31 @@ public void testMultipleWildcards() {
             SystemIndexAccessLevel.NONE
         );
         assertThat(
-            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*X*")),
+            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*X*", ALL_APPLICABLE)),
             equalTo(resolvedExpressionsSet("testXXX", "testXXY", "testXYY"))
         );
         assertThat(
-            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*X*Y")),
+            newHashSet(
+                IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*X*Y", ALL_APPLICABLE)
+            ),
             equalTo(resolvedExpressionsSet("testXXY", "testXYY"))
         );
         assertThat(
-            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "kuku*Y*")),
+            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "kuku*Y*", ALL_APPLICABLE)),
             equalTo(resolvedExpressionsSet("kukuYYY"))
         );
         assertThat(
-            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*Y*")),
+            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*Y*", ALL_APPLICABLE)),
             equalTo(resolvedExpressionsSet("testXXY", "testXYY", "testYYY", "kukuYYY"))
         );
         assertThat(
-            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*Y*X")).size(),
+            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*Y*X", ALL_APPLICABLE))
+                .size(),
             equalTo(0)
         );
         assertThat(
-            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*Y*X")).size(),
+            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*Y*X", ALL_APPLICABLE))
+                .size(),
             equalTo(0)
         );
     }
@@ -156,15 +164,35 @@ public void testAll() {
             .put(indexBuilder("testYYY"));
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
 
-        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(
-            state,
-            IndicesOptions.lenientExpandOpen(),
-            SystemIndexAccessLevel.NONE
-        );
-        assertThat(
-            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)),
-            equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY"))
-        );
+        {
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(
+                state,
+                IndicesOptions.lenientExpandOpen(),
+                SystemIndexAccessLevel.NONE
+            );
+            assertThat(
+                newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, ALL_APPLICABLE)),
+                equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY"))
+            );
+        }
+
+        {
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(
+                state,
+                IndicesOptions.builder(IndicesOptions.lenientExpandOpen())
+                    .gatekeeperOptions(
+                        IndicesOptions.GatekeeperOptions.builder(IndicesOptions.lenientExpandOpen().gatekeeperOptions())
+                            .allowSelectors(false)
+                            .build()
+                    )
+                    .build(),
+                SystemIndexAccessLevel.NONE
+            );
+            assertThat(
+                newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, null)),
+                equalTo(resolvedExpressionsNoSelectorSet("testXXX", "testXYY", "testYYY"))
+            );
+        }
     }
 
     public void testAllAliases() {
@@ -184,7 +212,10 @@ public void testAllAliases() {
                 IndicesOptions.lenientExpandOpen(), // don't include hidden
                 SystemIndexAccessLevel.NONE
             );
-            assertThat(newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)), equalTo(newHashSet()));
+            assertThat(
+                newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, ALL_APPLICABLE)),
+                equalTo(newHashSet())
+            );
         }
 
         {
@@ -204,7 +235,7 @@ public void testAllAliases() {
                 SystemIndexAccessLevel.NONE
             );
             assertThat(
-                newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)),
+                newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, ALL_APPLICABLE)),
                 equalTo(resolvedExpressionsSet("index-visible-alias"))
             );
         }
@@ -215,6 +246,7 @@ public void testAllDataStreams() {
         String dataStreamName = "foo_logs";
         long epochMillis = randomLongBetween(1580536800000L, 1583042400000L);
         IndexMetadata firstBackingIndexMetadata = createBackingIndex(dataStreamName, 1, epochMillis).build();
+        IndexMetadata firstFailureIndexMetadata = createFailureStore(dataStreamName, 1, epochMillis).build();
 
         IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(
             randomBoolean(),
@@ -231,7 +263,14 @@ public void testAllDataStreams() {
             // visible data streams should be returned by _all even show backing indices are hidden
             Metadata.Builder mdBuilder = Metadata.builder()
                 .put(firstBackingIndexMetadata, true)
-                .put(DataStreamTestHelper.newInstance(dataStreamName, List.of(firstBackingIndexMetadata.getIndex())));
+                .put(firstFailureIndexMetadata, true)
+                .put(
+                    DataStreamTestHelper.newInstance(
+                        dataStreamName,
+                        List.of(firstBackingIndexMetadata.getIndex()),
+                        List.of(firstFailureIndexMetadata.getIndex())
+                    )
+                );
 
             ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
 
@@ -247,16 +286,33 @@ public void testAllDataStreams() {
             );
 
             assertThat(
-                newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)),
+                newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, DATA)),
                 equalTo(resolvedExpressionsSet(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis)))
             );
+            assertThat(
+                newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, ALL_APPLICABLE)),
+                equalTo(
+                    resolvedExpressionsSet(
+                        DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis),
+                        DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis)
+                    )
+                )
+            );
         }
 
         {
             // if data stream itself is hidden, backing indices should not be returned
-            var dataStream = DataStream.builder(dataStreamName, List.of(firstBackingIndexMetadata.getIndex())).setHidden(true).build();
+            var dataStream = DataStream.builder(dataStreamName, List.of(firstBackingIndexMetadata.getIndex()))
+                .setFailureIndices(
+                    DataStream.DataStreamIndices.failureIndicesBuilder(List.of(firstFailureIndexMetadata.getIndex())).build()
+                )
+                .setHidden(true)
+                .build();
 
-            Metadata.Builder mdBuilder = Metadata.builder().put(firstBackingIndexMetadata, true).put(dataStream);
+            Metadata.Builder mdBuilder = Metadata.builder()
+                .put(firstBackingIndexMetadata, true)
+                .put(firstFailureIndexMetadata, true)
+                .put(dataStream);
 
             ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
 
@@ -271,7 +327,11 @@ public void testAllDataStreams() {
                 NONE
             );
 
-            assertThat(newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)), equalTo(newHashSet()));
+            assertThat(newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, DATA)), equalTo(Set.of()));
+            assertThat(
+                newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, ALL_APPLICABLE)),
+                equalTo(Set.of())
+            );
         }
     }
 
@@ -394,51 +454,57 @@ public void testResolveAliases() {
         {
             Collection<ResolvedExpression> indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
                 indicesAndAliasesContext,
-                "foo_a*"
+                "foo_a*",
+                ALL_APPLICABLE
             );
-            assertThat(indices, containsInAnyOrder(new ResolvedExpression("foo_index"), new ResolvedExpression("bar_index")));
+            assertThat(indices, containsInAnyOrder(new ResolvedExpression("foo_index", DATA), new ResolvedExpression("bar_index", DATA)));
         }
         {
             Collection<ResolvedExpression> indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
                 skipAliasesLenientContext,
-                "foo_a*"
+                "foo_a*",
+                ALL_APPLICABLE
             );
             assertEquals(0, indices.size());
         }
         {
             Set<ResolvedExpression> indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
                 skipAliasesStrictContext,
-                "foo_a*"
+                "foo_a*",
+                ALL_APPLICABLE
             );
             assertThat(indices, empty());
         }
         {
             Collection<ResolvedExpression> indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
                 indicesAndAliasesContext,
-                "foo*"
+                "foo*",
+                ALL_APPLICABLE
             );
             assertThat(
                 indices,
                 containsInAnyOrder(
-                    new ResolvedExpression("foo_foo"),
-                    new ResolvedExpression("foo_index"),
-                    new ResolvedExpression("bar_index")
+                    new ResolvedExpression("foo_foo", DATA),
+                    new ResolvedExpression("foo_index", DATA),
+                    new ResolvedExpression("bar_index", DATA)
                 )
             );
         }
         {
             Collection<ResolvedExpression> indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
                 skipAliasesLenientContext,
-                "foo*"
+                "foo*",
+                ALL_APPLICABLE
             );
-            assertThat(indices, containsInAnyOrder(new ResolvedExpression("foo_foo"), new ResolvedExpression("foo_index")));
+            assertThat(indices, containsInAnyOrder(new ResolvedExpression("foo_foo", DATA), new ResolvedExpression("foo_index", DATA)));
         }
         {
             Collection<ResolvedExpression> indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
                 skipAliasesStrictContext,
-                "foo*"
+                "foo*",
+                ALL_APPLICABLE
             );
-            assertThat(indices, containsInAnyOrder(new ResolvedExpression("foo_foo"), new ResolvedExpression("foo_index")));
+            assertThat(indices, containsInAnyOrder(new ResolvedExpression("foo_foo", DATA), new ResolvedExpression("foo_index", DATA)));
         }
     }
 
@@ -447,6 +513,8 @@ public void testResolveDataStreams() {
         long epochMillis = randomLongBetween(1580536800000L, 1583042400000L);
         IndexMetadata firstBackingIndexMetadata = createBackingIndex(dataStreamName, 1, epochMillis).build();
         IndexMetadata secondBackingIndexMetadata = createBackingIndex(dataStreamName, 2, epochMillis).build();
+        IndexMetadata firstFailureIndexMetadata = createFailureStore(dataStreamName, 1, epochMillis).build();
+        IndexMetadata secondFailureIndexMetadata = createFailureStore(dataStreamName, 2, epochMillis).build();
 
         Metadata.Builder mdBuilder = Metadata.builder()
             .put(indexBuilder("foo_foo").state(State.OPEN))
@@ -454,11 +522,14 @@ public void testResolveDataStreams() {
             .put(indexBuilder("foo_index").state(State.OPEN).putAlias(AliasMetadata.builder("foo_alias")))
             .put(indexBuilder("bar_index").state(State.OPEN).putAlias(AliasMetadata.builder("foo_alias")))
             .put(firstBackingIndexMetadata, true)
+            .put(firstFailureIndexMetadata, true)
             .put(secondBackingIndexMetadata, true)
+            .put(secondFailureIndexMetadata, true)
             .put(
                 DataStreamTestHelper.newInstance(
                     dataStreamName,
-                    List.of(firstBackingIndexMetadata.getIndex(), secondBackingIndexMetadata.getIndex())
+                    List.of(firstBackingIndexMetadata.getIndex(), secondBackingIndexMetadata.getIndex()),
+                    List.of(firstFailureIndexMetadata.getIndex(), secondFailureIndexMetadata.getIndex())
                 )
             );
 
@@ -484,20 +555,25 @@ public void testResolveDataStreams() {
             // data streams are not included but expression matches the data stream
             Collection<ResolvedExpression> indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
                 indicesAndAliasesContext,
-                "foo_*"
+                "foo_*",
+                ALL_APPLICABLE
             );
             assertThat(
                 indices,
                 containsInAnyOrder(
-                    new ResolvedExpression("foo_index"),
-                    new ResolvedExpression("foo_foo"),
-                    new ResolvedExpression("bar_index")
+                    new ResolvedExpression("foo_index", DATA),
+                    new ResolvedExpression("foo_foo", DATA),
+                    new ResolvedExpression("bar_index", DATA)
                 )
             );
 
             // data streams are not included and expression doesn't match the data steram
-            indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(indicesAndAliasesContext, "bar_*");
-            assertThat(indices, containsInAnyOrder(new ResolvedExpression("bar_bar"), new ResolvedExpression("bar_index")));
+            indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
+                indicesAndAliasesContext,
+                "bar_*",
+                ALL_APPLICABLE
+            );
+            assertThat(indices, containsInAnyOrder(new ResolvedExpression("bar_bar", DATA), new ResolvedExpression("bar_index", DATA)));
         }
 
         {
@@ -525,33 +601,55 @@ public void testResolveDataStreams() {
             // data stream's corresponding backing indices are resolved
             Collection<ResolvedExpression> indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
                 indicesAliasesAndDataStreamsContext,
-                "foo_*"
+                "foo_*",
+                ALL_APPLICABLE
             );
             assertThat(
                 indices,
                 containsInAnyOrder(
-                    new ResolvedExpression("foo_index"),
-                    new ResolvedExpression("bar_index"),
-                    new ResolvedExpression("foo_foo"),
-                    new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis)),
-                    new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis))
+                    new ResolvedExpression("foo_index", DATA),
+                    new ResolvedExpression("bar_index", DATA),
+                    new ResolvedExpression("foo_foo", DATA),
+                    new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), DATA),
+                    new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis), DATA),
+                    new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis), DATA),
+                    new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis), DATA)
+                )
+            );
+
+            // data stream's corresponding failure indices are resolved
+            indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
+                indicesAliasesAndDataStreamsContext,
+                "foo_*",
+                FAILURES
+            );
+            assertThat(
+                newHashSet(indices),
+                equalTo(
+                    resolvedExpressionsSet(
+                        DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis),
+                        DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis)
+                    )
                 )
             );
 
             // include all wildcard adds the data stream's backing indices
             indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
                 indicesAliasesAndDataStreamsContext,
-                "*"
+                "*",
+                ALL_APPLICABLE
             );
             assertThat(
                 indices,
                 containsInAnyOrder(
-                    new ResolvedExpression("foo_index"),
-                    new ResolvedExpression("bar_index"),
-                    new ResolvedExpression("foo_foo"),
-                    new ResolvedExpression("bar_bar"),
-                    new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis)),
-                    new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis))
+                    new ResolvedExpression("foo_index", DATA),
+                    new ResolvedExpression("bar_index", DATA),
+                    new ResolvedExpression("foo_foo", DATA),
+                    new ResolvedExpression("bar_bar", DATA),
+                    new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), DATA),
+                    new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis), DATA),
+                    new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis), DATA),
+                    new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis), DATA)
                 )
             );
         }
@@ -582,33 +680,116 @@ public void testResolveDataStreams() {
             // data stream's corresponding backing indices are resolved
             Collection<ResolvedExpression> indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
                 indicesAliasesDataStreamsAndHiddenIndices,
-                "foo_*"
+                "foo_*",
+                ALL_APPLICABLE
             );
             assertThat(
                 indices,
                 containsInAnyOrder(
-                    new ResolvedExpression("foo_index"),
-                    new ResolvedExpression("bar_index"),
-                    new ResolvedExpression("foo_foo"),
-                    new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis)),
-                    new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis))
+                    new ResolvedExpression("foo_index", DATA),
+                    new ResolvedExpression("bar_index", DATA),
+                    new ResolvedExpression("foo_foo", DATA),
+                    new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), DATA),
+                    new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis), DATA),
+                    new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis), DATA),
+                    new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis), DATA)
+                )
+            );
+
+            // only data stream's corresponding failure indices are resolved
+            indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
+                indicesAliasesDataStreamsAndHiddenIndices,
+                "foo_*",
+                FAILURES
+            );
+            assertThat(
+                newHashSet(indices),
+                equalTo(
+                    resolvedExpressionsSet(
+                        DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis),
+                        DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis)
+                    )
+                )
+            );
+
+            // Resolve both backing and failure indices
+            indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
+                indicesAliasesDataStreamsAndHiddenIndices,
+                "foo_*",
+                ALL_APPLICABLE
+            );
+            assertThat(
+                newHashSet(indices),
+                equalTo(
+                    resolvedExpressionsSet(
+                        "foo_index",
+                        "bar_index",
+                        "foo_foo",
+                        DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis),
+                        DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis),
+                        DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis),
+                        DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis)
+                    )
                 )
             );
 
             // include all wildcard adds the data stream's backing indices
             indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
                 indicesAliasesDataStreamsAndHiddenIndices,
-                "*"
+                "*",
+                ALL_APPLICABLE
             );
             assertThat(
-                indices,
-                containsInAnyOrder(
-                    new ResolvedExpression("foo_index"),
-                    new ResolvedExpression("bar_index"),
-                    new ResolvedExpression("foo_foo"),
-                    new ResolvedExpression("bar_bar"),
-                    new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis)),
-                    new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis))
+                newHashSet(indices),
+                equalTo(
+                    resolvedExpressionsSet(
+                        "foo_index",
+                        "bar_index",
+                        "foo_foo",
+                        "bar_bar",
+                        DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis),
+                        DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis),
+                        DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis),
+                        DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis)
+                    )
+                )
+            );
+
+            // include all wildcard adds the data stream's failure indices
+            indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
+                indicesAliasesDataStreamsAndHiddenIndices,
+                "*",
+                FAILURES
+            );
+            assertThat(
+                newHashSet(indices),
+                equalTo(
+                    resolvedExpressionsSet(
+                        DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis),
+                        DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis)
+                    )
+                )
+            );
+
+            // include all wildcard adds the data stream's backing and failure indices
+            indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
+                indicesAliasesDataStreamsAndHiddenIndices,
+                "*",
+                ALL_APPLICABLE
+            );
+            assertThat(
+                newHashSet(indices),
+                equalTo(
+                    resolvedExpressionsSet(
+                        "foo_index",
+                        "bar_index",
+                        "foo_foo",
+                        "bar_bar",
+                        DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis),
+                        DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis),
+                        DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis),
+                        DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis)
+                    )
                 )
             );
         }
@@ -642,34 +823,47 @@ public void testMatchesConcreteIndicesWildcardAndAliases() {
 
         Collection<ResolvedExpression> matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
             indicesAndAliasesContext,
-            "*"
+            "*",
+            ALL_APPLICABLE
         );
         assertThat(
             matches,
             containsInAnyOrder(
-                new ResolvedExpression("bar_bar"),
-                new ResolvedExpression("foo_foo"),
-                new ResolvedExpression("foo_index"),
-                new ResolvedExpression("bar_index")
+                new ResolvedExpression("bar_bar", DATA),
+                new ResolvedExpression("foo_foo", DATA),
+                new ResolvedExpression("foo_index", DATA),
+                new ResolvedExpression("bar_index", DATA)
             )
         );
-        matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(onlyIndicesContext, "*");
+        matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(onlyIndicesContext, "*", ALL_APPLICABLE);
         assertThat(
             matches,
             containsInAnyOrder(
-                new ResolvedExpression("bar_bar"),
-                new ResolvedExpression("foo_foo"),
-                new ResolvedExpression("foo_index"),
-                new ResolvedExpression("bar_index")
+                new ResolvedExpression("bar_bar", DATA),
+                new ResolvedExpression("foo_foo", DATA),
+                new ResolvedExpression("foo_index", DATA),
+                new ResolvedExpression("bar_index", DATA)
             )
         );
-        matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(indicesAndAliasesContext, "foo*");
+        matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
+            indicesAndAliasesContext,
+            "foo*",
+            ALL_APPLICABLE
+        );
         assertThat(
             matches,
-            containsInAnyOrder(new ResolvedExpression("foo_foo"), new ResolvedExpression("foo_index"), new ResolvedExpression("bar_index"))
+            containsInAnyOrder(
+                new ResolvedExpression("foo_foo", DATA),
+                new ResolvedExpression("foo_index", DATA),
+                new ResolvedExpression("bar_index", DATA)
+            )
         );
-        matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(onlyIndicesContext, "foo*");
-        assertThat(matches, containsInAnyOrder(new ResolvedExpression("foo_foo"), new ResolvedExpression("foo_index")));
+        matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(
+            onlyIndicesContext,
+            "foo*",
+            ALL_APPLICABLE
+        );
+        assertThat(matches, containsInAnyOrder(new ResolvedExpression("foo_foo", DATA), new ResolvedExpression("foo_index", DATA)));
     }
 
     private static IndexMetadata.Builder indexBuilder(String index, boolean hidden) {
@@ -682,10 +876,17 @@ private static IndexMetadata.Builder indexBuilder(String index) {
     }
 
     private static void assertWildcardResolvesToEmpty(IndexNameExpressionResolver.Context context, String wildcardExpression) {
-        assertThat(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, wildcardExpression), empty());
+        assertThat(
+            IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, wildcardExpression, DATA),
+            empty()
+        );
     }
 
     private Set<ResolvedExpression> resolvedExpressionsSet(String... expressions) {
-        return Arrays.stream(expressions).map(ResolvedExpression::new).collect(Collectors.toSet());
+        return Arrays.stream(expressions).map(e -> new ResolvedExpression(e, DATA)).collect(Collectors.toSet());
+    }
+
+    private Set<ResolvedExpression> resolvedExpressionsNoSelectorSet(String... expressions) {
+        return Arrays.stream(expressions).map(e -> new ResolvedExpression(e)).collect(Collectors.toSet());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java
index 331b5d92ca94e..a91cef576df33 100644
--- a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java
@@ -31,6 +31,8 @@
 import static java.util.Collections.emptySet;
 import static org.elasticsearch.test.NodeRoles.nonRemoteClusterClientNode;
 import static org.elasticsearch.test.NodeRoles.remoteClusterClientNode;
+import static org.elasticsearch.test.TransportVersionUtils.getPreviousVersion;
+import static org.elasticsearch.test.TransportVersionUtils.randomVersionBetween;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -250,4 +252,61 @@ public void testDiscoveryNodeToString() {
         assertThat(toString, containsString("{" + node.getBuildVersion() + "}"));
         assertThat(toString, containsString("{test-attr=val}"));// attributes
     }
+
+    public void testDiscoveryNodeMinReadOnlyVersionSerialization() throws Exception {
+        var node = DiscoveryNodeUtils.create("_id", buildNewFakeTransportAddress(), VersionInformation.CURRENT);
+
+        {
+            try (var out = new BytesStreamOutput()) {
+                out.setTransportVersion(TransportVersion.current());
+                node.writeTo(out);
+
+                try (var in = StreamInput.wrap(out.bytes().array())) {
+                    in.setTransportVersion(TransportVersion.current());
+
+                    var deserialized = new DiscoveryNode(in);
+                    assertThat(deserialized.getId(), equalTo(node.getId()));
+                    assertThat(deserialized.getAddress(), equalTo(node.getAddress()));
+                    assertThat(deserialized.getMinIndexVersion(), equalTo(node.getMinIndexVersion()));
+                    assertThat(deserialized.getMaxIndexVersion(), equalTo(node.getMaxIndexVersion()));
+                    assertThat(deserialized.getMinReadOnlyIndexVersion(), equalTo(node.getMinReadOnlyIndexVersion()));
+                    assertThat(deserialized.getVersionInformation(), equalTo(node.getVersionInformation()));
+                }
+            }
+        }
+
+        {
+            var oldVersion = randomVersionBetween(
+                random(),
+                TransportVersions.MINIMUM_COMPATIBLE,
+                getPreviousVersion(TransportVersions.NODE_VERSION_INFORMATION_WITH_MIN_READ_ONLY_INDEX_VERSION)
+            );
+            try (var out = new BytesStreamOutput()) {
+                out.setTransportVersion(oldVersion);
+                node.writeTo(out);
+
+                try (var in = StreamInput.wrap(out.bytes().array())) {
+                    in.setTransportVersion(oldVersion);
+
+                    var deserialized = new DiscoveryNode(in);
+                    assertThat(deserialized.getId(), equalTo(node.getId()));
+                    assertThat(deserialized.getAddress(), equalTo(node.getAddress()));
+                    assertThat(deserialized.getMinIndexVersion(), equalTo(node.getMinIndexVersion()));
+                    assertThat(deserialized.getMaxIndexVersion(), equalTo(node.getMaxIndexVersion()));
+                    assertThat(deserialized.getMinReadOnlyIndexVersion(), equalTo(node.getMinIndexVersion()));
+                    assertThat(
+                        deserialized.getVersionInformation(),
+                        equalTo(
+                            new VersionInformation(
+                                node.getBuildVersion(),
+                                node.getMinIndexVersion(),
+                                node.getMinIndexVersion(),
+                                node.getMaxIndexVersion()
+                            )
+                        )
+                    );
+                }
+            }
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java
index 943fb6fd63b0b..1db6192eee80e 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java
@@ -16,6 +16,8 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.shard.ShardId;
@@ -511,7 +513,7 @@ public void testRoutingPathUpdate() throws IOException {
             IllegalArgumentException.class,
             () -> routing.updateShard(randomAlphaOfLength(5), randomBoolean() ? null : randomAlphaOfLength(5))
         );
-        assertThat(e.getMessage(), equalTo("update is not supported because the destination index [test] is in time series mode"));
+        assertThat(e.getMessage(), equalTo("update is not supported because the destination index [test] is in time_series mode"));
     }
 
     public void testRoutingIndexWithRouting() throws IOException {
@@ -525,7 +527,7 @@ public void testRoutingIndexWithRouting() throws IOException {
         );
         assertThat(
             e.getMessage(),
-            equalTo("specifying routing is not supported because the destination index [test] is in time series mode")
+            equalTo("specifying routing is not supported because the destination index [test] is in time_series mode")
         );
     }
 
@@ -534,7 +536,7 @@ public void testRoutingPathCollectSearchWithRouting() throws IOException {
         Exception e = expectThrows(IllegalArgumentException.class, () -> routing.collectSearchShards(randomAlphaOfLength(5), null));
         assertThat(
             e.getMessage(),
-            equalTo("searching with a specified routing is not supported because the destination index [test] is in time series mode")
+            equalTo("searching with a specified routing is not supported because the destination index [test] is in time_series mode")
         );
     }
 
@@ -647,14 +649,42 @@ public void testRoutingPathReadWithInvalidString() throws IOException {
         int shards = between(2, 1000);
         IndexRouting indexRouting = indexRoutingForPath(shards, "foo");
         Exception e = expectThrows(ResourceNotFoundException.class, () -> shardIdForReadFromSourceExtracting(indexRouting, "!@#"));
-        assertThat(e.getMessage(), equalTo("invalid id [!@#] for index [test] in time series mode"));
+        assertThat(e.getMessage(), equalTo("invalid id [!@#] for index [test] in time_series mode"));
     }
 
     public void testRoutingPathReadWithShortString() throws IOException {
         int shards = between(2, 1000);
         IndexRouting indexRouting = indexRoutingForPath(shards, "foo");
         Exception e = expectThrows(ResourceNotFoundException.class, () -> shardIdForReadFromSourceExtracting(indexRouting, ""));
-        assertThat(e.getMessage(), equalTo("invalid id [] for index [test] in time series mode"));
+        assertThat(e.getMessage(), equalTo("invalid id [] for index [test] in time_series mode"));
+    }
+
+    public void testRoutingPathLogsdb() throws IOException {
+        int shards = between(2, 1000);
+        IndexRouting routing = IndexRouting.fromIndexMetadata(
+            IndexMetadata.builder("test")
+                .settings(
+                    settings(IndexVersion.current()).put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo")
+                        .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB)
+                        .build()
+                )
+                .numberOfShards(shards)
+                .numberOfReplicas(1)
+                .build()
+        );
+
+        IndexRequest req = new IndexRequest();
+        routing.preProcess(req);
+        assertNull(req.id());
+
+        // Verify that routing uses the field name and value in the routing path.
+        int expectedShard = Math.floorMod(hash(List.of("foo", "A")), shards);
+        BytesReference sourceBytes = source(Map.of("foo", "A", "bar", "B"));
+        assertEquals(expectedShard, routing.indexShard(null, null, XContentType.JSON, sourceBytes));
+
+        // Verify that the request id gets updated to contain the routing hash.
+        routing.postProcess(req);
+        assertEquals(expectedShard, routing.getShard(req.id(), null));
     }
 
     /**
@@ -673,7 +703,11 @@ private IndexRouting indexRoutingForPath(int shards, String path) {
     private IndexRouting indexRoutingForPath(IndexVersion createdVersion, int shards, String path) {
         return IndexRouting.fromIndexMetadata(
             IndexMetadata.builder("test")
-                .settings(settings(createdVersion).put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), path))
+                .settings(
+                    settings(createdVersion).put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), path)
+                        .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
+                        .build()
+                )
                 .numberOfShards(shards)
                 .numberOfReplicas(1)
                 .build()
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
index e81b85e4fec8f..88633b14ac461 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
@@ -638,6 +638,7 @@ public void testRemainingDelayCalculationsWithUnrelatedShutdowns() {
             shutdowns = shutdowns.putSingleNodeMetadata(
                 SingleNodeShutdownMetadata.builder()
                     .setNodeId(randomValueOtherThan(lastNodeId, () -> randomAlphaOfLengthBetween(5, 10)))
+                    .setNodeEphemeralId(randomValueOtherThan(lastNodeId, () -> randomAlphaOfLengthBetween(5, 10)))
                     .setReason(this.getTestName())
                     .setStartedAtMillis(randomNonNegativeLong())
                     .setType(type)
@@ -658,6 +659,7 @@ public void testRemainingDelayCalculationWhenNodeIsShuttingDownForRemoval() {
             NodesShutdownMetadata shutdowns = NodesShutdownMetadata.EMPTY.putSingleNodeMetadata(
                 SingleNodeShutdownMetadata.builder()
                     .setNodeId(lastNodeId)
+                    .setNodeEphemeralId(lastNodeId)
                     .setReason(this.getTestName())
                     .setStartedAtMillis(randomNonNegativeLong())
                     .setType(type)
@@ -678,6 +680,7 @@ public void testRemainingDelayCalculationWhenNodeIsKnownToBeRestartingWithCustom
         NodesShutdownMetadata shutdowns = NodesShutdownMetadata.EMPTY.putSingleNodeMetadata(
             SingleNodeShutdownMetadata.builder()
                 .setNodeId(lastNodeId)
+                .setNodeEphemeralId(lastNodeId)
                 .setReason(this.getTestName())
                 .setStartedAtMillis(randomNonNegativeLong())
                 .setType(SingleNodeShutdownMetadata.Type.RESTART)
@@ -700,6 +703,7 @@ public void testRemainingDelayCalculationWhenNodeIsKnownToBeRestartingWithDefaul
         NodesShutdownMetadata shutdowns = NodesShutdownMetadata.EMPTY.putSingleNodeMetadata(
             SingleNodeShutdownMetadata.builder()
                 .setNodeId(lastNodeId)
+                .setNodeEphemeralId(lastNodeId)
                 .setReason(this.getTestName())
                 .setStartedAtMillis(randomNonNegativeLong())
                 .setType(SingleNodeShutdownMetadata.Type.RESTART)
@@ -728,6 +732,7 @@ public void testRemainingDelayUsesIndexLevelDelayIfNodeWasNotRestartingWhenShard
         NodesShutdownMetadata shutdowns = NodesShutdownMetadata.EMPTY.putSingleNodeMetadata(
             SingleNodeShutdownMetadata.builder()
                 .setNodeId(lastNodeId)
+                .setNodeEphemeralId(lastNodeId)
                 .setReason(this.getTestName())
                 .setStartedAtMillis(randomNonNegativeLong())
                 .setType(SingleNodeShutdownMetadata.Type.RESTART)
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsServiceTests.java
index 35f1780464659..2a4d91437df0b 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsServiceTests.java
@@ -84,7 +84,7 @@ public void testShardStats() {
                 clusterService,
                 () -> clusterInfo,
                 createShardAllocator(),
-                new NodeAllocationStatsProvider(TEST_WRITE_LOAD_FORECASTER, ClusterSettings.createBuiltInClusterSettings())
+                new NodeAllocationStatsAndWeightsCalculator(TEST_WRITE_LOAD_FORECASTER, ClusterSettings.createBuiltInClusterSettings())
             );
             assertThat(
                 service.stats(),
@@ -125,7 +125,7 @@ public void testRelocatingShardIsOnlyCountedOnceOnTargetNode() {
                 clusterService,
                 EmptyClusterInfoService.INSTANCE,
                 createShardAllocator(),
-                new NodeAllocationStatsProvider(TEST_WRITE_LOAD_FORECASTER, ClusterSettings.createBuiltInClusterSettings())
+                new NodeAllocationStatsAndWeightsCalculator(TEST_WRITE_LOAD_FORECASTER, ClusterSettings.createBuiltInClusterSettings())
             );
             assertThat(
                 service.stats(),
@@ -182,7 +182,7 @@ public DesiredBalance getDesiredBalance() {
                         );
                     }
                 },
-                new NodeAllocationStatsProvider(TEST_WRITE_LOAD_FORECASTER, ClusterSettings.createBuiltInClusterSettings())
+                new NodeAllocationStatsAndWeightsCalculator(TEST_WRITE_LOAD_FORECASTER, ClusterSettings.createBuiltInClusterSettings())
             );
             assertThat(
                 service.stats(),
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
index 201fbc9e3ca18..5a6e8e65495a0 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
@@ -909,6 +909,7 @@ protected void updateIndicesReadOnly(Set<String> indicesToUpdate, Releasable onC
                                 sourceNode,
                                 SingleNodeShutdownMetadata.builder()
                                     .setNodeId(sourceNode)
+                                    .setNodeEphemeralId(sourceNode)
                                     .setReason("testing")
                                     .setType(SingleNodeShutdownMetadata.Type.REPLACE)
                                     .setTargetNodeName(targetNode)
@@ -1293,6 +1294,7 @@ public void testSkipDiskThresholdMonitorWhenStateNotRecovered() {
                         "node1",
                         SingleNodeShutdownMetadata.builder()
                             .setNodeId("node1")
+                            .setNodeEphemeralId("node1")
                             .setReason("testing")
                             .setType(SingleNodeShutdownMetadata.Type.REPLACE)
                             .setTargetNodeName("node3")
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java
index 54f4f0ffb6e15..cd94c87bb4b57 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java
@@ -922,6 +922,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocat
                             "node-0",
                             SingleNodeShutdownMetadata.builder()
                                 .setNodeId("node-0")
+                                .setNodeEphemeralId("node-0")
                                 .setType(SingleNodeShutdownMetadata.Type.REPLACE)
                                 .setTargetNodeName("node-2")
                                 .setStartedAtMillis(System.currentTimeMillis())
@@ -1280,6 +1281,7 @@ public void testShouldLogOnTooManyUndesiredAllocations() {
             var builder = SingleNodeShutdownMetadata.builder()
                 .setType(type)
                 .setNodeId("data-node-1")
+                .setNodeEphemeralId("data-node-1")
                 .setStartedAtMillis(randomNonNegativeLong())
                 .setReason("test");
             if (type.equals(SingleNodeShutdownMetadata.Type.SIGTERM)) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java
index 9caf89d4d7613..21d547c1593b8 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java
@@ -868,6 +868,7 @@ public void resetDesiredBalance() {
             final var shutdownType = randomFrom(Type.SIGTERM, Type.REMOVE, Type.REPLACE);
             final var singleShutdownMetadataBuilder = SingleNodeShutdownMetadata.builder()
                 .setNodeId(node2.getId())
+                .setNodeEphemeralId(node2.getEphemeralId())
                 .setReason("test")
                 .setType(shutdownType)
                 .setStartedAtMillis(randomNonNegativeLong());
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceTests.java
index 760900817780a..2c15addfe217b 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceTests.java
@@ -46,19 +46,19 @@ public void testShardMovements() {
         );
 
         assertThat(
-            "1 shard movements when existing shard is moved and new shard copy is unassigned",
+            "1 shard movements when an existing shard copy is moved and new shard copy is unassigned",
             shardMovements(new ShardAssignment(Set.of("a", "b"), 2, 0, 0), new ShardAssignment(Set.of("a", "c"), 3, 1, 0)),
             equalTo(1)
         );
 
         assertThat(
-            "1 shard movement",
+            "1 shard movement when an existing shard copy is moved",
             shardMovements(new ShardAssignment(Set.of("a", "b"), 2, 0, 0), new ShardAssignment(Set.of("a", "c"), 2, 0, 0)),
             equalTo(1)
         );
 
         assertThat(
-            "2 shard movement",
+            "2 shard movements when both shard copies are move to new nodes",
             shardMovements(new ShardAssignment(Set.of("a", "b"), 2, 0, 0), new ShardAssignment(Set.of("c", "d"), 2, 0, 0)),
             equalTo(2)
         );
@@ -77,10 +77,10 @@ public void testShardMovements() {
     }
 
     private static int shardMovements(ShardAssignment old, ShardAssignment updated) {
-        return DesiredBalance.shardMovements(of(old), of(updated));
+        return DesiredBalance.shardMovements(createDesiredBalanceWith(old), createDesiredBalanceWith(updated));
     }
 
-    private static DesiredBalance of(ShardAssignment assignment) {
+    private static DesiredBalance createDesiredBalanceWith(ShardAssignment assignment) {
         return new DesiredBalance(1, Map.of(new ShardId("index", "_na_", 0), assignment));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDeciderTests.java
index e8ac6d22698dd..7750a01a13a84 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDeciderTests.java
@@ -420,6 +420,7 @@ private NodesShutdownMetadata createNodeShutdownReplacementMetadata(String sourc
         return new NodesShutdownMetadata(new HashMap<>()).putSingleNodeMetadata(
             SingleNodeShutdownMetadata.builder()
                 .setNodeId(sourceNodeId)
+                .setNodeEphemeralId(sourceNodeId)
                 .setTargetNodeName(targetNodeName)
                 .setType(SingleNodeShutdownMetadata.Type.REPLACE)
                 .setReason(this.getTestName())
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java
index e81e27a7343f3..d795c80bf5b96 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java
@@ -227,6 +227,7 @@ private NodesShutdownMetadata createNodesShutdownMetadata(SingleNodeShutdownMeta
         return new NodesShutdownMetadata(new HashMap<>()).putSingleNodeMetadata(
             SingleNodeShutdownMetadata.builder()
                 .setNodeId(nodeId)
+                .setNodeEphemeralId(nodeId)
                 .setType(shutdownType)
                 .setReason(this.getTestName())
                 .setStartedAtMillis(1L)
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDeciderTests.java
index 0ef5fcc624e1f..7f42beb5181ec 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDeciderTests.java
@@ -216,6 +216,7 @@ public void testYesWhenSnapshotInProgressButShardIsPausedDueToShutdown() {
                             nodeId,
                             SingleNodeShutdownMetadata.builder()
                                 .setNodeId(nodeId)
+                                .setNodeEphemeralId(nodeId)
                                 .setType(SingleNodeShutdownMetadata.Type.REMOVE)
                                 .setStartedAtMillis(randomNonNegativeLong())
                                 .setReason("test")
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java
index a6c8064ad551b..586f088d554a5 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java
@@ -2254,6 +2254,7 @@ private static ClusterState createClusterStateWith(
                         it -> it.nodeId,
                         it -> SingleNodeShutdownMetadata.builder()
                             .setNodeId(it.nodeId)
+                            .setNodeEphemeralId(it.nodeId)
                             .setType(it.type)
                             .setReason("test")
                             .setNodeSeen(true)
diff --git a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
index bad2b07ba7972..0c7af66a7190e 100644
--- a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
@@ -30,12 +30,12 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.shard.IndexLongFieldRange;
 import org.elasticsearch.index.shard.ShardLongFieldRange;
@@ -51,7 +51,6 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
@@ -70,7 +69,7 @@ public void testClusterStateSerialization() throws Exception {
             .settings(settings(IndexVersion.current()))
             .numberOfShards(10)
             .numberOfReplicas(1)
-            .eventIngestedRange(eventIngestedRangeInput, TransportVersions.V_8_15_0);
+            .eventIngestedRange(eventIngestedRangeInput);
 
         ClusterStateTestRecord result = createAndSerializeClusterState(indexMetadataBuilder, TransportVersion.current());
 
@@ -86,63 +85,6 @@ public void testClusterStateSerialization() throws Exception {
         }
     }
 
-    public void testClusterStateSerializationWithTimestampRangesWithOlderTransportVersion() throws Exception {
-        TransportVersion versionBeforeEventIngestedInClusterState = randomFrom(
-            TransportVersions.V_8_0_0,
-            TransportVersionUtils.getPreviousVersion(TransportVersions.V_8_15_0)
-        );
-        {
-            IndexLongFieldRange eventIngestedRangeInput = randomFrom(
-                IndexLongFieldRange.UNKNOWN,
-                IndexLongFieldRange.NO_SHARDS,
-                IndexLongFieldRange.EMPTY,
-                IndexLongFieldRange.NO_SHARDS.extendWithShardRange(0, 1, ShardLongFieldRange.of(100000, 200000))
-            );
-
-            IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder("test")
-                .settings(settings(IndexVersion.current()))
-                .numberOfShards(10)
-                .numberOfReplicas(1)
-                .eventIngestedRange(eventIngestedRangeInput, versionBeforeEventIngestedInClusterState);
-
-            ClusterStateTestRecord result = createAndSerializeClusterState(indexMetadataBuilder, versionBeforeEventIngestedInClusterState);
-
-            assertThat(result.serializedClusterState().getClusterName().value(), equalTo(result.clusterState().getClusterName().value()));
-            assertThat(result.serializedClusterState().routingTable().toString(), equalTo(result.clusterState().routingTable().toString()));
-
-            IndexLongFieldRange eventIngestedRangeOutput = result.serializedClusterState()
-                .getMetadata()
-                .index("test")
-                .getEventIngestedRange();
-            // should always come back as UNKNOWN when an older transport version is passed in
-            assertSame(IndexLongFieldRange.UNKNOWN, eventIngestedRangeOutput);
-        }
-        {
-            // UNKNOWN is the only allowed state for event.ingested range in older versions, so this serialization test should fail
-            IndexLongFieldRange eventIngestedRangeInput = randomFrom(
-                IndexLongFieldRange.NO_SHARDS,
-                IndexLongFieldRange.EMPTY,
-                IndexLongFieldRange.NO_SHARDS.extendWithShardRange(0, 1, ShardLongFieldRange.of(100000, 200000))
-            );
-
-            IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder("test")
-                .settings(settings(IndexVersion.current()))
-                .numberOfShards(10)
-                .numberOfReplicas(1)
-                .eventIngestedRange(eventIngestedRangeInput, TransportVersion.current());
-
-            AssertionError assertionError = expectThrows(
-                AssertionError.class,
-                () -> createAndSerializeClusterState(indexMetadataBuilder, versionBeforeEventIngestedInClusterState)
-            );
-
-            assertThat(
-                assertionError.getMessage(),
-                containsString("eventIngestedRange should be UNKNOWN until all nodes are on the new version")
-            );
-        }
-    }
-
     /**
      * @param clusterState original ClusterState created by helper method
      * @param serializedClusterState serialized version of the clusterState
@@ -397,8 +339,12 @@ public void writeTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-            return ChunkedToXContent.builder(params).object(b -> b.field("custom_string_object", strObject));
+        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+            return Iterators.concat(
+                Iterators.single((builder, params) -> builder.startObject()),
+                Iterators.single((builder, params) -> builder.field("custom_string_object", strObject)),
+                Iterators.single((builder, params) -> builder.endObject())
+            );
         }
 
         @Override
@@ -436,8 +382,12 @@ public void writeTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-            return ChunkedToXContent.builder(params).object(b -> b.field("custom_integer_object", intObject));
+        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+            return Iterators.concat(
+                Iterators.single((builder, params) -> builder.startObject()),
+                Iterators.single((builder, params) -> builder.field("custom_integer_object", intObject)),
+                Iterators.single((builder, params) -> builder.endObject())
+            );
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java
index 7c1c954e7b4e9..e6f50ef42365e 100644
--- a/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java
@@ -66,6 +66,12 @@ public long relativeTimeInMillis() {
                 assertThat(Thread.currentThread().getName(), containsString(ClusterApplierService.CLUSTER_UPDATE_THREAD_NAME));
                 return currentTimeMillis;
             }
+
+            @Override
+            public long rawRelativeTimeInMillis() {
+                assertThat(Thread.currentThread().getName(), containsString(ClusterApplierService.CLUSTER_UPDATE_THREAD_NAME));
+                return currentTimeMillis;
+            }
         };
         clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         allowClusterStateApplicationFailure = false;
@@ -207,15 +213,33 @@ public void testLongClusterStateUpdateLogging() throws Exception {
             );
             mockLog.addExpectation(
                 new MockLog.SeenEventExpectation(
-                    "test4",
+                    "test3",
                     ClusterApplierService.class.getCanonicalName(),
                     Level.WARN,
                     "*cluster state applier task [test3] took [34s] which is above the warn threshold of [*]: "
                         + "[running task [test3]] took [*"
                 )
             );
+            mockLog.addExpectation(
+                new MockLog.SeenEventExpectation(
+                    "test4",
+                    ClusterApplierService.class.getCanonicalName(),
+                    Level.WARN,
+                    "*cluster state applier task [test4] took [36s] which is above the warn threshold of [*]: "
+                        + "[running task [test4]] took [*"
+                )
+            );
+            mockLog.addExpectation(
+                new MockLog.SeenEventExpectation(
+                    "test5",
+                    ClusterApplierService.class.getCanonicalName(),
+                    Level.WARN,
+                    "*cluster state applier task [test5] took [38s] which is above the warn threshold of [*]: "
+                        + "[running task [test5]] took [*"
+                )
+            );
 
-            final CountDownLatch latch = new CountDownLatch(4);
+            final CountDownLatch latch = new CountDownLatch(6);
             final CountDownLatch processedFirstTask = new CountDownLatch(1);
             currentTimeMillis = randomLongBetween(0L, Long.MAX_VALUE / 2);
             clusterApplierService.runOnApplierThread(
@@ -266,9 +290,39 @@ public void onFailure(Exception e) {
                     }
                 }
             );
+            clusterApplierService.runOnApplierThread("test4", Priority.HIGH, currentState -> {
+                // do nothing (testing that onResponse is included in timing)
+            }, new ActionListener<>() {
+
+                @Override
+                public void onResponse(Void unused) {
+                    advanceTime(TimeValue.timeValueSeconds(36).millis());
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    fail();
+                }
+            });
+            clusterApplierService.runOnApplierThread("test5", Priority.HIGH, currentState -> {
+                throw new IllegalArgumentException("Testing that onFailure is included in timing");
+            }, new ActionListener<>() {
+
+                @Override
+                public void onResponse(Void unused) {
+                    fail();
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    advanceTime(TimeValue.timeValueSeconds(38).millis());
+                    latch.countDown();
+                }
+            });
             // Additional update task to make sure all previous logging made it to the loggerName
             // We don't check logging for this on since there is no guarantee that it will occur before our check
-            clusterApplierService.runOnApplierThread("test4", Priority.HIGH, currentState -> {}, new ActionListener<>() {
+            clusterApplierService.runOnApplierThread("test6", Priority.HIGH, currentState -> {}, new ActionListener<>() {
                 @Override
                 public void onResponse(Void ignored) {
                     latch.countDown();
diff --git a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java
index f9d8173e15be4..0fecccc3cb5e6 100644
--- a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java
@@ -46,6 +46,8 @@
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.StoppableExecutorServiceWrapper;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
@@ -259,9 +261,42 @@ public void clusterStatePublished(ClusterState newClusterState) {
         assertThat(registeredActions.toString(), registeredActions, contains(MasterService.STATE_UPDATE_ACTION_NAME));
     }
 
-    public void testThreadContext() throws InterruptedException {
+    public void testThreadContext() {
         try (var master = createMasterService(true)) {
-            final CountDownLatch latch = new CountDownLatch(1);
+
+            master.setClusterStatePublisher((clusterStatePublicationEvent, publishListener, ackListener) -> {
+                ClusterServiceUtils.setAllElapsedMillis(clusterStatePublicationEvent);
+                try (var ignored = threadPool.getThreadContext().newEmptyContext()) {
+                    if (randomBoolean()) {
+                        randomExecutor(threadPool).execute(() -> publishListener.onResponse(null));
+                        randomExecutor(threadPool).execute(() -> ackListener.onCommit(TimeValue.timeValueMillis(randomInt(10000))));
+                        randomExecutor(threadPool).execute(
+                            () -> ackListener.onNodeAck(
+                                clusterStatePublicationEvent.getNewState().nodes().getMasterNode(),
+                                randomBoolean() ? null : new RuntimeException("simulated ack failure")
+                            )
+                        );
+                    } else {
+                        randomExecutor(threadPool).execute(
+                            () -> publishListener.onFailure(new FailedToCommitClusterStateException("simulated publish failure"))
+                        );
+                    }
+                }
+            });
+
+            final Releasable onPublishComplete;
+            final Releasable onAckingComplete;
+            final Runnable awaitComplete;
+            {
+                final var publishLatch = new CountDownLatch(1);
+                final var ackingLatch = new CountDownLatch(1);
+                onPublishComplete = Releasables.assertOnce(publishLatch::countDown);
+                onAckingComplete = Releasables.assertOnce(ackingLatch::countDown);
+                awaitComplete = () -> {
+                    safeAwait(publishLatch);
+                    safeAwait(ackingLatch);
+                };
+            }
 
             try (ThreadContext.StoredContext ignored = threadPool.getThreadContext().stashContext()) {
 
@@ -272,15 +307,12 @@ public void testThreadContext() throws InterruptedException {
                         expectedHeaders.put(copiedHeader, randomIdentifier());
                     }
                 }
-
-                final Map<String, List<String>> expectedResponseHeaders = Collections.singletonMap(
-                    "testResponse",
-                    Collections.singletonList("testResponse")
-                );
                 threadPool.getThreadContext().putHeader(expectedHeaders);
 
-                final TimeValue ackTimeout = randomBoolean() ? TimeValue.ZERO : TimeValue.timeValueMillis(randomInt(10000));
-                final TimeValue masterTimeout = randomBoolean() ? TimeValue.ZERO : TimeValue.timeValueMillis(randomInt(10000));
+                final Map<String, List<String>> expectedResponseHeaders = Map.of("testResponse", List.of(randomIdentifier()));
+
+                final TimeValue ackTimeout = randomBoolean() ? TimeValue.MINUS_ONE : TimeValue.timeValueMillis(randomInt(10000));
+                final TimeValue masterTimeout = randomBoolean() ? TimeValue.MINUS_ONE : TimeValue.timeValueMillis(randomInt(10000));
 
                 master.submitUnbatchedStateUpdateTask(
                     "test",
@@ -289,8 +321,9 @@ public void testThreadContext() throws InterruptedException {
                         public ClusterState execute(ClusterState currentState) {
                             assertTrue(threadPool.getThreadContext().isSystemContext());
                             assertEquals(Collections.emptyMap(), threadPool.getThreadContext().getHeaders());
-                            threadPool.getThreadContext().addResponseHeader("testResponse", "testResponse");
-                            assertEquals(expectedResponseHeaders, threadPool.getThreadContext().getResponseHeaders());
+                            expectedResponseHeaders.forEach(
+                                (name, values) -> values.forEach(v -> threadPool.getThreadContext().addResponseHeader(name, v))
+                            );
 
                             if (randomBoolean()) {
                                 return ClusterState.builder(currentState).build();
@@ -303,44 +336,44 @@ public ClusterState execute(ClusterState currentState) {
 
                         @Override
                         public void onFailure(Exception e) {
-                            assertFalse(threadPool.getThreadContext().isSystemContext());
-                            assertEquals(expectedHeaders, threadPool.getThreadContext().getHeaders());
-                            assertEquals(expectedResponseHeaders, threadPool.getThreadContext().getResponseHeaders());
-                            latch.countDown();
+                            assertExpectedThreadContext(
+                                e instanceof ProcessClusterEventTimeoutException ? Map.of() : expectedResponseHeaders
+                            );
+                            onPublishComplete.close();
+                            onAckingComplete.close(); // no acking takes place if publication failed
                         }
 
                         @Override
                         public void clusterStateProcessed(ClusterState oldState, ClusterState newState) {
-                            assertFalse(threadPool.getThreadContext().isSystemContext());
-                            assertEquals(expectedHeaders, threadPool.getThreadContext().getHeaders());
-                            assertEquals(expectedResponseHeaders, threadPool.getThreadContext().getResponseHeaders());
-                            latch.countDown();
+                            assertExpectedThreadContext(expectedResponseHeaders);
+                            onPublishComplete.close();
                         }
 
                         @Override
                         public void onAllNodesAcked() {
-                            assertFalse(threadPool.getThreadContext().isSystemContext());
-                            assertEquals(expectedHeaders, threadPool.getThreadContext().getHeaders());
-                            assertEquals(expectedResponseHeaders, threadPool.getThreadContext().getResponseHeaders());
-                            latch.countDown();
+                            onAckCompletion();
                         }
 
                         @Override
                         public void onAckFailure(Exception e) {
-                            assertFalse(threadPool.getThreadContext().isSystemContext());
-                            assertEquals(expectedHeaders, threadPool.getThreadContext().getHeaders());
-                            assertEquals(expectedResponseHeaders, threadPool.getThreadContext().getResponseHeaders());
-                            latch.countDown();
+                            onAckCompletion();
                         }
 
                         @Override
                         public void onAckTimeout() {
+                            onAckCompletion();
+                        }
+
+                        private void onAckCompletion() {
+                            assertExpectedThreadContext(expectedResponseHeaders);
+                            onAckingComplete.close();
+                        }
+
+                        private void assertExpectedThreadContext(Map<String, List<String>> expectedResponseHeaders) {
                             assertFalse(threadPool.getThreadContext().isSystemContext());
                             assertEquals(expectedHeaders, threadPool.getThreadContext().getHeaders());
                             assertEquals(expectedResponseHeaders, threadPool.getThreadContext().getResponseHeaders());
-                            latch.countDown();
                         }
-
                     }
                 );
 
@@ -349,7 +382,7 @@ public void onAckTimeout() {
                 assertEquals(Collections.emptyMap(), threadPool.getThreadContext().getResponseHeaders());
             }
 
-            assertTrue(latch.await(10, TimeUnit.SECONDS));
+            awaitComplete.run();
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/common/TimeBasedUUIDGeneratorTests.java b/server/src/test/java/org/elasticsearch/common/TimeBasedUUIDGeneratorTests.java
index 964683a1972ba..fa51f89dea9c3 100644
--- a/server/src/test/java/org/elasticsearch/common/TimeBasedUUIDGeneratorTests.java
+++ b/server/src/test/java/org/elasticsearch/common/TimeBasedUUIDGeneratorTests.java
@@ -9,12 +9,14 @@
 
 package org.elasticsearch.common;
 
+import org.elasticsearch.common.util.ByteUtils;
 import org.elasticsearch.test.ESTestCase;
 
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.Base64;
 import java.util.HashSet;
+import java.util.OptionalInt;
 import java.util.Set;
 import java.util.function.Supplier;
 import java.util.stream.IntStream;
@@ -106,6 +108,12 @@ public void testUUIDEncodingDecodingWithRandomValues() {
         );
     }
 
+    public void testUUIDEncodingDecodingWithHash() {
+        int hash = randomInt();
+        byte[] decoded = Base64.getUrlDecoder().decode(UUIDs.base64TimeBasedKOrderedUUIDWithHash(OptionalInt.of(hash)));
+        assertEquals(hash, ByteUtils.readIntLE(decoded, decoded.length - 9));
+    }
+
     private void testUUIDEncodingDecodingHelper(final long timestamp, final int sequenceId, final byte[] macAddress) {
         final TestTimeBasedKOrderedUUIDDecoder decoder = new TestTimeBasedKOrderedUUIDDecoder(
             createKOrderedGenerator(() -> timestamp, () -> sequenceId, () -> macAddress).getBase64UUID()
diff --git a/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java b/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java
index 70a1735c7b1c5..463d0d4514bac 100644
--- a/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java
+++ b/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java
@@ -271,6 +271,21 @@ public void testEpochMillisParser() {
             IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> formatter.parse("12345.0."));
             assertThat(e.getMessage(), is("failed to parse date field [12345.0.] with format [epoch_millis]"));
         }
+        {
+            Instant instant = Instant.from(formatter.parse("-86400000"));
+            assertThat(instant.getEpochSecond(), is(-86400L));
+            assertThat(instant.getNano(), is(0));
+            assertThat(formatter.format(instant), is("-86400000"));
+            assertThat(Instant.from(formatter.parse(formatter.format(instant))), is(instant));
+        }
+        {
+            Instant instant = Instant.from(formatter.parse("-86400000.999999"));
+            assertThat(instant.getEpochSecond(), is(-86401L));
+            assertThat(instant.getNano(), is(999000001));
+            assertThat(formatter.format(instant), is("-86400000.999999"));
+            assertThat(Instant.from(formatter.parse(formatter.format(instant))), is(instant));
+        }
+
     }
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/common/time/EpochTimeTests.java b/server/src/test/java/org/elasticsearch/common/time/EpochTimeTests.java
new file mode 100644
index 0000000000000..59bf3958a3b26
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/common/time/EpochTimeTests.java
@@ -0,0 +1,291 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.common.time;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.test.ESTestCase;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.function.LongSupplier;
+
+import static org.elasticsearch.common.time.EpochTime.MILLIS_FORMATTER;
+import static org.hamcrest.Matchers.is;
+
+public class EpochTimeTests extends ESTestCase {
+
+    public void testNegativeEpochMillis() {
+        DateFormatter formatter = MILLIS_FORMATTER;
+
+        // validate that negative epoch millis around rounded appropriately by the parser
+        LongSupplier supplier = () -> 0L;
+        {
+            Instant instant = formatter.toDateMathParser().parse("0", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00.000999999Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-0", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00.000999999Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("0", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-0", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00Z", instant.toString());
+        }
+
+        {
+            Instant instant = formatter.toDateMathParser().parse("1", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00.001999999Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-1", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1969-12-31T23:59:59.999999999Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("1", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00.001Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-1", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1969-12-31T23:59:59.999Z", instant.toString());
+        }
+
+        {
+            Instant instant = formatter.toDateMathParser().parse("0.999999", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00.000999999Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-0.999999", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1969-12-31T23:59:59.999000001Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("0.999999", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00.000999999Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-0.999999", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1969-12-31T23:59:59.999000001Z", instant.toString());
+        }
+
+        {
+            Instant instant = formatter.toDateMathParser().parse("6250000430768", supplier, true, ZoneId.of("UTC"));
+            assertEquals("2168-01-20T23:13:50.768999999Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-6250000430768", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1771-12-12T00:46:09.232999999Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("6250000430768", supplier, false, ZoneId.of("UTC"));
+            assertEquals("2168-01-20T23:13:50.768Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-6250000430768", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1771-12-12T00:46:09.232Z", instant.toString());
+        }
+
+        {
+            Instant instant = formatter.toDateMathParser().parse("0.123450", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00.000123450Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-0.123450", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1969-12-31T23:59:59.999876550Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("0.123450", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00.000123450Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-0.123450", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1969-12-31T23:59:59.999876550Z", instant.toString());
+        }
+
+        {
+            Instant instant = formatter.toDateMathParser().parse("0.123456", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00.000123456Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-0.123456", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1969-12-31T23:59:59.999876544Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("0.123456", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00.000123456Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-0.123456", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1969-12-31T23:59:59.999876544Z", instant.toString());
+        }
+
+        {
+            Instant instant = formatter.toDateMathParser().parse("86400000", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1970-01-02T00:00:00.000999999Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-86400000", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1969-12-31T00:00:00.000999999Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("86400000", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1970-01-02T00:00:00Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-86400000", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1969-12-31T00:00:00Z", instant.toString());
+        }
+
+        {
+            Instant instant = formatter.toDateMathParser().parse("86400000.999999", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1970-01-02T00:00:00.000999999Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-86400000.999999", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1969-12-30T23:59:59.999000001Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("86400000.999999", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1970-01-02T00:00:00.000999999Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-86400000.999999", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1969-12-30T23:59:59.999000001Z", instant.toString());
+        }
+
+        {
+            Instant instant = formatter.toDateMathParser().parse("200.89", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00.200890Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-200.89", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1969-12-31T23:59:59.799110Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("200.89", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00.200890Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-200.89", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1969-12-31T23:59:59.799110Z", instant.toString());
+        }
+
+        {
+            Instant instant = formatter.toDateMathParser().parse("200.", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00.200Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-200.", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1969-12-31T23:59:59.800Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("200.", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00.200Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-200.", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1969-12-31T23:59:59.800Z", instant.toString());
+        }
+
+        {
+            Instant instant = formatter.toDateMathParser().parse("0.200", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00.000200Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-0.200", supplier, true, ZoneId.of("UTC"));
+            assertEquals("1969-12-31T23:59:59.999800Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("0.200", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1970-01-01T00:00:00.000200Z", instant.toString());
+        }
+        {
+            Instant instant = formatter.toDateMathParser().parse("-0.200", supplier, false, ZoneId.of("UTC"));
+            assertEquals("1969-12-31T23:59:59.999800Z", instant.toString());
+        }
+
+        {
+            ElasticsearchParseException e = expectThrows(
+                ElasticsearchParseException.class,
+                () -> formatter.toDateMathParser().parse(".200", supplier, true, ZoneId.of("UTC"))
+            );
+            assertThat(e.getMessage().split(":")[0], is("failed to parse date field [.200] with format [epoch_millis]"));
+        }
+        {
+            ElasticsearchParseException e = expectThrows(
+                ElasticsearchParseException.class,
+                () -> formatter.toDateMathParser().parse("-.200", supplier, true, ZoneId.of("UTC"))
+            );
+            assertThat(e.getMessage().split(":")[0], is("failed to parse date field [-.200] with format [epoch_millis]"));
+        }
+        {
+            ElasticsearchParseException e = expectThrows(
+                ElasticsearchParseException.class,
+                () -> formatter.toDateMathParser().parse(".200", supplier, false, ZoneId.of("UTC"))
+            );
+            assertThat(e.getMessage().split(":")[0], is("failed to parse date field [.200] with format [epoch_millis]"));
+        }
+        {
+            ElasticsearchParseException e = expectThrows(
+                ElasticsearchParseException.class,
+                () -> formatter.toDateMathParser().parse("-.200", supplier, false, ZoneId.of("UTC"))
+            );
+            assertThat(e.getMessage().split(":")[0], is("failed to parse date field [-.200] with format [epoch_millis]"));
+        }
+
+        // tilda was included in the parsers at one point for delineating negative and positive infinity rounding and we want to
+        // ensure it doesn't show up unexpectedly in the parser with its original "~" value
+        {
+            ElasticsearchParseException e = expectThrows(
+                ElasticsearchParseException.class,
+                () -> formatter.toDateMathParser().parse("~-0.200", supplier, false, ZoneId.of("UTC"))
+            );
+            assertThat(e.getMessage().split(":")[0], is("failed to parse date field [~-0.200] with format [epoch_millis]"));
+        }
+        {
+            ElasticsearchParseException e = expectThrows(
+                ElasticsearchParseException.class,
+                () -> formatter.toDateMathParser().parse("~0.200", supplier, false, ZoneId.of("UTC"))
+            );
+            assertThat(e.getMessage().split(":")[0], is("failed to parse date field [~0.200] with format [epoch_millis]"));
+        }
+        {
+            ElasticsearchParseException e = expectThrows(
+                ElasticsearchParseException.class,
+                () -> formatter.toDateMathParser().parse("~-1", supplier, false, ZoneId.of("UTC"))
+            );
+            assertThat(e.getMessage().split(":")[0], is("failed to parse date field [~-1] with format [epoch_millis]"));
+        }
+        {
+            ElasticsearchParseException e = expectThrows(
+                ElasticsearchParseException.class,
+                () -> formatter.toDateMathParser().parse("~1", supplier, false, ZoneId.of("UTC"))
+            );
+            assertThat(e.getMessage().split(":")[0], is("failed to parse date field [~1] with format [epoch_millis]"));
+        }
+        {
+            ElasticsearchParseException e = expectThrows(
+                ElasticsearchParseException.class,
+                () -> formatter.toDateMathParser().parse("~-1.", supplier, false, ZoneId.of("UTC"))
+            );
+            assertThat(e.getMessage().split(":")[0], is("failed to parse date field [~-1.] with format [epoch_millis]"));
+        }
+        {
+            ElasticsearchParseException e = expectThrows(
+                ElasticsearchParseException.class,
+                () -> formatter.toDateMathParser().parse("~1.", supplier, false, ZoneId.of("UTC"))
+            );
+            assertThat(e.getMessage().split(":")[0], is("failed to parse date field [~1.] with format [epoch_millis]"));
+        }
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilderTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilderTests.java
deleted file mode 100644
index ff811f5d6d736..0000000000000
--- a/server/src/test/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilderTests.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.common.xcontent;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.collect.Iterators;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xcontent.ToXContent;
-
-import java.util.function.IntFunction;
-import java.util.stream.IntStream;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class ChunkedToXContentBuilderTests extends ESTestCase {
-
-    public void testFieldWithInnerChunkedObject() {
-
-        ToXContent innerXContent = (b, p) -> {
-            b.startObject();
-            b.field("field1", 10);
-            b.field("field2", "aaa");
-            b.endObject();
-            return b;
-        };
-
-        ToXContent outerXContent = (b, p) -> b.field("field3", 10).field("field4", innerXContent);
-
-        String expectedContent = Strings.toString(outerXContent);
-
-        ChunkedToXContentObject innerChunkedContent = params -> new ChunkedToXContentBuilder(params).object(
-            o -> o.field("field1", 10).field("field2", "aaa")
-        );
-
-        ChunkedToXContent outerChunkedContent = params -> new ChunkedToXContentBuilder(params).field("field3", 10)
-            .field("field4", innerChunkedContent);
-
-        assertThat(Strings.toString(outerChunkedContent), equalTo(expectedContent));
-    }
-
-    public void testFieldWithInnerChunkedArray() {
-
-        ToXContent innerXContent = (b, p) -> {
-            b.startArray();
-            b.value(10);
-            b.value(20);
-            b.endArray();
-            return b;
-        };
-
-        ToXContent outerXContent = (b, p) -> b.field("field3", 10).field("field4", innerXContent);
-
-        String expectedContent = Strings.toString(outerXContent);
-
-        IntFunction<ToXContent> value = v -> (b, p) -> b.value(v);
-
-        ChunkedToXContentObject innerChunkedContent = params -> new ChunkedToXContentBuilder(params).array(
-            IntStream.of(10, 20).mapToObj(value).iterator()
-        );
-
-        ChunkedToXContent outerChunkedContent = params -> new ChunkedToXContentBuilder(params).field("field3", 10)
-            .field("field4", innerChunkedContent);
-
-        assertThat(Strings.toString(outerChunkedContent), equalTo(expectedContent));
-    }
-
-    public void testFieldWithInnerChunkedField() {
-
-        ToXContent innerXContent = (b, p) -> b.value(10);
-        ToXContent outerXContent = (b, p) -> b.field("field3", 10).field("field4", innerXContent);
-
-        String expectedContent = Strings.toString(outerXContent);
-
-        ChunkedToXContentObject innerChunkedContent = params -> Iterators.single((b, p) -> b.value(10));
-
-        ChunkedToXContent outerChunkedContent = params -> new ChunkedToXContentBuilder(params).field("field3", 10)
-            .field("field4", innerChunkedContent);
-
-        assertThat(Strings.toString(outerChunkedContent), equalTo(expectedContent));
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java
index de1990361e766..8f62f8d33eacd 100644
--- a/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java
+++ b/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.core.RestApiVersion;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
@@ -116,8 +115,6 @@ public static NewSubObject parse(XContentParser parser) {
         }
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
-    @AwaitsFix(bugUrl = "this can be re-enabled once our rest api version is bumped to V_9")
     public void testNotCompatibleRequest() throws IOException {
         NamedXContentRegistry registry = new NamedXContentRegistry(
             List.of(
diff --git a/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java b/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java
index 22308e15f4845..eccdd1c6ffea7 100644
--- a/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java
@@ -43,7 +43,7 @@ private BuildVersion randomBuildVersion() {
     }
 
     private IndexVersion randomIndexVersion() {
-        return rarely() ? IndexVersion.fromId(randomInt()) : IndexVersionUtils.randomVersion(random());
+        return rarely() ? IndexVersion.fromId(randomInt()) : IndexVersionUtils.randomVersion();
     }
 
     public void testEqualsHashcodeSerialization() {
diff --git a/server/src/test/java/org/elasticsearch/features/FeatureServiceTests.java b/server/src/test/java/org/elasticsearch/features/FeatureServiceTests.java
index 874a6a96313e4..a64303f376b20 100644
--- a/server/src/test/java/org/elasticsearch/features/FeatureServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/features/FeatureServiceTests.java
@@ -9,8 +9,14 @@
 
 package org.elasticsearch.features;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.node.VersionInformation;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.List;
@@ -69,6 +75,12 @@ public void testStateHasFeatures() {
         );
 
         ClusterState state = ClusterState.builder(ClusterName.DEFAULT)
+            .nodes(
+                DiscoveryNodes.builder()
+                    .add(DiscoveryNodeUtils.create("node1"))
+                    .add(DiscoveryNodeUtils.create("node2"))
+                    .add(DiscoveryNodeUtils.create("node3"))
+            )
             .nodeFeatures(
                 Map.of("node1", Set.of("f1", "f2", "nf1"), "node2", Set.of("f1", "f2", "nf2"), "node3", Set.of("f1", "f2", "nf1"))
             )
@@ -81,4 +93,33 @@ public void testStateHasFeatures() {
         assertFalse(service.clusterHasFeature(state, new NodeFeature("nf2")));
         assertFalse(service.clusterHasFeature(state, new NodeFeature("nf3")));
     }
+
+    private static Version nextMajor() {
+        return Version.fromId((Version.CURRENT.major + 1) * 1_000_000 + 99);
+    }
+
+    public void testStateHasAssumedFeatures() {
+        List<FeatureSpecification> specs = List.of(
+            new TestFeatureSpecification(Set.of(new NodeFeature("f1"), new NodeFeature("f2"), new NodeFeature("af1", true)))
+        );
+
+        ClusterState state = ClusterState.builder(ClusterName.DEFAULT)
+            .nodes(
+                DiscoveryNodes.builder()
+                    .add(DiscoveryNodeUtils.create("node1"))
+                    .add(DiscoveryNodeUtils.create("node2"))
+                    .add(
+                        DiscoveryNodeUtils.builder("node3")
+                            .version(new VersionInformation(nextMajor(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current()))
+                            .build()
+                    )
+            )
+            .nodeFeatures(Map.of("node1", Set.of("f1", "af1"), "node2", Set.of("f1", "f2", "af1"), "node3", Set.of("f1", "f2")))
+            .build();
+
+        FeatureService service = new FeatureService(specs);
+        assertTrue(service.clusterHasFeature(state, new NodeFeature("f1")));
+        assertFalse(service.clusterHasFeature(state, new NodeFeature("f2")));
+        assertTrue(service.clusterHasFeature(state, new NodeFeature("af1", true)));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
index a161794e35b91..fd0718e5280fe 100644
--- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
@@ -264,7 +264,11 @@ private static class MockIndexMetadataVerifier extends IndexMetadataVerifier {
         }
 
         @Override
-        public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) {
+        public IndexMetadata verifyIndexMetadata(
+            IndexMetadata indexMetadata,
+            IndexVersion minimumIndexCompatibilityVersion,
+            IndexVersion minimumReadOnlyIndexCompatibilityVersion
+        ) {
             return upgrade ? IndexMetadata.builder(indexMetadata).build() : indexMetadata;
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java
index 4428a7e078510..7e761cb10d4d6 100644
--- a/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java
@@ -54,7 +54,6 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.env.BuildVersion;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
@@ -1392,7 +1391,7 @@ public void testLimitsFileCount() throws IOException {
         }
     }
 
-    public void testOverrideLuceneVersion() throws IOException {
+    public void testOverrideNodeVersion() throws IOException {
         try (NodeEnvironment nodeEnvironment = newNodeEnvironment(createDataPaths())) {
             final PersistedClusterStateService persistedClusterStateService = newPersistedClusterStateService(nodeEnvironment);
             final String clusterUUID = UUIDs.randomBase64UUID(random());
@@ -1415,9 +1414,7 @@ public void testOverrideLuceneVersion() throws IOException {
                 assertThat(clusterState.metadata().version(), equalTo(version));
 
             }
-            @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_FOUNDATIONS)
             BuildVersion overrideVersion = BuildVersion.fromVersionId(Version.V_8_0_0.id);
-
             NodeMetadata prevMetadata = PersistedClusterStateService.nodeMetadata(persistedClusterStateService.getDataPaths());
             assertEquals(BuildVersion.current(), prevMetadata.nodeVersion());
             PersistedClusterStateService.overrideVersion(overrideVersion, persistedClusterStateService.getDataPaths());
diff --git a/server/src/test/java/org/elasticsearch/health/HealthIndicatorResultTests.java b/server/src/test/java/org/elasticsearch/health/HealthIndicatorResultTests.java
index cba0dacccd8bd..91936ea8b9092 100644
--- a/server/src/test/java/org/elasticsearch/health/HealthIndicatorResultTests.java
+++ b/server/src/test/java/org/elasticsearch/health/HealthIndicatorResultTests.java
@@ -198,8 +198,8 @@ public void testChunkCount() {
         diagnosisList.add(diagnosis2);
         HealthIndicatorResult result = new HealthIndicatorResult(name, status, symptom, details, impacts, diagnosisList);
 
-        // -> each Diagnosis yields 6 chunks => 12 chunks from both diagnosis
-        // -> HealthIndicatorResult surrounds the diagnosis list by 5 chunks
-        AbstractChunkedSerializingTestCase.assertChunkCount(result, ignored -> (6 * 2) + 5);
+        // -> each Diagnosis yields 5 chunks => 10 chunks from both diagnosis
+        // -> HealthIndicatorResult surrounds the diagnosis list by 2 chunks
+        AbstractChunkedSerializingTestCase.assertChunkCount(result, ignored -> 12);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/health/node/LocalHealthMonitorTests.java b/server/src/test/java/org/elasticsearch/health/node/LocalHealthMonitorTests.java
index 7a63934b4810b..eca30fdce2cf8 100644
--- a/server/src/test/java/org/elasticsearch/health/node/LocalHealthMonitorTests.java
+++ b/server/src/test/java/org/elasticsearch/health/node/LocalHealthMonitorTests.java
@@ -24,8 +24,6 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.RelativeByteSizeValue;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.features.FeatureService;
-import org.elasticsearch.health.HealthFeatures;
 import org.elasticsearch.health.HealthStatus;
 import org.elasticsearch.health.metadata.HealthMetadata;
 import org.elasticsearch.health.node.selection.HealthNode;
@@ -119,18 +117,9 @@ public void setUp() throws Exception {
 
         client = mock(Client.class);
 
-        FeatureService featureService = new FeatureService(List.of(new HealthFeatures()));
-
         mockHealthTracker = new MockHealthTracker();
 
-        localHealthMonitor = LocalHealthMonitor.create(
-            Settings.EMPTY,
-            clusterService,
-            threadPool,
-            client,
-            featureService,
-            List.of(mockHealthTracker)
-        );
+        localHealthMonitor = LocalHealthMonitor.create(Settings.EMPTY, clusterService, threadPool, client, List.of(mockHealthTracker));
     }
 
     @After
diff --git a/server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutorTests.java b/server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutorTests.java
index 97f44f7480a72..3069589f9556c 100644
--- a/server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutorTests.java
+++ b/server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutorTests.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.features.FeatureService;
-import org.elasticsearch.health.HealthFeatures;
 import org.elasticsearch.persistent.PersistentTaskState;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksService;
@@ -77,8 +76,8 @@ public void setUp() throws Exception {
         clusterService = createClusterService(threadPool);
         localNodeId = clusterService.localNode().getId();
         persistentTasksService = mock(PersistentTasksService.class);
-        featureService = new FeatureService(List.of(new HealthFeatures()));
         settings = Settings.builder().build();
+        featureService = new FeatureService(List.of());
         clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
     }
 
@@ -196,6 +195,7 @@ private ClusterState stateWithNodeShuttingDown(ClusterState clusterState, Single
                 localNodeId,
                 SingleNodeShutdownMetadata.builder()
                     .setNodeId(localNodeId)
+                    .setNodeEphemeralId(localNodeId)
                     .setReason("shutdown for a unit test")
                     .setType(type)
                     .setStartedAtMillis(randomNonNegativeLong())
diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java
index 441ad8a5a225a..7221d69b74d46 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java
@@ -160,6 +160,20 @@ public void testSortingAgainstAliases() {
         assertEquals("Cannot use alias [field] as an index sort field", e.getMessage());
     }
 
+    public void testSortingAgainstAliasesPre713() {
+        IndexSettings indexSettings = indexSettings(
+            Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersions.V_7_12_0).put("index.sort.field", "field").build()
+        );
+        MappedFieldType aliased = new KeywordFieldMapper.KeywordFieldType("aliased");
+        Sort sort = buildIndexSort(indexSettings, Map.of("field", aliased));
+        assertThat(sort.getSort(), arrayWithSize(1));
+        assertThat(sort.getSort()[0].getField(), equalTo("aliased"));
+        assertWarnings(
+            "Index sort for index [test] defined on field [field] which resolves to field [aliased]. "
+                + "You will not be able to define an index sort over aliased fields in new indexes"
+        );
+    }
+
     public void testTimeSeriesMode() {
         IndexSettings indexSettings = indexSettings(
             Settings.builder()
diff --git a/server/src/test/java/org/elasticsearch/index/IndexVersionTests.java b/server/src/test/java/org/elasticsearch/index/IndexVersionTests.java
index 2a425c9256c31..8575b87c36799 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexVersionTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexVersionTests.java
@@ -11,7 +11,6 @@
 
 import org.apache.lucene.util.Version;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.index.IndexVersionUtils;
 import org.hamcrest.Matchers;
@@ -151,9 +150,7 @@ public void testMax() {
         }
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
-    @AwaitsFix(bugUrl = "believe this fails because index version has not yet been bumped to 9.0")
-    public void testMinimumCompatibleVersion() {
+    public void testGetMinimumCompatibleIndexVersion() {
         assertThat(IndexVersion.getMinimumCompatibleIndexVersion(7170099), equalTo(IndexVersion.fromId(6000099)));
         assertThat(IndexVersion.getMinimumCompatibleIndexVersion(8000099), equalTo(IndexVersion.fromId(7000099)));
         assertThat(IndexVersion.getMinimumCompatibleIndexVersion(10000000), equalTo(IndexVersion.fromId(9000000)));
@@ -193,8 +190,6 @@ public void testParseLenient() {
         }
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
-    @AwaitsFix(bugUrl = "can be unmuted once lucene is bumped to version 10")
     public void testLuceneVersionOnUnknownVersions() {
         // between two known versions, should use the lucene version of the previous version
         IndexVersion previousVersion = IndexVersionUtils.getPreviousVersion();
@@ -207,7 +202,7 @@ public void testLuceneVersionOnUnknownVersions() {
 
         // too old version, major should be the oldest supported lucene version minus 1
         IndexVersion oldVersion = IndexVersion.fromId(5020199);
-        assertThat(oldVersion.luceneVersion().major, equalTo(IndexVersionUtils.getFirstVersion().luceneVersion().major - 1));
+        assertThat(oldVersion.luceneVersion().major, equalTo(IndexVersionUtils.getLowestReadCompatibleVersion().luceneVersion().major - 1));
 
         // future version, should be the same version as today
         IndexVersion futureVersion = IndexVersion.fromId(currentVersion.id() + 100);
diff --git a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
index c626be7983c46..ce8b0e457b762 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
@@ -81,7 +81,7 @@ public void testLevelPrecedence() {
         String uuid = UUIDs.randomBase64UUID();
         IndexMetadata metadata = createIndexMetadata("index-precedence", settings(uuid));
         IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        IndexingSlowLog log = new IndexingSlowLog(settings, mock(SlowLogFieldProvider.class));
+        IndexingSlowLog log = new IndexingSlowLog(settings, mock(SlowLogFields.class));
 
         ParsedDocument doc = EngineTestCase.createParsedDoc("1", null);
         Engine.Index index = new Engine.Index(Uid.encodeId("doc_id"), randomNonNegativeLong(), doc);
@@ -142,7 +142,7 @@ public void testTwoLoggersDifferentLevel() {
             ),
             Settings.EMPTY
         );
-        IndexingSlowLog log1 = new IndexingSlowLog(index1Settings, mock(SlowLogFieldProvider.class));
+        IndexingSlowLog log1 = new IndexingSlowLog(index1Settings, mock(SlowLogFields.class));
 
         IndexSettings index2Settings = new IndexSettings(
             createIndexMetadata(
@@ -155,7 +155,7 @@ public void testTwoLoggersDifferentLevel() {
             ),
             Settings.EMPTY
         );
-        IndexingSlowLog log2 = new IndexingSlowLog(index2Settings, mock(SlowLogFieldProvider.class));
+        IndexingSlowLog log2 = new IndexingSlowLog(index2Settings, mock(SlowLogFields.class));
 
         ParsedDocument doc = EngineTestCase.createParsedDoc("1", null);
         Engine.Index index = new Engine.Index(Uid.encodeId("doc_id"), randomNonNegativeLong(), doc);
@@ -179,12 +179,12 @@ public void testMultipleSlowLoggersUseSingleLog4jLogger() {
         LoggerContext context = (LoggerContext) LogManager.getContext(false);
 
         IndexSettings index1Settings = new IndexSettings(createIndexMetadata("index1", settings(UUIDs.randomBase64UUID())), Settings.EMPTY);
-        IndexingSlowLog log1 = new IndexingSlowLog(index1Settings, mock(SlowLogFieldProvider.class));
+        IndexingSlowLog log1 = new IndexingSlowLog(index1Settings, mock(SlowLogFields.class));
 
         int numberOfLoggersBefore = context.getLoggers().size();
 
         IndexSettings index2Settings = new IndexSettings(createIndexMetadata("index2", settings(UUIDs.randomBase64UUID())), Settings.EMPTY);
-        IndexingSlowLog log2 = new IndexingSlowLog(index2Settings, mock(SlowLogFieldProvider.class));
+        IndexingSlowLog log2 = new IndexingSlowLog(index2Settings, mock(SlowLogFields.class));
         context = (LoggerContext) LogManager.getContext(false);
 
         int numberOfLoggersAfter = context.getLoggers().size();
@@ -355,7 +355,7 @@ public void testReformatSetting() {
                 .build()
         );
         IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        IndexingSlowLog log = new IndexingSlowLog(settings, mock(SlowLogFieldProvider.class));
+        IndexingSlowLog log = new IndexingSlowLog(settings, mock(SlowLogFields.class));
         assertFalse(log.isReformat());
         settings.updateIndexMetadata(
             newIndexMeta("index", Settings.builder().put(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING.getKey(), "true").build())
@@ -372,7 +372,7 @@ public void testReformatSetting() {
 
         metadata = newIndexMeta("index", Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build());
         settings = new IndexSettings(metadata, Settings.EMPTY);
-        log = new IndexingSlowLog(settings, mock(SlowLogFieldProvider.class));
+        log = new IndexingSlowLog(settings, mock(SlowLogFields.class));
         assertTrue(log.isReformat());
         try {
             settings.updateIndexMetadata(
@@ -405,7 +405,7 @@ public void testSetLevels() {
                 .build()
         );
         IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        IndexingSlowLog log = new IndexingSlowLog(settings, mock(SlowLogFieldProvider.class));
+        IndexingSlowLog log = new IndexingSlowLog(settings, mock(SlowLogFields.class));
         assertEquals(TimeValue.timeValueMillis(100).nanos(), log.getIndexTraceThreshold());
         assertEquals(TimeValue.timeValueMillis(200).nanos(), log.getIndexDebugThreshold());
         assertEquals(TimeValue.timeValueMillis(300).nanos(), log.getIndexInfoThreshold());
@@ -436,7 +436,7 @@ public void testSetLevels() {
         assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getIndexWarnThreshold());
 
         settings = new IndexSettings(metadata, Settings.EMPTY);
-        log = new IndexingSlowLog(settings, mock(SlowLogFieldProvider.class));
+        log = new IndexingSlowLog(settings, mock(SlowLogFields.class));
 
         assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getIndexTraceThreshold());
         assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getIndexDebugThreshold());
diff --git a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
index 23fc788a89bde..6f3077defa4a3 100644
--- a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.index.IndexVersionUtils;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -25,11 +26,89 @@ public void testLogsIndexModeSetting() {
     public void testDefaultHostNameSortField() {
         final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings());
         assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB));
-        final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
+        boolean sortOnHostName = randomBoolean();
+        final IndexSettings settings = new IndexSettings(
+            metadata,
+            Settings.builder().put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), sortOnHostName).build()
+        );
+        assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(sortOnHostName));
+    }
+
+    public void testDefaultHostNameSortFieldAndMapping() {
+        final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings());
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB));
+        final IndexSettings settings = new IndexSettings(
+            metadata,
+            Settings.builder()
+                .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true)
+                .put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true)
+                .build()
+        );
         assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(true));
         assertThat(IndexMode.LOGSDB.getDefaultMapping(settings).string(), containsString("host.name"));
     }
 
+    public void testDefaultHostNameSortFieldBwc() {
+        final IndexMetadata metadata = IndexMetadata.builder("test")
+            .settings(
+                indexSettings(IndexVersionUtils.getPreviousVersion(IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME), 1, 1).put(
+                    buildSettings()
+                )
+            )
+            .build();
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB));
+        final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
+        assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(true));
+    }
+
+    public void testDefaultHostNameSortWithOrder() {
+        final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings());
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB));
+        var exception = expectThrows(
+            IllegalArgumentException.class,
+            () -> new IndexSettings(
+                metadata,
+                Settings.builder()
+                    .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), randomBoolean())
+                    .put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "desc")
+                    .build()
+            )
+        );
+        assertEquals("index.sort.fields:[] index.sort.order:[desc], size mismatch", exception.getMessage());
+    }
+
+    public void testDefaultHostNameSortWithMode() {
+        final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings());
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB));
+        var exception = expectThrows(
+            IllegalArgumentException.class,
+            () -> new IndexSettings(
+                metadata,
+                Settings.builder()
+                    .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), randomBoolean())
+                    .put(IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey(), "MAX")
+                    .build()
+            )
+        );
+        assertEquals("index.sort.fields:[] index.sort.mode:[MAX], size mismatch", exception.getMessage());
+    }
+
+    public void testDefaultHostNameSortWithMissing() {
+        final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings());
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB));
+        var exception = expectThrows(
+            IllegalArgumentException.class,
+            () -> new IndexSettings(
+                metadata,
+                Settings.builder()
+                    .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), randomBoolean())
+                    .put(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_first")
+                    .build()
+            )
+        );
+        assertEquals("index.sort.fields:[] index.sort.missing:[_first], size mismatch", exception.getMessage());
+    }
+
     public void testCustomSortField() {
         final Settings sortSettings = Settings.builder()
             .put(buildSettings())
diff --git a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
index 50e3269a6b9ba..359118c7cb5a1 100644
--- a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
+++ b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
@@ -103,7 +103,7 @@ public void testLevelPrecedence() {
         try (SearchContext ctx = searchContextWithSourceAndTask(createIndex("index"))) {
             String uuid = UUIDs.randomBase64UUID();
             IndexSettings settings = new IndexSettings(createIndexMetadata("index", settings(uuid)), Settings.EMPTY);
-            SearchSlowLog log = new SearchSlowLog(settings, mock(SlowLogFieldProvider.class));
+            SearchSlowLog log = new SearchSlowLog(settings, mock(SlowLogFields.class));
 
             // For this test, when level is not breached, the level below should be used.
             {
@@ -187,7 +187,7 @@ public void testTwoLoggersDifferentLevel() {
                 ),
                 Settings.EMPTY
             );
-            SearchSlowLog log1 = new SearchSlowLog(settings1, mock(SlowLogFieldProvider.class));
+            SearchSlowLog log1 = new SearchSlowLog(settings1, mock(SlowLogFields.class));
 
             IndexSettings settings2 = new IndexSettings(
                 createIndexMetadata(
@@ -200,7 +200,7 @@ public void testTwoLoggersDifferentLevel() {
                 ),
                 Settings.EMPTY
             );
-            SearchSlowLog log2 = new SearchSlowLog(settings2, mock(SlowLogFieldProvider.class));
+            SearchSlowLog log2 = new SearchSlowLog(settings2, mock(SlowLogFields.class));
 
             {
                 // threshold set on WARN only, should not log
@@ -223,7 +223,7 @@ public void testMultipleSlowLoggersUseSingleLog4jLogger() {
 
         try (SearchContext ctx1 = searchContextWithSourceAndTask(createIndex("index-1"))) {
             IndexSettings settings1 = new IndexSettings(createIndexMetadata("index-1", settings(UUIDs.randomBase64UUID())), Settings.EMPTY);
-            SearchSlowLog log1 = new SearchSlowLog(settings1, mock(SlowLogFieldProvider.class));
+            SearchSlowLog log1 = new SearchSlowLog(settings1, mock(SlowLogFields.class));
             int numberOfLoggersBefore = context.getLoggers().size();
 
             try (SearchContext ctx2 = searchContextWithSourceAndTask(createIndex("index-2"))) {
@@ -231,7 +231,7 @@ public void testMultipleSlowLoggersUseSingleLog4jLogger() {
                     createIndexMetadata("index-2", settings(UUIDs.randomBase64UUID())),
                     Settings.EMPTY
                 );
-                SearchSlowLog log2 = new SearchSlowLog(settings2, mock(SlowLogFieldProvider.class));
+                SearchSlowLog log2 = new SearchSlowLog(settings2, mock(SlowLogFields.class));
 
                 int numberOfLoggersAfter = context.getLoggers().size();
                 assertThat(numberOfLoggersAfter, equalTo(numberOfLoggersBefore));
@@ -323,7 +323,7 @@ public void testSetQueryLevels() {
                 .build()
         );
         IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        SearchSlowLog log = new SearchSlowLog(settings, mock(SlowLogFieldProvider.class));
+        SearchSlowLog log = new SearchSlowLog(settings, mock(SlowLogFields.class));
         assertEquals(TimeValue.timeValueMillis(100).nanos(), log.getQueryTraceThreshold());
         assertEquals(TimeValue.timeValueMillis(200).nanos(), log.getQueryDebugThreshold());
         assertEquals(TimeValue.timeValueMillis(300).nanos(), log.getQueryInfoThreshold());
@@ -354,7 +354,7 @@ public void testSetQueryLevels() {
         assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getQueryWarnThreshold());
 
         settings = new IndexSettings(metadata, Settings.EMPTY);
-        log = new SearchSlowLog(settings, mock(SlowLogFieldProvider.class));
+        log = new SearchSlowLog(settings, mock(SlowLogFields.class));
 
         assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getQueryTraceThreshold());
         assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getQueryDebugThreshold());
@@ -429,7 +429,7 @@ public void testSetFetchLevels() {
                 .build()
         );
         IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        SearchSlowLog log = new SearchSlowLog(settings, mock(SlowLogFieldProvider.class));
+        SearchSlowLog log = new SearchSlowLog(settings, mock(SlowLogFields.class));
         assertEquals(TimeValue.timeValueMillis(100).nanos(), log.getFetchTraceThreshold());
         assertEquals(TimeValue.timeValueMillis(200).nanos(), log.getFetchDebugThreshold());
         assertEquals(TimeValue.timeValueMillis(300).nanos(), log.getFetchInfoThreshold());
@@ -460,7 +460,7 @@ public void testSetFetchLevels() {
         assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getFetchWarnThreshold());
 
         settings = new IndexSettings(metadata, Settings.EMPTY);
-        log = new SearchSlowLog(settings, mock(SlowLogFieldProvider.class));
+        log = new SearchSlowLog(settings, mock(SlowLogFields.class));
 
         assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getFetchTraceThreshold());
         assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getFetchDebugThreshold());
diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java
index d0fa037079255..93dedbb355fbd 100644
--- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java
@@ -173,14 +173,7 @@ public void testRoutingPathEqualsObjectNameError() {
             b.startObject("dim").field("type", "keyword").field("time_series_dimension", true).endObject();
             b.endObject().endObject();
         })));
-        assertThat(
-            e.getMessage(),
-            equalTo(
-                "All fields that match routing_path must be configured with [time_series_dimension: true] "
-                    + "or flattened fields with a list of dimensions in [time_series_dimensions] and "
-                    + "without the [script] parameter. [dim.o] was [object]."
-            )
-        );
+        assertThat(e.getMessage(), equalTo("All fields that match routing_path must be flattened fields. [dim.o] was [object]."));
     }
 
     public void testRoutingPathMatchesNonDimensionKeyword() {
diff --git a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java
index 04170030c1173..db780f0640986 100644
--- a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java
+++ b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java
@@ -106,7 +106,7 @@ public void setUp() throws Exception {
     }
 
     public void testDefaultAnalyzers() throws IOException {
-        IndexVersion version = IndexVersionUtils.randomVersion(random());
+        IndexVersion version = IndexVersionUtils.randomVersion();
         Settings settings = Settings.builder()
             .put(IndexMetadata.SETTING_VERSION_CREATED, version)
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
@@ -120,7 +120,7 @@ public void testDefaultAnalyzers() throws IOException {
     }
 
     public void testOverrideDefaultAnalyzer() throws IOException {
-        IndexVersion version = IndexVersionUtils.randomVersion(random());
+        IndexVersion version = IndexVersionUtils.randomVersion();
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
         IndexAnalyzers indexAnalyzers = AnalysisRegistry.build(
             IndexCreationContext.CREATE_INDEX,
@@ -137,7 +137,7 @@ public void testOverrideDefaultAnalyzer() throws IOException {
     }
 
     public void testOverrideDefaultAnalyzerWithoutAnalysisModeAll() throws IOException {
-        IndexVersion version = IndexVersionUtils.randomVersion(random());
+        IndexVersion version = IndexVersionUtils.randomVersion();
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("index", settings);
         TokenFilterFactory tokenFilter = new AbstractTokenFilterFactory("my_filter") {
@@ -216,7 +216,7 @@ public void testOverrideDefaultIndexAnalyzerIsUnsupported() {
     }
 
     public void testOverrideDefaultSearchAnalyzer() {
-        IndexVersion version = IndexVersionUtils.randomVersion(random());
+        IndexVersion version = IndexVersionUtils.randomVersion();
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
         IndexAnalyzers indexAnalyzers = AnalysisRegistry.build(
             IndexCreationContext.CREATE_INDEX,
@@ -319,8 +319,8 @@ public void testBuiltInAnalyzersAreCached() throws IOException {
         }
     }
 
-    public void testNoTypeOrTokenizerErrorMessage() throws IOException {
-        IndexVersion version = IndexVersionUtils.randomVersion(random());
+    public void testNoTypeOrTokenizerErrorMessage() {
+        IndexVersion version = IndexVersionUtils.randomVersion();
         Settings settings = Settings.builder()
             .put(IndexMetadata.SETTING_VERSION_CREATED, version)
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
diff --git a/server/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerTests.java b/server/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerTests.java
index 0a7bd495f2f22..7f3399cb24a15 100644
--- a/server/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerTests.java
+++ b/server/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerTests.java
@@ -55,6 +55,10 @@ public void testThatInstancesAreTheSameAlwaysForKeywordAnalyzer() {
             PreBuiltAnalyzers.KEYWORD.getAnalyzer(IndexVersion.current()),
             is(PreBuiltAnalyzers.KEYWORD.getAnalyzer(IndexVersions.MINIMUM_COMPATIBLE))
         );
+        assertThat(
+            PreBuiltAnalyzers.KEYWORD.getAnalyzer(IndexVersion.current()),
+            is(PreBuiltAnalyzers.KEYWORD.getAnalyzer(IndexVersions.MINIMUM_READONLY_COMPATIBLE))
+        );
     }
 
     public void testThatInstancesAreCachedAndReused() {
@@ -63,7 +67,7 @@ public void testThatInstancesAreCachedAndReused() {
             PreBuiltAnalyzers.STANDARD.getAnalyzer(IndexVersion.current())
         );
         // same index version should be cached
-        IndexVersion v = IndexVersionUtils.randomVersion(random());
+        IndexVersion v = IndexVersionUtils.randomVersion();
         assertSame(PreBuiltAnalyzers.STANDARD.getAnalyzer(v), PreBuiltAnalyzers.STANDARD.getAnalyzer(v));
         assertNotSame(
             PreBuiltAnalyzers.STANDARD.getAnalyzer(IndexVersion.current()),
@@ -71,7 +75,7 @@ public void testThatInstancesAreCachedAndReused() {
         );
 
         // Same Lucene version should be cached:
-        IndexVersion v1 = IndexVersionUtils.randomVersion(random());
+        IndexVersion v1 = IndexVersionUtils.randomVersion();
         IndexVersion v2 = new IndexVersion(v1.id() - 1, v1.luceneVersion());
         assertSame(PreBuiltAnalyzers.STOP.getAnalyzer(v1), PreBuiltAnalyzers.STOP.getAnalyzer(v2));
     }
@@ -81,7 +85,7 @@ public void testThatAnalyzersAreUsedInMapping() throws IOException {
         PreBuiltAnalyzers randomPreBuiltAnalyzer = PreBuiltAnalyzers.values()[randomInt];
         String analyzerName = randomPreBuiltAnalyzer.name().toLowerCase(Locale.ROOT);
 
-        IndexVersion randomVersion = IndexVersionUtils.randomVersion(random());
+        IndexVersion randomVersion = IndexVersionUtils.randomWriteVersion();
         Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, randomVersion).build();
 
         NamedAnalyzer namedAnalyzer = new PreBuiltAnalyzerProvider(
diff --git a/server/src/test/java/org/elasticsearch/index/analysis/PreConfiguredTokenFilterTests.java b/server/src/test/java/org/elasticsearch/index/analysis/PreConfiguredTokenFilterTests.java
index 40b37452990c7..a1a91ef2373f3 100644
--- a/server/src/test/java/org/elasticsearch/index/analysis/PreConfiguredTokenFilterTests.java
+++ b/server/src/test/java/org/elasticsearch/index/analysis/PreConfiguredTokenFilterTests.java
@@ -41,7 +41,7 @@ public boolean incrementToken() {
 
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", Settings.EMPTY);
 
-        IndexVersion version1 = IndexVersionUtils.randomVersion(random());
+        IndexVersion version1 = IndexVersionUtils.randomVersion();
         Settings settings1 = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version1).build();
         TokenFilterFactory tff_v1_1 = pctf.get(indexSettings, TestEnvironment.newEnvironment(emptyNodeSettings), "singleton", settings1);
         TokenFilterFactory tff_v1_2 = pctf.get(indexSettings, TestEnvironment.newEnvironment(emptyNodeSettings), "singleton", settings1);
@@ -66,7 +66,7 @@ public boolean incrementToken() {
             }
         );
 
-        IndexVersion version1 = IndexVersionUtils.randomVersion(random());
+        IndexVersion version1 = IndexVersionUtils.randomVersion();
         IndexSettings indexSettings1 = IndexSettingsModule.newIndexSettings(
             "test",
             Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version1).build()
@@ -133,7 +133,7 @@ public boolean incrementToken() {
         );
         assertSame(tff_v1_1, tff_v1_2);
 
-        IndexVersion version2 = IndexVersionUtils.getPreviousMajorVersion(IndexVersionUtils.getFirstVersion());
+        IndexVersion version2 = IndexVersionUtils.getPreviousMajorVersion(IndexVersionUtils.getLowestReadCompatibleVersion());
         IndexSettings indexSettings2 = IndexSettingsModule.newIndexSettings(
             "test",
             Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version2).build()
diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java
index 4d97235c5fae5..61bd5323b5b43 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java
@@ -437,32 +437,35 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex(
         float[] centroid,
         float cDotC
     ) throws IOException {
-        long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES);
-        final IndexOutput tempQuantizedVectorData = segmentWriteState.directory.createTempOutput(
-            binarizedVectorData.getName(),
-            "temp",
-            segmentWriteState.context
-        );
-        final IndexOutput tempScoreQuantizedVectorData = segmentWriteState.directory.createTempOutput(
-            binarizedVectorData.getName(),
-            "score_temp",
-            segmentWriteState.context
-        );
-        IndexInput binarizedDataInput = null;
-        IndexInput binarizedScoreDataInput = null;
-        boolean success = false;
-        int descritizedDimension = BQVectorUtils.discretize(fieldInfo.getVectorDimension(), 64);
-        BinaryQuantizer quantizer = new BinaryQuantizer(
+        final long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES);
+        final int descritizedDimension = BQVectorUtils.discretize(fieldInfo.getVectorDimension(), 64);
+        final BinaryQuantizer quantizer = new BinaryQuantizer(
             fieldInfo.getVectorDimension(),
             descritizedDimension,
             fieldInfo.getVectorSimilarityFunction()
         );
+
+        IndexOutput tempQuantizedVectorData = null;
+        IndexOutput tempScoreQuantizedVectorData = null;
+        final DocsWithFieldSet docsWithField;
+        boolean success = false;
+
         try {
+            tempQuantizedVectorData = segmentWriteState.directory.createTempOutput(
+                binarizedVectorData.getName(),
+                "temp",
+                segmentWriteState.context
+            );
+            tempScoreQuantizedVectorData = segmentWriteState.directory.createTempOutput(
+                binarizedVectorData.getName(),
+                "score_temp",
+                segmentWriteState.context
+            );
             FloatVectorValues floatVectorValues = KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState);
             if (fieldInfo.getVectorSimilarityFunction() == COSINE) {
                 floatVectorValues = new NormalizedFloatVectorValues(floatVectorValues);
             }
-            DocsWithFieldSet docsWithField = writeBinarizedVectorAndQueryData(
+            docsWithField = writeBinarizedVectorAndQueryData(
                 tempQuantizedVectorData,
                 tempScoreQuantizedVectorData,
                 floatVectorValues,
@@ -470,13 +473,30 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex(
                 quantizer
             );
             CodecUtil.writeFooter(tempQuantizedVectorData);
-            IOUtils.close(tempQuantizedVectorData);
+            CodecUtil.writeFooter(tempScoreQuantizedVectorData);
+            success = true;
+        } finally {
+            if (success) {
+                IOUtils.close(tempQuantizedVectorData, tempScoreQuantizedVectorData);
+            } else {
+                IOUtils.closeWhileHandlingException(tempQuantizedVectorData, tempScoreQuantizedVectorData);
+                if (tempQuantizedVectorData != null) {
+                    IOUtils.deleteFilesIgnoringExceptions(segmentWriteState.directory, tempQuantizedVectorData.getName());
+                }
+                if (tempScoreQuantizedVectorData != null) {
+                    IOUtils.deleteFilesIgnoringExceptions(segmentWriteState.directory, tempScoreQuantizedVectorData.getName());
+                }
+            }
+        }
+
+        IndexInput binarizedDataInput = null;
+        IndexInput binarizedScoreDataInput = null;
+        success = false;
+        try {
             binarizedDataInput = segmentWriteState.directory.openInput(tempQuantizedVectorData.getName(), segmentWriteState.context);
             binarizedVectorData.copyBytes(binarizedDataInput, binarizedDataInput.length() - CodecUtil.footerLength());
-            long vectorDataLength = binarizedVectorData.getFilePointer() - vectorDataOffset;
+            final long vectorDataLength = binarizedVectorData.getFilePointer() - vectorDataOffset;
             CodecUtil.retrieveChecksum(binarizedDataInput);
-            CodecUtil.writeFooter(tempScoreQuantizedVectorData);
-            IOUtils.close(tempScoreQuantizedVectorData);
             binarizedScoreDataInput = segmentWriteState.directory.openInput(
                 tempScoreQuantizedVectorData.getName(),
                 segmentWriteState.context
@@ -490,10 +510,9 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex(
                 cDotC,
                 docsWithField
             );
-            success = true;
             final IndexInput finalBinarizedDataInput = binarizedDataInput;
             final IndexInput finalBinarizedScoreDataInput = binarizedScoreDataInput;
-            OffHeapBinarizedVectorValues vectorValues = new OffHeapBinarizedVectorValues.DenseOffHeapVectorValues(
+            final OffHeapBinarizedVectorValues vectorValues = new OffHeapBinarizedVectorValues.DenseOffHeapVectorValues(
                 fieldInfo.getVectorDimension(),
                 docsWithField.cardinality(),
                 centroid,
@@ -503,7 +522,7 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex(
                 vectorsScorer,
                 finalBinarizedDataInput
             );
-            RandomVectorScorerSupplier scorerSupplier = vectorsScorer.getRandomVectorScorerSupplier(
+            final RandomVectorScorerSupplier scorerSupplier = vectorsScorer.getRandomVectorScorerSupplier(
                 fieldInfo.getVectorSimilarityFunction(),
                 new OffHeapBinarizedQueryVectorValues(
                     finalBinarizedScoreDataInput,
@@ -513,22 +532,20 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex(
                 ),
                 vectorValues
             );
+            final String tempQuantizedVectorDataName = tempQuantizedVectorData.getName();
+            final String tempScoreQuantizedVectorDataName = tempScoreQuantizedVectorData.getName();
+            success = true;
             return new BinarizedCloseableRandomVectorScorerSupplier(scorerSupplier, vectorValues, () -> {
                 IOUtils.close(finalBinarizedDataInput, finalBinarizedScoreDataInput);
                 IOUtils.deleteFilesIgnoringExceptions(
                     segmentWriteState.directory,
-                    tempQuantizedVectorData.getName(),
-                    tempScoreQuantizedVectorData.getName()
+                    tempQuantizedVectorDataName,
+                    tempScoreQuantizedVectorDataName
                 );
             });
         } finally {
             if (success == false) {
-                IOUtils.closeWhileHandlingException(
-                    tempQuantizedVectorData,
-                    tempScoreQuantizedVectorData,
-                    binarizedDataInput,
-                    binarizedScoreDataInput
-                );
+                IOUtils.closeWhileHandlingException(binarizedDataInput, binarizedScoreDataInput);
                 IOUtils.deleteFilesIgnoringExceptions(
                     segmentWriteState.directory,
                     tempQuantizedVectorData.getName(),
diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 3e3be6a315af2..26de6a7897786 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -3568,7 +3568,8 @@ public void testRecoverFromForeignTranslog() throws IOException {
             new TranslogDeletionPolicy(),
             () -> SequenceNumbers.NO_OPS_PERFORMED,
             primaryTerm::get,
-            seqNo -> {}
+            seqNo -> {},
+            TranslogOperationAsserter.DEFAULT
         );
         translog.add(TranslogOperationsUtils.indexOp("SomeBogusId", 0, primaryTerm.get()));
         assertEquals(generation.translogFileGeneration(), translog.currentFileGeneration());
@@ -6645,7 +6646,7 @@ public void testStoreHonorsLuceneVersion() throws IOException {
         for (IndexVersion createdVersion : List.of(
             IndexVersion.current(),
             lowestCompatiblePreviousVersion,
-            IndexVersionUtils.getFirstVersion()
+            IndexVersionUtils.getLowestWriteCompatibleVersion()
         )) {
             Settings settings = Settings.builder().put(indexSettings()).put(IndexMetadata.SETTING_VERSION_CREATED, createdVersion).build();
             IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", settings);
diff --git a/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java b/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java
index 2a6c3428d6d45..a5d5d9b210e33 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java
@@ -19,14 +19,12 @@
 
 import java.io.IOException;
 
-import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING;
-
 public class LuceneSyntheticSourceChangesSnapshotTests extends SearchBasedChangesSnapshotTests {
     @Override
     protected Settings indexSettings() {
         return Settings.builder()
             .put(super.indexSettings())
-            .put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name())
+            .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name())
             .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true)
             .build();
     }
diff --git a/server/src/test/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicyTests.java b/server/src/test/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicyTests.java
index 74d6e83aff266..b8600842effe4 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicyTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicyTests.java
@@ -44,7 +44,7 @@
 import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 public class RecoverySourcePruneMergePolicyTests extends ESTestCase {
 
@@ -191,7 +191,7 @@ public void testPruneSome() throws IOException {
                                     }
                                     assertEquals(i, extra_source.docID());
                                     if (syntheticRecoverySource) {
-                                        assertThat(extra_source.longValue(), greaterThan(10L));
+                                        assertThat(extra_source.longValue(), greaterThanOrEqualTo(10L));
                                     } else {
                                         assertThat(extra_source.longValue(), equalTo(1L));
                                     }
diff --git a/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java b/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java
new file mode 100644
index 0000000000000..d0455c14bd784
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.engine;
+
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+
+public class TranslogOperationAsserterTests extends EngineTestCase {
+
+    @Override
+    protected Settings indexSettings() {
+        return Settings.builder()
+            .put(super.indexSettings())
+            .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
+            .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name())
+            .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true)
+            .build();
+    }
+
+    Translog.Index toIndexOp(String source) throws IOException {
+        XContentParser parser = createParser(XContentType.JSON.xContent(), source);
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.copyCurrentStructure(parser);
+        return new Translog.Index(
+            "1",
+            0,
+            1,
+            1,
+            new BytesArray(Strings.toString(builder)),
+            null,
+            IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP
+        );
+    }
+
+    EngineConfig engineConfig(boolean useSyntheticSource) {
+        EngineConfig config = engine.config();
+        Settings.Builder settings = Settings.builder().put(config.getIndexSettings().getSettings());
+        if (useSyntheticSource) {
+            settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name());
+            settings.put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true);
+        } else {
+            settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.name());
+            settings.put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), false);
+        }
+        IndexMetadata imd = IndexMetadata.builder(config.getIndexSettings().getIndexMetadata()).settings(settings).build();
+        return config(
+            new IndexSettings(imd, Settings.EMPTY),
+            config.getStore(),
+            config.getTranslogConfig().getTranslogPath(),
+            config.getMergePolicy(),
+            null
+        );
+    }
+
+    public void testBasic() throws Exception {
+        TranslogOperationAsserter syntheticAsserter = TranslogOperationAsserter.withEngineConfig(engineConfig(true));
+        TranslogOperationAsserter regularAsserter = TranslogOperationAsserter.withEngineConfig(engineConfig(false));
+        {
+            var o1 = toIndexOp("""
+                {
+                    "value": "value-1"
+                }
+                """);
+            var o2 = toIndexOp("""
+                {
+                    "value": [ "value-1" ]
+                }
+                """);
+            assertTrue(syntheticAsserter.assertSameIndexOperation(o1, o2));
+            assertFalse(regularAsserter.assertSameIndexOperation(o1, o2));
+        }
+        {
+            var o1 = toIndexOp("""
+                {
+                    "value": [ "value-1", "value-2" ]
+                }
+                """);
+            var o2 = toIndexOp("""
+                {
+                    "value": [ "value-1", "value-2" ]
+                }
+                """);
+            assertTrue(syntheticAsserter.assertSameIndexOperation(o1, o2));
+            assertTrue(regularAsserter.assertSameIndexOperation(o1, o2));
+        }
+        {
+            var o1 = toIndexOp("""
+                {
+                    "value": [ "value-2", "value-1" ]
+                }
+                """);
+            var o2 = toIndexOp("""
+                {
+                    "value": [ "value-1", "value-2" ]
+                }
+                """);
+            assertTrue(syntheticAsserter.assertSameIndexOperation(o1, o2));
+            assertFalse(regularAsserter.assertSameIndexOperation(o1, o2));
+        }
+        {
+            var o1 = toIndexOp("""
+                {
+                    "value": [ "value-1", "value-2" ]
+                }
+                """);
+            var o2 = toIndexOp("""
+                {
+                    "value": [ "value-1", "value-2", "value-2" ]
+                }
+                """);
+            assertTrue(syntheticAsserter.assertSameIndexOperation(o1, o2));
+            assertFalse(regularAsserter.assertSameIndexOperation(o1, o2));
+        }
+        {
+            var o1 = toIndexOp("""
+                {
+                    "value": [ "value-1", "value-2" ]
+                }
+                """);
+            var o2 = toIndexOp("""
+                {
+                    "value": [ "value-1", "value-2", "value-3" ]
+                }
+                """);
+            assertFalse(syntheticAsserter.assertSameIndexOperation(o1, o2));
+            assertFalse(regularAsserter.assertSameIndexOperation(o1, o2));
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java
new file mode 100644
index 0000000000000..130c10130c4f3
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.geo.Orientation;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.geo.GeometryTestUtils;
+import org.elasticsearch.geo.ShapeTestUtils;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.Rectangle;
+import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor;
+import org.elasticsearch.lucene.spatial.BinaryShapeDocValuesField;
+import org.elasticsearch.lucene.spatial.CartesianShapeIndexer;
+import org.elasticsearch.lucene.spatial.CoordinateEncoder;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.hamcrest.RectangleMatcher;
+import org.elasticsearch.test.hamcrest.WellKnownBinaryBytesRefMatcher;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.IntStream;
+
+public class AbstractShapeGeometryFieldMapperTests extends ESTestCase {
+    public void testCartesianBoundsBlockLoader() throws IOException {
+        testBoundsBlockLoaderAux(
+            CoordinateEncoder.CARTESIAN,
+            () -> ShapeTestUtils.randomGeometryWithoutCircle(0, false),
+            CartesianShapeIndexer::new,
+            SpatialEnvelopeVisitor::visitCartesian
+        );
+    }
+
+    // TODO when we turn this optimization on for geo, this test should pass.
+    public void ignoreTestGeoBoundsBlockLoader() throws IOException {
+        testBoundsBlockLoaderAux(
+            CoordinateEncoder.GEO,
+            () -> GeometryTestUtils.randomGeometryWithoutCircle(0, false),
+            field -> new GeoShapeIndexer(Orientation.RIGHT, field),
+            g -> SpatialEnvelopeVisitor.visitGeo(g, SpatialEnvelopeVisitor.WrapLongitude.WRAP)
+        );
+    }
+
+    private static void testBoundsBlockLoaderAux(
+        CoordinateEncoder encoder,
+        Supplier<Geometry> generator,
+        Function<String, ShapeIndexer> indexerFactory,
+        Function<Geometry, Optional<Rectangle>> visitor
+    ) throws IOException {
+        var geometries = IntStream.range(0, 50).mapToObj(i -> generator.get()).toList();
+        var loader = new AbstractShapeGeometryFieldMapper.AbstractShapeGeometryFieldType.BoundsBlockLoader("field", encoder);
+        try (Directory directory = newDirectory()) {
+            try (var iw = new RandomIndexWriter(random(), directory)) {
+                for (Geometry geometry : geometries) {
+                    var shape = new BinaryShapeDocValuesField("field", encoder);
+                    shape.add(indexerFactory.apply("field").indexShape(geometry), geometry);
+                    var doc = new Document();
+                    doc.add(shape);
+                    iw.addDocument(doc);
+                }
+            }
+
+            var expected = new ArrayList<Rectangle>();
+            var byteRefResults = new ArrayList<BytesRef>();
+            int currentIndex = 0;
+            try (DirectoryReader reader = DirectoryReader.open(directory)) {
+                for (var leaf : reader.leaves()) {
+                    LeafReader leafReader = leaf.reader();
+                    int numDocs = leafReader.numDocs();
+                    // We specifically check just the even indices, to verify the loader can skip documents correctly.
+                    int[] array = evenArray(numDocs);
+                    for (int i = 0; i < array.length; i += 1) {
+                        expected.add(visitor.apply(geometries.get(array[i] + currentIndex)).get());
+                    }
+                    try (var block = (TestBlock) loader.reader(leaf).read(TestBlock.factory(leafReader.numDocs()), TestBlock.docs(array))) {
+                        for (int i = 0; i < block.size(); i++) {
+                            byteRefResults.add((BytesRef) block.get(i));
+                        }
+                    }
+                    currentIndex += numDocs;
+                }
+            }
+
+            for (int i = 0; i < expected.size(); i++) {
+                Rectangle rectangle = expected.get(i);
+                var geoString = rectangle.toString();
+                assertThat(
+                    Strings.format("geometry '%s' wasn't extracted correctly", geoString),
+                    byteRefResults.get(i),
+                    WellKnownBinaryBytesRefMatcher.encodes(RectangleMatcher.closeToFloat(rectangle, 1e-3, encoder))
+                );
+            }
+        }
+    }
+
+    private static int[] evenArray(int maxIndex) {
+        return IntStream.range(0, maxIndex / 2).map(x -> x * 2).toArray();
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
index 134d21ba475b7..d5d9d84673a0e 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
@@ -60,6 +61,7 @@
 import java.util.Set;
 import java.util.function.Function;
 
+import static org.elasticsearch.index.mapper.CompletionFieldMapper.COMPLETION_CONTEXTS_LIMIT;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
@@ -757,7 +759,7 @@ public void testLimitOfContextMappings() throws Throwable {
             .startObject("suggest")
             .field("type", "completion")
             .startArray("contexts");
-        for (int i = 0; i < CompletionFieldMapper.COMPLETION_CONTEXTS_LIMIT + 1; i++) {
+        for (int i = 0; i < COMPLETION_CONTEXTS_LIMIT + 1; i++) {
             mappingBuilder.startObject();
             mappingBuilder.field("name", Integer.toString(i));
             mappingBuilder.field("type", "category");
@@ -769,7 +771,7 @@ public void testLimitOfContextMappings() throws Throwable {
         MapperParsingException e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
             b.field("type", "completion");
             b.startArray("contexts");
-            for (int i = 0; i < CompletionFieldMapper.COMPLETION_CONTEXTS_LIMIT + 1; i++) {
+            for (int i = 0; i < COMPLETION_CONTEXTS_LIMIT + 1; i++) {
                 b.startObject();
                 b.field("name", Integer.toString(i));
                 b.field("type", "category");
@@ -779,8 +781,29 @@ public void testLimitOfContextMappings() throws Throwable {
         })));
         assertTrue(
             e.getMessage(),
-            e.getMessage()
-                .contains("Limit of completion field contexts [" + CompletionFieldMapper.COMPLETION_CONTEXTS_LIMIT + "] has been exceeded")
+            e.getMessage().contains("Limit of completion field contexts [" + COMPLETION_CONTEXTS_LIMIT + "] has been exceeded")
+        );
+
+        // test pre-8 deprecation warnings
+        createDocumentMapper(IndexVersions.V_7_0_0, fieldMapping(b -> {
+            b.field("type", "completion");
+            b.startArray("contexts");
+            for (int i = 0; i < COMPLETION_CONTEXTS_LIMIT + 1; i++) {
+                b.startObject();
+                b.field("name", Integer.toString(i));
+                b.field("type", "category");
+                b.endObject();
+            }
+            b.endArray();
+        }));
+        assertCriticalWarnings(
+            "You have defined more than ["
+                + COMPLETION_CONTEXTS_LIMIT
+                + "] completion contexts"
+                + " in the mapping for field [field]. The maximum allowed number of completion contexts in a mapping will be limited to "
+                + "["
+                + COMPLETION_CONTEXTS_LIMIT
+                + "] starting in version [8.0]."
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
index 54f6143d5cb30..fcadc7b238a43 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
@@ -20,7 +20,9 @@
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType;
 import org.elasticsearch.script.DateFieldScript;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.test.index.IndexVersionUtils;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -44,6 +46,7 @@
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.mockito.Mockito.mock;
 
 public class DateFieldMapperTests extends MapperTestCase {
 
@@ -245,6 +248,10 @@ public void testBadNullValue() throws IOException {
                     + "failed to parse date field [foo] with format [strict_date_optional_time||epoch_millis]"
             )
         );
+
+        createDocumentMapper(IndexVersions.V_7_9_0, fieldMapping(b -> b.field("type", "date").field("null_value", "foo")));
+
+        assertWarnings("Error parsing [foo] as date in [null_value] on field [field]); [null_value] will be ignored");
     }
 
     public void testNullConfigValuesFail() {
@@ -757,4 +764,51 @@ public void testLegacyField() throws Exception {
         assertNotEquals(DEFAULT_DATE_TIME_FORMATTER, ((DateFieldType) service.fieldType("mydate")).dateTimeFormatter);
     }
 
+    public void testLegacyDateFormatName() {
+        DateFieldMapper.Builder builder = new DateFieldMapper.Builder(
+            "format",
+            DateFieldMapper.Resolution.MILLISECONDS,
+            null,
+            mock(ScriptService.class),
+            true,
+            // BWC compatible index, e.g 7.x
+            IndexVersionUtils.randomVersionBetween(
+                random(),
+                IndexVersions.V_7_0_0,
+                IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0)
+            )
+        );
+
+        // Check that we allow the use of camel case date formats on 7.x indices
+        @SuppressWarnings("unchecked")
+        FieldMapper.Parameter<String> formatParam = (FieldMapper.Parameter<String>) builder.getParameters()[3];
+        formatParam.parse("date_time_format", mock(MappingParserContext.class), "strictDateOptionalTime");
+        builder.buildFormatter(); // shouldn't throw exception
+
+        formatParam.parse("date_time_format", mock(MappingParserContext.class), "strictDateOptionalTime||strictDateOptionalTimeNanos");
+        builder.buildFormatter(); // shouldn't throw exception
+
+        DateFieldMapper.Builder newFieldBuilder = new DateFieldMapper.Builder(
+            "format",
+            DateFieldMapper.Resolution.MILLISECONDS,
+            null,
+            mock(ScriptService.class),
+            true,
+            IndexVersion.current()
+        );
+
+        @SuppressWarnings("unchecked")
+        final FieldMapper.Parameter<String> newFormatParam = (FieldMapper.Parameter<String>) newFieldBuilder.getParameters()[3];
+
+        // Check that we don't allow the use of camel case date formats on 8.x indices
+        assertEquals(
+            "Error parsing [format] on field [format]: Invalid format: [strictDateOptionalTime]: Unknown pattern letter: t",
+            expectThrows(IllegalArgumentException.class, () -> {
+                newFormatParam.parse("date_time_format", mock(MappingParserContext.class), "strictDateOptionalTime");
+                assertEquals("strictDateOptionalTime", newFormatParam.getValue());
+                newFieldBuilder.buildFormatter();
+            }).getMessage()
+        );
+
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java
index 7e9a196faaa26..c77b434168492 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java
@@ -21,8 +21,10 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.plugins.internal.XContentMeteringParserDecorator;
 import org.elasticsearch.test.XContentTestUtils;
+import org.elasticsearch.test.index.IndexVersionUtils;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
@@ -1376,6 +1378,42 @@ public void testSubobjectsFalseWithInnerNestedFromDynamicTemplate() {
         );
     }
 
+    /**
+     * test that bad mapping is accepted for indices created before 8.0.0.
+     * We still need to test this in v9 because of N-2 read compatibility
+     */
+    public void testMalformedDynamicMapping_v7() throws IOException {
+        String mapping = """
+                {
+                    "_doc": {
+                        "dynamic_templates": [
+                            {
+                                "my_template": {
+                                    "mapping": {
+                                        "ignore_malformed": true,
+                                        "type": "keyword"
+                                    },
+                                    "path_match": "*"
+                                }
+                            }
+                        ]
+                    }
+                }
+            """;
+        MapperParsingException ex = expectThrows(MapperParsingException.class, () -> createMapperService(mapping));
+        assertThat(ex.getMessage(), containsString("dynamic template [my_template] has invalid content"));
+
+        // the previous exception should be ignored by indices with v7 index versions
+        IndexVersion indexVersion = IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.V_8_0_0);
+        Settings settings = Settings.builder().put("number_of_shards", 1).put("index.version.created", indexVersion).build();
+        MapperService mapperService = createMapperService(indexVersion, settings, () -> randomBoolean());
+        merge(mapperService, mapping);
+        assertWarnings(
+            "Parameter [ignore_malformed] is used in a dynamic template mapping and has no effect on type [keyword]. "
+                + "Usage will result in an error in future major versions and should be removed."
+        );
+    }
+
     public void testSubobjectsAutoFlatPaths() throws IOException {
         assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled());
         MapperService mapperService = createDynamicTemplateAutoSubobjects();
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java
index 907a1a15721dc..4b0fac2cf2e0f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java
@@ -54,7 +54,7 @@ public void putMappings() {
     }
 
     public void testGetMappings() {
-        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings().get();
+        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get();
         assertExpectedMappings(getMappingsResponse.mappings());
     }
 
@@ -71,7 +71,7 @@ public void testGetFieldMappings() {
         assertFieldMappings(mappings.get("filtered"), FILTERED_FLAT_FIELDS);
         // double check that submitting the filtered mappings to an unfiltered index leads to the same get field mappings output
         // as the one coming from a filtered index with same mappings
-        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("filtered").get();
+        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "filtered").get();
         MappingMetadata filtered = getMappingsResponse.getMappings().get("filtered");
         assertAcked(indicesAdmin().prepareCreate("test").setMapping(filtered.getSourceAsMap()));
         GetFieldMappingsResponse response = indicesAdmin().prepareGetFieldMappings("test").setFields("*").get();
@@ -98,7 +98,7 @@ public void testFieldCapabilities() {
         assertFieldCaps(filtered, filteredFields);
         // double check that submitting the filtered mappings to an unfiltered index leads to the same field_caps output
         // as the one coming from a filtered index with same mappings
-        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("filtered").get();
+        GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "filtered").get();
         MappingMetadata filteredMapping = getMappingsResponse.getMappings().get("filtered");
         assertAcked(indicesAdmin().prepareCreate("test").setMapping(filteredMapping.getSourceAsMap()));
         FieldCapabilitiesResponse test = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("test")).actionGet();
@@ -155,7 +155,7 @@ private void assertExpectedMappings(Map<String, MappingMetadata> mappings) {
     private void assertMappingsAreValid(Map<String, Object> sourceAsMap) {
         // check that the returned filtered mappings are still valid mappings by submitting them and retrieving them back
         assertAcked(indicesAdmin().prepareCreate("test").setMapping(sourceAsMap));
-        GetMappingsResponse testMappingsResponse = indicesAdmin().prepareGetMappings("test").get();
+        GetMappingsResponse testMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get();
         assertEquals(1, testMappingsResponse.getMappings().size());
         // the mappings are returned unfiltered for this index, yet they are the same as the previous ones that were returned filtered
         assertFiltered(testMappingsResponse.getMappings().get("test"));
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java
index 8646e1b66dcb0..1ba5f423d4b03 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java
@@ -12,6 +12,7 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -130,7 +131,7 @@ private MapperService mapperServiceWithCustomSettings(
         for (var entry : customSettings.entrySet()) {
             settings.put(entry.getKey(), entry.getValue());
         }
-        settings.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC);
+        settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC);
         return createMapperService(settings.build(), mapping(mapping));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
index 86c1157259790..1b8a2d68cd930 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.script.IpFieldScript;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -208,6 +209,12 @@ public void testNullValue() throws IOException {
             e.getMessage(),
             "Failed to parse mapping: Error parsing [null_value] on field [field]: ':1' is not an IP string literal."
         );
+
+        createDocumentMapper(IndexVersions.V_7_9_0, fieldMapping(b -> {
+            b.field("type", "ip");
+            b.field("null_value", ":1");
+        }));
+        assertWarnings("Error parsing [:1] as IP in [null_value] on field [field]); [null_value] will be ignored");
     }
 
     public void testDimension() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupInferenceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupInferenceFieldMapperTests.java
index b1470c1ee5b3b..755b83e8eb7ad 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupInferenceFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupInferenceFieldMapperTests.java
@@ -105,11 +105,6 @@ public InferenceFieldMetadata getMetadata(Set<String> sourcePaths) {
             return new InferenceFieldMetadata(fullPath(), INFERENCE_ID, SEARCH_INFERENCE_ID, sourcePaths.toArray(new String[0]));
         }
 
-        @Override
-        public Object getOriginalValue(Map<String, Object> sourceAsMap) {
-            return null;
-        }
-
         @Override
         protected void parseCreateField(DocumentParserContext context) {}
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java
index e0f58b8922be2..b87ab09c530d6 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java
@@ -322,7 +322,11 @@ public void testBlankFieldName() throws Exception {
     }
 
     public void testBlankFieldNameBefore8_6_0() throws Exception {
-        IndexVersion version = IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.V_8_5_0);
+        IndexVersion version = IndexVersionUtils.randomVersionBetween(
+            random(),
+            IndexVersions.MINIMUM_READONLY_COMPATIBLE,
+            IndexVersions.V_8_5_0
+        );
         TransportVersion transportVersion = TransportVersionUtils.randomVersionBetween(
             random(),
             TransportVersions.MINIMUM_COMPATIBLE,
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTestCase.java
index 6ea41763cbcc2..b669c717719b0 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTestCase.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTestCase.java
@@ -43,7 +43,7 @@ public void testCreateExplicitMappingSucceeds() throws Exception {
             """, getMapping());
         var resp = client().admin().indices().prepareCreate("test").setMapping(mapping).get();
         assertTrue(resp.isAcknowledged());
-        var mappingsResp = client().admin().indices().prepareGetMappings("test").get();
+        var mappingsResp = client().admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get();
         var mappingMetadata = mappingsResp.getMappings().get("test");
         var fieldType = XContentMapValues.extractValue("properties.field.type", mappingMetadata.getSourceAsMap());
         assertThat(fieldType, equalTo(getTypeName()));
@@ -149,7 +149,7 @@ public void testCreateExplicitMappingInIndexTemplateSucceeds() throws Exception
         var resp = client().prepareIndex("test1").setSource("field", "hello world").get();
         assertThat(resp.status(), equalTo(RestStatus.CREATED));
 
-        var mappingsResp = client().admin().indices().prepareGetMappings("test1").get();
+        var mappingsResp = client().admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test1").get();
         var mappingMetadata = mappingsResp.getMappings().get("test1");
         var fieldType = XContentMapValues.extractValue("properties.field.type", mappingMetadata.getSourceAsMap());
         assertThat(fieldType, equalTo(getTypeName()));
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
index 378920d0e6db5..bc560d94b8f52 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
@@ -427,7 +427,7 @@ public void testRecoverySourceWitInvalidSettings() {
 
         {
             Settings settings = Settings.builder()
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString())
+                .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString())
                 .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true)
                 .build();
             IllegalArgumentException exc = expectThrows(
@@ -465,66 +465,12 @@ public void testRecoverySourceWitInvalidSettings() {
                 )
             );
         }
-        {
-            Settings settings = Settings.builder()
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString())
-                .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true)
-                .build();
-            IllegalArgumentException exc = expectThrows(
-                IllegalArgumentException.class,
-                () -> createMapperService(
-                    IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BACKPORT),
-                    settings,
-                    () -> false,
-                    topMapping(b -> {})
-                )
-            );
-            assertThat(
-                exc.getMessage(),
-                containsString(
-                    String.format(
-                        Locale.ROOT,
-                        "The setting [%s] is unavailable on this cluster",
-                        IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey()
-                    )
-                )
-            );
-        }
-        {
-            Settings settings = Settings.builder()
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString())
-                .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true)
-                .build();
-            IllegalArgumentException exc = expectThrows(
-                IllegalArgumentException.class,
-                () -> createMapperService(
-                    IndexVersionUtils.randomVersionBetween(
-                        random(),
-                        IndexVersions.UPGRADE_TO_LUCENE_10_0_0,
-                        IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER
-                    ),
-                    settings,
-                    () -> false,
-                    topMapping(b -> {})
-                )
-            );
-            assertThat(
-                exc.getMessage(),
-                containsString(
-                    String.format(
-                        Locale.ROOT,
-                        "The setting [%s] is unavailable on this cluster",
-                        IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey()
-                    )
-                )
-            );
-        }
     }
 
     public void testRecoverySourceWithSyntheticSource() throws IOException {
         {
             Settings settings = Settings.builder()
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString())
+                .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString())
                 .build();
             MapperService mapperService = createMapperService(settings, topMapping(b -> {}));
             DocumentMapper docMapper = mapperService.documentMapper();
@@ -534,7 +480,7 @@ public void testRecoverySourceWithSyntheticSource() throws IOException {
         }
         {
             Settings settings = Settings.builder()
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString())
+                .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString())
                 .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true)
                 .build();
             MapperService mapperService = createMapperService(settings, topMapping(b -> {}));
@@ -593,7 +539,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException {
             final XContentBuilder mappings = topMapping(b -> {});
             final Settings settings = Settings.builder()
                 .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name())
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
+                .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
                 .build();
             final MapperService mapperService = createMapperService(settings, mappings);
             DocumentMapper docMapper = mapperService.documentMapper();
@@ -603,7 +549,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException {
             final XContentBuilder mappings = topMapping(b -> {});
             final Settings settings = Settings.builder()
                 .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name())
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
+                .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
                 .build();
             final MapperService mapperService = createMapperService(settings, mappings);
             final DocumentMapper docMapper = mapperService.documentMapper();
@@ -613,7 +559,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException {
             final XContentBuilder mappings = topMapping(b -> {});
             final Settings settings = Settings.builder()
                 .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name())
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
+                .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
                 .build();
             final MapperService mapperService = createMapperService(settings, mappings);
             final DocumentMapper docMapper = mapperService.documentMapper();
@@ -625,7 +571,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException {
             final XContentBuilder mappings = topMapping(b -> {});
             final Settings settings = Settings.builder()
                 .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
+                .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
                 .build();
             final MapperService mapperService = createMapperService(settings, mappings);
             DocumentMapper docMapper = mapperService.documentMapper();
@@ -635,7 +581,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException {
             final XContentBuilder mappings = topMapping(b -> {});
             final Settings settings = Settings.builder()
                 .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
+                .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
                 .build();
             final MapperService mapperService = createMapperService(settings, mappings);
             final DocumentMapper docMapper = mapperService.documentMapper();
@@ -645,7 +591,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException {
             final XContentBuilder mappings = topMapping(b -> {});
             final Settings settings = Settings.builder()
                 .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
+                .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
                 .build();
             var ex = expectThrows(MapperParsingException.class, () -> createMapperService(settings, mappings));
             assertEquals("Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode", ex.getMessage());
@@ -667,7 +613,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException {
                 """;
             final Settings settings = Settings.builder()
                 .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
+                .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
                 .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
                 .build();
             final MapperService mapperService = createMapperService(settings, mappings);
@@ -689,7 +635,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException {
                 """;
             final Settings settings = Settings.builder()
                 .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
+                .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
                 .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
                 .build();
             final MapperService mapperService = createMapperService(settings, mappings);
@@ -711,7 +657,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException {
                 """;
             final Settings settings = Settings.builder()
                 .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
+                .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
                 .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
                 .build();
             var ex = expectThrows(MapperParsingException.class, () -> createMapperService(settings, mappings));
@@ -722,7 +668,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException {
         {
             final XContentBuilder mappings = topMapping(b -> {});
             final Settings settings = Settings.builder()
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
+                .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
                 .build();
             final MapperService mapperService = createMapperService(settings, mappings);
             DocumentMapper docMapper = mapperService.documentMapper();
@@ -731,7 +677,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException {
         {
             final XContentBuilder mappings = topMapping(b -> {});
             final Settings settings = Settings.builder()
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
+                .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
                 .build();
             final MapperService mapperService = createMapperService(settings, mappings);
             final DocumentMapper docMapper = mapperService.documentMapper();
@@ -740,7 +686,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException {
         {
             final XContentBuilder mappings = topMapping(b -> {});
             final Settings settings = Settings.builder()
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
+                .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
                 .build();
             final MapperService mapperService = createMapperService(settings, mappings);
             final DocumentMapper docMapper = mapperService.documentMapper();
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java
new file mode 100644
index 0000000000000..40e5829b5b12e
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper.blockloader;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.mapper.BlockLoaderTestCase;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+public class KeywordFieldBlockLoaderTests extends BlockLoaderTestCase {
+    public KeywordFieldBlockLoaderTests() {
+        super(FieldType.KEYWORD);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    protected Object expected(Map<String, Object> fieldMapping, Object value, boolean syntheticSource) {
+        if (value == null) {
+            return null;
+        }
+
+        var ignoreAbove = fieldMapping.get("ignore_above") == null
+            ? Integer.MAX_VALUE
+            : ((Number) fieldMapping.get("ignore_above")).intValue();
+
+        if (value instanceof String s) {
+            return convert(s, ignoreAbove);
+        }
+
+        Function<Stream<String>, Stream<BytesRef>> convertValues = s -> s.map(v -> convert(v, ignoreAbove)).filter(Objects::nonNull);
+
+        if ((boolean) fieldMapping.getOrDefault("doc_values", false)) {
+            // Sorted and no duplicates
+
+            var values = new HashSet<>((List<String>) value);
+            var resultList = convertValues.compose(s -> values.stream().filter(Objects::nonNull).sorted())
+                .andThen(Stream::toList)
+                .apply(values.stream());
+            return maybeFoldList(resultList);
+        }
+
+        // store: "true" and source
+        var resultList = convertValues.andThen(Stream::toList).apply(((List<String>) value).stream());
+        return maybeFoldList(resultList);
+    }
+
+    private Object maybeFoldList(List<?> list) {
+        if (list.isEmpty()) {
+            return null;
+        }
+
+        if (list.size() == 1) {
+            return list.get(0);
+        }
+
+        return list;
+    }
+
+    private BytesRef convert(String value, int ignoreAbove) {
+        if (value == null) {
+            return null;
+        }
+
+        return value.length() <= ignoreAbove ? new BytesRef(value) : null;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java
index 342d61b78defd..3f574a29469c2 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java
@@ -1514,19 +1514,19 @@ public void testVectorSimilarity() {
         );
         assertEquals(
             VectorSimilarityFunction.EUCLIDEAN,
-            VectorSimilarity.L2_NORM.vectorSimilarityFunction(IndexVersionUtils.randomVersion(random()), ElementType.BYTE)
+            VectorSimilarity.L2_NORM.vectorSimilarityFunction(IndexVersionUtils.randomVersion(), ElementType.BYTE)
         );
         assertEquals(
             VectorSimilarityFunction.EUCLIDEAN,
-            VectorSimilarity.L2_NORM.vectorSimilarityFunction(IndexVersionUtils.randomVersion(random()), ElementType.FLOAT)
+            VectorSimilarity.L2_NORM.vectorSimilarityFunction(IndexVersionUtils.randomVersion(), ElementType.FLOAT)
         );
         assertEquals(
             VectorSimilarityFunction.DOT_PRODUCT,
-            VectorSimilarity.DOT_PRODUCT.vectorSimilarityFunction(IndexVersionUtils.randomVersion(random()), ElementType.BYTE)
+            VectorSimilarity.DOT_PRODUCT.vectorSimilarityFunction(IndexVersionUtils.randomVersion(), ElementType.BYTE)
         );
         assertEquals(
             VectorSimilarityFunction.DOT_PRODUCT,
-            VectorSimilarity.DOT_PRODUCT.vectorSimilarityFunction(IndexVersionUtils.randomVersion(random()), ElementType.FLOAT)
+            VectorSimilarity.DOT_PRODUCT.vectorSimilarityFunction(IndexVersionUtils.randomVersion(), ElementType.FLOAT)
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java
index d37b4a4bacb4e..5c067cb2d0a27 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java
@@ -36,6 +36,7 @@
 import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.BBQ_MIN_DIMS;
 import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType.BYTE;
 import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType.FLOAT;
+import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.OVERSAMPLE_LIMIT;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -456,19 +457,18 @@ public void testRescoreOversampleModifiesNumCandidates() {
         );
 
         // Total results is k, internal k is multiplied by oversample
-        checkRescoreQueryParameters(fieldType, 10, 200, 2.5F, null, 500, 10);
+        checkRescoreQueryParameters(fieldType, 10, 200, 2.5F, 25, 200, 10);
         // If numCands < k, update numCands to k
-        checkRescoreQueryParameters(fieldType, 10, 20, 2.5F, null, 50, 10);
-        // Oversampling limits for num candidates
-        checkRescoreQueryParameters(fieldType, 1000, 1000, 11.0F, null, 10000, 1000);
-        checkRescoreQueryParameters(fieldType, 5000, 7500, 2.5F, null, 10000, 5000);
+        checkRescoreQueryParameters(fieldType, 10, 20, 2.5F, 25, 25, 10);
+        // Oversampling limits for k
+        checkRescoreQueryParameters(fieldType, 1000, 1000, 11.0F, OVERSAMPLE_LIMIT, OVERSAMPLE_LIMIT, 1000);
     }
 
     private static void checkRescoreQueryParameters(
         DenseVectorFieldType fieldType,
-        Integer k,
+        int k,
         int candidates,
-        float numCandsFactor,
+        float oversample,
         Integer expectedK,
         int expectedCandidates,
         int expectedResults
@@ -477,7 +477,7 @@ private static void checkRescoreQueryParameters(
             VectorData.fromFloats(new float[] { 1, 4, 10 }),
             k,
             candidates,
-            numCandsFactor,
+            oversample,
             null,
             null,
             null
diff --git a/server/src/test/java/org/elasticsearch/index/query/functionscore/ScoreFunctionBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/functionscore/ScoreFunctionBuilderTests.java
index 8d060d94e4c21..b58ac513a6449 100644
--- a/server/src/test/java/org/elasticsearch/index/query/functionscore/ScoreFunctionBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/functionscore/ScoreFunctionBuilderTests.java
@@ -66,7 +66,6 @@ public void testRandomScoreFunctionWithSeedNoField() throws Exception {
         Mockito.when(context.getFieldType(IdFieldMapper.NAME)).thenReturn(new KeywordFieldMapper.KeywordFieldType(IdFieldMapper.NAME));
         Mockito.when(context.isFieldMapped(IdFieldMapper.NAME)).thenReturn(true);
         builder.toFunction(context);
-        assertWarnings("As of version 7.0 Elasticsearch will require that a [field] parameter is provided when a [seed] is set");
     }
 
     public void testRandomScoreFunctionWithSeed() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/index/replication/RetentionLeasesReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/RetentionLeasesReplicationTests.java
index 8f45a15c73bb6..1f82d7998257e 100644
--- a/server/src/test/java/org/elasticsearch/index/replication/RetentionLeasesReplicationTests.java
+++ b/server/src/test/java/org/elasticsearch/index/replication/RetentionLeasesReplicationTests.java
@@ -147,7 +147,7 @@ protected void syncRetentionLeases(ShardId id, RetentionLeases leases, ActionLis
     public void testTurnOffTranslogRetentionAfterAllShardStarted() throws Exception {
         final Settings.Builder settings = Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
         if (randomBoolean()) {
-            settings.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomCompatibleVersion(random()));
+            settings.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomCompatibleWriteVersion(random()));
         }
         try (ReplicationGroup group = createGroup(between(1, 2), settings.build())) {
             group.startAll();
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index eacb4cf35a422..0bbf63b07c4cb 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -84,6 +84,7 @@
 import org.elasticsearch.index.engine.InternalEngineFactory;
 import org.elasticsearch.index.engine.ReadOnlyEngine;
 import org.elasticsearch.index.engine.Segment;
+import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.FieldDataStats;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -789,6 +790,21 @@ public void onFailure(final Exception e) {
                 }
             }, TimeValue.timeValueSeconds(30));
             latch.await();
+
+            // It's possible to acquire permits if we skip the primary mode check
+            var permitAcquiredLatch = new CountDownLatch(1);
+            indexShard.acquirePrimaryOperationPermit(ActionListener.wrap(r -> {
+                r.close();
+                permitAcquiredLatch.countDown();
+            }, Assert::assertNotNull), EsExecutors.DIRECT_EXECUTOR_SERVICE, false, IndexShard.PrimaryPermitCheck.NONE);
+            safeAwait(permitAcquiredLatch);
+
+            var allPermitsAcquiredLatch = new CountDownLatch(1);
+            indexShard.acquireAllPrimaryOperationsPermits(ActionListener.wrap(r -> {
+                r.close();
+                allPermitsAcquiredLatch.countDown();
+            }, Assert::assertNotNull), TimeValue.timeValueSeconds(30), IndexShard.PrimaryPermitCheck.NONE);
+            safeAwait(allPermitsAcquiredLatch);
         }
 
         if (Assertions.ENABLED && indexShard.routingEntry().isRelocationTarget() == false) {
@@ -1729,6 +1745,86 @@ public String[] listAll() throws IOException {
         }
     }
 
+    public void testIndexingErrors() throws IOException {
+        AtomicBoolean throwOnIndex = new AtomicBoolean();
+        IndexShard shard = newStartedShard(randomBoolean(), Settings.EMPTY, config -> new InternalEngine(config) {
+            @Override
+            public IndexResult index(Index index) throws IOException {
+                if (throwOnIndex.get()) {
+                    throw new IOException("test indexing errors");
+                } else {
+                    return super.index(index);
+                }
+            }
+        });
+        long nbIndexedDocs = randomIntBetween(1, 10);
+        AtomicLong nbFailed = new AtomicLong();
+        for (int id = 0; id < nbIndexedDocs; id++) {
+            throwOnIndex.set(randomBoolean());
+            if (throwOnIndex.get()) {
+                nbFailed.incrementAndGet();
+                int finalId = id;
+                expectThrows(IOException.class, () -> indexDoc(shard, "_doc", "test" + finalId));
+            } else {
+                Engine.IndexResult indexResult = indexDoc(shard, "_doc", "test" + id);
+                assertThat(indexResult.isCreated(), is(true));
+            }
+        }
+        assertThat(shard.indexingStats().getTotal().getIndexFailedCount(), equalTo(nbFailed.get()));
+        assertThat(shard.indexingStats().getTotal().getIndexFailedDueToVersionConflictCount(), equalTo(0L));
+        assertThat(shard.indexingStats().getTotal().getIndexCount(), equalTo(nbIndexedDocs - nbFailed.get()));
+        closeShards(shard);
+    }
+
+    public void testIndexingErrorsDueToVersionConflict() throws IOException {
+        AtomicBoolean throwOnIndex = new AtomicBoolean();
+        IndexShard shard = newStartedShard(true, Settings.EMPTY, config -> new InternalEngine(config) {
+            @Override
+            public IndexResult index(Index index) throws IOException {
+                if (throwOnIndex.get()) {
+                    throw new IOException("test indexing errors");
+                } else {
+                    return super.index(index);
+                }
+            }
+        });
+        long nbIndexedDocs = randomIntBetween(1, 10);
+        AtomicLong indexingFailedCount = new AtomicLong();
+        AtomicLong indexingFailedWithVersionConflictCount = new AtomicLong();
+        AtomicLong indexingSuccessCount = new AtomicLong();
+        for (int id = 0; id < nbIndexedDocs; id++) {
+            if (randomBoolean()) {
+                // version conflict
+                throwOnIndex.set(false);
+                indexingFailedWithVersionConflictCount.incrementAndGet();
+                indexingFailedCount.incrementAndGet();
+                Engine.IndexResult indexResult = indexDoc(shard, "test" + id, 10L, "{}", XContentType.JSON, null);
+                assertThat(indexResult.isCreated(), is(false));
+                assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
+            } else {
+                throwOnIndex.set(randomBoolean());
+                int finalId = id;
+                if (throwOnIndex.get()) {
+                    // indexing failure
+                    indexingFailedCount.incrementAndGet();
+                    expectThrows(IOException.class, () -> indexDoc(shard, "_doc", "test" + finalId));
+                } else {
+                    // indexing successful
+                    indexingSuccessCount.incrementAndGet();
+                    Engine.IndexResult indexResult = indexDoc(shard, "_doc", "test" + id);
+                    assertThat(indexResult.isCreated(), is(true));
+                }
+            }
+        }
+        assertThat(shard.indexingStats().getTotal().getIndexCount(), equalTo(indexingSuccessCount.get()));
+        assertThat(shard.indexingStats().getTotal().getIndexFailedCount(), equalTo(indexingFailedCount.get()));
+        assertThat(
+            shard.indexingStats().getTotal().getIndexFailedDueToVersionConflictCount(),
+            equalTo(indexingFailedWithVersionConflictCount.get())
+        );
+        closeShards(shard);
+    }
+
     public void testRefreshMetric() throws IOException {
         IndexShard shard = newStartedShard();
         // refresh on: finalize and end of recovery
diff --git a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java
index f6d7b3d1d65f3..ecb942492af53 100644
--- a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java
@@ -74,7 +74,7 @@ public float score(float freq, long norm) {
         };
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> SimilarityService.validateSimilarity(IndexVersions.MINIMUM_COMPATIBLE, negativeScoresSim)
+            () -> SimilarityService.validateSimilarity(IndexVersions.MINIMUM_READONLY_COMPATIBLE, negativeScoresSim)
         );
         assertThat(e.getMessage(), Matchers.containsString("Similarities should not return negative scores"));
 
@@ -99,7 +99,7 @@ public float score(float freq, long norm) {
         };
         e = expectThrows(
             IllegalArgumentException.class,
-            () -> SimilarityService.validateSimilarity(IndexVersions.MINIMUM_COMPATIBLE, decreasingScoresWithFreqSim)
+            () -> SimilarityService.validateSimilarity(IndexVersions.MINIMUM_READONLY_COMPATIBLE, decreasingScoresWithFreqSim)
         );
         assertThat(e.getMessage(), Matchers.containsString("Similarity scores should not decrease when term frequency increases"));
 
@@ -124,7 +124,7 @@ public float score(float freq, long norm) {
         };
         e = expectThrows(
             IllegalArgumentException.class,
-            () -> SimilarityService.validateSimilarity(IndexVersions.MINIMUM_COMPATIBLE, increasingScoresWithNormSim)
+            () -> SimilarityService.validateSimilarity(IndexVersions.MINIMUM_READONLY_COMPATIBLE, increasingScoresWithNormSim)
         );
         assertThat(e.getMessage(), Matchers.containsString("Similarity scores should not increase when norm increases"));
     }
diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java
index f72152bd7ff8b..9aa847c837e96 100644
--- a/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java
+++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.engine.TranslogOperationAsserter;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 import org.mockito.Mockito;
@@ -95,6 +96,7 @@ private Tuple<List<TranslogReader>, TranslogWriter> createReadersAndWriter() thr
                 BigArrays.NON_RECYCLING_INSTANCE,
                 TranslogTests.RANDOMIZING_IO_BUFFERS,
                 TranslogConfig.NOOP_OPERATION_LISTENER,
+                TranslogOperationAsserter.DEFAULT,
                 true
             );
             writer = Mockito.spy(writer);
diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index 97f49df41d099..99f2e2a562eec 100644
--- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -53,6 +53,7 @@
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.Engine.Operation.Origin;
+import org.elasticsearch.index.engine.TranslogOperationAsserter;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.LuceneDocument;
 import org.elasticsearch.index.mapper.ParsedDocument;
@@ -224,7 +225,8 @@ protected Translog createTranslog(TranslogConfig config) throws IOException {
             new TranslogDeletionPolicy(),
             () -> SequenceNumbers.NO_OPS_PERFORMED,
             primaryTerm::get,
-            getPersistedSeqNoConsumer()
+            getPersistedSeqNoConsumer(),
+            TranslogOperationAsserter.DEFAULT
         );
     }
 
@@ -235,7 +237,8 @@ protected Translog openTranslog(TranslogConfig config, String translogUUID) thro
             new TranslogDeletionPolicy(),
             () -> SequenceNumbers.NO_OPS_PERFORMED,
             primaryTerm::get,
-            getPersistedSeqNoConsumer()
+            getPersistedSeqNoConsumer(),
+            TranslogOperationAsserter.DEFAULT
         );
     }
 
@@ -270,7 +273,8 @@ private Translog create(Path path) throws IOException {
             new TranslogDeletionPolicy(),
             () -> globalCheckpoint.get(),
             primaryTerm::get,
-            getPersistedSeqNoConsumer()
+            getPersistedSeqNoConsumer(),
+            TranslogOperationAsserter.DEFAULT
         );
     }
 
@@ -1444,7 +1448,8 @@ public int write(ByteBuffer src) throws IOException {
                 new TranslogDeletionPolicy(),
                 () -> SequenceNumbers.NO_OPS_PERFORMED,
                 primaryTerm::get,
-                persistedSeqNos::add
+                persistedSeqNos::add,
+                TranslogOperationAsserter.DEFAULT
             ) {
                 @Override
                 ChannelFactory getChannelFactory() {
@@ -1559,7 +1564,8 @@ public void force(boolean metaData) throws IOException {
                 new TranslogDeletionPolicy(),
                 () -> SequenceNumbers.NO_OPS_PERFORMED,
                 primaryTerm::get,
-                persistedSeqNos::add
+                persistedSeqNos::add,
+                TranslogOperationAsserter.DEFAULT
             ) {
                 @Override
                 ChannelFactory getChannelFactory() {
@@ -1746,7 +1752,8 @@ public void testBasicRecovery() throws IOException {
                 translog.getDeletionPolicy(),
                 () -> SequenceNumbers.NO_OPS_PERFORMED,
                 primaryTerm::get,
-                seqNo -> {}
+                seqNo -> {},
+                TranslogOperationAsserter.DEFAULT
             );
             assertEquals(
                 "lastCommitted must be 1 less than current",
@@ -1803,7 +1810,8 @@ public void testRecoveryUncommitted() throws IOException {
                 deletionPolicy,
                 () -> SequenceNumbers.NO_OPS_PERFORMED,
                 primaryTerm::get,
-                seqNo -> {}
+                seqNo -> {},
+                TranslogOperationAsserter.DEFAULT
             )
         ) {
             assertNotNull(translogGeneration);
@@ -1830,7 +1838,8 @@ public void testRecoveryUncommitted() throws IOException {
                     deletionPolicy,
                     () -> SequenceNumbers.NO_OPS_PERFORMED,
                     primaryTerm::get,
-                    seqNo -> {}
+                    seqNo -> {},
+                    TranslogOperationAsserter.DEFAULT
                 )
             ) {
                 assertNotNull(translogGeneration);
@@ -1894,7 +1903,8 @@ public void testRecoveryUncommittedFileExists() throws IOException {
                 deletionPolicy,
                 () -> SequenceNumbers.NO_OPS_PERFORMED,
                 primaryTerm::get,
-                seqNo -> {}
+                seqNo -> {},
+                TranslogOperationAsserter.DEFAULT
             )
         ) {
             assertNotNull(translogGeneration);
@@ -1922,7 +1932,8 @@ public void testRecoveryUncommittedFileExists() throws IOException {
                     deletionPolicy,
                     () -> SequenceNumbers.NO_OPS_PERFORMED,
                     primaryTerm::get,
-                    seqNo -> {}
+                    seqNo -> {},
+                    TranslogOperationAsserter.DEFAULT
                 )
             ) {
                 assertNotNull(translogGeneration);
@@ -1984,7 +1995,15 @@ public void testRecoveryUncommittedCorruptedCheckpoint() throws IOException {
         final TranslogDeletionPolicy deletionPolicy = translog.getDeletionPolicy();
         final TranslogCorruptedException translogCorruptedException = expectThrows(
             TranslogCorruptedException.class,
-            () -> new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {})
+            () -> new Translog(
+                config,
+                translogUUID,
+                deletionPolicy,
+                () -> SequenceNumbers.NO_OPS_PERFORMED,
+                primaryTerm::get,
+                seqNo -> {},
+                TranslogOperationAsserter.DEFAULT
+            )
         );
         assertThat(
             translogCorruptedException.getMessage(),
@@ -2010,7 +2029,8 @@ public void testRecoveryUncommittedCorruptedCheckpoint() throws IOException {
                 deletionPolicy,
                 () -> SequenceNumbers.NO_OPS_PERFORMED,
                 primaryTerm::get,
-                seqNo -> {}
+                seqNo -> {},
+                TranslogOperationAsserter.DEFAULT
             )
         ) {
             assertNotNull(translogGeneration);
@@ -2293,7 +2313,8 @@ public void testOpenForeignTranslog() throws IOException {
                 new TranslogDeletionPolicy(),
                 () -> SequenceNumbers.NO_OPS_PERFORMED,
                 primaryTerm::get,
-                seqNo -> {}
+                seqNo -> {},
+                TranslogOperationAsserter.DEFAULT
             );
             fail("translog doesn't belong to this UUID");
         } catch (TranslogCorruptedException ex) {
@@ -2305,7 +2326,8 @@ public void testOpenForeignTranslog() throws IOException {
             deletionPolicy,
             () -> SequenceNumbers.NO_OPS_PERFORMED,
             primaryTerm::get,
-            seqNo -> {}
+            seqNo -> {},
+            TranslogOperationAsserter.DEFAULT
         );
         try (Translog.Snapshot snapshot = this.translog.newSnapshot(randomLongBetween(0, firstUncommitted), Long.MAX_VALUE)) {
             for (int i = firstUncommitted; i < translogOperations; i++) {
@@ -2503,7 +2525,8 @@ public void testFailFlush() throws IOException {
                 deletionPolicy,
                 () -> SequenceNumbers.NO_OPS_PERFORMED,
                 primaryTerm::get,
-                seqNo -> {}
+                seqNo -> {},
+                TranslogOperationAsserter.DEFAULT
             )
         ) {
             assertEquals(
@@ -2649,7 +2672,8 @@ protected void afterAdd() throws IOException {
                     new TranslogDeletionPolicy(),
                     () -> SequenceNumbers.NO_OPS_PERFORMED,
                     primaryTerm::get,
-                    seqNo -> {}
+                    seqNo -> {},
+                    TranslogOperationAsserter.DEFAULT
                 );
                 Translog.Snapshot snapshot = tlog.newSnapshot()
             ) {
@@ -2708,7 +2732,8 @@ public void testRecoveryFromAFutureGenerationCleansUp() throws IOException {
             deletionPolicy,
             () -> SequenceNumbers.NO_OPS_PERFORMED,
             primaryTerm::get,
-            seqNo -> {}
+            seqNo -> {},
+            TranslogOperationAsserter.DEFAULT
         );
         assertThat(translog.getMinFileGeneration(), equalTo(1L));
         // no trimming done yet, just recovered
@@ -2771,7 +2796,8 @@ public void testRecoveryFromFailureOnTrimming() throws IOException {
                 deletionPolicy,
                 () -> SequenceNumbers.NO_OPS_PERFORMED,
                 primaryTerm::get,
-                seqNo -> {}
+                seqNo -> {},
+                TranslogOperationAsserter.DEFAULT
             )
         ) {
             // we don't know when things broke exactly
@@ -2875,7 +2901,15 @@ private Translog getFailableTranslog(
                 primaryTerm.get()
             );
         }
-        return new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}) {
+        return new Translog(
+            config,
+            translogUUID,
+            deletionPolicy,
+            () -> SequenceNumbers.NO_OPS_PERFORMED,
+            primaryTerm::get,
+            seqNo -> {},
+            TranslogOperationAsserter.DEFAULT
+        ) {
             @Override
             ChannelFactory getChannelFactory() {
                 return channelFactory;
@@ -3019,7 +3053,8 @@ public void testFailWhileCreateWriteWithRecoveredTLogs() throws IOException {
                 new TranslogDeletionPolicy(),
                 () -> SequenceNumbers.NO_OPS_PERFORMED,
                 primaryTerm::get,
-                seqNo -> {}
+                seqNo -> {},
+                TranslogOperationAsserter.DEFAULT
             ) {
                 @Override
                 protected TranslogWriter createWriter(
@@ -3087,7 +3122,8 @@ public void testRecoverWithUnbackedNextGenInIllegalState() throws IOException {
                 translog.getDeletionPolicy(),
                 () -> SequenceNumbers.NO_OPS_PERFORMED,
                 primaryTerm::get,
-                seqNo -> {}
+                seqNo -> {},
+                TranslogOperationAsserter.DEFAULT
             )
         );
         assertEquals(ex.getMessage(), "failed to create new translog file");
@@ -3114,7 +3150,8 @@ public void testRecoverWithUnbackedNextGenAndFutureFile() throws IOException {
                 deletionPolicy,
                 () -> SequenceNumbers.NO_OPS_PERFORMED,
                 primaryTerm::get,
-                seqNo -> {}
+                seqNo -> {},
+                TranslogOperationAsserter.DEFAULT
             )
         ) {
             assertFalse(tlog.syncNeeded());
@@ -3130,7 +3167,15 @@ public void testRecoverWithUnbackedNextGenAndFutureFile() throws IOException {
 
         TranslogException ex = expectThrows(
             TranslogException.class,
-            () -> new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {})
+            () -> new Translog(
+                config,
+                translogUUID,
+                deletionPolicy,
+                () -> SequenceNumbers.NO_OPS_PERFORMED,
+                primaryTerm::get,
+                seqNo -> {},
+                TranslogOperationAsserter.DEFAULT
+            )
         );
         assertEquals(ex.getMessage(), "failed to create new translog file");
         assertEquals(ex.getCause().getClass(), FileAlreadyExistsException.class);
@@ -3256,7 +3301,8 @@ public void testWithRandomException() throws IOException {
                     deletionPolicy,
                     () -> SequenceNumbers.NO_OPS_PERFORMED,
                     primaryTerm::get,
-                    seqNo -> {}
+                    seqNo -> {},
+                    TranslogOperationAsserter.DEFAULT
                 );
                 Translog.Snapshot snapshot = translog.newSnapshot(localCheckpointOfSafeCommit + 1, Long.MAX_VALUE)
             ) {
@@ -3351,7 +3397,8 @@ public void testPendingDelete() throws IOException {
             deletionPolicy,
             () -> SequenceNumbers.NO_OPS_PERFORMED,
             primaryTerm::get,
-            seqNo -> {}
+            seqNo -> {},
+            TranslogOperationAsserter.DEFAULT
         );
         translog.add(TranslogOperationsUtils.indexOp("2", 1, primaryTerm.get()));
         translog.rollGeneration();
@@ -3365,7 +3412,8 @@ public void testPendingDelete() throws IOException {
             deletionPolicy,
             () -> SequenceNumbers.NO_OPS_PERFORMED,
             primaryTerm::get,
-            seqNo -> {}
+            seqNo -> {},
+            TranslogOperationAsserter.DEFAULT
         );
     }
 
@@ -3713,7 +3761,15 @@ class MisbehavingTranslog extends Translog {
                 LongSupplier globalCheckpointSupplier,
                 LongSupplier primaryTermSupplier
             ) throws IOException {
-                super(config, translogUUID, deletionPolicy, globalCheckpointSupplier, primaryTermSupplier, seqNo -> {});
+                super(
+                    config,
+                    translogUUID,
+                    deletionPolicy,
+                    globalCheckpointSupplier,
+                    primaryTermSupplier,
+                    seqNo -> {},
+                    TranslogOperationAsserter.DEFAULT
+                );
             }
 
             void callCloseDirectly() throws IOException {
@@ -3855,7 +3911,8 @@ public void copy(Path source, Path target, CopyOption... options) throws IOExcep
                     brokenTranslog.getDeletionPolicy(),
                     () -> SequenceNumbers.NO_OPS_PERFORMED,
                     primaryTerm::get,
-                    seqNo -> {}
+                    seqNo -> {},
+                    TranslogOperationAsserter.DEFAULT
                 )
             ) {
                 recoveredTranslog.rollGeneration();
@@ -3889,7 +3946,8 @@ public void testSyncConcurrently() throws Exception {
                 new TranslogDeletionPolicy(),
                 globalCheckpointSupplier,
                 primaryTerm::get,
-                persistedSeqNos::add
+                persistedSeqNos::add,
+                TranslogOperationAsserter.DEFAULT
             )
         ) {
             Thread[] threads = new Thread[between(2, 8)];
@@ -3974,7 +4032,8 @@ public void force(boolean metaData) throws IOException {
             new TranslogDeletionPolicy(),
             () -> SequenceNumbers.NO_OPS_PERFORMED,
             primaryTerm::get,
-            seqNo -> {}
+            seqNo -> {},
+            TranslogOperationAsserter.DEFAULT
         ) {
             @Override
             ChannelFactory getChannelFactory() {
@@ -4040,7 +4099,8 @@ public void testDisabledFsync() throws IOException {
                 new TranslogDeletionPolicy(),
                 () -> SequenceNumbers.NO_OPS_PERFORMED,
                 primaryTerm::get,
-                getPersistedSeqNoConsumer()
+                getPersistedSeqNoConsumer(),
+                TranslogOperationAsserter.DEFAULT
             ) {
                 @Override
                 ChannelFactory getChannelFactory() {
diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
index ab65d56557ad9..0e333491588a6 100644
--- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
@@ -9,7 +9,6 @@
 
 package org.elasticsearch.indices;
 
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper;
@@ -98,8 +97,6 @@ public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
         DataStreamTimestampFieldMapper.NAME,
         FieldNamesFieldMapper.NAME };
 
-    @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_FOUNDATIONS)
-    @AwaitsFix(bugUrl = "test is referencing 7.x index versions so needs to be updated for 9.0 bump")
     public void testBuiltinMappers() {
         IndicesModule module = new IndicesModule(Collections.emptyList());
         {
@@ -239,14 +236,14 @@ public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
 
     public void testFieldNamesIsLast() {
         IndicesModule module = new IndicesModule(Collections.emptyList());
-        IndexVersion version = IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current());
+        IndexVersion version = IndexVersionUtils.randomVersion();
         List<String> fieldNames = new ArrayList<>(module.getMapperRegistry().getMetadataMapperParsers(version).keySet());
         assertEquals(FieldNamesFieldMapper.NAME, fieldNames.get(fieldNames.size() - 1));
     }
 
     public void testFieldNamesIsLastWithPlugins() {
         IndicesModule module = new IndicesModule(fakePlugins);
-        IndexVersion version = IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current());
+        IndexVersion version = IndexVersionUtils.randomVersion();
         List<String> fieldNames = new ArrayList<>(module.getMapperRegistry().getMetadataMapperParsers(version).keySet());
         assertEquals(FieldNamesFieldMapper.NAME, fieldNames.get(fieldNames.size() - 1));
     }
diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
index 17975b7d18dd8..b56afadb14924 100644
--- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.SlowLogFieldProvider;
+import org.elasticsearch.index.SlowLogFields;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineConfig;
 import org.elasticsearch.index.engine.EngineFactory;
@@ -209,16 +210,18 @@ static void setFields(Map<String, String> fields) {
         }
 
         @Override
-        public void init(IndexSettings indexSettings) {}
-
-        @Override
-        public Map<String, String> indexSlowLogFields() {
-            return fields;
-        }
-
-        @Override
-        public Map<String, String> searchSlowLogFields() {
-            return fields;
+        public SlowLogFields create(IndexSettings indexSettings) {
+            return new SlowLogFields() {
+                @Override
+                public Map<String, String> indexFields() {
+                    return fields;
+                }
+
+                @Override
+                public Map<String, String> searchFields() {
+                    return fields;
+                }
+            };
         }
     }
 
@@ -231,16 +234,18 @@ static void setFields(Map<String, String> fields) {
         }
 
         @Override
-        public void init(IndexSettings indexSettings) {}
-
-        @Override
-        public Map<String, String> indexSlowLogFields() {
-            return fields;
-        }
-
-        @Override
-        public Map<String, String> searchSlowLogFields() {
-            return fields;
+        public SlowLogFields create(IndexSettings indexSettings) {
+            return new SlowLogFields() {
+                @Override
+                public Map<String, String> indexFields() {
+                    return fields;
+                }
+
+                @Override
+                public Map<String, String> searchFields() {
+                    return fields;
+                }
+            };
         }
     }
 
@@ -805,33 +810,34 @@ public void testLoadSlowLogFieldProvider() {
         TestAnotherSlowLogFieldProvider.setFields(Map.of("key2", "value2"));
 
         var indicesService = getIndicesService();
-        SlowLogFieldProvider fieldProvider = indicesService.loadSlowLogFieldProvider();
+        SlowLogFieldProvider fieldProvider = indicesService.slowLogFieldProvider;
+        SlowLogFields fields = fieldProvider.create(null);
 
         // The map of fields from the two providers are merged to a single map of fields
-        assertEquals(Map.of("key1", "value1", "key2", "value2"), fieldProvider.searchSlowLogFields());
-        assertEquals(Map.of("key1", "value1", "key2", "value2"), fieldProvider.indexSlowLogFields());
+        assertEquals(Map.of("key1", "value1", "key2", "value2"), fields.searchFields());
+        assertEquals(Map.of("key1", "value1", "key2", "value2"), fields.indexFields());
 
         TestSlowLogFieldProvider.setFields(Map.of("key1", "value1"));
         TestAnotherSlowLogFieldProvider.setFields(Map.of("key1", "value2"));
 
         // There is an overlap of field names, since this isn't deterministic and probably a
         // programming error (two providers provide the same field) throw an exception
-        assertThrows(IllegalStateException.class, fieldProvider::searchSlowLogFields);
-        assertThrows(IllegalStateException.class, fieldProvider::indexSlowLogFields);
+        assertThrows(IllegalStateException.class, fields::searchFields);
+        assertThrows(IllegalStateException.class, fields::indexFields);
 
         TestSlowLogFieldProvider.setFields(Map.of("key1", "value1"));
         TestAnotherSlowLogFieldProvider.setFields(Map.of());
 
         // One provider has no fields
-        assertEquals(Map.of("key1", "value1"), fieldProvider.searchSlowLogFields());
-        assertEquals(Map.of("key1", "value1"), fieldProvider.indexSlowLogFields());
+        assertEquals(Map.of("key1", "value1"), fields.searchFields());
+        assertEquals(Map.of("key1", "value1"), fields.indexFields());
 
         TestSlowLogFieldProvider.setFields(Map.of());
         TestAnotherSlowLogFieldProvider.setFields(Map.of());
 
         // Both providers have no fields
-        assertEquals(Map.of(), fieldProvider.searchSlowLogFields());
-        assertEquals(Map.of(), fieldProvider.indexSlowLogFields());
+        assertEquals(Map.of(), fields.searchFields());
+        assertEquals(Map.of(), fields.indexFields());
     }
 
     public void testWithTempIndexServiceHandlesExistingIndex() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
index cf6941b84b791..abaab1ac8983b 100644
--- a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.index.IndexService.IndexCreationContext;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.Analysis;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.analysis.CharFilterFactory;
@@ -186,6 +187,34 @@ public void testUnderscoreInAnalyzerName() throws IOException {
         }
     }
 
+    public void testStandardFilterBWC() throws IOException {
+        // standard tokenfilter should have been removed entirely in the 7x line. However, a
+        // cacheing bug meant that it was still possible to create indexes using a standard
+        // filter until 7.6
+        {
+            IndexVersion version = IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_6_0, IndexVersion.current());
+            final Settings settings = Settings.builder()
+                .put("index.analysis.analyzer.my_standard.tokenizer", "standard")
+                .put("index.analysis.analyzer.my_standard.filter", "standard")
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .put(IndexMetadata.SETTING_VERSION_CREATED, version)
+                .build();
+            IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> getIndexAnalyzers(settings));
+            assertThat(exc.getMessage(), equalTo("The [standard] token filter has been removed."));
+        }
+        {
+            IndexVersion version = IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_5_2);
+            final Settings settings = Settings.builder()
+                .put("index.analysis.analyzer.my_standard.tokenizer", "standard")
+                .put("index.analysis.analyzer.my_standard.filter", "standard")
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .put(IndexMetadata.SETTING_VERSION_CREATED, version)
+                .build();
+            getIndexAnalyzers(settings);
+            assertWarnings("The [standard] token filter is deprecated and will be removed in a future version.");
+        }
+    }
+
     /**
      * Tests that plugins can register pre-configured char filters that vary in behavior based on Elasticsearch version, Lucene version,
      * and that do not vary based on version at all.
@@ -233,7 +262,7 @@ public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
             new StablePluginsRegistry()
         ).getAnalysisRegistry();
 
-        IndexVersion version = IndexVersionUtils.randomVersion(random());
+        IndexVersion version = IndexVersionUtils.randomVersion();
         IndexAnalyzers analyzers = getIndexAnalyzers(
             registry,
             Settings.builder()
@@ -302,7 +331,7 @@ public List<PreConfiguredTokenFilter> getPreConfiguredTokenFilters() {
             new StablePluginsRegistry()
         ).getAnalysisRegistry();
 
-        IndexVersion version = IndexVersionUtils.randomVersion(random());
+        IndexVersion version = IndexVersionUtils.randomVersion();
         IndexAnalyzers analyzers = getIndexAnalyzers(
             registry,
             Settings.builder()
@@ -389,7 +418,7 @@ public List<PreConfiguredTokenizer> getPreConfiguredTokenizers() {
             new StablePluginsRegistry()
         ).getAnalysisRegistry();
 
-        IndexVersion version = IndexVersionUtils.randomVersion(random());
+        IndexVersion version = IndexVersionUtils.randomVersion();
         IndexAnalyzers analyzers = getIndexAnalyzers(
             registry,
             Settings.builder()
diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/IncorrectSetupStablePluginsTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/IncorrectSetupStablePluginsTests.java
index ca9184bca75da..181d3ec44f2b3 100644
--- a/server/src/test/java/org/elasticsearch/indices/analysis/IncorrectSetupStablePluginsTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/analysis/IncorrectSetupStablePluginsTests.java
@@ -63,7 +63,7 @@ public void testIncorrectlyAnnotatedSettingsClass() throws IOException {
                 Settings.builder()
                     .put("index.analysis.analyzer.char_filter_test.tokenizer", "standard")
                     .put("index.analysis.analyzer.char_filter_test.char_filter", "incorrectlyAnnotatedSettings")
-                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion(random()))
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion())
                     .build(),
                 Map.of(
                     "incorrectlyAnnotatedSettings",
@@ -90,7 +90,7 @@ public void testIncorrectlyAnnotatedConstructor() throws IOException {
                 Settings.builder()
                     .put("index.analysis.analyzer.char_filter_test.tokenizer", "standard")
                     .put("index.analysis.analyzer.char_filter_test.char_filter", "noInjectCharFilter")
-                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion(random()))
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion())
                     .build(),
                 Map.of("noInjectCharFilter", new PluginInfo("noInjectCharFilter", NoInjectCharFilter.class.getName(), classLoader))
             )
@@ -112,7 +112,7 @@ public void testMultiplePublicConstructors() throws IOException {
                 Settings.builder()
                     .put("index.analysis.analyzer.char_filter_test.tokenizer", "standard")
                     .put("index.analysis.analyzer.char_filter_test.char_filter", "multipleConstructors")
-                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion(random()))
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion())
                     .build(),
                 Map.of("multipleConstructors", new PluginInfo("multipleConstructors", MultipleConstructors.class.getName(), classLoader))
             )
diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsNoSettingsTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsNoSettingsTests.java
index 7cbda0e7086cb..6eac3847efa43 100644
--- a/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsNoSettingsTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsNoSettingsTests.java
@@ -61,7 +61,7 @@ public IndexAnalyzers getIndexAnalyzers(Settings settings) throws IOException {
     }
 
     public void testStablePlugins() throws IOException {
-        IndexVersion version = IndexVersionUtils.randomVersion(random());
+        IndexVersion version = IndexVersionUtils.randomVersion();
         IndexAnalyzers analyzers = getIndexAnalyzers(
             Settings.builder()
                 .put("index.analysis.analyzer.char_filter_test.tokenizer", "standard")
diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsWithSettingsTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsWithSettingsTests.java
index acde315b140ab..82f49888e911d 100644
--- a/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsWithSettingsTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsWithSettingsTests.java
@@ -72,7 +72,7 @@ public void testCharFilters() throws IOException {
                 .put("index.analysis.analyzer.char_filter_with_defaults_test.tokenizer", "standard")
                 .put("index.analysis.analyzer.char_filter_with_defaults_test.char_filter", "stableCharFilterFactory")
 
-                .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion(random()))
+                .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion())
                 .build()
         );
         assertTokenStreamContents(analyzers.get("char_filter_test").tokenStream("", "t#st"), new String[] { "t3st" });
@@ -88,7 +88,7 @@ public void testTokenFilters() throws IOException {
 
                 .put("index.analysis.analyzer.token_filter_test.tokenizer", "standard")
                 .put("index.analysis.analyzer.token_filter_test.filter", "my_token_filter")
-                .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion(random()))
+                .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion())
                 .build()
         );
         assertTokenStreamContents(
@@ -109,7 +109,7 @@ public void testTokenizer() throws IOException {
                 .putList("index.analysis.tokenizer.my_tokenizer.tokenizer_list_of_chars", "_", " ")
 
                 .put("index.analysis.analyzer.tokenizer_test.tokenizer", "my_tokenizer")
-                .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion(random()))
+                .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion())
                 .build()
         );
         assertTokenStreamContents(analyzers.get("tokenizer_test").tokenStream("", "x_y z"), new String[] { "x", "y", "z" });
@@ -124,7 +124,7 @@ public void testAnalyzer() throws IOException {
                 .put("index.analysis.analyzer.analyzer_provider_test.old_char", "#")
                 .put("index.analysis.analyzer.analyzer_provider_test.new_number", 3)
                 .put("index.analysis.analyzer.analyzer_provider_test.analyzerUseTokenListOfChars", true)
-                .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion(random()))
+                .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion())
                 .build()
         );
         assertTokenStreamContents(analyzers.get("analyzer_provider_test").tokenStream("", "1x_y_#z"), new String[] { "y", "3z" });
diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
index 82154848ea039..39c327ddee228 100644
--- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
+++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
@@ -252,7 +252,11 @@ public Transport.Connection getConnection(DiscoveryNode node) {
         ) {
             // metadata upgrader should do nothing
             @Override
-            public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) {
+            public IndexMetadata verifyIndexMetadata(
+                IndexMetadata indexMetadata,
+                IndexVersion minimumIndexCompatibilityVersion,
+                IndexVersion minimumReadOnlyIndexCompatibilityVersion
+            ) {
                 return indexMetadata;
             }
         };
diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
index d039c265c98ae..d9b2936dc30c0 100644
--- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
@@ -721,7 +721,7 @@ public void testThrowExceptionOnPrimaryRelocatedBeforePhase1Started() throws IOE
 
         final IndexMetadata.Builder indexMetadata = IndexMetadata.builder("test")
             .settings(
-                indexSettings(IndexVersionUtils.randomVersion(random()), between(1, 5), between(0, 5)).put(
+                indexSettings(IndexVersionUtils.randomVersion(), between(1, 5), between(0, 5)).put(
                     IndexMetadata.SETTING_INDEX_UUID,
                     UUIDs.randomBase64UUID(random())
                 )
diff --git a/server/src/test/java/org/elasticsearch/inference/InferenceServiceConfigurationTestUtils.java b/server/src/test/java/org/elasticsearch/inference/InferenceServiceConfigurationTestUtils.java
index 8d145202f7165..2d7fe0dcb5558 100644
--- a/server/src/test/java/org/elasticsearch/inference/InferenceServiceConfigurationTestUtils.java
+++ b/server/src/test/java/org/elasticsearch/inference/InferenceServiceConfigurationTestUtils.java
@@ -9,8 +9,8 @@
 
 package org.elasticsearch.inference;
 
+import java.util.EnumSet;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength;
@@ -19,14 +19,16 @@
 public class InferenceServiceConfigurationTestUtils {
 
     public static InferenceServiceConfiguration getRandomServiceConfigurationField() {
-        return new InferenceServiceConfiguration.Builder().setProvider(randomAlphaOfLength(10))
-            .setTaskTypes(getRandomTaskTypeConfiguration())
-            .setConfiguration(getRandomServiceConfiguration(10))
+        return new InferenceServiceConfiguration.Builder().setService(randomAlphaOfLength(10))
+            .setName(randomAlphaOfLength(6))
+            .setTaskTypes(getRandomTaskTypes())
+            .setConfigurations(getRandomServiceConfiguration(10))
             .build();
     }
 
-    private static List<TaskSettingsConfiguration> getRandomTaskTypeConfiguration() {
-        return List.of(TaskSettingsConfigurationTestUtils.getRandomTaskSettingsConfigurationField());
+    private static EnumSet<TaskType> getRandomTaskTypes() {
+        TaskType[] values = TaskType.values();
+        return EnumSet.of(values[randomInt(values.length - 1)]);
     }
 
     private static Map<String, SettingsConfiguration> getRandomServiceConfiguration(int numFields) {
diff --git a/server/src/test/java/org/elasticsearch/inference/InferenceServiceConfigurationTests.java b/server/src/test/java/org/elasticsearch/inference/InferenceServiceConfigurationTests.java
index 7d97f85360c57..490ed68ab3e66 100644
--- a/server/src/test/java/org/elasticsearch/inference/InferenceServiceConfigurationTests.java
+++ b/server/src/test/java/org/elasticsearch/inference/InferenceServiceConfigurationTests.java
@@ -28,139 +28,26 @@ public class InferenceServiceConfigurationTests extends ESTestCase {
     public void testToXContent() throws IOException {
         String content = XContentHelper.stripWhitespace("""
             {
-               "provider": "some_provider",
-               "task_types": [
-                  {
-               "task_type": "text_embedding",
-               "configuration": {
+               "service": "some_provider",
+               "name": "Some Provider",
+               "task_types": ["text_embedding", "completion"],
+               "configurations": {
                     "text_field_configuration": {
-                        "default_value": null,
-                        "depends_on": [
-                            {
-                                "field": "some_field",
-                                "value": true
-                            }
-                        ],
-                        "display": "textbox",
+                        "description": "Wow, this tooltip is useful.",
                         "label": "Very important field",
-                        "options": [],
-                        "order": 4,
                         "required": true,
                         "sensitive": true,
-                        "tooltip": "Wow, this tooltip is useful.",
-                        "type": "str",
-                        "ui_restrictions": [],
-                        "validations": null,
-                        "value": ""
+                        "updatable": false,
+                        "type": "str"
                     },
                     "numeric_field_configuration": {
                         "default_value": 3,
-                        "depends_on": null,
-                        "display": "numeric",
+                        "description": "Wow, this tooltip is useful.",
                         "label": "Very important numeric field",
-                        "options": [],
-                        "order": 2,
                         "required": true,
                         "sensitive": false,
-                        "tooltip": "Wow, this tooltip is useful.",
-                        "type": "int",
-                        "ui_restrictions": [],
-                        "validations": [
-                            {
-                                "constraint": 0,
-                                "type": "greater_than"
-                            }
-                        ],
-                        "value": ""
-                    }
-               }
-            },
-            {
-               "task_type": "completion",
-               "configuration": {
-                    "text_field_configuration": {
-                        "default_value": null,
-                        "depends_on": [
-                            {
-                                "field": "some_field",
-                                "value": true
-                            }
-                        ],
-                        "display": "textbox",
-                        "label": "Very important field",
-                        "options": [],
-                        "order": 4,
-                        "required": true,
-                        "sensitive": true,
-                        "tooltip": "Wow, this tooltip is useful.",
-                        "type": "str",
-                        "ui_restrictions": [],
-                        "validations": null,
-                        "value": ""
-                    },
-                    "numeric_field_configuration": {
-                        "default_value": 3,
-                        "depends_on": null,
-                        "display": "numeric",
-                        "label": "Very important numeric field",
-                        "options": [],
-                        "order": 2,
-                        "required": true,
-                        "sensitive": false,
-                        "tooltip": "Wow, this tooltip is useful.",
-                        "type": "int",
-                        "ui_restrictions": [],
-                        "validations": [
-                            {
-                                "constraint": 0,
-                                "type": "greater_than"
-                            }
-                        ],
-                        "value": ""
-                    }
-               }
-            }
-               ],
-               "configuration": {
-                    "text_field_configuration": {
-                        "default_value": null,
-                        "depends_on": [
-                            {
-                                "field": "some_field",
-                                "value": true
-                            }
-                        ],
-                        "display": "textbox",
-                        "label": "Very important field",
-                        "options": [],
-                        "order": 4,
-                        "required": true,
-                        "sensitive": true,
-                        "tooltip": "Wow, this tooltip is useful.",
-                        "type": "str",
-                        "ui_restrictions": [],
-                        "validations": null,
-                        "value": ""
-                    },
-                    "numeric_field_configuration": {
-                        "default_value": 3,
-                        "depends_on": null,
-                        "display": "numeric",
-                        "label": "Very important numeric field",
-                        "options": [],
-                        "order": 2,
-                        "required": true,
-                        "sensitive": false,
-                        "tooltip": "Wow, this tooltip is useful.",
-                        "type": "int",
-                        "ui_restrictions": [],
-                        "validations": [
-                            {
-                                "constraint": 0,
-                                "type": "greater_than"
-                            }
-                        ],
-                        "value": ""
+                        "updatable": true,
+                        "type": "int"
                     }
                }
             }
@@ -183,8 +70,9 @@ public void testToMap() {
         InferenceServiceConfiguration configField = InferenceServiceConfigurationTestUtils.getRandomServiceConfigurationField();
         Map<String, Object> configFieldAsMap = configField.toMap();
 
-        assertThat(configFieldAsMap.get("provider"), equalTo(configField.getProvider()));
+        assertThat(configFieldAsMap.get("service"), equalTo(configField.getService()));
+        assertThat(configFieldAsMap.get("name"), equalTo(configField.getName()));
         assertThat(configFieldAsMap.get("task_types"), equalTo(configField.getTaskTypes()));
-        assertThat(configFieldAsMap.get("configuration"), equalTo(configField.getConfiguration()));
+        assertThat(configFieldAsMap.get("configurations"), equalTo(configField.getConfigurations()));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/inference/SettingsConfigurationTestUtils.java b/server/src/test/java/org/elasticsearch/inference/SettingsConfigurationTestUtils.java
index 728dafc5383c1..ed78baeb9abe6 100644
--- a/server/src/test/java/org/elasticsearch/inference/SettingsConfigurationTestUtils.java
+++ b/server/src/test/java/org/elasticsearch/inference/SettingsConfigurationTestUtils.java
@@ -9,14 +9,7 @@
 
 package org.elasticsearch.inference;
 
-import org.elasticsearch.inference.configuration.SettingsConfigurationDependency;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption;
-import org.elasticsearch.inference.configuration.SettingsConfigurationValidation;
-import org.elasticsearch.inference.configuration.SettingsConfigurationValidationType;
-
-import java.util.List;
 
 import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength;
 import static org.elasticsearch.test.ESTestCase.randomBoolean;
@@ -25,50 +18,18 @@
 public class SettingsConfigurationTestUtils {
 
     public static SettingsConfiguration getRandomSettingsConfigurationField() {
-        return new SettingsConfiguration.Builder().setCategory(randomAlphaOfLength(10))
-            .setDefaultValue(randomAlphaOfLength(10))
-            .setDependsOn(List.of(getRandomSettingsConfigurationDependency()))
-            .setDisplay(getRandomSettingsConfigurationDisplayType())
+        return new SettingsConfiguration.Builder().setDefaultValue(randomAlphaOfLength(10))
+            .setDescription(randomAlphaOfLength(10))
             .setLabel(randomAlphaOfLength(10))
-            .setOptions(List.of(getRandomSettingsConfigurationSelectOption(), getRandomSettingsConfigurationSelectOption()))
-            .setOrder(randomInt())
-            .setPlaceholder(randomAlphaOfLength(10))
             .setRequired(randomBoolean())
             .setSensitive(randomBoolean())
-            .setTooltip(randomAlphaOfLength(10))
+            .setUpdatable(randomBoolean())
             .setType(getRandomConfigurationFieldType())
-            .setUiRestrictions(List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)))
-            .setValidations(List.of(getRandomSettingsConfigurationValidation()))
-            .setValue(randomAlphaOfLength(10))
-            .build();
-    }
-
-    private static SettingsConfigurationDependency getRandomSettingsConfigurationDependency() {
-        return new SettingsConfigurationDependency.Builder().setField(randomAlphaOfLength(10)).setValue(randomAlphaOfLength(10)).build();
-    }
-
-    private static SettingsConfigurationSelectOption getRandomSettingsConfigurationSelectOption() {
-        return new SettingsConfigurationSelectOption.Builder().setLabel(randomAlphaOfLength(10)).setValue(randomAlphaOfLength(10)).build();
-    }
-
-    private static SettingsConfigurationValidation getRandomSettingsConfigurationValidation() {
-        return new SettingsConfigurationValidation.Builder().setConstraint(randomAlphaOfLength(10))
-            .setType(getRandomConfigurationValidationType())
             .build();
     }
 
-    public static SettingsConfigurationDisplayType getRandomSettingsConfigurationDisplayType() {
-        SettingsConfigurationDisplayType[] values = SettingsConfigurationDisplayType.values();
-        return values[randomInt(values.length - 1)];
-    }
-
     public static SettingsConfigurationFieldType getRandomConfigurationFieldType() {
         SettingsConfigurationFieldType[] values = SettingsConfigurationFieldType.values();
         return values[randomInt(values.length - 1)];
     }
-
-    public static SettingsConfigurationValidationType getRandomConfigurationValidationType() {
-        SettingsConfigurationValidationType[] values = SettingsConfigurationValidationType.values();
-        return values[randomInt(values.length - 1)];
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/inference/SettingsConfigurationTests.java b/server/src/test/java/org/elasticsearch/inference/SettingsConfigurationTests.java
index e1293366a1152..551a25fe52f18 100644
--- a/server/src/test/java/org/elasticsearch/inference/SettingsConfigurationTests.java
+++ b/server/src/test/java/org/elasticsearch/inference/SettingsConfigurationTests.java
@@ -12,16 +12,12 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDependency;
-import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption;
-import org.elasticsearch.inference.configuration.SettingsConfigurationValidation;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
@@ -33,29 +29,12 @@ public class SettingsConfigurationTests extends ESTestCase {
     public void testToXContent() throws IOException {
         String content = XContentHelper.stripWhitespace("""
             {
-               "default_value": null,
-               "depends_on": [
-                 {
-                   "field": "some_field",
-                   "value": true
-                 }
-               ],
-               "display": "textbox",
+               "description": "Wow, this tooltip is useful.",
                "label": "Very important field",
-               "options": [],
-               "order": 4,
                "required": true,
                "sensitive": false,
-               "tooltip": "Wow, this tooltip is useful.",
-               "type": "str",
-               "ui_restrictions": [],
-               "validations": [
-                  {
-                     "constraint": 0,
-                     "type": "greater_than"
-                  }
-               ],
-               "value": ""
+               "updatable": true,
+               "type": "str"
             }
             """);
 
@@ -72,38 +51,12 @@ public void testToXContent() throws IOException {
     public void testToXContent_WithNumericSelectOptions() throws IOException {
         String content = XContentHelper.stripWhitespace("""
             {
-               "default_value": null,
-               "depends_on": [
-                 {
-                   "field": "some_field",
-                   "value": true
-                 }
-               ],
-               "display": "textbox",
+               "description": "Wow, this tooltip is useful.",
                "label": "Very important field",
-               "options": [
-                 {
-                   "label": "five",
-                   "value": 5
-                 },
-                 {
-                   "label": "ten",
-                   "value": 10
-                 }
-               ],
-               "order": 4,
                "required": true,
                "sensitive": false,
-               "tooltip": "Wow, this tooltip is useful.",
-               "type": "str",
-               "ui_restrictions": [],
-               "validations": [
-                  {
-                     "constraint": 0,
-                     "type": "greater_than"
-                  }
-               ],
-               "value": ""
+               "updatable": true,
+               "type": "str"
             }
             """);
 
@@ -135,153 +88,28 @@ public void testToXContentCrawlerConfig_WithNullValue() throws IOException {
         assertToXContentEquivalent(originalBytes, toXContent(parsed, XContentType.JSON, humanReadable), XContentType.JSON);
     }
 
-    public void testToXContentWithMultipleConstraintTypes() throws IOException {
-        String content = XContentHelper.stripWhitespace("""
-            {
-               "default_value": null,
-               "depends_on": [
-                 {
-                   "field": "some_field",
-                   "value": true
-                 }
-               ],
-               "display": "textbox",
-               "label": "Very important field",
-               "options": [],
-               "order": 4,
-               "required": true,
-               "sensitive": false,
-               "tooltip": "Wow, this tooltip is useful.",
-               "type": "str",
-               "ui_restrictions": [],
-               "validations": [
-                   {
-                       "constraint": 32,
-                       "type": "less_than"
-                   },
-                   {
-                       "constraint": "^\\\\\\\\d{4}-\\\\\\\\d{2}-\\\\\\\\d{2}$",
-                       "type": "regex"
-                   },
-                   {
-                       "constraint": "int",
-                       "type": "list_type"
-                   },
-                   {
-                       "constraint": [
-                           1,
-                           2,
-                           3
-                       ],
-                       "type": "included_in"
-                   },
-                   {
-                       "constraint": [
-                           "string_1",
-                           "string_2",
-                           "string_3"
-                       ],
-                       "type": "included_in"
-                   }
-               ],
-               "value": ""
-            }
-            """);
-
-        SettingsConfiguration configuration = SettingsConfiguration.fromXContentBytes(new BytesArray(content), XContentType.JSON);
-        boolean humanReadable = true;
-        BytesReference originalBytes = toShuffledXContent(configuration, XContentType.JSON, ToXContent.EMPTY_PARAMS, humanReadable);
-        SettingsConfiguration parsed;
-        try (XContentParser parser = createParser(XContentType.JSON.xContent(), originalBytes)) {
-            parsed = SettingsConfiguration.fromXContent(parser);
-        }
-        assertToXContentEquivalent(originalBytes, toXContent(parsed, XContentType.JSON, humanReadable), XContentType.JSON);
-    }
-
     public void testToMap() {
         SettingsConfiguration configField = SettingsConfigurationTestUtils.getRandomSettingsConfigurationField();
         Map<String, Object> configFieldAsMap = configField.toMap();
 
-        if (configField.getCategory() != null) {
-            assertThat(configFieldAsMap.get("category"), equalTo(configField.getCategory()));
-        } else {
-            assertFalse(configFieldAsMap.containsKey("category"));
-        }
-
         assertThat(configFieldAsMap.get("default_value"), equalTo(configField.getDefaultValue()));
 
-        if (configField.getDependsOn() != null) {
-            List<Map<String, Object>> dependsOnAsList = configField.getDependsOn()
-                .stream()
-                .map(SettingsConfigurationDependency::toMap)
-                .toList();
-            assertThat(configFieldAsMap.get("depends_on"), equalTo(dependsOnAsList));
-        } else {
-            assertFalse(configFieldAsMap.containsKey("depends_on"));
-        }
-
-        if (configField.getDisplay() != null) {
-            assertThat(configFieldAsMap.get("display"), equalTo(configField.getDisplay().toString()));
+        if (configField.getDescription() != null) {
+            assertThat(configFieldAsMap.get("description"), equalTo(configField.getDescription()));
         } else {
-            assertFalse(configFieldAsMap.containsKey("display"));
+            assertFalse(configFieldAsMap.containsKey("description"));
         }
 
         assertThat(configFieldAsMap.get("label"), equalTo(configField.getLabel()));
 
-        if (configField.getOptions() != null) {
-            List<Map<String, Object>> optionsAsList = configField.getOptions()
-                .stream()
-                .map(SettingsConfigurationSelectOption::toMap)
-                .toList();
-            assertThat(configFieldAsMap.get("options"), equalTo(optionsAsList));
-        } else {
-            assertFalse(configFieldAsMap.containsKey("options"));
-        }
-
-        if (configField.getOrder() != null) {
-            assertThat(configFieldAsMap.get("order"), equalTo(configField.getOrder()));
-        } else {
-            assertFalse(configFieldAsMap.containsKey("order"));
-        }
-
-        if (configField.getPlaceholder() != null) {
-            assertThat(configFieldAsMap.get("placeholder"), equalTo(configField.getPlaceholder()));
-        } else {
-            assertFalse(configFieldAsMap.containsKey("placeholder"));
-        }
-
         assertThat(configFieldAsMap.get("required"), equalTo(configField.isRequired()));
         assertThat(configFieldAsMap.get("sensitive"), equalTo(configField.isSensitive()));
-
-        if (configField.getTooltip() != null) {
-            assertThat(configFieldAsMap.get("tooltip"), equalTo(configField.getTooltip()));
-        } else {
-            assertFalse(configFieldAsMap.containsKey("tooltip"));
-        }
+        assertThat(configFieldAsMap.get("updatable"), equalTo(configField.isUpdatable()));
 
         if (configField.getType() != null) {
             assertThat(configFieldAsMap.get("type"), equalTo(configField.getType().toString()));
         } else {
             assertFalse(configFieldAsMap.containsKey("type"));
         }
-
-        if (configField.getUiRestrictions() != null) {
-            assertThat(configFieldAsMap.get("ui_restrictions"), equalTo(configField.getUiRestrictions()));
-        } else {
-            assertFalse(configFieldAsMap.containsKey("ui_restrictions"));
-        }
-
-        if (configField.getValidations() != null) {
-            List<Map<String, Object>> validationsAsList = configField.getValidations()
-                .stream()
-                .map(SettingsConfigurationValidation::toMap)
-                .toList();
-            assertThat(configFieldAsMap.get("validations"), equalTo(validationsAsList));
-        } else {
-            assertFalse(configFieldAsMap.containsKey("validations"));
-        }
-
-        assertThat(configFieldAsMap.get("value"), equalTo(configField.getValue()));
-
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/inference/TaskSettingsConfigurationTestUtils.java b/server/src/test/java/org/elasticsearch/inference/TaskSettingsConfigurationTestUtils.java
deleted file mode 100644
index 81abeaefd9f1a..0000000000000
--- a/server/src/test/java/org/elasticsearch/inference/TaskSettingsConfigurationTestUtils.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.inference;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength;
-import static org.elasticsearch.test.ESTestCase.randomInt;
-
-public class TaskSettingsConfigurationTestUtils {
-
-    public static TaskSettingsConfiguration getRandomTaskSettingsConfigurationField() {
-        return new TaskSettingsConfiguration.Builder().setTaskType(getRandomTaskType())
-            .setConfiguration(getRandomServiceConfiguration(10))
-            .build();
-    }
-
-    private static TaskType getRandomTaskType() {
-        TaskType[] values = TaskType.values();
-        return values[randomInt(values.length - 1)];
-    }
-
-    private static Map<String, SettingsConfiguration> getRandomServiceConfiguration(int numFields) {
-        var numConfigFields = randomInt(numFields);
-        Map<String, SettingsConfiguration> configuration = new HashMap<>();
-        for (int i = 0; i < numConfigFields; i++) {
-            configuration.put(randomAlphaOfLength(10), SettingsConfigurationTestUtils.getRandomSettingsConfigurationField());
-        }
-
-        return configuration;
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/inference/TaskSettingsConfigurationTests.java b/server/src/test/java/org/elasticsearch/inference/TaskSettingsConfigurationTests.java
deleted file mode 100644
index d37fffc78ebd6..0000000000000
--- a/server/src/test/java/org/elasticsearch/inference/TaskSettingsConfigurationTests.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.inference;
-
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.XContentType;
-
-import java.io.IOException;
-import java.util.Map;
-
-import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
-import static org.hamcrest.CoreMatchers.equalTo;
-
-public class TaskSettingsConfigurationTests extends ESTestCase {
-    public void testToXContent() throws IOException {
-        String content = XContentHelper.stripWhitespace("""
-            {
-               "task_type": "text_embedding",
-               "configuration": {
-                    "text_field_configuration": {
-                        "default_value": null,
-                        "depends_on": [
-                            {
-                                "field": "some_field",
-                                "value": true
-                            }
-                        ],
-                        "display": "textbox",
-                        "label": "Very important field",
-                        "options": [],
-                        "order": 4,
-                        "required": true,
-                        "sensitive": true,
-                        "tooltip": "Wow, this tooltip is useful.",
-                        "type": "str",
-                        "ui_restrictions": [],
-                        "validations": null,
-                        "value": ""
-                    },
-                    "numeric_field_configuration": {
-                        "default_value": 3,
-                        "depends_on": null,
-                        "display": "numeric",
-                        "label": "Very important numeric field",
-                        "options": [],
-                        "order": 2,
-                        "required": true,
-                        "sensitive": false,
-                        "tooltip": "Wow, this tooltip is useful.",
-                        "type": "int",
-                        "ui_restrictions": [],
-                        "validations": [
-                            {
-                                "constraint": 0,
-                                "type": "greater_than"
-                            }
-                        ],
-                        "value": ""
-                    }
-               }
-            }
-            """);
-
-        TaskSettingsConfiguration configuration = TaskSettingsConfiguration.fromXContentBytes(new BytesArray(content), XContentType.JSON);
-        boolean humanReadable = true;
-        BytesReference originalBytes = toShuffledXContent(configuration, XContentType.JSON, ToXContent.EMPTY_PARAMS, humanReadable);
-        TaskSettingsConfiguration parsed;
-        try (XContentParser parser = createParser(XContentType.JSON.xContent(), originalBytes)) {
-            parsed = TaskSettingsConfiguration.fromXContent(parser);
-        }
-        assertToXContentEquivalent(originalBytes, toXContent(parsed, XContentType.JSON, humanReadable), XContentType.JSON);
-    }
-
-    public void testToMap() {
-        TaskSettingsConfiguration configField = TaskSettingsConfigurationTestUtils.getRandomTaskSettingsConfigurationField();
-        Map<String, Object> configFieldAsMap = configField.toMap();
-
-        assertThat(configFieldAsMap.get("task_type"), equalTo(configField.getTaskType()));
-        assertThat(configFieldAsMap.get("configuration"), equalTo(configField.getConfiguration()));
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/inference/configuration/SettingsConfigurationDisplayTypeTests.java b/server/src/test/java/org/elasticsearch/inference/configuration/SettingsConfigurationDisplayTypeTests.java
deleted file mode 100644
index 603ea9480783c..0000000000000
--- a/server/src/test/java/org/elasticsearch/inference/configuration/SettingsConfigurationDisplayTypeTests.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.inference.configuration;
-
-import org.elasticsearch.inference.SettingsConfigurationTestUtils;
-import org.elasticsearch.test.ESTestCase;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class SettingsConfigurationDisplayTypeTests extends ESTestCase {
-
-    public void testDisplayType_WithValidConfigurationDisplayTypeString() {
-        SettingsConfigurationDisplayType displayType = SettingsConfigurationTestUtils.getRandomSettingsConfigurationDisplayType();
-        assertThat(SettingsConfigurationDisplayType.displayType(displayType.toString()), equalTo(displayType));
-    }
-
-    public void testDisplayType_WithInvalidConfigurationDisplayTypeString_ExpectIllegalArgumentException() {
-        expectThrows(
-            IllegalArgumentException.class,
-            () -> SettingsConfigurationDisplayType.displayType("invalid configuration display type")
-        );
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidationTypeTests.java b/server/src/test/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidationTypeTests.java
deleted file mode 100644
index d35968004ea0d..0000000000000
--- a/server/src/test/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidationTypeTests.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.inference.configuration;
-
-import org.elasticsearch.inference.SettingsConfigurationTestUtils;
-import org.elasticsearch.test.ESTestCase;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class SettingsConfigurationValidationTypeTests extends ESTestCase {
-
-    public void testValidationType_WithValidConfigurationValidationTypeString() {
-        SettingsConfigurationValidationType validationType = SettingsConfigurationTestUtils.getRandomConfigurationValidationType();
-
-        assertThat(SettingsConfigurationValidationType.validationType(validationType.toString()), equalTo(validationType));
-    }
-
-    public void testValidationType_WithInvalidConfigurationValidationTypeString_ExpectIllegalArgumentException() {
-        assertThrows(IllegalArgumentException.class, () -> SettingsConfigurationValidationType.validationType("invalid validation type"));
-    }
-
-}
diff --git a/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java
index 126641037fde7..2c400fe7b9173 100644
--- a/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java
+++ b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java
@@ -157,7 +157,7 @@ private void assertHighlightOneDoc(
                     noMatchSize,
                     expectedPassages.length,
                     maxAnalyzedOffset,
-                    queryMaxAnalyzedOffset,
+                    QueryMaxAnalyzedOffset.create(queryMaxAnalyzedOffset, maxAnalyzedOffset),
                     true,
                     true
                 );
diff --git a/server/src/test/java/org/elasticsearch/lucene/util/automaton/MinimizationOperationsTests.java b/server/src/test/java/org/elasticsearch/lucene/util/automaton/MinimizationOperationsTests.java
new file mode 100644
index 0000000000000..0f231ecd57c13
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/lucene/util/automaton/MinimizationOperationsTests.java
@@ -0,0 +1,73 @@
+/*
+ * @notice
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Sourced from: https://github.com/apache/lucene/blob/main/lucene/core/src/test/org/apache/lucene/util/automaton/TestMinimize.java
+ */
+
+package org.elasticsearch.lucene.util.automaton;
+
+import org.apache.lucene.tests.util.automaton.AutomatonTestUtil;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.Operations;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.apache.lucene.tests.util.automaton.AutomatonTestUtil.subsetOf;
+
+public class MinimizationOperationsTests extends ESTestCase {
+
+    /** the minimal and non-minimal are compared to ensure they are the same. */
+    public void testBasic() {
+        int num = atLeast(200);
+        for (int i = 0; i < num; i++) {
+            Automaton a = AutomatonTestUtil.randomAutomaton(random());
+            Automaton la = Operations.determinize(Operations.removeDeadStates(a), Integer.MAX_VALUE);
+            Automaton lb = MinimizationOperations.minimize(a, Integer.MAX_VALUE);
+            assertTrue(sameLanguage(la, lb));
+        }
+    }
+
+    /**
+     * compare minimized against minimized with a slower, simple impl. we check not only that they are
+     * the same, but that #states/#transitions are the same.
+     */
+    public void testAgainstBrzozowski() {
+        int num = atLeast(200);
+        for (int i = 0; i < num; i++) {
+            Automaton a = AutomatonTestUtil.randomAutomaton(random());
+            a = AutomatonTestUtil.minimizeSimple(a);
+            Automaton b = MinimizationOperations.minimize(a, Integer.MAX_VALUE);
+            assertTrue(sameLanguage(a, b));
+            assertEquals(a.getNumStates(), b.getNumStates());
+            int numStates = a.getNumStates();
+
+            int sum1 = 0;
+            for (int s = 0; s < numStates; s++) {
+                sum1 += a.getNumTransitions(s);
+            }
+            int sum2 = 0;
+            for (int s = 0; s < numStates; s++) {
+                sum2 += b.getNumTransitions(s);
+            }
+
+            assertEquals(sum1, sum2);
+        }
+    }
+
+    private static boolean sameLanguage(Automaton a1, Automaton a2) {
+        if (a1 == a2) {
+            return true;
+        }
+        return subsetOf(a2, a1) && subsetOf(a1, a2);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java b/server/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
index 33801dfb98417..ace8499d8ffd0 100644
--- a/server/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
+++ b/server/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
@@ -243,7 +243,7 @@ private static NodeInfo createNodeInfo() {
         return new NodeInfo(
             randomAlphaOfLengthBetween(6, 32),
             new CompatibilityVersions(TransportVersionUtils.randomVersion(random()), Map.of()),
-            IndexVersionUtils.randomVersion(random()),
+            IndexVersionUtils.randomVersion(),
             componentVersions,
             build,
             node,
diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java
index c568f6a38a5fb..433270cae15d9 100644
--- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java
@@ -628,6 +628,7 @@ public void testTasksNotAssignedToShuttingDownNodes() {
                         DiscoveryNode::getId,
                         node -> SingleNodeShutdownMetadata.builder()
                             .setNodeId(node.getId())
+                            .setNodeEphemeralId(node.getEphemeralId())
                             .setReason("shutdown for a unit test")
                             .setType(type)
                             .setStartedAtMillis(randomNonNegativeLong())
@@ -966,7 +967,7 @@ private ClusterState insignificantChange(ClusterState clusterState) {
         }
         // Just add a random index - that shouldn't change anything
         IndexMetadata indexMetadata = IndexMetadata.builder(randomAlphaOfLength(10))
-            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion(random())))
+            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion()))
             .numberOfShards(1)
             .numberOfReplicas(1)
             .build();
@@ -1044,7 +1045,7 @@ private ClusterState initialState() {
 
     private void changeRoutingTable(Metadata.Builder metadata, RoutingTable.Builder routingTable) {
         IndexMetadata indexMetadata = IndexMetadata.builder(randomAlphaOfLength(10))
-            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion(random())))
+            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion()))
             .numberOfShards(1)
             .numberOfReplicas(1)
             .build();
diff --git a/server/src/test/java/org/elasticsearch/readiness/ReadinessServiceTests.java b/server/src/test/java/org/elasticsearch/readiness/ReadinessServiceTests.java
index 8644dfc033c95..8a84b0ec59b51 100644
--- a/server/src/test/java/org/elasticsearch/readiness/ReadinessServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/readiness/ReadinessServiceTests.java
@@ -32,7 +32,6 @@
 import org.elasticsearch.http.HttpInfo;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.HttpStats;
-import org.elasticsearch.reservedstate.service.FileSettingsFeatures;
 import org.elasticsearch.reservedstate.service.FileSettingsService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLog;
@@ -48,7 +47,6 @@
 import java.nio.channels.ServerSocketChannel;
 import java.util.Collections;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 import static org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata.ErrorKind.TRANSIENT;
@@ -60,7 +58,6 @@ public class ReadinessServiceTests extends ESTestCase implements ReadinessClient
     private ThreadPool threadpool;
     private Environment env;
     private FakeHttpTransport httpTransport;
-    private static final Set<String> nodeFeatures = Set.of(FileSettingsFeatures.FILE_SETTINGS_SUPPORTED.id());
 
     private static Metadata emptyReservedStateMetadata;
     static {
@@ -246,6 +243,7 @@ public void testStatusChange() throws Exception {
                                 httpTransport.node.getId(),
                                 SingleNodeShutdownMetadata.builder()
                                     .setNodeId(httpTransport.node.getId())
+                                    .setNodeEphemeralId(httpTransport.node.getEphemeralId())
                                     .setReason("testing")
                                     .setType(SingleNodeShutdownMetadata.Type.RESTART)
                                     .setStartedAtMillis(randomNonNegativeLong())
@@ -309,26 +307,6 @@ public void testFileSettingsUpdateError() throws Exception {
         readinessService.close();
     }
 
-    public void testFileSettingsMixedCluster() throws Exception {
-        readinessService.start();
-
-        // initially the service isn't ready because initial cluster state has not been applied yet
-        assertFalse(readinessService.ready());
-
-        ClusterState noFileSettingsState = ClusterState.builder(noFileSettingsState())
-            // the master node is upgraded to support file settings, but existing node2 is not
-            .nodeFeatures(Map.of(httpTransport.node.getId(), nodeFeatures))
-            .build();
-        ClusterChangedEvent event = new ClusterChangedEvent("test", noFileSettingsState, emptyState());
-        readinessService.clusterChanged(event);
-
-        // when upgrading from nodes before file settings exist, readiness should return true once a master is elected
-        assertTrue(readinessService.ready());
-
-        readinessService.stop();
-        readinessService.close();
-    }
-
     private ClusterState emptyState() {
         return ClusterState.builder(new ClusterName("cluster"))
             .nodes(
@@ -346,7 +324,6 @@ private ClusterState noFileSettingsState() {
                     .masterNodeId(httpTransport.node.getId())
                     .localNodeId(httpTransport.node.getId())
             )
-            .nodeFeatures(Map.of(httpTransport.node.getId(), nodeFeatures, "node2", nodeFeatures))
             .build();
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java b/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java
index 040ab9fd5c2e9..028438b5e9267 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java
@@ -210,12 +210,15 @@ public void testObsoleteVersion() {
         assertThat(
             e.getMessage(),
             equalTo(
-                "A compatible version is required on both Content-Type and Accept headers if either one has requested a "
-                    + "compatible version and the compatible versions must match. "
-                    + "Accept="
-                    + acceptHeader(PREVIOUS_VERSION)
-                    + ", Content-Type="
-                    + contentTypeHeader(OBSOLETE_VERSION)
+                "Content-Type version must be either version "
+                    + CURRENT_VERSION
+                    + " or "
+                    + PREVIOUS_VERSION
+                    + ", but found "
+                    + OBSOLETE_VERSION
+                    + ". "
+                    + "Content-Type="
+                    + acceptHeader(OBSOLETE_VERSION)
             )
         );
     }
diff --git a/server/src/test/java/org/elasticsearch/rest/RestResponseTests.java b/server/src/test/java/org/elasticsearch/rest/RestResponseTests.java
index b85ad31288c8c..bd810cea216fc 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestResponseTests.java
@@ -51,6 +51,7 @@
 import static org.elasticsearch.ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE;
 import static org.elasticsearch.ElasticsearchExceptionTests.assertDeepEquals;
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.rest.RestController.ERROR_TRACE_DEFAULT;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -180,7 +181,7 @@ public void testStackTrace() throws IOException {
                     } else {
                         assertThat(response.status(), is(RestStatus.BAD_REQUEST));
                     }
-                    boolean traceExists = request.paramAsBoolean("error_trace", false) && channel.detailedErrorsEnabled();
+                    boolean traceExists = request.paramAsBoolean("error_trace", ERROR_TRACE_DEFAULT) && channel.detailedErrorsEnabled();
                     if (traceExists) {
                         assertThat(response.content().utf8ToString(), containsString(ElasticsearchException.STACK_TRACE));
                     } else {
diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java
index 29857ef4a519f..b1da067e2f7e6 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java
@@ -11,7 +11,7 @@
 
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
-import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.cluster.ClusterState;
@@ -23,6 +23,7 @@
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.Table;
+import org.elasticsearch.index.shard.IndexingStats;
 import org.elasticsearch.index.shard.ShardPath;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
@@ -40,11 +41,91 @@
 
 public class RestShardsActionTests extends ESTestCase {
 
+    private DiscoveryNode localNode;
+    private List<ShardRouting> shardRoutings;
+    private ClusterStateResponse clusterStateResponse;
+    private IndicesStatsResponse indicesStatsResponse;
+
     public void testBuildTable() {
-        final int numShards = randomIntBetween(1, 5);
-        DiscoveryNode localNode = DiscoveryNodeUtils.create("local");
+        mockShardStats(randomBoolean());
+
+        final RestShardsAction action = new RestShardsAction();
+        final Table table = action.buildTable(new FakeRestRequest(), clusterStateResponse, indicesStatsResponse);
+
+        // now, verify the table is correct
+        List<Table.Cell> headers = table.getHeaders();
+        assertThat(headers.get(0).value, equalTo("index"));
+        assertThat(headers.get(1).value, equalTo("shard"));
+        assertThat(headers.get(2).value, equalTo("prirep"));
+        assertThat(headers.get(3).value, equalTo("state"));
+        assertThat(headers.get(4).value, equalTo("docs"));
+        assertThat(headers.get(5).value, equalTo("store"));
+        assertThat(headers.get(6).value, equalTo("dataset"));
+        assertThat(headers.get(7).value, equalTo("ip"));
+        assertThat(headers.get(8).value, equalTo("id"));
+        assertThat(headers.get(9).value, equalTo("node"));
+        assertThat(headers.get(10).value, equalTo("unassigned.reason"));
+
+        final List<List<Table.Cell>> rows = table.getRows();
+        assertThat(rows.size(), equalTo(shardRoutings.size()));
+
+        Iterator<ShardRouting> shardRoutingsIt = shardRoutings.iterator();
+        for (final List<Table.Cell> row : rows) {
+            ShardRouting shardRouting = shardRoutingsIt.next();
+            ShardStats shardStats = indicesStatsResponse.asMap().get(shardRouting);
+            assertThat(row.get(0).value, equalTo(shardRouting.getIndexName()));
+            assertThat(row.get(1).value, equalTo(shardRouting.getId()));
+            assertThat(row.get(2).value, equalTo(shardRouting.primary() ? "p" : "r"));
+            assertThat(row.get(3).value, equalTo(shardRouting.state()));
+            assertThat(row.get(7).value, equalTo(localNode.getHostAddress()));
+            assertThat(row.get(8).value, equalTo(localNode.getId()));
+            assertThat(row.get(70).value, equalTo(shardStats.getDataPath()));
+            assertThat(row.get(71).value, equalTo(shardStats.getStatePath()));
+        }
+    }
+
+    public void testShardStatsForIndexing() {
+        mockShardStats(true);
+
+        final RestShardsAction action = new RestShardsAction();
+        final Table table = action.buildTable(new FakeRestRequest(), clusterStateResponse, indicesStatsResponse);
+
+        // now, verify the table is correct
+        List<Table.Cell> headers = table.getHeaders();
+        assertThat(headers.get(29).value, equalTo("indexing.delete_current"));
+        assertThat(headers.get(30).value, equalTo("indexing.delete_time"));
+        assertThat(headers.get(31).value, equalTo("indexing.delete_total"));
+        assertThat(headers.get(32).value, equalTo("indexing.index_current"));
+        assertThat(headers.get(33).value, equalTo("indexing.index_time"));
+        assertThat(headers.get(34).value, equalTo("indexing.index_total"));
+        assertThat(headers.get(35).value, equalTo("indexing.index_failed"));
+        assertThat(headers.get(36).value, equalTo("indexing.index_failed_due_to_version_conflict"));
+
+        final List<List<Table.Cell>> rows = table.getRows();
+        assertThat(rows.size(), equalTo(shardRoutings.size()));
 
-        List<ShardRouting> shardRoutings = new ArrayList<>(numShards);
+        Iterator<ShardRouting> shardRoutingsIt = shardRoutings.iterator();
+        for (final List<Table.Cell> row : rows) {
+            ShardRouting shardRouting = shardRoutingsIt.next();
+            ShardStats shardStats = indicesStatsResponse.asMap().get(shardRouting);
+            assertThat(row.get(29).value, equalTo(shardStats.getStats().getIndexing().getTotal().getDeleteCurrent()));
+            assertThat(row.get(30).value, equalTo(shardStats.getStats().getIndexing().getTotal().getDeleteTime()));
+            assertThat(row.get(31).value, equalTo(shardStats.getStats().getIndexing().getTotal().getDeleteCount()));
+            assertThat(row.get(32).value, equalTo(shardStats.getStats().getIndexing().getTotal().getIndexCurrent()));
+            assertThat(row.get(33).value, equalTo(shardStats.getStats().getIndexing().getTotal().getIndexTime()));
+            assertThat(row.get(34).value, equalTo(shardStats.getStats().getIndexing().getTotal().getIndexCount()));
+            assertThat(row.get(35).value, equalTo(shardStats.getStats().getIndexing().getTotal().getIndexFailedCount()));
+            assertThat(
+                row.get(36).value,
+                equalTo(shardStats.getStats().getIndexing().getTotal().getIndexFailedDueToVersionConflictCount())
+            );
+        }
+    }
+
+    private void mockShardStats(boolean includeCommonStats) {
+        final int numShards = randomIntBetween(1, 5);
+        this.localNode = DiscoveryNodeUtils.create("local");
+        this.shardRoutings = new ArrayList<>(numShards);
         Map<ShardRouting, ShardStats> shardStatsMap = new HashMap<>();
         String index = "index";
         for (int i = 0; i < numShards; i++) {
@@ -53,10 +134,33 @@ public void testBuildTable() {
             Path path = createTempDir().resolve("indices")
                 .resolve(shardRouting.shardId().getIndex().getUUID())
                 .resolve(String.valueOf(shardRouting.shardId().id()));
+            CommonStats commonStats = null;
+            if (includeCommonStats) {
+                commonStats = new CommonStats(randomFrom(CommonStatsFlags.ALL, new CommonStatsFlags(CommonStatsFlags.Flag.Indexing)));
+                commonStats.indexing.add(
+                    new IndexingStats(
+                        new IndexingStats.Stats(
+                            randomNonNegativeLong(),
+                            randomNonNegativeLong(),
+                            randomNonNegativeLong(),
+                            randomNonNegativeLong(),
+                            randomNonNegativeLong(),
+                            randomNonNegativeLong(),
+                            randomNonNegativeLong(),
+                            randomNonNegativeLong(),
+                            randomNonNegativeLong(),
+                            randomBoolean(),
+                            randomNonNegativeLong(),
+                            randomNonNegativeLong(),
+                            randomNonNegativeLong()
+                        )
+                    )
+                );
+            }
             ShardStats shardStats = new ShardStats(
                 shardRouting,
                 new ShardPath(false, path, path, shardRouting.shardId()),
-                null,
+                commonStats,
                 null,
                 null,
                 null,
@@ -67,56 +171,18 @@ public void testBuildTable() {
             shardRoutings.add(shardRouting);
         }
 
-        IndexStats indexStats = mock(IndexStats.class);
-        when(indexStats.getPrimaries()).thenReturn(new CommonStats());
-        when(indexStats.getTotal()).thenReturn(new CommonStats());
-
-        IndicesStatsResponse stats = mock(IndicesStatsResponse.class);
-        when(stats.asMap()).thenReturn(shardStatsMap);
+        this.indicesStatsResponse = mock(IndicesStatsResponse.class);
+        when(this.indicesStatsResponse.asMap()).thenReturn(shardStatsMap);
 
         DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class);
         when(discoveryNodes.get(localNode.getId())).thenReturn(localNode);
 
-        ClusterStateResponse state = mock(ClusterStateResponse.class);
+        this.clusterStateResponse = mock(ClusterStateResponse.class);
         RoutingTable routingTable = mock(RoutingTable.class);
         when(routingTable.allShardsIterator()).thenReturn(shardRoutings);
         ClusterState clusterState = mock(ClusterState.class);
         when(clusterState.routingTable()).thenReturn(routingTable);
         when(clusterState.nodes()).thenReturn(discoveryNodes);
-        when(state.getState()).thenReturn(clusterState);
-
-        final RestShardsAction action = new RestShardsAction();
-        final Table table = action.buildTable(new FakeRestRequest(), state, stats);
-
-        // now, verify the table is correct
-        List<Table.Cell> headers = table.getHeaders();
-        assertThat(headers.get(0).value, equalTo("index"));
-        assertThat(headers.get(1).value, equalTo("shard"));
-        assertThat(headers.get(2).value, equalTo("prirep"));
-        assertThat(headers.get(3).value, equalTo("state"));
-        assertThat(headers.get(4).value, equalTo("docs"));
-        assertThat(headers.get(5).value, equalTo("store"));
-        assertThat(headers.get(6).value, equalTo("dataset"));
-        assertThat(headers.get(7).value, equalTo("ip"));
-        assertThat(headers.get(8).value, equalTo("id"));
-        assertThat(headers.get(9).value, equalTo("node"));
-        assertThat(headers.get(10).value, equalTo("unassigned.reason"));
-
-        final List<List<Table.Cell>> rows = table.getRows();
-        assertThat(rows.size(), equalTo(numShards));
-
-        Iterator<ShardRouting> shardRoutingsIt = shardRoutings.iterator();
-        for (final List<Table.Cell> row : rows) {
-            ShardRouting shardRouting = shardRoutingsIt.next();
-            ShardStats shardStats = shardStatsMap.get(shardRouting);
-            assertThat(row.get(0).value, equalTo(shardRouting.getIndexName()));
-            assertThat(row.get(1).value, equalTo(shardRouting.getId()));
-            assertThat(row.get(2).value, equalTo(shardRouting.primary() ? "p" : "r"));
-            assertThat(row.get(3).value, equalTo(shardRouting.state()));
-            assertThat(row.get(7).value, equalTo(localNode.getHostAddress()));
-            assertThat(row.get(8).value, equalTo(localNode.getId()));
-            assertThat(row.get(69).value, equalTo(shardStats.getDataPath()));
-            assertThat(row.get(70).value, equalTo(shardStats.getStatePath()));
-        }
+        when(clusterStateResponse.getState()).thenReturn(clusterState);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestActionTests.java
index c29ce51ecc01b..ac6c66a13b507 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestActionTests.java
@@ -175,6 +175,14 @@ public void testSimulateIngestRestToXContentListener() throws Exception {
                     "executed_pipelines" : [
                       "pipeline1",
                       "pipeline2"
+                    ],
+                    "ignored_fields" : [
+                      {
+                        "field" : "abc"
+                      },
+                      {
+                        "field" : "def"
+                      }
                     ]
                   }
                 },
@@ -199,6 +207,14 @@ public void testSimulateIngestRestToXContentListener() throws Exception {
                     "executed_pipelines" : [
                       "pipeline1",
                       "pipeline2"
+                    ],
+                    "ignored_fields" : [
+                      {
+                        "field" : "abc"
+                      },
+                      {
+                        "field" : "def"
+                      }
                     ]
                   }
                 }
@@ -228,6 +244,7 @@ private BulkItemResponse getSuccessBulkItemResponse(String id, String source) {
                 BytesReference.fromByteBuffers(sourceByteBuffer),
                 XContentType.JSON,
                 List.of("pipeline1", "pipeline2"),
+                List.of("abc", "def"),
                 null
             )
         );
diff --git a/server/src/test/java/org/elasticsearch/script/ScriptCacheStatsTests.java b/server/src/test/java/org/elasticsearch/script/ScriptCacheStatsTests.java
index 991967c2db885..9a4d6fdef4671 100644
--- a/server/src/test/java/org/elasticsearch/script/ScriptCacheStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/script/ScriptCacheStatsTests.java
@@ -31,9 +31,7 @@ public void testXContentChunkedWithGeneralMode() throws IOException {
         var scriptCacheStats = new ScriptCacheStats(stats);
 
         builder.startObject();
-        for (var it = scriptCacheStats.toXContentChunked(ToXContent.EMPTY_PARAMS); it.hasNext();) {
-            it.next().toXContent(builder, ToXContent.EMPTY_PARAMS);
-        }
+        scriptCacheStats.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
 
         String expected = """
@@ -63,9 +61,7 @@ public void testXContentChunkedWithContextMode() throws IOException {
         );
 
         builder.startObject();
-        for (var it = scriptCacheStats.toXContentChunked(ToXContent.EMPTY_PARAMS); it.hasNext();) {
-            it.next().toXContent(builder, ToXContent.EMPTY_PARAMS);
-        }
+        scriptCacheStats.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
 
         String expected = """
diff --git a/server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java b/server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java
index dcaa64ede9e89..48d09c75cb2d1 100644
--- a/server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java
+++ b/server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java
@@ -51,12 +51,12 @@ public void testFloatVectorClassBindings() throws IOException {
                 BinaryDenseVectorScriptDocValuesTests.wrap(
                     new float[][] { docVector },
                     ElementType.FLOAT,
-                    IndexVersions.MINIMUM_COMPATIBLE
+                    IndexVersions.MINIMUM_READONLY_COMPATIBLE
                 ),
                 "test",
                 ElementType.FLOAT,
                 dims,
-                IndexVersions.MINIMUM_COMPATIBLE
+                IndexVersions.MINIMUM_READONLY_COMPATIBLE
             ),
             new BinaryDenseVectorDocValuesField(
                 BinaryDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.FLOAT, IndexVersion.current()),
@@ -303,12 +303,12 @@ public void testByteVsFloatSimilarity() throws IOException {
                 BinaryDenseVectorScriptDocValuesTests.wrap(
                     new float[][] { docVector },
                     ElementType.FLOAT,
-                    IndexVersions.MINIMUM_COMPATIBLE
+                    IndexVersions.MINIMUM_READONLY_COMPATIBLE
                 ),
                 "field0",
                 ElementType.FLOAT,
                 dims,
-                IndexVersions.MINIMUM_COMPATIBLE
+                IndexVersions.MINIMUM_READONLY_COMPATIBLE
             ),
             new BinaryDenseVectorDocValuesField(
                 BinaryDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.FLOAT, IndexVersion.current()),
diff --git a/server/src/test/java/org/elasticsearch/script/field/vectors/DenseVectorTests.java b/server/src/test/java/org/elasticsearch/script/field/vectors/DenseVectorTests.java
index 8a5298777ede0..63d502a248aa8 100644
--- a/server/src/test/java/org/elasticsearch/script/field/vectors/DenseVectorTests.java
+++ b/server/src/test/java/org/elasticsearch/script/field/vectors/DenseVectorTests.java
@@ -69,7 +69,11 @@ public void testFloatVsListQueryVector() {
         assertEquals(knn.cosineSimilarity(arrayQV), knn.cosineSimilarity(listQV), 0.001f);
         assertEquals(knn.cosineSimilarity((Object) listQV), knn.cosineSimilarity((Object) arrayQV), 0.001f);
 
-        for (IndexVersion indexVersion : List.of(IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current())) {
+        for (IndexVersion indexVersion : List.of(
+            IndexVersions.MINIMUM_READONLY_COMPATIBLE,
+            IndexVersions.MINIMUM_COMPATIBLE,
+            IndexVersion.current()
+        )) {
             BytesRef value = BinaryDenseVectorScriptDocValuesTests.mockEncodeDenseVector(docVector, ElementType.FLOAT, indexVersion);
             BinaryDenseVector bdv = new BinaryDenseVector(docVector, value, dims, indexVersion);
 
diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
index a474c1dc38c50..82737b13a1a46 100644
--- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
+++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.tests.store.BaseDirectoryWrapper;
@@ -204,7 +205,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) {
             );
 
             // resultWindow greater than maxResultWindow and scrollContext isn't null
-            when(shardSearchRequest.scroll()).thenReturn(new Scroll(TimeValue.timeValueMillis(randomInt(1000))));
+            when(shardSearchRequest.scroll()).thenReturn(TimeValue.timeValueMillis(randomInt(1000)));
             ReaderContext readerContext = new LegacyReaderContext(
                 newContextId(),
                 indexService,
@@ -245,7 +246,10 @@ protected Engine.Searcher acquireSearcherInternal(String source) {
                 // resultWindow not greater than maxResultWindow and both rescore and sort are not null
                 context1.from(0);
                 DocValueFormat docValueFormat = mock(DocValueFormat.class);
-                SortAndFormats sortAndFormats = new SortAndFormats(new Sort(), new DocValueFormat[] { docValueFormat });
+                SortAndFormats sortAndFormats = new SortAndFormats(
+                    new Sort(new SortField[] { SortField.FIELD_DOC }),
+                    new DocValueFormat[] { docValueFormat }
+                );
                 context1.sort(sortAndFormats);
 
                 RescoreContext rescoreContext = mock(RescoreContext.class);
@@ -287,7 +291,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) {
                 @Override
                 public ScrollContext scrollContext() {
                     ScrollContext scrollContext = new ScrollContext();
-                    scrollContext.scroll = new Scroll(TimeValue.timeValueSeconds(5));
+                    scrollContext.scroll = TimeValue.timeValueSeconds(5);
                     return scrollContext;
                 }
             };
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceSingleNodeTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceSingleNodeTests.java
index 02593e41f5d84..16e066e486de9 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceSingleNodeTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceSingleNodeTests.java
@@ -376,8 +376,7 @@ public void testSearchWhileIndexDeleted() throws InterruptedException {
         try {
             final int rounds = scaledRandomIntBetween(100, 10000);
             SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true);
-            SearchRequest scrollSearchRequest = new SearchRequest().allowPartialSearchResults(true)
-                .scroll(new Scroll(TimeValue.timeValueMinutes(1)));
+            SearchRequest scrollSearchRequest = new SearchRequest().allowPartialSearchResults(true).scroll(TimeValue.timeValueMinutes(1));
             for (int i = 0; i < rounds; i++) {
                 try {
                     try {
@@ -1188,8 +1187,7 @@ public void testSearchWhileIndexDeletedDoesNotLeakSearchContext() throws Executi
         });
 
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true);
-        SearchRequest scrollSearchRequest = new SearchRequest().allowPartialSearchResults(true)
-            .scroll(new Scroll(TimeValue.timeValueMinutes(1)));
+        SearchRequest scrollSearchRequest = new SearchRequest().allowPartialSearchResults(true).scroll(TimeValue.timeValueMinutes(1));
 
         // the scrolls are not explicitly freed, but should all be gone when the test finished.
         // for completeness, we also randomly test the regular search path.
@@ -1657,7 +1655,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) {
     }
 
     private static class ShardScrollRequestTest extends ShardSearchRequest {
-        private Scroll scroll;
+        private final TimeValue scroll;
 
         ShardScrollRequestTest(ShardId shardId) {
             super(
@@ -1671,11 +1669,11 @@ private static class ShardScrollRequestTest extends ShardSearchRequest {
                 -1,
                 null
             );
-            this.scroll = new Scroll(TimeValue.timeValueMinutes(1));
+            this.scroll = TimeValue.timeValueMinutes(1);
         }
 
         @Override
-        public Scroll scroll() {
+        public TimeValue scroll() {
             return this.scroll;
         }
     }
@@ -2684,7 +2682,8 @@ public void testDfsQueryPhaseRewrite() {
         service.executeQueryPhase(
             new QuerySearchRequest(null, context.id(), request, new AggregatedDfs(Map.of(), Map.of(), 10)),
             new SearchShardTask(42L, "", "", "", null, emptyMap()),
-            plainActionFuture
+            plainActionFuture,
+            TransportVersion.current()
         );
 
         plainActionFuture.actionGet();
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 31bcab31ca8a7..d041121b8a96b 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -13,6 +13,8 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -53,9 +55,14 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.BiFunction;
 import java.util.function.Predicate;
 
+import static org.elasticsearch.search.SearchService.maybeWrapListenerForStackTrace;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.Matchers.not;
+
 public class SearchServiceTests extends IndexShardTestCase {
 
     public void testCanMatchMatchAll() throws IOException {
@@ -117,6 +124,33 @@ public Type getType() {
         doTestCanMatch(searchRequest, sortField, true, null, false);
     }
 
+    public void testMaybeWrapListenerForStackTrace() {
+        // Tests that the same listener has stack trace if is not wrapped or does not have stack trace if it is wrapped.
+        AtomicBoolean isWrapped = new AtomicBoolean(false);
+        ActionListener<SearchPhaseResult> listener = new ActionListener<>() {
+            @Override
+            public void onResponse(SearchPhaseResult searchPhaseResult) {
+                // noop - we only care about failure scenarios
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                if (isWrapped.get()) {
+                    assertThat(e.getStackTrace().length, is(0));
+                } else {
+                    assertThat(e.getStackTrace().length, is(not(0)));
+                }
+            }
+        };
+        Exception e = new Exception();
+        e.fillInStackTrace();
+        assertThat(e.getStackTrace().length, is(not(0)));
+        listener.onFailure(e);
+        listener = maybeWrapListenerForStackTrace(listener, TransportVersion.current(), threadPool);
+        isWrapped.set(true);
+        listener.onFailure(e);
+    }
+
     private void doTestCanMatch(
         SearchRequest searchRequest,
         SortField sortField,
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
index 38294fb030ed4..bf26326abafbf 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
@@ -83,9 +83,9 @@ public class DateHistogramAggregatorTests extends DateHistogramAggregatorTestCas
         "2017-12-12T22:55:46"
     );
 
-    public void testBooleanFieldDeprecated() throws IOException {
+    public void testBooleanFieldUnsupported() throws IOException {
         final String fieldName = "bogusBoolean";
-        testCase(iw -> {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> testCase(iw -> {
             Document d = new Document();
             d.add(new SortedNumericDocValuesField(fieldName, 0));
             iw.addDocument(d);
@@ -95,8 +95,8 @@ public void testBooleanFieldDeprecated() throws IOException {
                 new DateHistogramAggregationBuilder("name").calendarInterval(DateHistogramInterval.HOUR).field(fieldName),
                 new BooleanFieldMapper.BooleanFieldType(fieldName)
             )
-        );
-        assertWarnings("Running DateHistogram aggregations on [boolean] fields is deprecated");
+        ));
+        assertThat(e.getMessage(), equalTo("Field [bogusBoolean] of type [boolean] is not supported for aggregation [date_histogram]"));
     }
 
     public void testMatchNoDocs() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java
index deada75279e33..971f3bab7b6a3 100644
--- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java
@@ -11,8 +11,12 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.memory.MemoryIndex;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.fetch.FetchContext;
@@ -190,6 +194,12 @@ private HitContext hitExecuteMultiple(
         when(fetchContext.getIndexName()).thenReturn("index");
         SearchExecutionContext sec = mock(SearchExecutionContext.class);
         when(sec.isSourceEnabled()).thenReturn(sourceBuilder != null);
+        IndexSettings indexSettings = new IndexSettings(
+            IndexMetadata.builder("index").settings(indexSettings(IndexVersion.current(), 1, 0)).build(),
+            Settings.EMPTY
+        );
+        when(sec.indexVersionCreated()).thenReturn(indexSettings.getIndexVersionCreated());
+        when(sec.getIndexSettings()).thenReturn(indexSettings);
         when(fetchContext.getSearchExecutionContext()).thenReturn(sec);
 
         final SearchHit searchHit = SearchHit.unpooled(1, null, nestedIdentity);
diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
index d1bbc1ec5910b..1bcc89b68c141 100644
--- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
@@ -577,10 +577,10 @@ public void testPreTagsWithoutPostTags() throws IOException {
     public void testInvalidMaxAnalyzedOffset() throws IOException {
         XContentParseException e = expectParseThrows(
             XContentParseException.class,
-            "{ \"max_analyzed_offset\" : " + randomIntBetween(-100, 0) + "}"
+            "{ \"max_analyzed_offset\" : " + randomIntBetween(-100, -2) + "}"
         );
         assertThat(e.getMessage(), containsString("[highlight] failed to parse field [" + MAX_ANALYZED_OFFSET_FIELD.toString() + "]"));
-        assertThat(e.getCause().getMessage(), containsString("[max_analyzed_offset] must be a positive integer"));
+        assertThat(e.getCause().getMessage(), containsString("[max_analyzed_offset] must be a positive integer, or -1"));
     }
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java
index da28b0eff441f..2724b86f9acd4 100644
--- a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java
+++ b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java
@@ -11,6 +11,7 @@
 
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Predicates;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.MatchNoneQueryBuilder;
@@ -88,10 +89,7 @@ protected KnnRetrieverBuilder createTestInstance() {
     protected KnnRetrieverBuilder doParseInstance(XContentParser parser) throws IOException {
         return (KnnRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder(
             parser,
-            new RetrieverParserContext(
-                new SearchUsage(),
-                nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED || nf == KnnRetrieverBuilder.KNN_RETRIEVER_SUPPORTED
-            )
+            new RetrieverParserContext(new SearchUsage(), Predicates.never())
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/search/retriever/RescorerRetrieverBuilderParsingTests.java b/server/src/test/java/org/elasticsearch/search/retriever/RescorerRetrieverBuilderParsingTests.java
new file mode 100644
index 0000000000000..fa83246d90cb2
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/retriever/RescorerRetrieverBuilderParsingTests.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.search.retriever;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.rescore.QueryRescorerBuilderTests;
+import org.elasticsearch.search.rescore.RescorerBuilder;
+import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.usage.SearchUsage;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.XContentParser;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+
+public class RescorerRetrieverBuilderParsingTests extends AbstractXContentTestCase<RescorerRetrieverBuilder> {
+    private static List<NamedXContentRegistry.Entry> xContentRegistryEntries;
+
+    @BeforeClass
+    public static void init() {
+        xContentRegistryEntries = new SearchModule(Settings.EMPTY, emptyList()).getNamedXContents();
+    }
+
+    @AfterClass
+    public static void afterClass() throws Exception {
+        xContentRegistryEntries = null;
+    }
+
+    @Override
+    protected RescorerRetrieverBuilder createTestInstance() {
+        int num = randomIntBetween(1, 3);
+        List<RescorerBuilder<?>> rescorers = new ArrayList<>();
+        for (int i = 0; i < num; i++) {
+            rescorers.add(QueryRescorerBuilderTests.randomRescoreBuilder());
+        }
+        return new RescorerRetrieverBuilder(TestRetrieverBuilder.createRandomTestRetrieverBuilder(), rescorers);
+    }
+
+    @Override
+    protected RescorerRetrieverBuilder doParseInstance(XContentParser parser) throws IOException {
+        return (RescorerRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder(
+            parser,
+            new RetrieverParserContext(new SearchUsage(), n -> true)
+        );
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(xContentRegistryEntries);
+        entries.add(
+            new NamedXContentRegistry.Entry(
+                RetrieverBuilder.class,
+                TestRetrieverBuilder.TEST_SPEC.getName(),
+                (p, c) -> TestRetrieverBuilder.TEST_SPEC.getParser().fromXContent(p, (RetrieverParserContext) c),
+                TestRetrieverBuilder.TEST_SPEC.getName().getForRestApiVersion()
+            )
+        );
+        return new NamedXContentRegistry(entries);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/retriever/RetrieverBuilderVersionTests.java b/server/src/test/java/org/elasticsearch/search/retriever/RetrieverBuilderVersionTests.java
deleted file mode 100644
index 6448d11de2e47..0000000000000
--- a/server/src/test/java/org/elasticsearch/search/retriever/RetrieverBuilderVersionTests.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.search.retriever;
-
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.features.NodeFeature;
-import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.json.JsonXContent;
-
-import java.io.IOException;
-import java.util.List;
-
-/** Tests retrievers validate on their own {@link NodeFeature} */
-public class RetrieverBuilderVersionTests extends ESTestCase {
-
-    public void testRetrieverVersions() throws IOException {
-        try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"retriever\":{\"standard\":{}}}")) {
-            SearchSourceBuilder ssb = new SearchSourceBuilder();
-            ParsingException iae = expectThrows(ParsingException.class, () -> ssb.parseXContent(parser, true, nf -> false));
-            assertEquals("Unknown key for a START_OBJECT in [retriever].", iae.getMessage());
-        }
-
-        try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"retriever\":{\"standard\":{}}}")) {
-            SearchSourceBuilder ssb = new SearchSourceBuilder();
-            ParsingException iae = expectThrows(
-                ParsingException.class,
-                () -> ssb.parseXContent(parser, true, nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED)
-            );
-            assertEquals("unknown retriever [standard]", iae.getMessage());
-        }
-
-        try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"retriever\":{\"standard\":{}}}")) {
-            SearchSourceBuilder ssb = new SearchSourceBuilder();
-            ssb.parseXContent(
-                parser,
-                true,
-                nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED || nf == StandardRetrieverBuilder.STANDARD_RETRIEVER_SUPPORTED
-            );
-        }
-
-        try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"retriever\":{\"knn\":{}}}")) {
-            SearchSourceBuilder ssb = new SearchSourceBuilder();
-            ParsingException iae = expectThrows(
-                ParsingException.class,
-                () -> ssb.parseXContent(parser, true, nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED)
-            );
-            assertEquals("unknown retriever [knn]", iae.getMessage());
-        }
-
-        try (
-            XContentParser parser = createParser(
-                JsonXContent.jsonXContent,
-                "{\"retriever\":{\"knn\":{\"field\": \"test\", \"k\": 2, \"num_candidates\": 5, \"query_vector\": [1, 2, 3]}}}"
-            )
-        ) {
-            SearchSourceBuilder ssb = new SearchSourceBuilder();
-            ssb.parseXContent(
-                parser,
-                true,
-                nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED || nf == KnnRetrieverBuilder.KNN_RETRIEVER_SUPPORTED
-            );
-        }
-    }
-
-    @Override
-    protected NamedXContentRegistry xContentRegistry() {
-        return new NamedXContentRegistry(new SearchModule(Settings.EMPTY, List.of()).getNamedXContents());
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java b/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java
index eacd949077bc4..979c588089f5c 100644
--- a/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java
+++ b/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Predicates;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.MatchNoneQueryBuilder;
@@ -100,10 +101,7 @@ protected StandardRetrieverBuilder createTestInstance() {
     protected StandardRetrieverBuilder doParseInstance(XContentParser parser) throws IOException {
         return (StandardRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder(
             parser,
-            new RetrieverParserContext(
-                new SearchUsage(),
-                nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED || nf == StandardRetrieverBuilder.STANDARD_RETRIEVER_SUPPORTED
-            )
+            new RetrieverParserContext(new SearchUsage(), Predicates.never())
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java b/server/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java
index 74329d9462d38..1e9b702d9b8e2 100644
--- a/server/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java
+++ b/server/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java
@@ -47,7 +47,7 @@ public void testBodyParamsOverrideQueryStringParams() throws Exception {
                 public void searchScroll(SearchScrollRequest request, ActionListener<SearchResponse> listener) {
                     scrollCalled.set(true);
                     assertThat(request.scrollId(), equalTo("BODY"));
-                    assertThat(request.scroll().keepAlive().getStringRep(), equalTo("1m"));
+                    assertThat(request.scroll().getStringRep(), equalTo("1m"));
                 }
             };
             RestSearchScrollAction action = new RestSearchScrollAction();
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java
index 375712ee60861..b3764d528ff0f 100644
--- a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java
@@ -46,7 +46,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.NUM_CANDS_OVERSAMPLE_LIMIT;
+import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.OVERSAMPLE_LIMIT;
 import static org.elasticsearch.search.SearchService.DEFAULT_SIZE;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -82,7 +82,7 @@ private void checkIndexTypeAndDimensions() {
 
     abstract KnnVectorQueryBuilder createKnnVectorQueryBuilder(
         String fieldName,
-        Integer k,
+        int k,
         int numCands,
         RescoreVectorBuilder rescoreVectorBuilder,
         Float similarity
@@ -138,8 +138,8 @@ protected void initializeAdditionalMappings(MapperService mapperService) throws
     @Override
     protected KnnVectorQueryBuilder doCreateTestQueryBuilder() {
         String fieldName = randomBoolean() ? VECTOR_FIELD : VECTOR_ALIAS_FIELD;
-        Integer k = randomBoolean() ? null : randomIntBetween(1, 100);
-        int numCands = randomIntBetween(k == null ? DEFAULT_SIZE : k + 20, 1000);
+        int k = randomIntBetween(1, 100);
+        int numCands = randomIntBetween(k + 20, 1000);
         KnnVectorQueryBuilder queryBuilder = createKnnVectorQueryBuilder(
             fieldName,
             k,
@@ -176,8 +176,13 @@ protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query que
             assertThat(((VectorSimilarityQuery) query).getSimilarity(), equalTo(queryBuilder.getVectorSimilarity()));
             query = ((VectorSimilarityQuery) query).getInnerKnnQuery();
         }
+        Integer k = queryBuilder.k();
+        if (k == null) {
+            k = context.requestSize() == null || context.requestSize() < 0 ? DEFAULT_SIZE : context.requestSize();
+        }
         if (queryBuilder.rescoreVectorBuilder() != null && isQuantizedElementType()) {
             RescoreKnnVectorQuery rescoreQuery = (RescoreKnnVectorQuery) query;
+            assertEquals(k.intValue(), (rescoreQuery.k()));
             query = rescoreQuery.innerQuery();
         }
         switch (elementType()) {
@@ -191,14 +196,11 @@ protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query que
         }
         BooleanQuery booleanQuery = builder.build();
         Query filterQuery = booleanQuery.clauses().isEmpty() ? null : booleanQuery;
-        // The field should always be resolved to the concrete field
-        Integer k = queryBuilder.k();
         Integer numCands = queryBuilder.numCands();
         if (queryBuilder.rescoreVectorBuilder() != null && isQuantizedElementType()) {
-            Float numCandsFactor = queryBuilder.rescoreVectorBuilder().numCandidatesFactor();
-            int minCands = k == null ? 1 : k;
-            numCands = Math.max(minCands, (int) Math.ceil(numCands * numCandsFactor));
-            numCands = Math.min(numCands, NUM_CANDS_OVERSAMPLE_LIMIT);
+            Float oversample = queryBuilder.rescoreVectorBuilder().oversample();
+            k = Math.min(OVERSAMPLE_LIMIT, (int) Math.ceil(k * oversample));
+            numCands = Math.max(numCands, k);
         }
 
         Query knnVectorQueryBuilt = switch (elementType()) {
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnByteVectorQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnByteVectorQueryBuilderTests.java
index f6c2e754cec63..26066389c63f1 100644
--- a/server/src/test/java/org/elasticsearch/search/vectors/KnnByteVectorQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnByteVectorQueryBuilderTests.java
@@ -20,7 +20,7 @@ DenseVectorFieldMapper.ElementType elementType() {
     @Override
     protected KnnVectorQueryBuilder createKnnVectorQueryBuilder(
         String fieldName,
-        Integer k,
+        int k,
         int numCands,
         RescoreVectorBuilder rescoreVectorBuilder,
         Float similarity
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnFloatVectorQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnFloatVectorQueryBuilderTests.java
index 6f67e4be29a06..70d29ab525ef1 100644
--- a/server/src/test/java/org/elasticsearch/search/vectors/KnnFloatVectorQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnFloatVectorQueryBuilderTests.java
@@ -20,7 +20,7 @@ DenseVectorFieldMapper.ElementType elementType() {
     @Override
     KnnVectorQueryBuilder createKnnVectorQueryBuilder(
         String fieldName,
-        Integer k,
+        int k,
         int numCands,
         RescoreVectorBuilder rescoreVectorBuilder,
         Float similarity
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java
index a39438af5b72a..8cca3f9ed8a21 100644
--- a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java
@@ -224,9 +224,9 @@ public void testToQueryBuilder() {
             builder.addFilterQuery(filter);
         }
 
-        QueryBuilder expected = new KnnVectorQueryBuilder(field, vector, null, numCands, rescoreVectorBuilder, similarity).addFilterQueries(
-            filterQueries
-        ).boost(boost);
+        QueryBuilder expected = new KnnVectorQueryBuilder(field, vector, numCands, numCands, rescoreVectorBuilder, similarity)
+            .addFilterQueries(filterQueries)
+            .boost(boost);
         assertEquals(expected, builder.toQueryBuilder());
     }
 
@@ -259,7 +259,7 @@ public void testInvalidRescoreVectorBuilder() {
             IllegalArgumentException.class,
             () -> new KnnSearchBuilder("field", randomVector(3), 10, 100, new RescoreVectorBuilder(0.99F), null)
         );
-        assertThat(e.getMessage(), containsString("[num_candidates_factor] must be >= 1.0"));
+        assertThat(e.getMessage(), containsString("[oversample] must be >= 1.0"));
     }
 
     public void testRewrite() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/RescoreKnnVectorQueryTests.java b/server/src/test/java/org/elasticsearch/search/vectors/RescoreKnnVectorQueryTests.java
index 7bbe7dcc155c5..861a8b11db567 100644
--- a/server/src/test/java/org/elasticsearch/search/vectors/RescoreKnnVectorQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/search/vectors/RescoreKnnVectorQueryTests.java
@@ -9,8 +9,6 @@
 
 package org.elasticsearch.search.vectors;
 
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.KnnFloatVectorField;
 import org.apache.lucene.index.DirectoryReader;
@@ -33,11 +31,9 @@
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.PriorityQueue;
 import java.util.stream.Collectors;
@@ -49,21 +45,11 @@
 public class RescoreKnnVectorQueryTests extends ESTestCase {
 
     public static final String FIELD_NAME = "float_vector";
-    private final int numDocs;
-    private final Integer k;
-
-    public RescoreKnnVectorQueryTests(boolean useK) {
-        this.numDocs = randomIntBetween(10, 100);
-        this.k = useK ? randomIntBetween(1, numDocs - 1) : null;
-    }
 
     public void testRescoreDocs() throws Exception {
+        int numDocs = randomIntBetween(10, 100);
         int numDims = randomIntBetween(5, 100);
-
-        Integer adjustedK = k;
-        if (k == null) {
-            adjustedK = numDocs;
-        }
+        int k = randomIntBetween(1, numDocs - 1);
 
         try (Directory d = newDirectory()) {
             addRandomDocuments(numDocs, d, numDims);
@@ -77,7 +63,7 @@ public void testRescoreDocs() throws Exception {
                     FIELD_NAME,
                     queryVector,
                     VectorSimilarityFunction.COSINE,
-                    adjustedK,
+                    k,
                     new MatchAllDocsQuery()
                 );
 
@@ -86,7 +72,7 @@ public void testRescoreDocs() throws Exception {
                 Map<Integer, Float> rescoredDocs = Arrays.stream(docs.scoreDocs)
                     .collect(Collectors.toMap(scoreDoc -> scoreDoc.doc, scoreDoc -> scoreDoc.score));
 
-                assertThat(rescoredDocs.size(), equalTo(adjustedK));
+                assertThat(rescoredDocs.size(), equalTo(k));
 
                 Collection<Float> rescoredScores = new HashSet<>(rescoredDocs.values());
 
@@ -113,7 +99,7 @@ public void testRescoreDocs() throws Exception {
                 assertThat(rescoredDocs.size(), equalTo(0));
 
                 // Check top scoring docs are contained in rescored docs
-                for (int i = 0; i < adjustedK; i++) {
+                for (int i = 0; i < k; i++) {
                     Float topScore = topK.poll();
                     if (rescoredScores.contains(topScore) == false) {
                         fail("Top score " + topScore + " not contained in rescored doc scores " + rescoredScores);
@@ -124,7 +110,9 @@ public void testRescoreDocs() throws Exception {
     }
 
     public void testProfiling() throws Exception {
+        int numDocs = randomIntBetween(10, 100);
         int numDims = randomIntBetween(5, 100);
+        int k = randomIntBetween(1, numDocs - 1);
 
         try (Directory d = newDirectory()) {
             addRandomDocuments(numDocs, d, numDims);
@@ -132,13 +120,13 @@ public void testProfiling() throws Exception {
             try (IndexReader reader = DirectoryReader.open(d)) {
                 float[] queryVector = randomVector(numDims);
 
-                checkProfiling(queryVector, reader, new MatchAllDocsQuery());
-                checkProfiling(queryVector, reader, new MockQueryProfilerProvider(randomIntBetween(1, 100)));
+                checkProfiling(k, numDocs, queryVector, reader, new MatchAllDocsQuery());
+                checkProfiling(k, numDocs, queryVector, reader, new MockQueryProfilerProvider(randomIntBetween(1, 100)));
             }
         }
     }
 
-    private void checkProfiling(float[] queryVector, IndexReader reader, Query innerQuery) throws IOException {
+    private void checkProfiling(int k, int numDocs, float[] queryVector, IndexReader reader, Query innerQuery) throws IOException {
         RescoreKnnVectorQuery rescoreKnnVectorQuery = new RescoreKnnVectorQuery(
             FIELD_NAME,
             queryVector,
@@ -229,13 +217,4 @@ private static void addRandomDocuments(int numDocs, Directory d, int numDims) th
             w.forceMerge(1);
         }
     }
-
-    @ParametersFactory
-    public static Iterable<Object[]> parameters() {
-        List<Object[]> params = new ArrayList<>();
-        params.add(new Object[] { true });
-        params.add(new Object[] { false });
-
-        return params;
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceTests.java
index cc21ade314715..cadd9d5196f69 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceTests.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.health.Diagnosis;
 import org.elasticsearch.health.Diagnosis.Resource.Type;
-import org.elasticsearch.health.HealthFeatures;
 import org.elasticsearch.health.HealthIndicatorDetails;
 import org.elasticsearch.health.HealthIndicatorResult;
 import org.elasticsearch.health.SimpleHealthIndicatorDetails;
@@ -37,7 +36,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.cluster.node.DiscoveryNode.DISCOVERY_NODE_COMPARATOR;
@@ -382,10 +380,7 @@ public void testMappedFieldsForTelemetry() {
     }
 
     private ClusterState createClusterStateWith(RepositoriesMetadata metadata) {
-        var features = Set.of(HealthFeatures.SUPPORTS_EXTENDED_REPOSITORY_INDICATOR.id());
-        var builder = ClusterState.builder(new ClusterName("test-cluster"))
-            .nodes(DiscoveryNodes.builder().add(node1).add(node2).build())
-            .nodeFeatures(Map.of(node1.getId(), features, node2.getId(), features));
+        var builder = ClusterState.builder(new ClusterName("test-cluster")).nodes(DiscoveryNodes.builder().add(node1).add(node2).build());
         if (metadata != null) {
             builder.metadata(Metadata.builder().putCustom(RepositoriesMetadata.TYPE, metadata));
         }
@@ -399,7 +394,7 @@ private static RepositoryMetadata createRepositoryMetadata(String name, boolean
     private RepositoryIntegrityHealthIndicatorService createRepositoryIntegrityHealthIndicatorService(ClusterState clusterState) {
         var clusterService = mock(ClusterService.class);
         when(clusterService.state()).thenReturn(clusterState);
-        return new RepositoryIntegrityHealthIndicatorService(clusterService, featureService);
+        return new RepositoryIntegrityHealthIndicatorService(clusterService);
     }
 
     private SimpleHealthIndicatorDetails createDetails(int total, int corruptedCount, List<String> corrupted, int unknown, int invalid) {
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
index b7f33151961ea..0fe886f37aa47 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
@@ -93,6 +93,7 @@
 import org.elasticsearch.cluster.coordination.LeaderHeartbeatService;
 import org.elasticsearch.cluster.coordination.Reconfigurator;
 import org.elasticsearch.cluster.coordination.StatefulPreVoteCollector;
+import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexMetadataVerifier;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -240,9 +241,7 @@
 import static org.hamcrest.Matchers.iterableWithSize;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 public class SnapshotResiliencyTests extends ESTestCase {
 
@@ -2110,8 +2109,6 @@ protected void connectToNodesAndWait(ClusterState newClusterState) {
                     }
                 );
                 recoverySettings = new RecoverySettings(settings, clusterSettings);
-                FeatureService mockFeatureService = mock(FeatureService.class);
-                when(mockFeatureService.clusterHasFeature(any(), any())).thenReturn(true);
                 mockTransport = new DisruptableMockTransport(node, deterministicTaskQueue) {
                     @Override
                     protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) {
@@ -2402,13 +2399,13 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() {
                             null,
                             FailureStoreMetrics.NOOP
                         ),
-                        mockFeatureService,
                         client,
                         actionFilters,
                         indexNameExpressionResolver,
                         new IndexingPressure(settings),
                         EmptySystemIndices.INSTANCE,
-                        FailureStoreMetrics.NOOP
+                        FailureStoreMetrics.NOOP,
+                        DataStreamFailureStoreSettings.create(ClusterSettings.createBuiltInClusterSettings())
                     )
                 );
                 final TransportShardBulkAction transportShardBulkAction = new TransportShardBulkAction(
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java
index a92d55f6d419c..741dfd2466430 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java
@@ -88,6 +88,7 @@ private ClusterState getClusterStateWithNodeShutdownMetadata(List<String> nodeId
                                 nodeId -> SingleNodeShutdownMetadata.builder()
                                     .setType(SingleNodeShutdownMetadata.Type.REMOVE)
                                     .setNodeId(nodeId)
+                                    .setNodeEphemeralId(nodeId)
                                     .setStartedAtMillis(randomNonNegativeLong())
                                     .setReason(getTestName())
                                     .build()
@@ -133,7 +134,7 @@ private Entry randomSnapshot() {
             shards,
             null,
             SnapshotInfoTestUtils.randomUserMetadata(),
-            IndexVersionUtils.randomVersion(random())
+            IndexVersionUtils.randomVersion()
         );
     }
 
@@ -476,6 +477,7 @@ public void testXContent() throws IOException {
                                 "node-id",
                                 SingleNodeShutdownMetadata.builder()
                                     .setNodeId("node-id")
+                                    .setNodeEphemeralId("node-id")
                                     .setType(SingleNodeShutdownMetadata.Type.REMOVE)
                                     .setStartedAtMillis(randomNonNegativeLong())
                                     .setReason("test")
diff --git a/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java b/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java
index 889afab00e830..c0185832d6122 100644
--- a/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java
@@ -27,6 +27,7 @@
 
 import static org.elasticsearch.common.bytes.ReleasableBytesReferenceStreamInputTests.wrapAsReleasable;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasItems;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -182,7 +183,7 @@ public void testDecodePreHeaderSizeVariableInt() throws IOException {
         }
     }
 
-    public void testDecodeHandshakeCompatibility() throws IOException {
+    public void testDecodeHandshakeV7Compatibility() throws IOException {
         String action = "test-request";
         long requestId = randomNonNegativeLong();
         final String headerKey = randomAlphaOfLength(10);
@@ -223,6 +224,55 @@ public void testDecodeHandshakeCompatibility() throws IOException {
 
     }
 
+    public void testDecodeHandshakeV8Compatibility() throws IOException {
+        doHandshakeCompatibilityTest(TransportHandshaker.REQUEST_HANDSHAKE_VERSION, null);
+        doHandshakeCompatibilityTest(TransportHandshaker.REQUEST_HANDSHAKE_VERSION, Compression.Scheme.DEFLATE);
+    }
+
+    public void testDecodeHandshakeV9Compatibility() throws IOException {
+        doHandshakeCompatibilityTest(TransportHandshaker.V9_HANDSHAKE_VERSION, null);
+        doHandshakeCompatibilityTest(TransportHandshaker.V9_HANDSHAKE_VERSION, Compression.Scheme.DEFLATE);
+    }
+
+    private void doHandshakeCompatibilityTest(TransportVersion transportVersion, Compression.Scheme compressionScheme) throws IOException {
+        String action = "test-request";
+        long requestId = randomNonNegativeLong();
+        final String headerKey = randomAlphaOfLength(10);
+        final String headerValue = randomAlphaOfLength(20);
+        threadContext.putHeader(headerKey, headerValue);
+        OutboundMessage message = new OutboundMessage.Request(
+            threadContext,
+            new TestRequest(randomAlphaOfLength(100)),
+            transportVersion,
+            action,
+            requestId,
+            true,
+            compressionScheme
+        );
+
+        try (RecyclerBytesStreamOutput os = new RecyclerBytesStreamOutput(recycler)) {
+            final BytesReference bytes = message.serialize(os);
+            int totalHeaderSize = TcpHeader.headerSize(transportVersion);
+
+            InboundDecoder decoder = new InboundDecoder(recycler);
+            final ArrayList<Object> fragments = new ArrayList<>();
+            final ReleasableBytesReference releasable1 = wrapAsReleasable(bytes);
+            int bytesConsumed = decoder.decode(releasable1, fragments::add);
+            assertThat(bytesConsumed, greaterThan(totalHeaderSize));
+            assertTrue(releasable1.hasReferences());
+
+            final Header header = (Header) fragments.get(0);
+            assertEquals(requestId, header.getRequestId());
+            assertEquals(transportVersion, header.getVersion());
+            assertEquals(compressionScheme == Compression.Scheme.DEFLATE, header.isCompressed());
+            assertTrue(header.isHandshake());
+            assertTrue(header.isRequest());
+            assertFalse(header.needsToReadVariableHeader());
+            assertEquals(headerValue, header.getRequestHeaders().get(headerKey));
+            fragments.clear();
+        }
+    }
+
     public void testClientChannelTypeFailsDecodingRequests() throws Exception {
         String action = "test-request";
         long requestId = randomNonNegativeLong();
@@ -488,23 +538,16 @@ public void testCheckVersionCompatibility() {
     }
 
     public void testCheckHandshakeCompatibility() {
-        try {
-            InboundDecoder.checkHandshakeVersionCompatibility(randomFrom(TransportHandshaker.ALLOWED_HANDSHAKE_VERSIONS));
-        } catch (IllegalStateException e) {
-            throw new AssertionError(e);
-        }
+        for (final var allowedHandshakeVersion : TransportHandshaker.ALLOWED_HANDSHAKE_VERSIONS) {
+            InboundDecoder.checkHandshakeVersionCompatibility(allowedHandshakeVersion); // should not throw
 
-        var invalid = TransportVersion.fromId(TransportHandshaker.EARLIEST_HANDSHAKE_VERSION.id() - 1);
-        try {
-            InboundDecoder.checkHandshakeVersionCompatibility(invalid);
-            fail();
-        } catch (IllegalStateException expected) {
+            var invalid = TransportVersion.fromId(allowedHandshakeVersion.id() + randomFrom(-1, +1));
             assertEquals(
                 "Received message from unsupported version: ["
                     + invalid
                     + "] allowed versions are: "
                     + TransportHandshaker.ALLOWED_HANDSHAKE_VERSIONS,
-                expected.getMessage()
+                expectThrows(IllegalStateException.class, () -> InboundDecoder.checkHandshakeVersionCompatibility(invalid)).getMessage()
             );
         }
     }
diff --git a/server/src/test/resources/org/elasticsearch/action/admin/cluster/stats/telemetry_test.json b/server/src/test/resources/org/elasticsearch/action/admin/cluster/stats/telemetry_test.json
index fe9c77cb2a183..a92bab739b37d 100644
--- a/server/src/test/resources/org/elasticsearch/action/admin/cluster/stats/telemetry_test.json
+++ b/server/src/test/resources/org/elasticsearch/action/admin/cluster/stats/telemetry_test.json
@@ -1,5 +1,4 @@
 {
-  "_search" : {
     "total" : 10,
     "success" : 20,
     "skipped" : 5,
@@ -63,5 +62,4 @@
         }
       }
     }
-  }
-}
\ No newline at end of file
+}
diff --git a/settings.gradle b/settings.gradle
index 747fbb3e439fe..df428139c92a0 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -18,6 +18,8 @@ plugins {
   id 'elasticsearch.java-toolchain'
 }
 
+enableFeaturePreview "STABLE_CONFIGURATION_CACHE"
+
 rootProject.name = "elasticsearch"
 
 dependencyResolutionManagement {
@@ -87,6 +89,7 @@ List projects = [
   'distribution:tools:ansi-console',
   'server',
   'test:framework',
+  'test:fixtures:aws-fixture-utils',
   'test:fixtures:aws-sts-fixture',
   'test:fixtures:azure-fixture',
   'test:fixtures:ec2-imds-fixture',
diff --git a/test/external-modules/apm-integration/build.gradle b/test/external-modules/apm-integration/build.gradle
index 7f64b33b81423..4a5d6d9d6d914 100644
--- a/test/external-modules/apm-integration/build.gradle
+++ b/test/external-modules/apm-integration/build.gradle
@@ -13,8 +13,8 @@ apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.internal-es-plugin'
 
 esplugin {
-  description 'Apm integration plugin'
-  classname 'org.elasticsearch.test.apmintegration.ApmIntegrationPlugin'
+  description = 'Apm integration plugin'
+  classname ='org.elasticsearch.test.apmintegration.ApmIntegrationPlugin'
 }
 
 // let the javaRestTest see the classpath of main
diff --git a/test/external-modules/build.gradle b/test/external-modules/build.gradle
index 3ba6b309071f3..47b909dbd708f 100644
--- a/test/external-modules/build.gradle
+++ b/test/external-modules/build.gradle
@@ -11,8 +11,8 @@ subprojects {
   apply plugin: 'elasticsearch.base-internal-es-plugin'
 
   esplugin {
-    name it.name
-    licenseFile rootProject.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
-    noticeFile rootProject.file('NOTICE.txt')
+    name = it.name
+    licenseFile = rootProject.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
+    noticeFile = rootProject.file('NOTICE.txt')
   }
 }
diff --git a/test/external-modules/delayed-aggs/build.gradle b/test/external-modules/delayed-aggs/build.gradle
index fae5e93b37fc3..5d1291c8ee503 100644
--- a/test/external-modules/delayed-aggs/build.gradle
+++ b/test/external-modules/delayed-aggs/build.gradle
@@ -14,8 +14,8 @@ tasks.named('yamlRestTest').configure {
 }
 
 esplugin {
-  description 'A test module that allows to delay aggregations on shards with a configurable time'
-  classname 'org.elasticsearch.test.delayedshard.DelayedShardAggregationPlugin'
+  description = 'A test module that allows to delay aggregations on shards with a configurable time'
+  classname ='org.elasticsearch.test.delayedshard.DelayedShardAggregationPlugin'
 }
 
 restResources {
diff --git a/test/external-modules/die-with-dignity/build.gradle b/test/external-modules/die-with-dignity/build.gradle
index 6c1da40406a5d..a6622997bf5db 100644
--- a/test/external-modules/die-with-dignity/build.gradle
+++ b/test/external-modules/die-with-dignity/build.gradle
@@ -13,8 +13,8 @@ apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.internal-es-plugin'
 
 esplugin {
-  description 'Die with dignity plugin'
-  classname 'org.elasticsearch.test.diewithdignity.DieWithDignityPlugin'
+  description = 'Die with dignity plugin'
+  classname ='org.elasticsearch.test.diewithdignity.DieWithDignityPlugin'
 }
 
 // let the javaRestTest see the classpath of main
diff --git a/test/external-modules/error-query/build.gradle b/test/external-modules/error-query/build.gradle
index 3c72145c11f8b..8ac47c339f21d 100644
--- a/test/external-modules/error-query/build.gradle
+++ b/test/external-modules/error-query/build.gradle
@@ -14,8 +14,8 @@ tasks.named('yamlRestTest').configure {
 }
 
 esplugin {
-  description 'A test module that exposes a way to simulate search shard failures and warnings'
-  classname 'org.elasticsearch.test.errorquery.ErrorQueryPlugin'
+  description = 'A test module that exposes a way to simulate search shard failures and warnings'
+  classname ='org.elasticsearch.test.errorquery.ErrorQueryPlugin'
 }
 
 restResources {
diff --git a/test/external-modules/esql-heap-attack/build.gradle b/test/external-modules/esql-heap-attack/build.gradle
index ba85e0dbd8693..fa8c43048a6d6 100644
--- a/test/external-modules/esql-heap-attack/build.gradle
+++ b/test/external-modules/esql-heap-attack/build.gradle
@@ -14,8 +14,8 @@ apply plugin: 'elasticsearch.internal-test-artifact'
 group = 'org.elasticsearch.plugin'
 
 esplugin {
-  description 'A test module that can trigger out of memory'
-  classname 'org.elasticsearch.test.esql.heap_attack.HeapAttackPlugin'
+  description = 'A test module that can trigger out of memory'
+  classname ='org.elasticsearch.test.esql.heap_attack.HeapAttackPlugin'
 }
 
 tasks.named('javaRestTest') {
diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java
index ace3db377664c..958132b3e4076 100644
--- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java
+++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java
@@ -194,6 +194,16 @@ private void assertCircuitBreaks(ThrowingRunnable r) throws IOException {
         );
     }
 
+    private void assertFoldCircuitBreaks(ThrowingRunnable r) throws IOException {
+        ResponseException e = expectThrows(ResponseException.class, r);
+        Map<?, ?> map = responseAsMap(e.getResponse());
+        logger.info("expected fold circuit breaking {}", map);
+        assertMap(
+            map,
+            matchesMap().entry("status", 400).entry("error", matchesMap().extraOk().entry("type", "fold_too_much_memory_exception"))
+        );
+    }
+
     private void assertParseFailure(ThrowingRunnable r) throws IOException {
         ResponseException e = expectThrows(ResponseException.class, r);
         Map<?, ?> map = responseAsMap(e.getResponse());
@@ -325,11 +335,23 @@ public void testManyConcatFromRow() throws IOException {
         assertManyStrings(resp, strings);
     }
 
+    /**
+     * Hits a circuit breaker by building many moderately long strings.
+     */
+    public void testHugeManyConcatFromRow() throws IOException {
+        assertFoldCircuitBreaks(
+            () -> manyConcat(
+                "ROW a=9999999999999, b=99999999999999999, c=99999999999999999, d=99999999999999999, e=99999999999999999",
+                5000
+            )
+        );
+    }
+
     /**
      * Fails to parse a huge huge query.
      */
     public void testHugeHugeManyConcatFromRow() throws IOException {
-        assertParseFailure(() -> manyConcat("ROW a=9999, b=9999, c=9999, d=9999, e=9999", 50000));
+        assertParseFailure(() -> manyConcat("ROW a=9999, b=9999, c=9999, d=9999, e=9999", 6000));
     }
 
     /**
@@ -387,13 +409,20 @@ public void testHugeManyRepeat() throws IOException {
      * Returns many moderately long strings.
      */
     public void testManyRepeatFromRow() throws IOException {
-        int strings = 10000;
+        int strings = 300;
         Response resp = manyRepeat("ROW a = 99", strings);
         assertManyStrings(resp, strings);
     }
 
     /**
-     * Fails to parse a huge huge query.
+     * Hits a circuit breaker by building many moderately long strings.
+     */
+    public void testHugeManyRepeatFromRow() throws IOException {
+        assertFoldCircuitBreaks(() -> manyRepeat("ROW a = 99", 400));
+    }
+
+    /**
+     * Fails to parse a huge, huge query.
      */
     public void testHugeHugeManyRepeatFromRow() throws IOException {
         assertParseFailure(() -> manyRepeat("ROW a = 99", 100000));
diff --git a/test/external-modules/jvm-crash/build.gradle b/test/external-modules/jvm-crash/build.gradle
index 0b06142e81939..bc1a96836889b 100644
--- a/test/external-modules/jvm-crash/build.gradle
+++ b/test/external-modules/jvm-crash/build.gradle
@@ -14,8 +14,8 @@ apply plugin: 'elasticsearch.internal-test-artifact'
 group = 'org.elasticsearch.plugin'
 
 esplugin {
-  description 'A test module that can trigger A JVM crash'
-  classname 'org.elasticsearch.test.jvm_crash.JvmCrashPlugin'
+  description = 'A test module that can trigger A JVM crash'
+  classname ='org.elasticsearch.test.jvm_crash.JvmCrashPlugin'
 }
 
 tasks.named('javaRestTest') {
diff --git a/test/external-modules/latency-simulating-directory/build.gradle b/test/external-modules/latency-simulating-directory/build.gradle
index 36437aaf6c4ec..70c23c32c8ad1 100644
--- a/test/external-modules/latency-simulating-directory/build.gradle
+++ b/test/external-modules/latency-simulating-directory/build.gradle
@@ -8,8 +8,8 @@
  */
 
 esplugin {
-  description 'A test module that simulates read latencies from an object-store based repository'
-  classname 'org.elasticsearch.test.simulatedlatencyrepo.LatencySimulatingRepositoryPlugin'
+  description = 'A test module that simulates read latencies from an object-store based repository'
+  classname ='org.elasticsearch.test.simulatedlatencyrepo.LatencySimulatingRepositoryPlugin'
 }
 
 apply plugin: 'elasticsearch.internal-cluster-test'
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/SignificanceHeuristicBuilder.java b/test/fixtures/aws-fixture-utils/build.gradle
similarity index 70%
rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/SignificanceHeuristicBuilder.java
rename to test/fixtures/aws-fixture-utils/build.gradle
index 4044c3a72a30b..5e2981b00d6e3 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/heuristic/SignificanceHeuristicBuilder.java
+++ b/test/fixtures/aws-fixture-utils/build.gradle
@@ -6,9 +6,11 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
+apply plugin: 'elasticsearch.java'
 
-package org.elasticsearch.search.aggregations.bucket.terms.heuristic;
+description = 'Utils for AWS-related fixtures'
 
-import org.elasticsearch.xcontent.ToXContentFragment;
-
-public interface SignificanceHeuristicBuilder extends ToXContentFragment {}
+dependencies {
+  implementation project(':server')
+  implementation project(':test:framework')
+}
diff --git a/test/fixtures/aws-fixture-utils/src/main/java/fixture/aws/AwsCredentialsUtils.java b/test/fixtures/aws-fixture-utils/src/main/java/fixture/aws/AwsCredentialsUtils.java
new file mode 100644
index 0000000000000..9000e06ca1350
--- /dev/null
+++ b/test/fixtures/aws-fixture-utils/src/main/java/fixture/aws/AwsCredentialsUtils.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package fixture.aws;
+
+import com.sun.net.httpserver.HttpExchange;
+
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.function.BiPredicate;
+import java.util.function.Supplier;
+
+import static fixture.aws.AwsFixtureUtils.sendError;
+
+public enum AwsCredentialsUtils {
+    ;
+
+    /**
+     * @return an authorization predicate that ensures the access key matches the given values.
+     */
+    public static BiPredicate<String, String> fixedAccessKey(String accessKey) {
+        return mutableAccessKey(() -> accessKey);
+    }
+
+    /**
+     * @return an authorization predicate that ensures the access key matches one supplied by the given supplier.
+     */
+    public static BiPredicate<String, String> mutableAccessKey(Supplier<String> accessKeySupplier) {
+        return (authorizationHeader, sessionTokenHeader) -> authorizationHeader != null
+            && authorizationHeader.contains(accessKeySupplier.get());
+    }
+
+    /**
+     * @return an authorization predicate that ensures the access key and session token both match the given values.
+     */
+    public static BiPredicate<String, String> fixedAccessKeyAndToken(String accessKey, String sessionToken) {
+        Objects.requireNonNull(sessionToken);
+        final var accessKeyPredicate = fixedAccessKey(accessKey);
+        return (authorizationHeader, sessionTokenHeader) -> accessKeyPredicate.test(authorizationHeader, sessionTokenHeader)
+            && sessionToken.equals(sessionTokenHeader);
+    }
+
+    /**
+     * Check the authorization headers of the given {@param exchange} against the given {@param authorizationPredicate}. If they match,
+     * returns {@code true}. If they do not match, sends a {@code 403 Forbidden} response and returns {@code false}.
+     */
+    public static boolean checkAuthorization(BiPredicate<String, String> authorizationPredicate, HttpExchange exchange) throws IOException {
+        if (authorizationPredicate.test(
+            exchange.getRequestHeaders().getFirst("Authorization"),
+            exchange.getRequestHeaders().getFirst("x-amz-security-token")
+        )) {
+            return true;
+        }
+
+        sendError(exchange, RestStatus.FORBIDDEN, "AccessDenied", "Access denied by " + authorizationPredicate);
+        return false;
+    }
+}
diff --git a/test/fixtures/aws-fixture-utils/src/main/java/fixture/aws/AwsFixtureUtils.java b/test/fixtures/aws-fixture-utils/src/main/java/fixture/aws/AwsFixtureUtils.java
new file mode 100644
index 0000000000000..d46a7e10c4fe0
--- /dev/null
+++ b/test/fixtures/aws-fixture-utils/src/main/java/fixture/aws/AwsFixtureUtils.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package fixture.aws;
+
+import com.sun.net.httpserver.Headers;
+import com.sun.net.httpserver.HttpExchange;
+
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.nio.charset.StandardCharsets;
+
+public enum AwsFixtureUtils {
+    ;
+
+    /**
+     * @return an {@link InetSocketAddress} for a test fixture running on {@code localhost} which binds to any available port.
+     */
+    public static InetSocketAddress getLocalFixtureAddress() {
+        try {
+            return new InetSocketAddress(InetAddress.getByName("localhost"), 0);
+        } catch (UnknownHostException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Send an XML-formatted error response typical of an AWS service.
+     */
+    public static void sendError(final HttpExchange exchange, final RestStatus status, final String errorCode, final String message)
+        throws IOException {
+        final Headers headers = exchange.getResponseHeaders();
+        headers.add("Content-Type", "application/xml");
+
+        final String requestId = exchange.getRequestHeaders().getFirst("x-amz-request-id");
+        if (requestId != null) {
+            headers.add("x-amz-request-id", requestId);
+        }
+
+        if (errorCode == null || "HEAD".equals(exchange.getRequestMethod())) {
+            exchange.sendResponseHeaders(status.getStatus(), -1L);
+            exchange.close();
+        } else {
+            final byte[] response = ("<?xml version=\"1.0\" encoding=\"UTF-8\"?><Error>"
+                + "<Code>"
+                + errorCode
+                + "</Code>"
+                + "<Message>"
+                + message
+                + "</Message>"
+                + "<RequestId>"
+                + requestId
+                + "</RequestId>"
+                + "</Error>").getBytes(StandardCharsets.UTF_8);
+            exchange.sendResponseHeaders(status.getStatus(), response.length);
+            exchange.getResponseBody().write(response);
+            exchange.close();
+        }
+    }
+}
diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/DynamicS3Credentials.java b/test/fixtures/aws-fixture-utils/src/main/java/fixture/aws/DynamicAwsCredentials.java
similarity index 88%
rename from test/fixtures/s3-fixture/src/main/java/fixture/s3/DynamicS3Credentials.java
rename to test/fixtures/aws-fixture-utils/src/main/java/fixture/aws/DynamicAwsCredentials.java
index 4e8f267ad3543..24a5b4ea018e3 100644
--- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/DynamicS3Credentials.java
+++ b/test/fixtures/aws-fixture-utils/src/main/java/fixture/aws/DynamicAwsCredentials.java
@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package fixture.s3;
+package fixture.aws;
 
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 
@@ -18,10 +18,9 @@
 /**
  * Allows dynamic creation of access-key/session-token credentials for accessing AWS services such as S3. Typically there's one service
  * (e.g. IMDS or STS) which creates credentials dynamically and registers them here using {@link #addValidCredentials}, and then the
- * {@link S3HttpFixture} uses {@link #isAuthorized} to validate the credentials it receives corresponds with some previously-generated
- * credentials.
+ * fixture uses {@link #isAuthorized} to validate the credentials it receives corresponds with some previously-generated credentials.
  */
-public class DynamicS3Credentials {
+public class DynamicAwsCredentials {
     private final Map<String, Set<String>> validCredentialsMap = ConcurrentCollections.newConcurrentMap();
 
     public boolean isAuthorized(String authorizationHeader, String sessionTokenHeader) {
diff --git a/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpHandler.java b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpHandler.java
index 0c58205ac8d60..7606e9261dd3e 100644
--- a/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpHandler.java
+++ b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpHandler.java
@@ -14,8 +14,11 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -45,21 +48,38 @@ public class Ec2ImdsHttpHandler implements HttpHandler {
 
     private final BiConsumer<String, String> newCredentialsConsumer;
     private final Map<String, String> instanceAddresses;
-    private final Set<String> validCredentialsEndpoints = ConcurrentCollections.newConcurrentSet();
+    private final Set<String> validCredentialsEndpoints;
+    private final boolean dynamicProfileNames;
     private final Supplier<String> availabilityZoneSupplier;
+    @Nullable // if instance identity document not available
+    private final ToXContent instanceIdentityDocument;
 
     public Ec2ImdsHttpHandler(
         Ec2ImdsVersion ec2ImdsVersion,
         BiConsumer<String, String> newCredentialsConsumer,
         Collection<String> alternativeCredentialsEndpoints,
         Supplier<String> availabilityZoneSupplier,
+        @Nullable ToXContent instanceIdentityDocument,
         Map<String, String> instanceAddresses
     ) {
         this.ec2ImdsVersion = Objects.requireNonNull(ec2ImdsVersion);
         this.newCredentialsConsumer = Objects.requireNonNull(newCredentialsConsumer);
         this.instanceAddresses = instanceAddresses;
-        this.validCredentialsEndpoints.addAll(alternativeCredentialsEndpoints);
+
+        if (alternativeCredentialsEndpoints.isEmpty()) {
+            dynamicProfileNames = true;
+            validCredentialsEndpoints = ConcurrentCollections.newConcurrentSet();
+        } else if (ec2ImdsVersion == Ec2ImdsVersion.V2) {
+            throw new IllegalArgumentException(
+                Strings.format("alternative credentials endpoints %s requires IMDSv1", alternativeCredentialsEndpoints)
+            );
+        } else {
+            dynamicProfileNames = false;
+            validCredentialsEndpoints = Set.copyOf(alternativeCredentialsEndpoints);
+        }
+
         this.availabilityZoneSupplier = availabilityZoneSupplier;
+        this.instanceIdentityDocument = instanceIdentityDocument;
     }
 
     @Override
@@ -78,6 +98,8 @@ public void handle(final HttpExchange exchange) throws IOException {
                         validImdsTokens.add(token);
                         final var responseBody = token.getBytes(StandardCharsets.UTF_8);
                         exchange.getResponseHeaders().add("Content-Type", "text/plain");
+                        exchange.getResponseHeaders()
+                            .add("x-aws-ec2-metadata-token-ttl-seconds", Long.toString(TimeValue.timeValueDays(1).seconds()));
                         exchange.sendResponseHeaders(RestStatus.OK.getStatus(), responseBody.length);
                         exchange.getResponseBody().write(responseBody);
                     }
@@ -98,7 +120,7 @@ public void handle(final HttpExchange exchange) throws IOException {
             }
 
             if ("GET".equals(requestMethod)) {
-                if (path.equals(IMDS_SECURITY_CREDENTIALS_PATH)) {
+                if (path.equals(IMDS_SECURITY_CREDENTIALS_PATH) && dynamicProfileNames) {
                     final var profileName = randomIdentifier();
                     validCredentialsEndpoints.add(IMDS_SECURITY_CREDENTIALS_PATH + profileName);
                     sendStringResponse(exchange, profileName);
@@ -107,6 +129,9 @@ public void handle(final HttpExchange exchange) throws IOException {
                     final var availabilityZone = availabilityZoneSupplier.get();
                     sendStringResponse(exchange, availabilityZone);
                     return;
+                } else if (instanceIdentityDocument != null && path.equals("/latest/dynamic/instance-identity/document")) {
+                    sendStringResponse(exchange, Strings.toString(instanceIdentityDocument));
+                    return;
                 } else if (validCredentialsEndpoints.contains(path)) {
                     final String accessKey = randomIdentifier();
                     final String sessionToken = randomIdentifier();
diff --git a/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsServiceBuilder.java b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsServiceBuilder.java
index 505c9978bc4fb..d70ee723942ec 100644
--- a/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsServiceBuilder.java
+++ b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsServiceBuilder.java
@@ -10,6 +10,7 @@
 package fixture.aws.imds;
 
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.ToXContent;
 
 import java.util.Collection;
 import java.util.HashMap;
@@ -24,6 +25,7 @@ public class Ec2ImdsServiceBuilder {
     private BiConsumer<String, String> newCredentialsConsumer = Ec2ImdsServiceBuilder::rejectNewCredentials;
     private Collection<String> alternativeCredentialsEndpoints = Set.of();
     private Supplier<String> availabilityZoneSupplier = Ec2ImdsServiceBuilder::rejectAvailabilityZone;
+    private ToXContent instanceIdentityDocument = null;
     private final Map<String, String> instanceAddresses = new HashMap<>();
 
     public Ec2ImdsServiceBuilder(Ec2ImdsVersion ec2ImdsVersion) {
@@ -64,8 +66,13 @@ public Ec2ImdsHttpHandler buildHandler() {
             newCredentialsConsumer,
             alternativeCredentialsEndpoints,
             availabilityZoneSupplier,
+            instanceIdentityDocument,
             Map.copyOf(instanceAddresses)
         );
     }
 
+    public Ec2ImdsServiceBuilder instanceIdentityDocument(ToXContent instanceIdentityDocument) {
+        this.instanceIdentityDocument = instanceIdentityDocument;
+        return this;
+    }
 }
diff --git a/test/fixtures/ec2-imds-fixture/src/test/java/fixture/aws/imds/Ec2ImdsHttpHandlerTests.java b/test/fixtures/ec2-imds-fixture/src/test/java/fixture/aws/imds/Ec2ImdsHttpHandlerTests.java
index 6d3eb3d14e9b2..0c0d02b32d4a5 100644
--- a/test/fixtures/ec2-imds-fixture/src/test/java/fixture/aws/imds/Ec2ImdsHttpHandlerTests.java
+++ b/test/fixtures/ec2-imds-fixture/src/test/java/fixture/aws/imds/Ec2ImdsHttpHandlerTests.java
@@ -52,16 +52,13 @@ public void testImdsV1() throws IOException {
         assertTrue(Strings.hasText(profileName));
 
         final var credentialsResponse = handleRequest(handler, "GET", SECURITY_CREDENTIALS_URI + profileName);
-        assertEquals(RestStatus.OK, credentialsResponse.status());
 
         assertThat(generatedCredentials, aMapWithSize(1));
-        final var accessKey = generatedCredentials.keySet().iterator().next();
-        final var sessionToken = generatedCredentials.values().iterator().next();
-
-        final var responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), credentialsResponse.body().streamInput(), false);
-        assertEquals(Set.of("AccessKeyId", "Expiration", "RoleArn", "SecretAccessKey", "Token"), responseMap.keySet());
-        assertEquals(accessKey, responseMap.get("AccessKeyId"));
-        assertEquals(sessionToken, responseMap.get("Token"));
+        assertValidCredentialsResponse(
+            credentialsResponse,
+            generatedCredentials.keySet().iterator().next(),
+            generatedCredentials.values().iterator().next()
+        );
     }
 
     public void testImdsV2Disabled() {
@@ -78,6 +75,7 @@ public void testImdsV2() throws IOException {
 
         final var tokenResponse = handleRequest(handler, "PUT", "/latest/api/token");
         assertEquals(RestStatus.OK, tokenResponse.status());
+        assertEquals(List.of("86400" /* seconds in a day */), tokenResponse.responseHeaders().get("x-aws-ec2-metadata-token-ttl-seconds"));
         final var token = tokenResponse.body().utf8ToString();
 
         final var roleResponse = checkImdsV2GetRequest(handler, SECURITY_CREDENTIALS_URI, token);
@@ -86,16 +84,13 @@ public void testImdsV2() throws IOException {
         assertTrue(Strings.hasText(profileName));
 
         final var credentialsResponse = checkImdsV2GetRequest(handler, SECURITY_CREDENTIALS_URI + profileName, token);
-        assertEquals(RestStatus.OK, credentialsResponse.status());
 
         assertThat(generatedCredentials, aMapWithSize(1));
-        final var accessKey = generatedCredentials.keySet().iterator().next();
-        final var sessionToken = generatedCredentials.values().iterator().next();
-
-        final var responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), credentialsResponse.body().streamInput(), false);
-        assertEquals(Set.of("AccessKeyId", "Expiration", "RoleArn", "SecretAccessKey", "Token"), responseMap.keySet());
-        assertEquals(accessKey, responseMap.get("AccessKeyId"));
-        assertEquals(sessionToken, responseMap.get("Token"));
+        assertValidCredentialsResponse(
+            credentialsResponse,
+            generatedCredentials.keySet().iterator().next(),
+            generatedCredentials.values().iterator().next()
+        );
     }
 
     public void testAvailabilityZone() {
@@ -113,7 +108,54 @@ public void testAvailabilityZone() {
         assertEquals(generatedAvailabilityZones, Set.of(availabilityZone));
     }
 
-    private record TestHttpResponse(RestStatus status, BytesReference body) {}
+    public void testAlternativeCredentialsEndpoint() throws IOException {
+        expectThrows(
+            IllegalArgumentException.class,
+            new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V2).alternativeCredentialsEndpoints(Set.of("/should-not-work"))::buildHandler
+        );
+
+        final var alternativePaths = randomList(1, 5, () -> "/" + randomIdentifier());
+        final Map<String, String> generatedCredentials = new HashMap<>();
+
+        final var handler = new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).alternativeCredentialsEndpoints(alternativePaths)
+            .newCredentialsConsumer(generatedCredentials::put)
+            .buildHandler();
+
+        final var credentialsResponse = handleRequest(handler, "GET", randomFrom(alternativePaths));
+
+        assertThat(generatedCredentials, aMapWithSize(1));
+        assertValidCredentialsResponse(
+            credentialsResponse,
+            generatedCredentials.keySet().iterator().next(),
+            generatedCredentials.values().iterator().next()
+        );
+    }
+
+    private static void assertValidCredentialsResponse(TestHttpResponse credentialsResponse, String accessKey, String sessionToken)
+        throws IOException {
+        assertEquals(RestStatus.OK, credentialsResponse.status());
+        final var responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), credentialsResponse.body().streamInput(), false);
+        assertEquals(Set.of("AccessKeyId", "Expiration", "RoleArn", "SecretAccessKey", "Token"), responseMap.keySet());
+        assertEquals(accessKey, responseMap.get("AccessKeyId"));
+        assertEquals(sessionToken, responseMap.get("Token"));
+    }
+
+    public void testInstanceIdentityDocument() {
+        final Set<String> generatedRegions = new HashSet<>();
+        final var handler = new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).instanceIdentityDocument((builder, params) -> {
+            final var newRegion = randomIdentifier();
+            generatedRegions.add(newRegion);
+            return builder.field("region", newRegion);
+        }).buildHandler();
+
+        final var instanceIdentityResponse = handleRequest(handler, "GET", "/latest/dynamic/instance-identity/document");
+        assertEquals(RestStatus.OK, instanceIdentityResponse.status());
+        final var instanceIdentityString = instanceIdentityResponse.body().utf8ToString();
+
+        assertEquals(Strings.format("{\"region\":\"%s\"}", generatedRegions.iterator().next()), instanceIdentityString);
+    }
+
+    private record TestHttpResponse(RestStatus status, Headers responseHeaders, BytesReference body) {}
 
     private static TestHttpResponse checkImdsV2GetRequest(Ec2ImdsHttpHandler handler, String uri, String token) {
         final var unauthorizedResponse = handleRequest(handler, "GET", uri, null);
@@ -145,7 +187,11 @@ private static TestHttpResponse handleRequest(Ec2ImdsHttpHandler handler, String
             fail(e);
         }
         assertNotEquals(0, httpExchange.getResponseCode());
-        return new TestHttpResponse(RestStatus.fromCode(httpExchange.getResponseCode()), httpExchange.getResponseBodyContents());
+        return new TestHttpResponse(
+            RestStatus.fromCode(httpExchange.getResponseCode()),
+            httpExchange.getResponseHeaders(),
+            httpExchange.getResponseBodyContents()
+        );
     }
 
     private static class TestHttpExchange extends HttpExchange {
diff --git a/test/fixtures/gcs-fixture/build.gradle b/test/fixtures/gcs-fixture/build.gradle
index e8f1a2e15a4e0..6cf2e1ee52c2c 100644
--- a/test/fixtures/gcs-fixture/build.gradle
+++ b/test/fixtures/gcs-fixture/build.gradle
@@ -9,7 +9,6 @@
 apply plugin: 'elasticsearch.java'
 
 description = 'Fixture for Google Cloud Storage service'
-tasks.named("test").configure { enabled = false }
 
 dependencies {
   api project(':server')
diff --git a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java
index f6b52a32a9a1d..b43c516d2b246 100644
--- a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java
+++ b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java
@@ -14,12 +14,10 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.bytes.CompositeBytesReference;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.rest.RestStatus;
@@ -40,13 +38,11 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.function.BiFunction;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.zip.GZIPInputStream;
 
+import static fixture.gcs.MockGcsBlobStore.failAndThrow;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.stream.Collectors.joining;
 import static org.elasticsearch.core.Strings.format;
@@ -58,13 +54,14 @@
 public class GoogleCloudStorageHttpHandler implements HttpHandler {
 
     private static final Logger logger = LogManager.getLogger(GoogleCloudStorageHttpHandler.class);
+    private static final String IF_GENERATION_MATCH = "ifGenerationMatch";
 
-    private final ConcurrentMap<String, BytesReference> blobs;
+    private final MockGcsBlobStore mockGcsBlobStore;
     private final String bucket;
 
     public GoogleCloudStorageHttpHandler(final String bucket) {
         this.bucket = Objects.requireNonNull(bucket);
-        this.blobs = new ConcurrentHashMap<>();
+        this.mockGcsBlobStore = new MockGcsBlobStore();
     }
 
     @Override
@@ -83,33 +80,30 @@ public void handle(final HttpExchange exchange) throws IOException {
                 exchange.sendResponseHeaders(RestStatus.OK.getStatus(), 0);
             } else if (Regex.simpleMatch("GET /storage/v1/b/" + bucket + "/o/*", request)) {
                 final String key = exchange.getRequestURI().getPath().replace("/storage/v1/b/" + bucket + "/o/", "");
-                final BytesReference blob = blobs.get(key);
-                if (blob == null) {
-                    exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1);
-                } else {
-                    final byte[] response = buildBlobInfoJson(key, blob.length()).getBytes(UTF_8);
-                    exchange.getResponseHeaders().add("Content-Type", "application/json; charset=utf-8");
-                    exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length);
-                    exchange.getResponseBody().write(response);
-                }
+                final Long ifGenerationMatch = parseOptionalLongParameter(exchange, IF_GENERATION_MATCH);
+                final MockGcsBlobStore.BlobVersion blob = mockGcsBlobStore.getBlob(key, ifGenerationMatch);
+                final byte[] response = buildBlobInfoJson(blob).getBytes(UTF_8);
+                exchange.getResponseHeaders().add("Content-Type", "application/json; charset=utf-8");
+                exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length);
+                exchange.getResponseBody().write(response);
             } else if (Regex.simpleMatch("GET /storage/v1/b/" + bucket + "/o*", request)) {
                 // List Objects https://cloud.google.com/storage/docs/json_api/v1/objects/list
                 final Map<String, String> params = new HashMap<>();
-                RestUtils.decodeQueryString(exchange.getRequestURI().getQuery(), 0, params);
+                RestUtils.decodeQueryString(exchange.getRequestURI(), params);
                 final String prefix = params.getOrDefault("prefix", "");
                 final String delimiter = params.get("delimiter");
 
                 final Set<String> prefixes = new HashSet<>();
                 final List<String> listOfBlobs = new ArrayList<>();
 
-                for (final Map.Entry<String, BytesReference> blob : blobs.entrySet()) {
+                for (final Map.Entry<String, MockGcsBlobStore.BlobVersion> blob : mockGcsBlobStore.listBlobs().entrySet()) {
                     final String blobName = blob.getKey();
                     if (prefix.isEmpty() || blobName.startsWith(prefix)) {
                         int delimiterPos = (delimiter != null) ? blobName.substring(prefix.length()).indexOf(delimiter) : -1;
                         if (delimiterPos > -1) {
                             prefixes.add("\"" + blobName.substring(0, prefix.length() + delimiterPos + 1) + "\"");
                         } else {
-                            listOfBlobs.add(buildBlobInfoJson(blobName, blob.getValue().length()));
+                            listOfBlobs.add(buildBlobInfoJson(blob.getValue()));
                         }
                     }
                 }
@@ -128,14 +122,16 @@ public void handle(final HttpExchange exchange) throws IOException {
 
             } else if (Regex.simpleMatch("GET /download/storage/v1/b/" + bucket + "/o/*", request)) {
                 // Download Object https://cloud.google.com/storage/docs/request-body
-                BytesReference blob = blobs.get(exchange.getRequestURI().getPath().replace("/download/storage/v1/b/" + bucket + "/o/", ""));
+                final String path = exchange.getRequestURI().getPath().replace("/download/storage/v1/b/" + bucket + "/o/", "");
+                final Long ifGenerationMatch = parseOptionalLongParameter(exchange, IF_GENERATION_MATCH);
+                final MockGcsBlobStore.BlobVersion blob = mockGcsBlobStore.getBlob(path, ifGenerationMatch);
                 if (blob != null) {
                     final String rangeHeader = exchange.getRequestHeaders().getFirst("Range");
                     final long offset;
                     final long end;
                     if (rangeHeader == null) {
                         offset = 0L;
-                        end = blob.length() - 1;
+                        end = blob.contents().length() - 1;
                     } else {
                         final HttpHeaderParser.Range range = HttpHeaderParser.parseRangeHeader(rangeHeader);
                         if (range == null) {
@@ -145,13 +141,13 @@ public void handle(final HttpExchange exchange) throws IOException {
                         end = range.end();
                     }
 
-                    if (offset >= blob.length()) {
+                    if (offset >= blob.contents().length()) {
                         exchange.getResponseHeaders().add("Content-Type", "application/octet-stream");
                         exchange.sendResponseHeaders(RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus(), -1);
                         return;
                     }
 
-                    BytesReference response = blob;
+                    BytesReference response = blob.contents();
                     exchange.getResponseHeaders().add("Content-Type", "application/octet-stream");
                     final int bufferedLength = response.length();
                     if (offset > 0 || bufferedLength > end) {
@@ -171,12 +167,12 @@ public void handle(final HttpExchange exchange) throws IOException {
                 final String uri = "/storage/v1/b/" + bucket + "/o/";
                 final StringBuilder batch = new StringBuilder();
                 for (String line : Streams.readAllLines(requestBody.streamInput())) {
-                    if (line.length() == 0 || line.startsWith("--") || line.toLowerCase(Locale.ROOT).startsWith("content")) {
+                    if (line.isEmpty() || line.startsWith("--") || line.toLowerCase(Locale.ROOT).startsWith("content")) {
                         batch.append(line).append("\r\n");
                     } else if (line.startsWith("DELETE")) {
                         final String name = line.substring(line.indexOf(uri) + uri.length(), line.lastIndexOf(" HTTP"));
                         if (Strings.hasText(name)) {
-                            blobs.remove(URLDecoder.decode(name, UTF_8));
+                            mockGcsBlobStore.deleteBlob(URLDecoder.decode(name, UTF_8));
                             batch.append("HTTP/1.1 204 NO_CONTENT").append("\r\n");
                             batch.append("\r\n");
                         }
@@ -191,11 +187,13 @@ public void handle(final HttpExchange exchange) throws IOException {
                 // Multipart upload
                 Optional<Tuple<String, BytesReference>> content = parseMultipartRequestBody(requestBody.streamInput());
                 if (content.isPresent()) {
-                    blobs.put(content.get().v1(), content.get().v2());
-
-                    byte[] response = String.format(Locale.ROOT, """
-                        {"bucket":"%s","name":"%s"}
-                        """, bucket, content.get().v1()).getBytes(UTF_8);
+                    final Long ifGenerationMatch = parseOptionalLongParameter(exchange, IF_GENERATION_MATCH);
+                    final MockGcsBlobStore.BlobVersion newBlobVersion = mockGcsBlobStore.updateBlob(
+                        content.get().v1(),
+                        ifGenerationMatch,
+                        content.get().v2()
+                    );
+                    byte[] response = buildBlobInfoJson(newBlobVersion).getBytes(UTF_8);
                     exchange.getResponseHeaders().add("Content-Type", "application/json");
                     exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length);
                     exchange.getResponseBody().write(response);
@@ -212,9 +210,13 @@ public void handle(final HttpExchange exchange) throws IOException {
             } else if (Regex.simpleMatch("POST /upload/storage/v1/b/" + bucket + "/*uploadType=resumable*", request)) {
                 // Resumable upload initialization https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload
                 final Map<String, String> params = new HashMap<>();
-                RestUtils.decodeQueryString(exchange.getRequestURI().getQuery(), 0, params);
+                RestUtils.decodeQueryString(exchange.getRequestURI(), params);
                 final String blobName = params.get("name");
-                blobs.put(blobName, BytesArray.EMPTY);
+                final Long ifGenerationMatch = parseOptionalLongParameter(exchange, IF_GENERATION_MATCH);
+                final MockGcsBlobStore.ResumableUpload resumableUpload = mockGcsBlobStore.createResumableUpload(
+                    blobName,
+                    ifGenerationMatch
+                );
 
                 byte[] response = requestBody.utf8ToString().getBytes(UTF_8);
                 exchange.getResponseHeaders().add("Content-Type", "application/json");
@@ -227,62 +229,72 @@ public void handle(final HttpExchange exchange) throws IOException {
                             + "/o?"
                             + "uploadType=resumable"
                             + "&upload_id="
-                            + UUIDs.randomBase64UUID()
-                            + "&test_blob_name="
-                            + blobName
-                    ); // not a Google Storage parameter, but it allows to pass the blob name
+                            + resumableUpload.uploadId()
+                    );
                 exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length);
                 exchange.getResponseBody().write(response);
 
             } else if (Regex.simpleMatch("PUT /upload/storage/v1/b/" + bucket + "/o?*uploadType=resumable*", request)) {
                 // Resumable upload https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload
                 final Map<String, String> params = new HashMap<>();
-                RestUtils.decodeQueryString(exchange.getRequestURI().getQuery(), 0, params);
+                RestUtils.decodeQueryString(exchange.getRequestURI(), params);
 
-                final String blobName = params.get("test_blob_name");
-                if (blobs.containsKey(blobName) == false) {
-                    exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1);
-                    return;
+                final String contentRangeValue = requireHeader(exchange, "Content-Range");
+                final HttpHeaderParser.ContentRange contentRange = HttpHeaderParser.parseContentRangeHeader(contentRangeValue);
+                if (contentRange == null) {
+                    throw failAndThrow("Invalid Content-Range: " + contentRangeValue);
                 }
-                BytesReference blob = blobs.get(blobName);
-                final String range = exchange.getRequestHeaders().getFirst("Content-Range");
-                final Integer limit = getContentRangeLimit(range);
-
-                blob = CompositeBytesReference.of(blob, requestBody);
-                blobs.put(blobName, blob);
-
-                if (limit == null) {
-                    if ("bytes */*".equals(range) == false) {
-                        final int start = getContentRangeStart(range);
-                        final int end = getContentRangeEnd(range);
-                        exchange.getResponseHeaders().add("Range", String.format(Locale.ROOT, "bytes=%d-%d", start, end));
-                    }
-                    exchange.getResponseHeaders().add("Content-Length", "0");
-                    exchange.sendResponseHeaders(308 /* Resume Incomplete */, -1);
-                } else {
-                    if (limit > blob.length()) {
-                        throw new AssertionError("Requesting more bytes than available for blob");
-                    }
-                    exchange.sendResponseHeaders(RestStatus.OK.getStatus(), -1);
+
+                final MockGcsBlobStore.UpdateResponse updateResponse = mockGcsBlobStore.updateResumableUpload(
+                    params.get("upload_id"),
+                    contentRange,
+                    requestBody
+                );
+
+                if (updateResponse.rangeHeader() != null) {
+                    exchange.getResponseHeaders().add("Range", updateResponse.rangeHeader().headerString());
                 }
+                exchange.getResponseHeaders().add("Content-Length", "0");
+                exchange.sendResponseHeaders(updateResponse.statusCode(), -1);
             } else {
                 exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1);
             }
+        } catch (MockGcsBlobStore.GcsRestException e) {
+            sendError(exchange, e);
         } finally {
-            int read = exchange.getRequestBody().read();
-            assert read == -1 : "Request body should have been fully read here but saw [" + read + "]";
             exchange.close();
         }
     }
 
-    private String buildBlobInfoJson(String blobName, int size) {
-        return String.format(Locale.ROOT, """
-            {"kind":"storage#object","bucket":"%s","name":"%s","id":"%s","size":"%s"}
-            """, bucket, blobName, blobName, size);
+    private void sendError(HttpExchange exchange, MockGcsBlobStore.GcsRestException e) throws IOException {
+        final String responseBody = Strings.format("""
+            {
+                "error": {
+                    "errors": [],
+                    "code": %d,
+                    "message": "%s"
+                }
+            }
+            """, e.getStatus().getStatus(), e.getMessage());
+        exchange.sendResponseHeaders(e.getStatus().getStatus(), responseBody.length());
+        exchange.getResponseBody().write(responseBody.getBytes(UTF_8));
+    }
+
+    private String buildBlobInfoJson(MockGcsBlobStore.BlobVersion blobReference) {
+        return String.format(
+            Locale.ROOT,
+            """
+                {"kind":"storage#object","bucket":"%s","name":"%s","id":"%s","size":"%s","generation":"%d"}""",
+            bucket,
+            blobReference.path(),
+            blobReference.path(),
+            blobReference.contents().length(),
+            blobReference.generation()
+        );
     }
 
     public Map<String, BytesReference> blobs() {
-        return blobs;
+        return Maps.transformValues(mockGcsBlobStore.listBlobs(), MockGcsBlobStore.BlobVersion::contents);
     }
 
     private static String httpServerUrl(final HttpExchange exchange) {
@@ -364,34 +376,24 @@ private static boolean isEndOfPart(BytesReference fullRequestBody, int endPos) {
         return true;
     }
 
-    private static final Pattern PATTERN_CONTENT_RANGE = Pattern.compile("bytes ([^/]*)/([0-9\\*]*)");
-    private static final Pattern PATTERN_CONTENT_RANGE_BYTES = Pattern.compile("([0-9]*)-([0-9]*)");
-
-    private static Integer parse(final Pattern pattern, final String contentRange, final BiFunction<String, String, Integer> fn) {
-        final Matcher matcher = pattern.matcher(contentRange);
-        if (matcher.matches() == false || matcher.groupCount() != 2) {
-            throw new IllegalArgumentException("Unable to parse content range header");
+    private static String requireHeader(HttpExchange exchange, String headerName) {
+        final String headerValue = exchange.getRequestHeaders().getFirst(headerName);
+        if (headerValue != null) {
+            return headerValue;
         }
-        return fn.apply(matcher.group(1), matcher.group(2));
+        throw failAndThrow("Missing required header: " + headerName);
     }
 
-    public static Integer getContentRangeLimit(final String contentRange) {
-        return parse(PATTERN_CONTENT_RANGE, contentRange, (bytes, limit) -> "*".equals(limit) ? null : Integer.parseInt(limit));
-    }
-
-    public static int getContentRangeStart(final String contentRange) {
-        return parse(
-            PATTERN_CONTENT_RANGE,
-            contentRange,
-            (bytes, limit) -> parse(PATTERN_CONTENT_RANGE_BYTES, bytes, (start, end) -> Integer.parseInt(start))
-        );
-    }
-
-    public static int getContentRangeEnd(final String contentRange) {
-        return parse(
-            PATTERN_CONTENT_RANGE,
-            contentRange,
-            (bytes, limit) -> parse(PATTERN_CONTENT_RANGE_BYTES, bytes, (start, end) -> Integer.parseInt(end))
-        );
+    private static Long parseOptionalLongParameter(HttpExchange exchange, String parameterName) {
+        final Map<String, String> params = new HashMap<>();
+        RestUtils.decodeQueryString(exchange.getRequestURI(), params);
+        if (params.containsKey(parameterName)) {
+            try {
+                return Long.parseLong(params.get(parameterName));
+            } catch (NumberFormatException e) {
+                throw failAndThrow("Invalid long parameter: " + parameterName);
+            }
+        }
+        return null;
     }
 }
diff --git a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/MockGcsBlobStore.java b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/MockGcsBlobStore.java
new file mode 100644
index 0000000000000..a5b055566f78d
--- /dev/null
+++ b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/MockGcsBlobStore.java
@@ -0,0 +1,202 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package fixture.gcs;
+
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.bytes.CompositeBytesReference;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.fixture.HttpHeaderParser;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class MockGcsBlobStore {
+
+    private static final int RESUME_INCOMPLETE = 308;
+    private final ConcurrentMap<String, BlobVersion> blobs = new ConcurrentHashMap<>();
+    private final ConcurrentMap<String, ResumableUpload> resumableUploads = new ConcurrentHashMap<>();
+
+    record BlobVersion(String path, long generation, BytesReference contents) {}
+
+    record ResumableUpload(String uploadId, String path, Long ifGenerationMatch, BytesReference contents, boolean completed) {
+
+        public ResumableUpload update(BytesReference contents, boolean completed) {
+            return new ResumableUpload(uploadId, path, ifGenerationMatch, contents, completed);
+        }
+    }
+
+    BlobVersion getBlob(String path, Long ifGenerationMatch) {
+        final BlobVersion blob = blobs.get(path);
+        if (blob == null) {
+            throw new BlobNotFoundException(path);
+        } else {
+            if (ifGenerationMatch != null) {
+                if (blob.generation != ifGenerationMatch) {
+                    throw new GcsRestException(
+                        RestStatus.PRECONDITION_FAILED,
+                        "Generation mismatch, expected " + ifGenerationMatch + " but got " + blob.generation
+                    );
+                }
+            }
+            return blob;
+        }
+    }
+
+    BlobVersion updateBlob(String path, Long ifGenerationMatch, BytesReference contents) {
+        return blobs.compute(path, (name, existing) -> {
+            if (existing != null) {
+                if (ifGenerationMatch != null) {
+                    if (ifGenerationMatch == 0) {
+                        throw new GcsRestException(
+                            RestStatus.PRECONDITION_FAILED,
+                            "Blob already exists at generation " + existing.generation
+                        );
+                    } else if (ifGenerationMatch != existing.generation) {
+                        throw new GcsRestException(
+                            RestStatus.PRECONDITION_FAILED,
+                            "Generation mismatch, expected " + ifGenerationMatch + ", got" + existing.generation
+                        );
+                    }
+                }
+                return new BlobVersion(path, existing.generation + 1, contents);
+            } else {
+                if (ifGenerationMatch != null && ifGenerationMatch != 0) {
+                    throw new GcsRestException(
+                        RestStatus.PRECONDITION_FAILED,
+                        "Blob does not exist, expected generation " + ifGenerationMatch
+                    );
+                }
+                return new BlobVersion(path, 1, contents);
+            }
+        });
+    }
+
+    ResumableUpload createResumableUpload(String path, Long ifGenerationMatch) {
+        final String uploadId = UUIDs.randomBase64UUID();
+        final ResumableUpload value = new ResumableUpload(uploadId, path, ifGenerationMatch, BytesArray.EMPTY, false);
+        resumableUploads.put(uploadId, value);
+        return value;
+    }
+
+    /**
+     * Update or query a resumable upload
+     *
+     * @see <a href="https://cloud.google.com/storage/docs/resumable-uploads">GCS Resumable Uploads</a>
+     * @param uploadId The upload ID
+     * @param contentRange The range being submitted
+     * @param requestBody The data for that range
+     * @return The response to the request
+     */
+    UpdateResponse updateResumableUpload(String uploadId, HttpHeaderParser.ContentRange contentRange, BytesReference requestBody) {
+        final AtomicReference<UpdateResponse> updateResponse = new AtomicReference<>();
+        resumableUploads.compute(uploadId, (uid, existing) -> {
+            if (existing == null) {
+                throw failAndThrow("Attempted to update a non-existent resumable: " + uid);
+            }
+
+            if (contentRange.hasRange() == false) {
+                // Content-Range: */... is a status check https://cloud.google.com/storage/docs/performing-resumable-uploads#status-check
+                if (existing.completed) {
+                    updateResponse.set(new UpdateResponse(RestStatus.OK.getStatus(), calculateRangeHeader(blobs.get(existing.path))));
+                } else {
+                    final HttpHeaderParser.Range range = calculateRangeHeader(existing);
+                    updateResponse.set(new UpdateResponse(RESUME_INCOMPLETE, range));
+                }
+                return existing;
+            } else {
+                if (contentRange.start() > contentRange.end()) {
+                    throw failAndThrow("Invalid content range " + contentRange);
+                }
+                if (contentRange.start() > existing.contents.length()) {
+                    throw failAndThrow(
+                        "Attempted to append after the end of the current content: size="
+                            + existing.contents.length()
+                            + ", start="
+                            + contentRange.start()
+                    );
+                }
+                if (contentRange.end() < existing.contents.length()) {
+                    throw failAndThrow("Attempted to upload no new data");
+                }
+                final int offset = Math.toIntExact(existing.contents.length() - contentRange.start());
+                final BytesReference updatedContent = CompositeBytesReference.of(
+                    existing.contents,
+                    requestBody.slice(offset, requestBody.length())
+                );
+                // We just received the last chunk, update the blob and remove the resumable upload from the map
+                if (contentRange.hasSize() && updatedContent.length() == contentRange.size()) {
+                    updateBlob(existing.path(), existing.ifGenerationMatch, updatedContent);
+                    updateResponse.set(new UpdateResponse(RestStatus.OK.getStatus(), null));
+                    return existing.update(BytesArray.EMPTY, true);
+                }
+                final ResumableUpload updated = existing.update(updatedContent, false);
+                updateResponse.set(new UpdateResponse(RESUME_INCOMPLETE, calculateRangeHeader(updated)));
+                return updated;
+            }
+        });
+        assert updateResponse.get() != null : "Should always produce an update response";
+        return updateResponse.get();
+    }
+
+    private static HttpHeaderParser.Range calculateRangeHeader(ResumableUpload resumableUpload) {
+        return resumableUpload.contents.length() > 0 ? new HttpHeaderParser.Range(0, resumableUpload.contents.length() - 1) : null;
+    }
+
+    private static HttpHeaderParser.Range calculateRangeHeader(BlobVersion blob) {
+        return blob.contents.length() > 0 ? new HttpHeaderParser.Range(0, blob.contents.length() - 1) : null;
+    }
+
+    record UpdateResponse(int statusCode, HttpHeaderParser.Range rangeHeader) {}
+
+    void deleteBlob(String path) {
+        blobs.remove(path);
+    }
+
+    Map<String, BlobVersion> listBlobs() {
+        return Map.copyOf(blobs);
+    }
+
+    static class BlobNotFoundException extends GcsRestException {
+
+        BlobNotFoundException(String path) {
+            super(RestStatus.NOT_FOUND, "Blob not found: " + path);
+        }
+    }
+
+    static class GcsRestException extends RuntimeException {
+
+        private final RestStatus status;
+
+        GcsRestException(RestStatus status, String errorMessage) {
+            super(errorMessage);
+            this.status = status;
+        }
+
+        public RestStatus getStatus() {
+            return status;
+        }
+    }
+
+    /**
+     * Fail the test with an assertion error and throw an exception in-line
+     *
+     * @param message The message to use on the {@link Throwable}s
+     * @return nothing, but claim to return an exception to help with static analysis
+     */
+    public static RuntimeException failAndThrow(String message) {
+        ExceptionsHelper.maybeDieOnAnotherThread(new AssertionError(message));
+        throw new IllegalStateException(message);
+    }
+}
diff --git a/test/fixtures/gcs-fixture/src/test/java/fixture/gcs/GoogleCloudStorageHttpHandlerTests.java b/test/fixtures/gcs-fixture/src/test/java/fixture/gcs/GoogleCloudStorageHttpHandlerTests.java
new file mode 100644
index 0000000000000..f83da9e1679a9
--- /dev/null
+++ b/test/fixtures/gcs-fixture/src/test/java/fixture/gcs/GoogleCloudStorageHttpHandlerTests.java
@@ -0,0 +1,793 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package fixture.gcs;
+
+import com.sun.net.httpserver.Headers;
+import com.sun.net.httpserver.HttpContext;
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpPrincipal;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.bytes.CompositeBytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.fixture.HttpHeaderParser;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPOutputStream;
+
+public class GoogleCloudStorageHttpHandlerTests extends ESTestCase {
+
+    private static final String HOST = "http://127.0.0.1:12345";
+    private static final int RESUME_INCOMPLETE = 308;
+    private static final Pattern GENERATION_PATTERN = Pattern.compile("\"generation\"\\s*:\\s*\"(\\d+)\"");
+
+    public void testRejectsBadUri() {
+        assertEquals(
+            RestStatus.NOT_FOUND.getStatus(),
+            handleRequest(new GoogleCloudStorageHttpHandler("bucket"), randomFrom("GET", "PUT", "POST", "DELETE", "HEAD"), "/not-in-bucket")
+                .status()
+        );
+    }
+
+    public void testCheckEndpoint() {
+        final var handler = new GoogleCloudStorageHttpHandler("bucket");
+
+        assertEquals(
+            RestStatus.OK,
+            handleRequest(handler, "GET", "/", BytesArray.EMPTY, Headers.of("Metadata-Flavor", "Google")).restStatus()
+        );
+    }
+
+    public void testSimpleObjectOperations() {
+        final var bucket = randomAlphaOfLength(10);
+        final var handler = new GoogleCloudStorageHttpHandler(bucket);
+        final var blobName = "path/" + randomAlphaOfLength(10);
+
+        assertEquals(RestStatus.NOT_FOUND, getBlobContents(handler, bucket, blobName, null, null).restStatus());
+
+        assertEquals(
+            new TestHttpResponse(RestStatus.OK, "{\"kind\":\"storage#objects\",\"items\":[],\"prefixes\":[]}"),
+            listBlobs(handler, bucket, null)
+        );
+
+        final var body = randomAlphaOfLength(50);
+        assertEquals(
+            RestStatus.OK,
+            executeUpload(handler, bucket, blobName, new BytesArray(body.getBytes(StandardCharsets.UTF_8)), null).restStatus()
+        );
+
+        assertEquals(new TestHttpResponse(RestStatus.OK, body), getBlobContents(handler, bucket, blobName, null, null));
+
+        assertEquals(new TestHttpResponse(RestStatus.OK, Strings.format("""
+            {"kind":"storage#objects","items":[{"kind":"storage#object","bucket":"%s","name":"%s","id":"%s","size":"50",\
+            "generation":"1"}],"prefixes":[]}""", bucket, blobName, blobName)), listBlobs(handler, bucket, null));
+
+        assertEquals(new TestHttpResponse(RestStatus.OK, Strings.format("""
+            {"kind":"storage#objects","items":[{"kind":"storage#object","bucket":"%s","name":"%s","id":"%s","size":"50",\
+            "generation":"1"}],"prefixes":[]}""", bucket, blobName, blobName)), listBlobs(handler, bucket, "path/"));
+
+        assertEquals(new TestHttpResponse(RestStatus.OK, """
+            {"kind":"storage#objects","items":[],"prefixes":[]}"""), listBlobs(handler, bucket, "some/other/path"));
+
+        assertEquals(
+            new TestHttpResponse(RestStatus.OK, """
+                --__END_OF_PART__d8b50acb-87dc-4630-a3d3-17d187132ebc__
+                Content-Length: 168
+                Content-Type: application/http
+                content-id: 1
+                content-transfer-encoding: binary
+
+                HTTP/1.1 204 NO_CONTENT
+
+
+
+
+                --__END_OF_PART__d8b50acb-87dc-4630-a3d3-17d187132ebc__
+                """.replaceAll("\n", "\r\n")),
+            handleRequest(
+                handler,
+                "POST",
+                "/batch/storage/v1",
+                createBatchDeleteRequest(bucket, blobName),
+                Headers.of("Content-Type", "mixed/multipart")
+            )
+        );
+        assertEquals(
+            RestStatus.OK,
+            handleRequest(
+                handler,
+                "POST",
+                "/batch/storage/v1",
+                createBatchDeleteRequest(bucket, blobName),
+                Headers.of("Content-Type", "mixed/multipart")
+            ).restStatus()
+        );
+
+        assertEquals(new TestHttpResponse(RestStatus.OK, """
+            {"kind":"storage#objects","items":[],"prefixes":[]}"""), listBlobs(handler, bucket, "path/"));
+    }
+
+    public void testGetWithBytesRange() {
+        final var bucket = randomIdentifier();
+        final var handler = new GoogleCloudStorageHttpHandler(bucket);
+        final var blobName = "blob_name_" + randomIdentifier();
+        final var blobBytes = randomBytesReference(256);
+
+        assertEquals(RestStatus.OK, executeUpload(handler, bucket, blobName, blobBytes, 0L).restStatus());
+
+        assertEquals(
+            "No Range",
+            new TestHttpResponse(RestStatus.OK, blobBytes, TestHttpExchange.EMPTY_HEADERS),
+            getBlobContents(handler, bucket, blobName, null, null)
+        );
+
+        var end = blobBytes.length() - 1;
+        assertEquals(
+            "Exact Range: bytes=0-" + end,
+            new TestHttpResponse(RestStatus.OK, blobBytes, TestHttpExchange.EMPTY_HEADERS),
+            getBlobContents(handler, bucket, blobName, null, new HttpHeaderParser.Range(0, end))
+        );
+
+        end = randomIntBetween(blobBytes.length() - 1, Integer.MAX_VALUE);
+        assertEquals(
+            "Larger Range: bytes=0-" + end,
+            new TestHttpResponse(RestStatus.OK, blobBytes, TestHttpExchange.EMPTY_HEADERS),
+            getBlobContents(handler, bucket, blobName, null, new HttpHeaderParser.Range(0, end))
+        );
+
+        var start = randomIntBetween(blobBytes.length(), Integer.MAX_VALUE - 1);
+        end = randomIntBetween(start, Integer.MAX_VALUE);
+        assertEquals(
+            "Invalid Range: bytes=" + start + '-' + end,
+            new TestHttpResponse(RestStatus.REQUESTED_RANGE_NOT_SATISFIED, BytesArray.EMPTY, TestHttpExchange.EMPTY_HEADERS),
+            getBlobContents(handler, bucket, blobName, null, new HttpHeaderParser.Range(start, end))
+        );
+
+        start = randomIntBetween(0, blobBytes.length() - 1);
+        var length = randomIntBetween(1, blobBytes.length() - start);
+        end = start + length - 1;
+        assertEquals(
+            "Range: bytes=" + start + '-' + end,
+            new TestHttpResponse(RestStatus.OK, blobBytes.slice(start, length), TestHttpExchange.EMPTY_HEADERS),
+            getBlobContents(handler, bucket, blobName, null, new HttpHeaderParser.Range(start, end))
+        );
+    }
+
+    public void testResumableUpload() {
+        final var bucket = randomIdentifier();
+        final var handler = new GoogleCloudStorageHttpHandler(bucket);
+        final var blobName = "blob_name_" + randomIdentifier();
+
+        final var createUploadResponse = handleRequest(
+            handler,
+            "POST",
+            "/upload/storage/v1/b/" + bucket + "/?uploadType=resumable&name=" + blobName
+        );
+        final var locationHeader = createUploadResponse.headers.getFirst("Location");
+        final var sessionURI = locationHeader.substring(locationHeader.indexOf(HOST) + HOST.length());
+        assertEquals(RestStatus.OK, createUploadResponse.restStatus());
+
+        // status check
+        assertEquals(
+            new TestHttpResponse(RESUME_INCOMPLETE, TestHttpExchange.EMPTY_HEADERS),
+            handleRequest(handler, "PUT", sessionURI, BytesArray.EMPTY, contentRangeHeader(null, null, null))
+        );
+
+        final var part1 = randomAlphaOfLength(50);
+        final var uploadPart1Response = handleRequest(handler, "PUT", sessionURI, part1, contentRangeHeader(0, 50, null));
+        assertEquals(new TestHttpResponse(RESUME_INCOMPLETE, rangeHeader(0, 49)), uploadPart1Response);
+
+        // status check
+        assertEquals(
+            new TestHttpResponse(RESUME_INCOMPLETE, rangeHeader(0, 49)),
+            handleRequest(handler, "PUT", sessionURI, BytesArray.EMPTY, contentRangeHeader(null, null, null))
+        );
+
+        final var part2 = randomAlphaOfLength(50);
+        final var uploadPart2Response = handleRequest(handler, "PUT", sessionURI, part2, contentRangeHeader(50, 99, null));
+        assertEquals(new TestHttpResponse(RESUME_INCOMPLETE, rangeHeader(0, 99)), uploadPart2Response);
+
+        // incomplete upload should not be visible yet
+        assertEquals(RestStatus.NOT_FOUND, getBlobContents(handler, bucket, blobName, null, null).restStatus());
+
+        final var part3 = randomAlphaOfLength(30);
+        final var uploadPart3Response = handleRequest(handler, "PUT", sessionURI, part3, contentRangeHeader(100, 129, 130));
+        assertEquals(new TestHttpResponse(RestStatus.OK, TestHttpExchange.EMPTY_HEADERS), uploadPart3Response);
+
+        // status check
+        assertEquals(
+            new TestHttpResponse(RestStatus.OK, rangeHeader(0, 129)),
+            handleRequest(handler, "PUT", sessionURI, BytesArray.EMPTY, contentRangeHeader(null, null, null))
+        );
+
+        // complete upload should be visible now
+
+        // can download contents
+        assertEquals(
+            new TestHttpResponse(RestStatus.OK, part1 + part2 + part3),
+            handleRequest(handler, "GET", "/download/storage/v1/b/" + bucket + "/o/" + blobName)
+        );
+
+        // can see in listing
+        assertEquals(
+            new TestHttpResponse(RestStatus.OK, Strings.format("""
+                {"kind":"storage#objects","items":[{"kind":"storage#object","bucket":"%s","name":"%s","id":"%s","size":"130",\
+                "generation":"1"}],"prefixes":[]}""", bucket, blobName, blobName)),
+            handleRequest(handler, "GET", "/storage/v1/b/" + bucket + "/o")
+        );
+
+        // can get metadata
+        assertEquals(
+            new TestHttpResponse(
+                RestStatus.OK,
+                Strings.format(
+                    """
+                        {"kind":"storage#object","bucket":"%s","name":"%s","id":"%s","size":"130","generation":"1"}""",
+                    bucket,
+                    blobName,
+                    blobName
+                )
+            ),
+            handleRequest(handler, "GET", "/storage/v1/b/" + bucket + "/o/" + blobName)
+        );
+    }
+
+    public void testIfGenerationMatch_MultipartUpload() {
+        final var bucket = randomIdentifier();
+        final var handler = new GoogleCloudStorageHttpHandler(bucket);
+        final var blobName = "blob_name_" + randomIdentifier();
+
+        assertEquals(
+            RestStatus.OK,
+            executeUpload(handler, bucket, blobName, randomBytesReference(randomIntBetween(100, 5_000)), null).restStatus()
+        );
+
+        // update, matched generation
+        assertEquals(
+            RestStatus.OK,
+            executeMultipartUpload(
+                handler,
+                bucket,
+                blobName,
+                randomBytesReference(randomIntBetween(100, 5_000)),
+                getCurrentGeneration(handler, bucket, blobName)
+            ).restStatus()
+        );
+
+        // update, mismatched generation
+        assertEquals(
+            RestStatus.PRECONDITION_FAILED,
+            executeMultipartUpload(
+                handler,
+                bucket,
+                blobName,
+                randomBytesReference(randomIntBetween(100, 5_000)),
+                randomValueOtherThan(getCurrentGeneration(handler, bucket, blobName), ESTestCase::randomNonNegativeLong)
+            ).restStatus()
+        );
+
+        // update, no generation
+        assertEquals(
+            RestStatus.OK,
+            executeMultipartUpload(handler, bucket, blobName, randomBytesReference(randomIntBetween(100, 5_000)), null).restStatus()
+        );
+
+        // update, zero generation
+        assertEquals(
+            RestStatus.PRECONDITION_FAILED,
+            executeMultipartUpload(handler, bucket, blobName, randomBytesReference(randomIntBetween(100, 5_000)), 0L).restStatus()
+        );
+
+        // new file, zero generation
+        assertEquals(
+            RestStatus.OK,
+            executeMultipartUpload(handler, bucket, blobName + "/new/1", randomBytesReference(randomIntBetween(100, 5_000)), 0L)
+                .restStatus()
+        );
+
+        // new file, non-zero generation
+        assertEquals(
+            RestStatus.PRECONDITION_FAILED,
+            executeMultipartUpload(
+                handler,
+                bucket,
+                blobName + "/new/2",
+                randomBytesReference(randomIntBetween(100, 5_000)),
+                randomLongBetween(1, Long.MAX_VALUE)
+            ).restStatus()
+        );
+    }
+
+    public void testIfGenerationMatch_ResumableUpload() {
+        final var bucket = randomIdentifier();
+        final var handler = new GoogleCloudStorageHttpHandler(bucket);
+        final var blobName = "blob_name_" + randomIdentifier();
+
+        assertEquals(
+            RestStatus.OK,
+            executeUpload(handler, bucket, blobName, randomBytesReference(randomIntBetween(100, 5_000)), null).restStatus()
+        );
+
+        // update, matched generation
+        assertEquals(
+            RestStatus.OK,
+            executeResumableUpload(
+                handler,
+                bucket,
+                blobName,
+                randomBytesReference(randomIntBetween(100, 5_000)),
+                getCurrentGeneration(handler, bucket, blobName)
+            ).restStatus()
+        );
+
+        // update, mismatched generation
+        assertEquals(
+            RestStatus.PRECONDITION_FAILED,
+            executeResumableUpload(
+                handler,
+                bucket,
+                blobName,
+                randomBytesReference(randomIntBetween(100, 5_000)),
+                randomValueOtherThan(getCurrentGeneration(handler, bucket, blobName), ESTestCase::randomNonNegativeLong)
+            ).restStatus()
+        );
+
+        // update, no generation
+        assertEquals(
+            RestStatus.OK,
+            executeResumableUpload(handler, bucket, blobName, randomBytesReference(randomIntBetween(100, 5_000)), null).restStatus()
+        );
+
+        // update, zero generation
+        assertEquals(
+            RestStatus.PRECONDITION_FAILED,
+            executeResumableUpload(handler, bucket, blobName, randomBytesReference(randomIntBetween(100, 5_000)), 0L).restStatus()
+        );
+
+        // new file, zero generation
+        assertEquals(
+            RestStatus.OK,
+            executeResumableUpload(handler, bucket, blobName + "/new/1", randomBytesReference(randomIntBetween(100, 5_000)), 0L)
+                .restStatus()
+        );
+
+        // new file, non-zero generation
+        assertEquals(
+            RestStatus.PRECONDITION_FAILED,
+            executeResumableUpload(
+                handler,
+                bucket,
+                blobName + "/new/2",
+                randomBytesReference(randomIntBetween(100, 5_000)),
+                randomLongBetween(1, Long.MAX_VALUE)
+            ).restStatus()
+        );
+    }
+
+    public void testIfGenerationMatch_GetObject() {
+        final var bucket = randomIdentifier();
+        final var handler = new GoogleCloudStorageHttpHandler(bucket);
+        final var blobName = "blob_name_" + randomIdentifier();
+
+        assertEquals(
+            RestStatus.OK,
+            executeUpload(handler, bucket, blobName, randomBytesReference(randomIntBetween(100, 5_000)), null).restStatus()
+        );
+
+        final long currentGeneration = getCurrentGeneration(handler, bucket, blobName);
+
+        // Get contents, matching generation
+        assertEquals(RestStatus.OK, getBlobContents(handler, bucket, blobName, currentGeneration, null).restStatus());
+
+        // Get contents, mismatched generation
+        assertEquals(
+            RestStatus.PRECONDITION_FAILED,
+            getBlobContents(handler, bucket, blobName, randomValueOtherThan(currentGeneration, ESTestCase::randomNonNegativeLong), null)
+                .restStatus()
+        );
+
+        // Get metadata, matching generation
+        assertEquals(RestStatus.OK, getBlobMetadata(handler, bucket, blobName, currentGeneration).restStatus());
+
+        // Get metadata, mismatched generation
+        assertEquals(
+            RestStatus.PRECONDITION_FAILED,
+            getBlobMetadata(handler, bucket, blobName, randomValueOtherThan(currentGeneration, ESTestCase::randomNonNegativeLong))
+                .restStatus()
+        );
+    }
+
+    private static TestHttpResponse executeUpload(
+        GoogleCloudStorageHttpHandler handler,
+        String bucket,
+        String blobName,
+        BytesReference bytes,
+        Long ifGenerationMatch
+    ) {
+        assert bytes.length() > 20;
+        if (randomBoolean()) {
+            return executeResumableUpload(handler, bucket, blobName, bytes, ifGenerationMatch);
+        } else {
+            return executeMultipartUpload(handler, bucket, blobName, bytes, ifGenerationMatch);
+        }
+    }
+
+    private static TestHttpResponse executeResumableUpload(
+        GoogleCloudStorageHttpHandler handler,
+        String bucket,
+        String blobName,
+        BytesReference bytes,
+        Long ifGenerationMatch
+    ) {
+        final var createUploadResponse = handleRequest(
+            handler,
+            "POST",
+            "/upload/storage/v1/b/"
+                + bucket
+                + "/?uploadType=resumable&name="
+                + blobName
+                + (ifGenerationMatch != null ? "&ifGenerationMatch=" + ifGenerationMatch : "")
+        );
+        final var locationHeader = createUploadResponse.headers.getFirst("Location");
+        final var sessionURI = locationHeader.substring(locationHeader.indexOf(HOST) + HOST.length());
+        assertEquals(RestStatus.OK, createUploadResponse.restStatus());
+
+        final int partBoundary = randomIntBetween(10, bytes.length() - 1);
+        final var part1 = bytes.slice(0, partBoundary);
+        final var uploadPart1Response = handleRequest(handler, "PUT", sessionURI, part1, contentRangeHeader(0, partBoundary - 1, null));
+        assertEquals(RESUME_INCOMPLETE, uploadPart1Response.status());
+
+        final var part2 = bytes.slice(partBoundary, bytes.length() - partBoundary);
+        return handleRequest(handler, "PUT", sessionURI, part2, contentRangeHeader(partBoundary, bytes.length() - 1, bytes.length()));
+    }
+
+    private static TestHttpResponse executeMultipartUpload(
+        GoogleCloudStorageHttpHandler handler,
+        String bucket,
+        String blobName,
+        BytesReference bytes,
+        Long ifGenerationMatch
+    ) {
+        return handleRequest(
+            handler,
+            "POST",
+            "/upload/storage/v1/b/"
+                + bucket
+                + "/?uploadType=multipart"
+                + (ifGenerationMatch != null ? "&ifGenerationMatch=" + ifGenerationMatch : ""),
+            createGzipCompressedMultipartUploadBody(bucket, blobName, bytes)
+        );
+    }
+
+    private static TestHttpResponse getBlobContents(
+        GoogleCloudStorageHttpHandler handler,
+        String bucket,
+        String blobName,
+        @Nullable Long ifGenerationMatch,
+        @Nullable HttpHeaderParser.Range range
+    ) {
+        return handleRequest(
+            handler,
+            "GET",
+            "/download/storage/v1/b/"
+                + bucket
+                + "/o/"
+                + blobName
+                + (ifGenerationMatch != null ? "?ifGenerationMatch=" + ifGenerationMatch : ""),
+            BytesArray.EMPTY,
+            range != null ? rangeHeader(range.start(), range.end()) : TestHttpExchange.EMPTY_HEADERS
+        );
+    }
+
+    private static TestHttpResponse getBlobMetadata(
+        GoogleCloudStorageHttpHandler handler,
+        String bucket,
+        String blobName,
+        @Nullable Long ifGenerationMatch
+    ) {
+        return handleRequest(
+            handler,
+            "GET",
+            "/storage/v1/b/" + bucket + "/o/" + blobName + (ifGenerationMatch != null ? "?ifGenerationMatch=" + ifGenerationMatch : "")
+        );
+    }
+
+    private static long getCurrentGeneration(GoogleCloudStorageHttpHandler handler, String bucket, String blobName) {
+        TestHttpResponse blobMetadata = getBlobMetadata(handler, bucket, blobName, null);
+        assertEquals(RestStatus.OK, blobMetadata.restStatus());
+        Matcher matcher = GENERATION_PATTERN.matcher(blobMetadata.body.utf8ToString());
+        assertTrue(matcher.find());
+        return Long.parseLong(matcher.group(1));
+    }
+
+    private static TestHttpResponse listBlobs(GoogleCloudStorageHttpHandler handler, String bucket, String prefix) {
+        return handleRequest(
+            handler,
+            "GET",
+            "/storage/v1/b/" + bucket + "/o" + (prefix != null ? "?prefix=" + URLEncoder.encode(prefix, StandardCharsets.UTF_8) : "")
+        );
+    }
+
+    private record TestHttpResponse(int status, BytesReference body, Headers headers) {
+        TestHttpResponse(RestStatus status, BytesReference body, Headers headers) {
+            this(status.getStatus(), body, headers);
+        }
+
+        TestHttpResponse(RestStatus status, String body) {
+            this(status.getStatus(), new BytesArray(body.getBytes(StandardCharsets.UTF_8)), TestHttpExchange.EMPTY_HEADERS);
+        }
+
+        TestHttpResponse(RestStatus status, Headers headers) {
+            this(status.getStatus(), BytesArray.EMPTY, headers);
+        }
+
+        TestHttpResponse(int statusCode, Headers headers) {
+            this(statusCode, BytesArray.EMPTY, headers);
+        }
+
+        RestStatus restStatus() {
+            return Objects.requireNonNull(RestStatus.fromCode(status));
+        }
+
+        @Override
+        public String toString() {
+            return "TestHttpResponse{" + "status=" + status + ", body={size=" + body.utf8ToString() + "}, headers=" + headers + '}';
+        }
+    }
+
+    private static TestHttpResponse handleRequest(GoogleCloudStorageHttpHandler handler, String method, String uri) {
+        return handleRequest(handler, method, uri, "");
+    }
+
+    private static TestHttpResponse handleRequest(GoogleCloudStorageHttpHandler handler, String method, String uri, String requestBody) {
+        return handleRequest(handler, method, uri, new BytesArray(requestBody.getBytes(StandardCharsets.UTF_8)));
+    }
+
+    private static TestHttpResponse handleRequest(
+        GoogleCloudStorageHttpHandler handler,
+        String method,
+        String uri,
+        String requestBody,
+        Headers headers
+    ) {
+        return handleRequest(handler, method, uri, new BytesArray(requestBody.getBytes(StandardCharsets.UTF_8)), headers);
+    }
+
+    private static TestHttpResponse handleRequest(
+        GoogleCloudStorageHttpHandler handler,
+        String method,
+        String uri,
+        BytesReference requestBody
+    ) {
+        return handleRequest(handler, method, uri, requestBody, TestHttpExchange.EMPTY_HEADERS);
+    }
+
+    private static TestHttpResponse handleRequest(
+        GoogleCloudStorageHttpHandler handler,
+        String method,
+        String uri,
+        BytesReference requestBody,
+        Headers requestHeaders
+    ) {
+        final var httpExchange = new TestHttpExchange(method, uri, requestBody, requestHeaders);
+        try {
+            handler.handle(httpExchange);
+        } catch (IOException e) {
+            fail(e);
+        }
+        assertNotEquals(0, httpExchange.getResponseCode());
+        var responseHeaders = new Headers();
+        httpExchange.getResponseHeaders().forEach((header, values) -> {
+            // com.sun.net.httpserver.Headers.Headers() normalize keys
+            if ("Range".equals(header) || "Content-range".equals(header) || "Location".equals(header)) {
+                responseHeaders.put(header, List.copyOf(values));
+            }
+        });
+        return new TestHttpResponse(httpExchange.getResponseCode(), httpExchange.getResponseBodyContents(), responseHeaders);
+    }
+
+    private static Headers contentRangeHeader(@Nullable Integer startInclusive, @Nullable Integer endInclusive, @Nullable Integer limit) {
+        final String rangeString = startInclusive != null && endInclusive != null ? startInclusive + "-" + endInclusive : "*";
+        final String limitString = limit == null ? "*" : limit.toString();
+        return Headers.of("Content-Range", "bytes " + rangeString + "/" + limitString);
+    }
+
+    private static Headers rangeHeader(long start, long end) {
+        return Headers.of("Range", Strings.format("bytes=%d-%d", start, end));
+    }
+
+    private static BytesReference createGzipCompressedMultipartUploadBody(String bucketName, String path, BytesReference content) {
+        final String metadataString = Strings.format("{\"bucket\":\"%s\", \"name\":\"%s\"}", bucketName, path);
+        final BytesReference header = new BytesArray(Strings.format("""
+            --__END_OF_PART__a607a67c-6df7-4b87-b8a1-81f639a75a97__
+            Content-Length: %d
+            Content-Type: application/json; charset=UTF-8
+            content-transfer-encoding: binary
+
+            %s
+            --__END_OF_PART__a607a67c-6df7-4b87-b8a1-81f639a75a97__
+            Content-Type: application/octet-stream
+            content-transfer-encoding: binary
+
+            """.replaceAll("\n", "\r\n"), metadataString.length(), metadataString).getBytes(StandardCharsets.UTF_8));
+
+        final BytesReference footer = new BytesArray("""
+
+            --__END_OF_PART__a607a67c-6df7-4b87-b8a1-81f639a75a97__--
+            """.replaceAll("\n", "\r\n"));
+        final ByteArrayOutputStream out = new ByteArrayOutputStream();
+        try (GZIPOutputStream gzipOutputStream = new GZIPOutputStream(out)) {
+            gzipOutputStream.write(BytesReference.toBytes(CompositeBytesReference.of(header, content, footer)));
+        } catch (IOException e) {
+            fail(e);
+        }
+        return new BytesArray(out.toByteArray());
+    }
+
+    private static String createBatchDeleteRequest(String bucketName, String... paths) {
+        final String deleteRequestTemplate = """
+            DELETE %s/storage/v1/b/%s/o/%s HTTP/1.1
+            Authorization: Bearer foo
+            x-goog-api-client: gl-java/23.0.0 gdcl/2.1.1 mac-os-x/15.2
+
+
+            """;
+        final String partTemplate = """
+            --__END_OF_PART__d8b50acb-87dc-4630-a3d3-17d187132ebc__
+            Content-Length: %d
+            Content-Type: application/http
+            content-id: %d
+            content-transfer-encoding: binary
+
+            %s
+            """;
+        StringBuilder builder = new StringBuilder();
+        AtomicInteger contentId = new AtomicInteger();
+        Arrays.stream(paths).forEach(p -> {
+            final String deleteRequest = Strings.format(deleteRequestTemplate, HOST, bucketName, p);
+            final String part = Strings.format(partTemplate, deleteRequest.length(), contentId.incrementAndGet(), deleteRequest);
+            builder.append(part);
+        });
+        builder.append("--__END_OF_PART__d8b50acb-87dc-4630-a3d3-17d187132ebc__");
+        return builder.toString();
+    }
+
+    private static class TestHttpExchange extends HttpExchange {
+
+        private static final Headers EMPTY_HEADERS = new Headers();
+
+        private final String method;
+        private final URI uri;
+        private final BytesReference requestBody;
+        private final Headers requestHeaders;
+
+        private final Headers responseHeaders = new Headers();
+        private final BytesStreamOutput responseBody = new BytesStreamOutput();
+        private int responseCode;
+
+        TestHttpExchange(String method, String uri, BytesReference requestBody, Headers requestHeaders) {
+            this.method = method;
+            this.uri = URI.create(uri);
+            this.requestBody = requestBody;
+            this.requestHeaders = new Headers(requestHeaders);
+            this.requestHeaders.add("Host", HOST);
+        }
+
+        @Override
+        public Headers getRequestHeaders() {
+            return requestHeaders;
+        }
+
+        @Override
+        public Headers getResponseHeaders() {
+            return responseHeaders;
+        }
+
+        @Override
+        public URI getRequestURI() {
+            return uri;
+        }
+
+        @Override
+        public String getRequestMethod() {
+            return method;
+        }
+
+        @Override
+        public HttpContext getHttpContext() {
+            return null;
+        }
+
+        @Override
+        public void close() {}
+
+        @Override
+        public InputStream getRequestBody() {
+            try {
+                return requestBody.streamInput();
+            } catch (IOException e) {
+                throw new AssertionError(e);
+            }
+        }
+
+        @Override
+        public OutputStream getResponseBody() {
+            return responseBody;
+        }
+
+        @Override
+        public void sendResponseHeaders(int rCode, long responseLength) {
+            this.responseCode = rCode;
+        }
+
+        @Override
+        public InetSocketAddress getRemoteAddress() {
+            return null;
+        }
+
+        @Override
+        public int getResponseCode() {
+            return responseCode;
+        }
+
+        public BytesReference getResponseBodyContents() {
+            return responseBody.bytes();
+        }
+
+        @Override
+        public InetSocketAddress getLocalAddress() {
+            return null;
+        }
+
+        @Override
+        public String getProtocol() {
+            return "HTTP/1.1";
+        }
+
+        @Override
+        public Object getAttribute(String name) {
+            return null;
+        }
+
+        @Override
+        public void setAttribute(String name, Object value) {
+            fail("setAttribute not implemented");
+        }
+
+        @Override
+        public void setStreams(InputStream i, OutputStream o) {
+            fail("setStreams not implemented");
+        }
+
+        @Override
+        public HttpPrincipal getPrincipal() {
+            fail("getPrincipal not implemented");
+            throw new UnsupportedOperationException("getPrincipal not implemented");
+        }
+    }
+}
diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle
index 9dc0263f49aee..857f497b29084 100644
--- a/test/fixtures/hdfs-fixture/build.gradle
+++ b/test/fixtures/hdfs-fixture/build.gradle
@@ -10,12 +10,10 @@
 apply plugin: 'elasticsearch.java'
 apply plugin: 'com.gradleup.shadow'
 
+
 import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
 
 configurations {
-//  all {
-//    transitive = true
-//  }
   hdfs2
   hdfs3
   consumable("shadowedHdfs2")
@@ -24,23 +22,79 @@ configurations {
 dependencies {
   compileOnly("org.apache.hadoop:hadoop-minicluster:2.8.5")
   api("com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}") {
-    transitive false
+    transitive = false
   }
   compileOnly "junit:junit:${versions.junit}"
-  hdfs2 "org.apache.hadoop:hadoop-minicluster:2.8.5"
-  hdfs3 "org.apache.hadoop:hadoop-minicluster:3.3.1"
 
+  def commonExcludes = [
+    [group: "org.apache.commons", module: "commons-compress"],
+    [group: "org.apache.hadoop", module: "hadoop-mapreduce-client-app"],
+    [group: "org.apache.hadoop", module: "hadoop-mapreduce-client-core"],
+    [group: "org.apache.hadoop", module: "hadoop-mapreduce-client-hs"],
+    [group: "org.apache.hadoop", module: "hadoop-mapreduce-client-jobclient"],
+    [group: "org.apache.hadoop", module: "hadoop-yarn-server-tests"],
+    [group: "org.apache.httpcomponents", module: "httpclient"],
+    [group: "org.apache.zookeeper", module: "zookeeper"],
+    [group: "org.apache.curator", module: "curator-recipes"],
+    [group: "org.apache.curator", module: "curator-client"],
+    [group: "org.apache.curator", module: "curator-framework"],
+    [group: "org.apache.avro", module: "avro"],
+    [group: "log4j", module: "log4j"],
+    [group: "io.netty", module: "netty-all"],
+    [group: "io.netty", module: "netty"],
+    [group: "com.squareup.okhttp", module: "okhttp"],
+    [group: "com.google.guava", module: "guava"],
+    [group: "com.google.code.gson", module: "gson"],
+    [group: "javax.servlet.jsp", module: "jsp-api"],
+    [group: "org.fusesource.leveldbjni", module: "leveldbjni-all"],
+    [group: "commons-cli", module: "commons-cli"],
+    [group: "org.mortbay.jetty", module: "servlet-api"],
+    [group: "commons-logging", module: "commons-logging"],
+    [group: "org.slf4j", module: "slf4j-log4j12"],
+    [group: "commons-codec", module: "commons-codec"],
+    [group: "com.sun.jersey", module: "jersey-core"],
+    [group: "com.sun.jersey", module: "jersey-json"],
+    [group: "com.google.code.findbugs", module: "jsr305"],
+    [group: "com.sun.jersey", module: "jersey-json"],
+    [group: "com.nimbusds", module: "nimbus-jose-jwt"],
+    [group: "com.jcraft", module: "jsch"],
+    [group: "org.slf4j", module: "slf4j-api"],
+  ]
+
+  hdfs2("org.apache.hadoop:hadoop-minicluster:2.8.5") {
+    commonExcludes.each { exclude it }
+    exclude group: "org.apache.commons", module: "commons-math3"
+    exclude group: "xmlenc", module: "xmlenc"
+    exclude group: "net.java.dev.jets3t", module: "jets3t"
+    exclude group: "org.apache.directory.server", module: "apacheds-i18n"
+    exclude group: "xerces", module: "xercesImpl"
+  }
+
+  hdfs3("org.apache.hadoop:hadoop-minicluster:3.3.1") {
+    commonExcludes.each { exclude it }
+    exclude group: "dnsjava", module: "dnsjava"
+    exclude group: "com.google.inject.extensions", module: "guice-servlet"
+    exclude group: "com.google.inject", module: "guice"
+    exclude group: "com.microsoft.sqlserver", module: "mssql-jdbc"
+    exclude group: "com.sun.jersey.contribs", module: "jersey-guice"
+    exclude group: "com.zaxxer", module: "HikariCP-java7"
+    exclude group: "com.sun.jersey", module: "jersey-server"
+    exclude group: "org.bouncycastle", module: "bcpkix-jdk15on"
+    exclude group: "org.bouncycastle", module: "bcprov-jdk15on"
+    exclude group: "org.ehcache", module: "ehcache"
+    exclude group: "org.apache.geronimo.specs", module: "geronimo-jcache_1.0_spec"
+    exclude group: "org.xerial.snappy", module: "snappy-java"
+  }
 }
 
 tasks.named("shadowJar").configure {
   archiveClassifier.set("hdfs3")
   // fix issues with signed jars
-
   relocate("org.apache.hadoop", "fixture.hdfs3.org.apache.hadoop") {
     exclude "org.apache.hadoop.hdfs.protocol.ClientProtocol"
     exclude "org.apache.hadoop.ipc.StandbyException"
   }
-  configurations << project.configurations.hdfs3
+  configurations.add(project.configurations.hdfs3)
 }
 
 def hdfs2Jar = tasks.register("hdfs2jar", ShadowJar) {
@@ -50,26 +104,15 @@ def hdfs2Jar = tasks.register("hdfs2jar", ShadowJar) {
   }
   archiveClassifier.set("hdfs2")
   from sourceSets.main.output
-  configurations << project.configurations.hdfs2
+  configurations.add(project.configurations.hdfs2)
 }
 
 tasks.withType(ShadowJar).configureEach {
   dependencies {
-//    exclude(dependency('commons-io:commons-io:2.8.0'))
     exclude(dependency("com.carrotsearch.randomizedtesting:randomizedtesting-runner:.*"))
     exclude(dependency("junit:junit:.*"))
-    exclude(dependency("org.slf4j:slf4j-api:.*"))
-    exclude(dependency("com.google.guava:guava:.*"))
-    exclude(dependency("org.apache.commons:commons-compress:.*"))
-    exclude(dependency("commons-logging:commons-logging:.*"))
-    exclude(dependency("commons-codec:commons-codec:.*"))
-    exclude(dependency("org.apache.httpcomponents:httpclient:.*"))
     exclude(dependency("org.apache.httpcomponents:httpcore:.*"))
     exclude(dependency("org.apache.logging.log4j:log4j-1.2-api:.*"))
-    exclude(dependency("log4j:log4j:.*"))
-    exclude(dependency("io.netty:.*:.*"))
-    exclude(dependency("com.nimbusds:nimbus-jose-jwt:.*"))
-    exclude(dependency("commons-cli:commons-cli:1.2"))
     exclude(dependency("net.java.dev.jna:jna:.*"))
     exclude(dependency("org.objenesis:objenesis:.*"))
     exclude(dependency('com.fasterxml.jackson.core:.*:.*'))
diff --git a/test/fixtures/krb5kdc-fixture/Dockerfile b/test/fixtures/krb5kdc-fixture/Dockerfile
index e862c7a71f2ba..47fc05d5aaf5b 100644
--- a/test/fixtures/krb5kdc-fixture/Dockerfile
+++ b/test/fixtures/krb5kdc-fixture/Dockerfile
@@ -1,9 +1,12 @@
-FROM ubuntu:14.04
-ADD . /fixture
+FROM alpine:3.21.0
+
+ADD src/main/resources /fixture
+RUN apk update && apk add -y --no-cache python3 krb5 krb5-server
+
 RUN echo kerberos.build.elastic.co > /etc/hostname
-RUN bash /fixture/src/main/resources/provision/installkdc.sh
+RUN sh /fixture/provision/installkdc.sh
 
 EXPOSE 88
 EXPOSE 88/udp
 
-CMD sleep infinity
+CMD ["sleep", "infinity"]
diff --git a/test/fixtures/krb5kdc-fixture/build.gradle b/test/fixtures/krb5kdc-fixture/build.gradle
index c9540011d80dd..887d6a2b68761 100644
--- a/test/fixtures/krb5kdc-fixture/build.gradle
+++ b/test/fixtures/krb5kdc-fixture/build.gradle
@@ -16,8 +16,8 @@ apply plugin: 'elasticsearch.deploy-test-fixtures'
 dockerFixtures {
   krb5dc {
     dockerContext = projectDir
-    version = "1.0"
-    baseImages = ["ubuntu:14.04"]
+    version = "1.1"
+    baseImages = ["alpine:3.21.0"]
   }
 }
 
diff --git a/test/fixtures/krb5kdc-fixture/src/main/java/org/elasticsearch/test/fixtures/krb5kdc/Krb5kDcContainer.java b/test/fixtures/krb5kdc-fixture/src/main/java/org/elasticsearch/test/fixtures/krb5kdc/Krb5kDcContainer.java
index cb1f86de51b1f..f44058d0ebcc4 100644
--- a/test/fixtures/krb5kdc-fixture/src/main/java/org/elasticsearch/test/fixtures/krb5kdc/Krb5kDcContainer.java
+++ b/test/fixtures/krb5kdc-fixture/src/main/java/org/elasticsearch/test/fixtures/krb5kdc/Krb5kDcContainer.java
@@ -29,7 +29,7 @@
 import java.util.List;
 
 public final class Krb5kDcContainer extends DockerEnvironmentAwareTestContainer {
-    public static final String DOCKER_BASE_IMAGE = "docker.elastic.co/elasticsearch-dev/krb5dc-fixture:1.0";
+    public static final String DOCKER_BASE_IMAGE = "docker.elastic.co/elasticsearch-dev/krb5dc-fixture:1.1";
     private final TemporaryFolder temporaryFolder = new TemporaryFolder();
     private final ProvisioningId provisioningId;
     private Path krb5ConfFile;
@@ -39,14 +39,14 @@ public final class Krb5kDcContainer extends DockerEnvironmentAwareTestContainer
     public enum ProvisioningId {
         HDFS(
             "hdfs",
-            "/fixture/src/main/resources/provision/hdfs.sh",
+            "/fixture/provision/hdfs.sh",
             "/fixture/build/keytabs/hdfs_hdfs.build.elastic.co.keytab",
             "/fixture/build/keytabs/elasticsearch.keytab",
             "hdfs/hdfs.build.elastic.co@BUILD.ELASTIC.CO"
         ),
         PEPPA(
             "peppa",
-            "/fixture/src/main/resources/provision/peppa.sh",
+            "/fixture/provision/peppa.sh",
             "/fixture/build/keytabs/peppa.keytab",
             "/fixture/build/keytabs/HTTP_localhost.keytab",
             "peppa@BUILD.ELASTIC.CO"
@@ -94,7 +94,7 @@ public Krb5kDcContainer(ProvisioningId provisioningId) {
         withNetworkAliases("kerberos.build.elastic.co", "build.elastic.co");
         withCopyFileToContainer(MountableFile.forHostPath("/dev/urandom"), "/dev/random");
         withExtraHost("kerberos.build.elastic.co", "127.0.0.1");
-        withCommand("bash", provisioningId.scriptPath);
+        withCommand("sh", provisioningId.scriptPath);
     }
 
     @Override
@@ -122,7 +122,7 @@ public String getConf() {
             .findFirst();
         String hostPortSpec = bindings.get().getHostPortSpec();
         String s = copyFileFromContainer("/fixture/build/krb5.conf.template", i -> IOUtils.toString(i, StandardCharsets.UTF_8));
-        return s.replace("${MAPPED_PORT}", hostPortSpec);
+        return s.replace("#KDC_DOCKER_HOST", "kdc = 127.0.0.1:" + hostPortSpec);
     }
 
     public Path getKeytab() {
diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/addprinc.sh b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/addprinc.sh
index 44bd7a841dedb..553bd8f85f70c 100755
--- a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/addprinc.sh
+++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/addprinc.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 
  # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  # or more contributor license agreements. Licensed under the "Elastic License
@@ -24,7 +24,7 @@ PASSWD="$2"
 USER=$(echo $PRINC | tr "/" "_")
 
 VDIR=/fixture
-RESOURCES=$VDIR/src/main/resources
+RESOURCES=$VDIR
 PROV_DIR=$RESOURCES/provision
 ENVPROP_FILE=$RESOURCES/env.properties
 BUILD_DIR=$VDIR/build
@@ -45,16 +45,16 @@ USER_KTAB=$LOCALSTATEDIR/$USER.keytab
 
 if [ -f $USER_KTAB ] && [ -z "$PASSWD" ]; then
   echo "Principal '${PRINC}@${REALM}' already exists. Re-copying keytab..."
-  sudo cp $USER_KTAB $KEYTAB_DIR/$USER.keytab
+  cp $USER_KTAB $KEYTAB_DIR/$USER.keytab
 else
   if [ -z "$PASSWD" ]; then
     echo "Provisioning '${PRINC}@${REALM}' principal and keytab..."
-    sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -randkey $USER_PRIN"
-    sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "ktadd -k $USER_KTAB $USER_PRIN"
-    sudo cp $USER_KTAB $KEYTAB_DIR/$USER.keytab
+    kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -randkey $USER_PRIN"
+    kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "ktadd -k $USER_KTAB $USER_PRIN"
+    cp $USER_KTAB $KEYTAB_DIR/$USER.keytab
   else
     echo "Provisioning '${PRINC}@${REALM}' principal with password..."
-    sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -pw $PASSWD $PRINC"
+    kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -pw $PASSWD $PRINC"
   fi
 fi
 
diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/hdfs.sh b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/hdfs.sh
index de08a52df3306..cf2eb5a1b7233 100644
--- a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/hdfs.sh
+++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/hdfs.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 
 set -e
 
diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/installkdc.sh b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/installkdc.sh
index 428747075ff36..a364349c56c68 100755
--- a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/installkdc.sh
+++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/installkdc.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 
  # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  # or more contributor license agreements. Licensed under the "Elastic License
@@ -12,8 +12,7 @@ set -e
 # KDC installation steps and considerations based on https://web.mit.edu/kerberos/krb5-latest/doc/admin/install_kdc.html
 # and helpful input from https://help.ubuntu.com/community/Kerberos
 
-VDIR=/fixture
-RESOURCES=$VDIR/src/main/resources
+RESOURCES=/fixture
 PROV_DIR=$RESOURCES/provision
 ENVPROP_FILE=$RESOURCES/env.properties
 LOCALSTATEDIR=/etc
@@ -49,33 +48,11 @@ touch $LOGDIR/kadmin.log
 touch $LOGDIR/krb5kdc.log
 touch $LOGDIR/krb5lib.log
 
-# Update package manager
-apt-get update -qqy
-
-# Installation asks a bunch of questions via debconf. Set the answers ahead of time
-debconf-set-selections <<< "krb5-config krb5-config/read_conf boolean true"
-debconf-set-selections <<< "krb5-config krb5-config/kerberos_servers string $KDC_NAME"
-debconf-set-selections <<< "krb5-config krb5-config/add_servers boolean true"
-debconf-set-selections <<< "krb5-config krb5-config/admin_server string $KDC_NAME"
-debconf-set-selections <<< "krb5-config krb5-config/add_servers_realm string $REALM_NAME"
-debconf-set-selections <<< "krb5-config krb5-config/default_realm string $REALM_NAME"
-debconf-set-selections <<< "krb5-admin-server krb5-admin-server/kadmind boolean true"
-debconf-set-selections <<< "krb5-admin-server krb5-admin-server/newrealm note"
-debconf-set-selections <<< "krb5-kdc krb5-kdc/debconf boolean true"
-debconf-set-selections <<< "krb5-kdc krb5-kdc/purge_data_too boolean false"
-
-# Install krb5 packages
-apt-get install -qqy krb5-{admin-server,kdc}
-
-# /dev/random produces output very slowly on Ubuntu VM's. Install haveged to increase entropy.
-apt-get install -qqy haveged
-haveged
-
 # Create kerberos database with stash file and garbage password
 kdb5_util create -s -r $REALM_NAME -P zyxwvutsrpqonmlk9876
 
 # Set up admin acls
-cat << EOF > /etc/krb5kdc/kadm5.acl
+cat << EOF > /var/lib/krb5kdc/kadm5.acl
 */admin@$REALM_NAME	*
 */*@$REALM_NAME		i
 EOF
diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template
index b66709968839a..e79caecbcf334 100644
--- a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template
+++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template
@@ -6,6 +6,7 @@
  # License v3.0 only", or the "Server Side Public License, v 1".
 
 [libdefaults]
+    spake_preauth_groups = edwards25519
     default_realm = ${REALM_NAME}
     dns_canonicalize_hostname = false
     dns_lookup_kdc = false
@@ -25,7 +26,7 @@
 [realms]
     ${REALM_NAME} = {
         kdc = 127.0.0.1:88
-        kdc = 127.0.0.1:${MAPPED_PORT}
+        #KDC_DOCKER_HOST
         admin_server = ${KDC_NAME}:749
         default_domain = ${BUILD_ZONE}
     }
diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/peppa.sh b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/peppa.sh
index da6480d891af7..24179da5882c7 100644
--- a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/peppa.sh
+++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/peppa.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 
 set -e
 
diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java
index 3ee18d71a5a79..56e702d2f76a0 100644
--- a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java
+++ b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java
@@ -15,7 +15,7 @@
 public final class MinioTestContainer extends DockerEnvironmentAwareTestContainer {
 
     private static final int servicePort = 9000;
-    public static final String DOCKER_BASE_IMAGE = "minio/minio:RELEASE.2021-03-01T04-20-55Z";
+    public static final String DOCKER_BASE_IMAGE = "minio/minio:RELEASE.2024-12-18T13-15-44Z";
     private final boolean enabled;
 
     public MinioTestContainer(boolean enabled, String accessKey, String secretKey, String bucketName) {
diff --git a/test/fixtures/s3-fixture/build.gradle b/test/fixtures/s3-fixture/build.gradle
index e4c35464608a8..c7792ce963a8d 100644
--- a/test/fixtures/s3-fixture/build.gradle
+++ b/test/fixtures/s3-fixture/build.gradle
@@ -16,4 +16,5 @@ dependencies {
     transitive = false
   }
   implementation project(':test:framework')
+  implementation project(':test:fixtures:aws-fixture-utils')
 }
diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixture.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixture.java
index ab70f043043cc..538c57bc2bdea 100644
--- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixture.java
+++ b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixture.java
@@ -13,16 +13,15 @@
 import com.sun.net.httpserver.HttpServer;
 
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.rest.RestStatus;
 import org.junit.rules.ExternalResource;
 
 import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
 import java.util.Objects;
 import java.util.function.BiPredicate;
-import java.util.function.Supplier;
+
+import static fixture.aws.AwsCredentialsUtils.checkAuthorization;
+import static fixture.aws.AwsCredentialsUtils.fixedAccessKey;
+import static fixture.aws.AwsFixtureUtils.getLocalFixtureAddress;
 
 public class S3HttpFixture extends ExternalResource {
 
@@ -49,14 +48,9 @@ protected HttpHandler createHandler() {
             @Override
             public void handle(final HttpExchange exchange) throws IOException {
                 try {
-                    if (authorizationPredicate.test(
-                        exchange.getRequestHeaders().getFirst("Authorization"),
-                        exchange.getRequestHeaders().getFirst("x-amz-security-token")
-                    ) == false) {
-                        sendError(exchange, RestStatus.FORBIDDEN, "AccessDenied", "Access denied by " + authorizationPredicate);
-                        return;
+                    if (checkAuthorization(authorizationPredicate, exchange)) {
+                        super.handle(exchange);
                     }
-                    super.handle(exchange);
                 } catch (Error e) {
                     // HttpServer catches Throwable, so we must throw errors on another thread
                     ExceptionsHelper.maybeDieOnAnotherThread(e);
@@ -76,10 +70,8 @@ public void stop(int delay) {
 
     protected void before() throws Throwable {
         if (enabled) {
-            InetSocketAddress inetSocketAddress = resolveAddress();
-            this.server = HttpServer.create(inetSocketAddress, 0);
-            HttpHandler handler = createHandler();
-            this.server.createContext("/", Objects.requireNonNull(handler));
+            this.server = HttpServer.create(getLocalFixtureAddress(), 0);
+            this.server.createContext("/", Objects.requireNonNull(createHandler()));
             server.start();
         }
     }
@@ -90,28 +82,4 @@ protected void after() {
             stop(0);
         }
     }
-
-    private static InetSocketAddress resolveAddress() {
-        try {
-            return new InetSocketAddress(InetAddress.getByName("localhost"), 0);
-        } catch (UnknownHostException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    public static BiPredicate<String, String> fixedAccessKey(String accessKey) {
-        return mutableAccessKey(() -> accessKey);
-    }
-
-    public static BiPredicate<String, String> mutableAccessKey(Supplier<String> accessKeySupplier) {
-        return (authorizationHeader, sessionTokenHeader) -> authorizationHeader != null
-            && authorizationHeader.contains(accessKeySupplier.get());
-    }
-
-    public static BiPredicate<String, String> fixedAccessKeyAndToken(String accessKey, String sessionToken) {
-        Objects.requireNonNull(sessionToken);
-        final var accessKeyPredicate = fixedAccessKey(accessKey);
-        return (authorizationHeader, sessionTokenHeader) -> accessKeyPredicate.test(authorizationHeader, sessionTokenHeader)
-            && sessionToken.equals(sessionTokenHeader);
-    }
 }
diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java
index bfc0428731c56..8477117305419 100644
--- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java
+++ b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java
@@ -8,7 +8,6 @@
  */
 package fixture.s3;
 
-import com.sun.net.httpserver.Headers;
 import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpHandler;
 
@@ -520,37 +519,6 @@ static List<String> extractPartEtags(BytesReference completeMultipartUploadBody)
         }
     }
 
-    public static void sendError(final HttpExchange exchange, final RestStatus status, final String errorCode, final String message)
-        throws IOException {
-        final Headers headers = exchange.getResponseHeaders();
-        headers.add("Content-Type", "application/xml");
-
-        final String requestId = exchange.getRequestHeaders().getFirst("x-amz-request-id");
-        if (requestId != null) {
-            headers.add("x-amz-request-id", requestId);
-        }
-
-        if (errorCode == null || "HEAD".equals(exchange.getRequestMethod())) {
-            exchange.sendResponseHeaders(status.getStatus(), -1L);
-            exchange.close();
-        } else {
-            final byte[] response = ("<?xml version=\"1.0\" encoding=\"UTF-8\"?><Error>"
-                + "<Code>"
-                + errorCode
-                + "</Code>"
-                + "<Message>"
-                + message
-                + "</Message>"
-                + "<RequestId>"
-                + requestId
-                + "</RequestId>"
-                + "</Error>").getBytes(StandardCharsets.UTF_8);
-            exchange.sendResponseHeaders(status.getStatus(), response.length);
-            exchange.getResponseBody().write(response);
-            exchange.close();
-        }
-    }
-
     MultipartUpload getUpload(String uploadId) {
         return uploads.get(uploadId);
     }
diff --git a/test/framework/src/integTest/java/org/elasticsearch/test/test/InternalClusterForbiddenSettingIT.java b/test/framework/src/integTest/java/org/elasticsearch/test/test/InternalClusterForbiddenSettingIT.java
index d13450fbb52dd..2033743354f34 100644
--- a/test/framework/src/integTest/java/org/elasticsearch/test/test/InternalClusterForbiddenSettingIT.java
+++ b/test/framework/src/integTest/java/org/elasticsearch/test/test/InternalClusterForbiddenSettingIT.java
@@ -26,7 +26,7 @@ protected boolean forbidPrivateIndexSettings() {
     }
 
     public void testRestart() throws Exception {
-        IndexVersion version = IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersion.current());
+        IndexVersion version = IndexVersionUtils.randomPreviousCompatibleWriteVersion(random(), IndexVersion.current());
         // create / delete an index with forbidden setting
         prepareCreate("test").setSettings(settings(version).build()).get();
         indicesAdmin().prepareDelete("test").get();
@@ -38,7 +38,7 @@ public void testRestart() throws Exception {
     }
 
     public void testRollingRestart() throws Exception {
-        IndexVersion version = IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersion.current());
+        IndexVersion version = IndexVersionUtils.randomPreviousCompatibleWriteVersion(random(), IndexVersion.current());
         // create / delete an index with forbidden setting
         prepareCreate("test").setSettings(settings(version).build()).get();
         indicesAdmin().prepareDelete("test").get();
diff --git a/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
index a11291e846ec1..950c54ddb1d22 100644
--- a/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
@@ -10,7 +10,6 @@
 package org.elasticsearch.action.support.replication;
 
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.TransportVersions;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -151,7 +150,7 @@ public static ClusterState state(
             .settings(indexSettings(IndexVersion.current(), 1, numberOfReplicas).put(SETTING_CREATION_DATE, System.currentTimeMillis()))
             .primaryTerm(0, primaryTerm)
             .timestampRange(timeFieldRange)
-            .eventIngestedRange(timeFieldRange, timeFieldRange == IndexLongFieldRange.UNKNOWN ? null : TransportVersions.V_8_15_0)
+            .eventIngestedRange(timeFieldRange)
             .build();
 
         IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
@@ -284,7 +283,7 @@ public static ClusterState state(final int numberOfNodes, final String[] indices
                 .settings(
                     indexSettings(IndexVersion.current(), numberOfPrimaries, 0).put(SETTING_CREATION_DATE, System.currentTimeMillis())
                 )
-                .eventIngestedRange(IndexLongFieldRange.UNKNOWN, randomFrom(TransportVersions.V_8_0_0, TransportVersions.V_8_15_0))
+                .eventIngestedRange(IndexLongFieldRange.UNKNOWN)
                 .build();
 
             IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(indexMetadata.getIndex());
@@ -390,7 +389,7 @@ public static ClusterState stateWithAssignedPrimariesAndReplicas(
                     )
                 )
                 .timestampRange(IndexLongFieldRange.UNKNOWN)
-                .eventIngestedRange(IndexLongFieldRange.UNKNOWN, null)
+                .eventIngestedRange(IndexLongFieldRange.UNKNOWN)
                 .build();
             metadataBuilder.put(indexMetadata, false).generateClusterUuidIfNeeded();
             IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(indexMetadata.getIndex());
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
index 75cd6da44724d..132dd4b119469 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.FailedShard;
-import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsProvider;
+import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
@@ -437,18 +437,16 @@ public void allocateUnassigned(
         }
     }
 
-    protected static final NodeAllocationStatsProvider EMPTY_NODE_ALLOCATION_STATS = new NodeAllocationStatsProvider(
-        WriteLoadForecaster.DEFAULT,
-        createBuiltInClusterSettings()
-    ) {
-        @Override
-        public Map<String, NodeAllocationAndClusterBalanceStats> stats(
-            Metadata metadata,
-            RoutingNodes routingNodes,
-            ClusterInfo clusterInfo,
-            @Nullable DesiredBalance desiredBalance
-        ) {
-            return Map.of();
-        }
-    };
+    protected static final NodeAllocationStatsAndWeightsCalculator EMPTY_NODE_ALLOCATION_STATS =
+        new NodeAllocationStatsAndWeightsCalculator(WriteLoadForecaster.DEFAULT, createBuiltInClusterSettings()) {
+            @Override
+            public Map<String, NodeAllocationStatsAndWeight> nodesAllocationStatsAndWeights(
+                Metadata metadata,
+                RoutingNodes routingNodes,
+                ClusterInfo clusterInfo,
+                @Nullable DesiredBalance desiredBalance
+            ) {
+                return Map.of();
+            }
+        };
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java
index 06e4e486e78b5..c3ce32d4ce333 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java
@@ -146,13 +146,35 @@ public static DataStream newInstance(
         boolean replicated,
         @Nullable DataStreamLifecycle lifecycle,
         List<Index> failureStores
+    ) {
+        return newInstance(
+            name,
+            indices,
+            generation,
+            metadata,
+            replicated,
+            lifecycle,
+            failureStores,
+            failureStores.isEmpty() ? DataStreamOptions.EMPTY : DataStreamOptions.FAILURE_STORE_ENABLED
+        );
+    }
+
+    public static DataStream newInstance(
+        String name,
+        List<Index> indices,
+        long generation,
+        Map<String, Object> metadata,
+        boolean replicated,
+        DataStreamLifecycle lifecycle,
+        List<Index> failureStores,
+        DataStreamOptions dataStreamOptions
     ) {
         return DataStream.builder(name, indices)
             .setGeneration(generation)
             .setMetadata(metadata)
             .setReplicated(replicated)
             .setLifecycle(lifecycle)
-            .setDataStreamOptions(failureStores.isEmpty() ? DataStreamOptions.EMPTY : DataStreamOptions.FAILURE_STORE_ENABLED)
+            .setDataStreamOptions(dataStreamOptions)
             .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build())
             .build();
     }
@@ -457,7 +479,7 @@ public static void getClusterStateWithDataStreams(
         Settings settings,
         int replicas,
         boolean replicated,
-        boolean storeFailures
+        Boolean storeFailures
     ) {
         builder.put(
             "template_1",
@@ -466,7 +488,7 @@ public static void getClusterStateWithDataStreams(
                 .template(
                     Template.builder()
                         .dataStreamOptions(
-                            DataStream.isFailureStoreFeatureFlagEnabled() && storeFailures ? createDataStreamOptionsTemplate(true) : null
+                            DataStream.isFailureStoreFeatureFlagEnabled() ? createDataStreamOptionsTemplate(storeFailures) : null
                         )
                 )
                 .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
@@ -484,7 +506,7 @@ public static void getClusterStateWithDataStreams(
             allIndices.addAll(backingIndices);
 
             List<IndexMetadata> failureStores = new ArrayList<>();
-            if (DataStream.isFailureStoreFeatureFlagEnabled() && storeFailures) {
+            if (DataStream.isFailureStoreFeatureFlagEnabled() && Boolean.TRUE.equals(storeFailures)) {
                 for (int failureStoreNumber = 1; failureStoreNumber <= dsTuple.v2(); failureStoreNumber++) {
                     failureStores.add(
                         createIndexMetadata(
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeUtils.java b/test/framework/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeUtils.java
index 64f8fa88762b8..20368753eac1d 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeUtils.java
@@ -76,6 +76,7 @@ public static class Builder {
         private BuildVersion buildVersion;
         private Version version;
         private IndexVersion minIndexVersion;
+        private IndexVersion minReadOnlyIndexVersion;
         private IndexVersion maxIndexVersion;
         private String externalId;
 
@@ -125,16 +126,23 @@ public Builder version(Version version, IndexVersion minIndexVersion, IndexVersi
             this.buildVersion = BuildVersion.fromVersionId(version.id());
             this.version = version;
             this.minIndexVersion = minIndexVersion;
+            this.minReadOnlyIndexVersion = minIndexVersion;
             this.maxIndexVersion = maxIndexVersion;
             return this;
         }
 
-        public Builder version(BuildVersion version, IndexVersion minIndexVersion, IndexVersion maxIndexVersion) {
+        public Builder version(
+            BuildVersion version,
+            IndexVersion minIndexVersion,
+            IndexVersion minReadOnlyIndexVersion,
+            IndexVersion maxIndexVersion
+        ) {
             // see comment in VersionInformation
             assert version.equals(BuildVersion.current());
             this.buildVersion = version;
             this.version = Version.CURRENT;
             this.minIndexVersion = minIndexVersion;
+            this.minReadOnlyIndexVersion = minReadOnlyIndexVersion;
             this.maxIndexVersion = maxIndexVersion;
             return this;
         }
@@ -143,6 +151,7 @@ public Builder version(VersionInformation versions) {
             this.buildVersion = versions.buildVersion();
             this.version = versions.nodeVersion();
             this.minIndexVersion = versions.minIndexVersion();
+            this.minReadOnlyIndexVersion = versions.minReadOnlyIndexVersion();
             this.maxIndexVersion = versions.maxIndexVersion();
             return this;
         }
@@ -170,10 +179,10 @@ public DiscoveryNode build() {
             }
 
             VersionInformation versionInfo;
-            if (minIndexVersion == null || maxIndexVersion == null) {
+            if (minIndexVersion == null || minReadOnlyIndexVersion == null || maxIndexVersion == null) {
                 versionInfo = VersionInformation.inferVersions(version);
             } else {
-                versionInfo = new VersionInformation(buildVersion, version, minIndexVersion, maxIndexVersion);
+                versionInfo = new VersionInformation(buildVersion, version, minIndexVersion, minReadOnlyIndexVersion, maxIndexVersion);
             }
 
             return new DiscoveryNode(name, id, ephemeralId, hostName, hostAddress, address, attributes, roles, versionInfo, externalId);
diff --git a/test/framework/src/main/java/org/elasticsearch/index/KnownIndexVersions.java b/test/framework/src/main/java/org/elasticsearch/index/KnownIndexVersions.java
index 47d239540814e..4f559a5f3eaef 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/KnownIndexVersions.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/KnownIndexVersions.java
@@ -16,7 +16,12 @@
  */
 public class KnownIndexVersions {
     /**
-     * A sorted list of all known transport versions
+     * A sorted list of all known index versions
      */
     public static final List<IndexVersion> ALL_VERSIONS = List.copyOf(IndexVersions.getAllVersions());
+
+    /**
+     * A sorted list of all known index versions that can be written to
+     */
+    public static final List<IndexVersion> ALL_WRITE_VERSIONS = List.copyOf(IndexVersions.getAllWriteVersions());
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
index 46f6a0b503bfb..9a160fffb965c 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
@@ -141,7 +141,6 @@
 import static org.elasticsearch.index.engine.Engine.Operation.Origin.PEER_RECOVERY;
 import static org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY;
 import static org.elasticsearch.index.engine.Engine.Operation.Origin.REPLICA;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -458,7 +457,13 @@ protected static ParsedDocument testParsedDocument(
     }
 
     public static ParsedDocument parseDocument(MapperService mapperService, String id, String routing) {
-        SourceToParse sourceToParse = new SourceToParse(id, new BytesArray("{ \"value\" : \"test\" }"), XContentType.JSON, routing);
+        String source = randomFrom(
+            "{ \"value\" : \"test-1\" }",
+            "{ \"value\" : [\"test-1\",\"test-2\"] }",
+            "{ \"value\" : [\"test-2\",\"test-1\"] }",
+            "{ \"value\" : [\"test-1\",\"test-2\",\"test-2\"] }"
+        );
+        SourceToParse sourceToParse = new SourceToParse(id, new BytesArray(source), XContentType.JSON, routing);
         return mapperService.documentMapper().parse(sourceToParse);
     }
 
@@ -492,7 +497,8 @@ protected Translog createTranslog(Path translogPath, LongSupplier primaryTermSup
             new TranslogDeletionPolicy(),
             () -> SequenceNumbers.NO_OPS_PERFORMED,
             primaryTermSupplier,
-            seqNo -> {}
+            seqNo -> {},
+            TranslogOperationAsserter.DEFAULT
         );
     }
 
@@ -1072,10 +1078,9 @@ public List<Engine.Operation> generateHistoryOnReplica(
             final int nestedValues = between(0, 3);
             final long startTime = threadPool.relativeTimeInNanos();
             final int copies = allowDuplicate && rarely() ? between(2, 4) : 1;
+            final var nonNestedDoc = parseDocument(mapperService, id, null);
             for (int copy = 0; copy < copies; copy++) {
-                final ParsedDocument doc = isNestedDoc
-                    ? nestedParsedDocFactory.apply(id, nestedValues)
-                    : parseDocument(engine.engineConfig.getMapperService(), id, null);
+                final ParsedDocument doc = isNestedDoc ? nestedParsedDocFactory.apply(id, nestedValues) : nonNestedDoc;
                 switch (opType) {
                     case INDEX -> operations.add(
                         new Engine.Index(
@@ -1345,6 +1350,7 @@ public static void assertConsistentHistoryBetweenTranslogAndLuceneIndex(Engine e
         } else {
             minSeqNoToRetain = engine.getMinRetainedSeqNo();
         }
+        TranslogOperationAsserter translogOperationAsserter = TranslogOperationAsserter.withEngineConfig(engine.engineConfig);
         for (Translog.Operation translogOp : translogOps) {
             final Translog.Operation luceneOp = luceneOps.get(translogOp.seqNo());
             if (luceneOp == null) {
@@ -1372,10 +1378,9 @@ public static void assertConsistentHistoryBetweenTranslogAndLuceneIndex(Engine e
             assertThat(luceneOp.opType(), equalTo(translogOp.opType()));
             if (luceneOp.opType() == Translog.Operation.Type.INDEX) {
                 if (engine.engineConfig.getIndexSettings().isRecoverySourceSyntheticEnabled()) {
-                    assertToXContentEquivalent(
-                        ((Translog.Index) luceneOp).source(),
-                        ((Translog.Index) translogOp).source(),
-                        XContentFactory.xContentType(((Translog.Index) luceneOp).source().array())
+                    assertTrue(
+                        "luceneOp=" + luceneOp + " != translogOp=" + translogOp,
+                        translogOperationAsserter.assertSameIndexOperation((Translog.Index) luceneOp, (Translog.Index) translogOp)
                     );
                 } else {
                     assertThat(((Translog.Index) luceneOp).source(), equalTo(((Translog.Index) translogOp).source()));
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java
new file mode 100644
index 0000000000000..8f5478e1181f1
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.fieldvisitor.StoredFieldLoader;
+import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification;
+import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+import org.elasticsearch.logsdb.datageneration.MappingGenerator;
+import org.elasticsearch.logsdb.datageneration.Template;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
+import org.elasticsearch.search.fetch.StoredFieldsSpec;
+import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public abstract class BlockLoaderTestCase extends MapperServiceTestCase {
+    private final String fieldName;
+    private final Template template;
+    private final MappingGenerator mappingGenerator;
+    private final FieldDataGenerator generator;
+
+    protected BlockLoaderTestCase(FieldType fieldType) {
+        this.fieldName = randomAlphaOfLengthBetween(5, 10);
+
+        // Disable all dynamic mapping
+        var specification = DataGeneratorSpecification.builder()
+            .withFullyDynamicMapping(false)
+            .withDataSourceHandlers(List.of(new DataSourceHandler() {
+                @Override
+                public DataSourceResponse.DynamicMappingGenerator handle(DataSourceRequest.DynamicMappingGenerator request) {
+                    return new DataSourceResponse.DynamicMappingGenerator(isObject -> false);
+                }
+            }))
+            .build();
+
+        this.template = new Template(Map.of(fieldName, new Template.Leaf(fieldName, fieldType)));
+        this.mappingGenerator = new MappingGenerator(specification);
+        this.generator = fieldType.generator(fieldName, specification.dataSource());
+    }
+
+    public void testBlockLoader() throws IOException {
+        var mapping = mappingGenerator.generate(template);
+        var mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent()).map(mapping.raw());
+
+        var syntheticSource = randomBoolean();
+        var mapperService = syntheticSource ? createSytheticSourceMapperService(mappingXContent) : createMapperService(mappingXContent);
+
+        var fieldValue = generator.generateValue();
+
+        Object blockLoaderResult = setupAndInvokeBlockLoader(mapperService, fieldValue);
+        Object expected = expected(mapping.lookup().get(fieldName), fieldValue, syntheticSource);
+        assertEquals(expected, blockLoaderResult);
+    }
+
+    protected abstract Object expected(Map<String, Object> fieldMapping, Object value, boolean syntheticSource);
+
+    private Object setupAndInvokeBlockLoader(MapperService mapperService, Object fieldValue) throws IOException {
+        try (Directory directory = newDirectory()) {
+            RandomIndexWriter iw = new RandomIndexWriter(random(), directory);
+
+            LuceneDocument doc = mapperService.documentMapper().parse(source(b -> {
+                b.field(fieldName);
+                b.value(fieldValue);
+            })).rootDoc();
+
+            iw.addDocument(doc);
+            iw.close();
+
+            try (DirectoryReader reader = DirectoryReader.open(directory)) {
+                LeafReaderContext context = reader.leaves().get(0);
+                return load(createBlockLoader(mapperService), context, mapperService);
+            }
+        }
+    }
+
+    private Object load(BlockLoader blockLoader, LeafReaderContext context, MapperService mapperService) throws IOException {
+        // `columnAtATimeReader` is tried first, we mimic `ValuesSourceReaderOperator`
+        var columnAtATimeReader = blockLoader.columnAtATimeReader(context);
+        if (columnAtATimeReader != null) {
+            var block = (TestBlock) columnAtATimeReader.read(TestBlock.factory(context.reader().numDocs()), TestBlock.docs(0));
+            return block.get(0);
+        }
+
+        StoredFieldsSpec storedFieldsSpec = blockLoader.rowStrideStoredFieldSpec();
+        SourceLoader.Leaf leafSourceLoader = null;
+        if (storedFieldsSpec.requiresSource()) {
+            var sourceLoader = mapperService.mappingLookup().newSourceLoader(null, SourceFieldMetrics.NOOP);
+            leafSourceLoader = sourceLoader.leaf(context.reader(), null);
+            storedFieldsSpec = storedFieldsSpec.merge(
+                new StoredFieldsSpec(true, storedFieldsSpec.requiresMetadata(), sourceLoader.requiredStoredFields())
+            );
+        }
+        BlockLoaderStoredFieldsFromLeafLoader storedFieldsLoader = new BlockLoaderStoredFieldsFromLeafLoader(
+            StoredFieldLoader.fromSpec(storedFieldsSpec).getLoader(context, null),
+            leafSourceLoader
+        );
+        storedFieldsLoader.advanceTo(0);
+
+        BlockLoader.Builder builder = blockLoader.builder(TestBlock.factory(context.reader().numDocs()), 1);
+        blockLoader.rowStrideReader(context).read(0, storedFieldsLoader, builder);
+        var block = (TestBlock) builder.build();
+        return block.get(0);
+    }
+
+    private BlockLoader createBlockLoader(MapperService mapperService) {
+        SearchLookup searchLookup = new SearchLookup(mapperService.mappingLookup().fieldTypesLookup()::get, null, null);
+
+        return mapperService.fieldType(fieldName).blockLoader(new MappedFieldType.BlockLoaderContext() {
+            @Override
+            public String indexName() {
+                return mapperService.getIndexSettings().getIndex().getName();
+            }
+
+            @Override
+            public IndexSettings indexSettings() {
+                return mapperService.getIndexSettings();
+            }
+
+            @Override
+            public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
+                // TODO randomize when adding support for fields that care about this
+                return MappedFieldType.FieldExtractPreference.NONE;
+            }
+
+            @Override
+            public SearchLookup lookup() {
+                return searchLookup;
+            }
+
+            @Override
+            public Set<String> sourcePaths(String name) {
+                return mapperService.mappingLookup().sourcePaths(name);
+            }
+
+            @Override
+            public String parentField(String field) {
+                return mapperService.mappingLookup().parentField(field);
+            }
+
+            @Override
+            public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
+                return (FieldNamesFieldMapper.FieldNamesFieldType) mapperService.fieldType(FieldNamesFieldMapper.NAME);
+            }
+        });
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java
index 2da2c5a08c177..b33abc2d781e8 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java
@@ -51,6 +51,7 @@
 import org.elasticsearch.index.fielddata.LeafFieldData;
 import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader;
 import org.elasticsearch.index.fieldvisitor.StoredFieldLoader;
+import org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.termvectors.TermVectorsService;
 import org.elasticsearch.index.translog.Translog;
@@ -87,8 +88,6 @@
 import java.util.stream.IntStream;
 
 import static java.util.stream.Collectors.toList;
-import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.DOC_VALUES;
-import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.NONE;
 import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.contains;
@@ -1181,7 +1180,7 @@ private void assertSyntheticSourceWithTranslogSnapshot(SyntheticSourceSupport su
         var firstExample = support.example(1);
         int maxDocs = randomIntBetween(20, 50);
         var settings = Settings.builder()
-            .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
+            .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
             .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true)
             .build();
         var mapperService = createMapperService(getVersion(), settings, () -> true, mapping(b -> {
@@ -1420,7 +1419,7 @@ public BlockReaderSupport(boolean columnAtATimeReader, MapperService mapper, Str
             this(columnAtATimeReader, true, mapper, loaderFieldName);
         }
 
-        private BlockLoader getBlockLoader(boolean columnReader) {
+        private BlockLoader getBlockLoader(FieldExtractPreference fieldExtractPreference) {
             SearchLookup searchLookup = new SearchLookup(mapper.mappingLookup().fieldTypesLookup()::get, null, null);
             return mapper.fieldType(loaderFieldName).blockLoader(new MappedFieldType.BlockLoaderContext() {
                 @Override
@@ -1434,8 +1433,8 @@ public IndexSettings indexSettings() {
                 }
 
                 @Override
-                public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
-                    return columnReader ? DOC_VALUES : NONE;
+                public FieldExtractPreference fieldExtractPreference() {
+                    return fieldExtractPreference;
                 }
 
                 @Override
@@ -1493,7 +1492,9 @@ protected final void testBlockLoader(
         BlockReaderSupport blockReaderSupport,
         SourceLoader sourceLoader
     ) throws IOException {
-        BlockLoader loader = blockReaderSupport.getBlockLoader(columnReader);
+        // EXTRACT_SPATIAL_BOUNDS is not currently supported in this test path.
+        var fieldExtractPreference = columnReader ? FieldExtractPreference.DOC_VALUES : FieldExtractPreference.NONE;
+        BlockLoader loader = blockReaderSupport.getBlockLoader(fieldExtractPreference);
         Function<Object, Object> valuesConvert = loadBlockExpected(blockReaderSupport, columnReader);
         if (valuesConvert == null) {
             assertNull(loader);
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MetadataMapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MetadataMapperTestCase.java
index 449ecc099412f..580eb6eacb27e 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MetadataMapperTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MetadataMapperTestCase.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.test.index.IndexVersionUtils;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -142,4 +143,88 @@ public final void testFixedMetaFieldsAreNotConfigurable() throws IOException {
         );
         assertEquals("Failed to parse mapping: " + fieldName() + " is not configurable", exception.getMessage());
     }
+
+    public void testTypeAndFriendsAreAcceptedBefore_8_6_0() throws IOException {
+        assumeTrue("Metadata field " + fieldName() + " isn't configurable", isConfigurable());
+        IndexVersion previousVersion = IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_6_0);
+        // we randomly also pick read-only versions to test that we can still parse the parameters for them
+        IndexVersion version = IndexVersionUtils.randomVersionBetween(
+            random(),
+            IndexVersionUtils.getLowestReadCompatibleVersion(),
+            previousVersion
+        );
+        assumeTrue("Metadata field " + fieldName() + " is not supported on version " + version, isSupportedOn(version));
+        MapperService mapperService = createMapperService(version, mapping(b -> {}));
+        // these parameters were previously silently ignored, they will still be ignored in existing indices
+        String[] unsupportedParameters = new String[] { "fields", "copy_to", "boost", "type" };
+        for (String param : unsupportedParameters) {
+            String mappingAsString = "{\n"
+                + "    \"_doc\" : {\n"
+                + "      \""
+                + fieldName()
+                + "\" : {\n"
+                + "        \""
+                + param
+                + "\" : \"any\"\n"
+                + "      }\n"
+                + "    }\n"
+                + "}";
+            assertNotNull(mapperService.parseMapping("_doc", MergeReason.MAPPING_UPDATE, new CompressedXContent(mappingAsString)));
+        }
+    }
+
+    public void testTypeAndFriendsAreDeprecatedFrom_8_6_0_TO_9_0_0() throws IOException {
+        assumeTrue("Metadata field " + fieldName() + " isn't configurable", isConfigurable());
+        IndexVersion previousVersion = IndexVersionUtils.getPreviousVersion(IndexVersions.UPGRADE_TO_LUCENE_10_0_0);
+        IndexVersion version = IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_8_6_0, previousVersion);
+        assumeTrue("Metadata field " + fieldName() + " is not supported on version " + version, isSupportedOn(version));
+        MapperService mapperService = createMapperService(version, mapping(b -> {}));
+        // these parameters were deprecated, they now should throw an error in new indices
+        String[] unsupportedParameters = new String[] { "fields", "copy_to", "boost", "type" };
+        for (String param : unsupportedParameters) {
+            String mappingAsString = "{\n"
+                + "    \"_doc\" : {\n"
+                + "      \""
+                + fieldName()
+                + "\" : {\n"
+                + "        \""
+                + param
+                + "\" : \"any\"\n"
+                + "      }\n"
+                + "    }\n"
+                + "}";
+            assertNotNull(mapperService.parseMapping("_doc", MergeReason.MAPPING_UPDATE, new CompressedXContent(mappingAsString)));
+            assertWarnings("Parameter [" + param + "] has no effect on metadata field [" + fieldName() + "] and will be removed in future");
+        }
+    }
+
+    public void testTypeAndFriendsThrow_After_9_0_0() throws IOException {
+        assumeTrue("Metadata field " + fieldName() + " isn't configurable", isConfigurable());
+        IndexVersion version = IndexVersionUtils.randomVersionBetween(
+            random(),
+            IndexVersions.UPGRADE_TO_LUCENE_10_0_0,
+            IndexVersion.current()
+        );
+        assumeTrue("Metadata field " + fieldName() + " is not supported on version " + version, isSupportedOn(version));
+        MapperService mapperService = createMapperService(version, mapping(b -> {}));
+        // these parameters were previously silently ignored, they are now deprecated in new indices
+        String[] unsupportedParameters = new String[] { "fields", "copy_to", "boost", "type" };
+        for (String param : unsupportedParameters) {
+            String mappingAsString = "{\n"
+                + "    \"_doc\" : {\n"
+                + "      \""
+                + fieldName()
+                + "\" : {\n"
+                + "        \""
+                + param
+                + "\" : \"any\"\n"
+                + "      }\n"
+                + "    }\n"
+                + "}";
+            expectThrows(
+                MapperParsingException.class,
+                () -> mapperService.parseMapping("_doc", MergeReason.MAPPING_UPDATE, new CompressedXContent(mappingAsString))
+            );
+        }
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
index 5b9a9e7a4efee..2ae4bb0343101 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
@@ -993,12 +993,23 @@ protected Engine.IndexResult indexDoc(IndexShard shard, String type, String id)
     }
 
     protected Engine.IndexResult indexDoc(IndexShard shard, String type, String id, String source) throws IOException {
-        return indexDoc(shard, id, source, XContentType.JSON, null);
+        return indexDoc(shard, id, Versions.MATCH_ANY, source, XContentType.JSON, null);
     }
 
-    // Uses an auto-generated ID if `id` is null/empty
     protected Engine.IndexResult indexDoc(IndexShard shard, String id, String source, XContentType xContentType, String routing)
         throws IOException {
+        return indexDoc(shard, id, Versions.MATCH_ANY, source, xContentType, routing);
+    }
+
+    // Uses an auto-generated ID if `id` is null/empty
+    protected Engine.IndexResult indexDoc(
+        IndexShard shard,
+        String id,
+        long version,
+        String source,
+        XContentType xContentType,
+        String routing
+    ) throws IOException {
         long autoGeneratedTimestamp = IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP;
         if (Strings.isEmpty(id)) {
             id = UUIDs.base64UUID();
@@ -1008,7 +1019,7 @@ protected Engine.IndexResult indexDoc(IndexShard shard, String id, String source
         Engine.IndexResult result;
         if (shard.routingEntry().primary()) {
             result = shard.applyIndexOperationOnPrimary(
-                Versions.MATCH_ANY,
+                version,
                 VersionType.INTERNAL,
                 sourceToParse,
                 SequenceNumbers.UNASSIGNED_SEQ_NO,
@@ -1024,7 +1035,7 @@ protected Engine.IndexResult indexDoc(IndexShard shard, String id, String source
                         .build()
                 );
                 result = shard.applyIndexOperationOnPrimary(
-                    Versions.MATCH_ANY,
+                    version,
                     VersionType.INTERNAL,
                     sourceToParse,
                     SequenceNumbers.UNASSIGNED_SEQ_NO,
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java
index 9bb2d34958a27..a46e07446052a 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java
@@ -19,9 +19,9 @@
  * @param template actual template data
  */
 public record Template(Map<String, Entry> template) {
-    sealed interface Entry permits Leaf, Object {}
+    public sealed interface Entry permits Leaf, Object {}
 
-    record Leaf(String name, FieldType type) implements Entry {}
+    public record Leaf(String name, FieldType type) implements Entry {}
 
-    record Object(String name, boolean nested, Map<String, Entry> children) implements Entry {}
+    public record Object(String name, boolean nested, Map<String, Entry> children) implements Entry {}
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java
index 4b33f3fefcf1b..b639108ea6ad2 100644
--- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java
@@ -65,6 +65,10 @@ private Supplier<Map<String, Object>> keywordMapping(
                 }
             }
 
+            if (ESTestCase.randomDouble() <= 0.2) {
+                injected.put("ignore_above", ESTestCase.randomIntBetween(1, 100));
+            }
+
             return injected;
         };
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index f057d35d6e7a9..9e2dee4d94212 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -112,7 +112,6 @@
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
-import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper;
 import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.shard.IndexShard;
@@ -206,7 +205,6 @@ public abstract class AggregatorTestCase extends ESTestCase {
     private static final List<String> TYPE_TEST_BLACKLIST = List.of(
         ObjectMapper.CONTENT_TYPE, // Cannot aggregate objects
         DenseVectorFieldMapper.CONTENT_TYPE, // Cannot aggregate dense vectors
-        RankVectorsFieldMapper.CONTENT_TYPE, // Cannot aggregate dense vectors
         SparseVectorFieldMapper.CONTENT_TYPE, // Sparse vectors are no longer supported
 
         NestedObjectMapper.CONTENT_TYPE, // TODO support for nested
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
index 29112b4bd8f5f..ee11530ab44b4 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
@@ -143,9 +143,8 @@ public void setupSuiteScopeCluster() throws Exception {
         }
         singleCentroid = computeCentroid(allSingleVal);
         multiCentroid = computeCentroid(allMultiVal);
-        assertAcked(prepareCreate(EMPTY_IDX_NAME).setMapping(SINGLE_VALUED_FIELD_NAME, "type=" + fieldTypeName()));
-
         assertAcked(
+            prepareCreate(EMPTY_IDX_NAME).setMapping(SINGLE_VALUED_FIELD_NAME, "type=" + fieldTypeName()),
             prepareCreate(DATELINE_IDX_NAME).setMapping(
                 SINGLE_VALUED_FIELD_NAME,
                 "type=" + fieldTypeName(),
diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
index a2bf70bf6e087..076453c8a48d8 100644
--- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
@@ -34,7 +34,6 @@
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.plugins.Plugin;
@@ -367,14 +366,12 @@ protected static Settings.Builder indexSettingsNoReplicas(int shards) {
     /**
      * Randomly write an empty snapshot of an older version to an empty repository to simulate an older repository metadata format.
      */
-    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION)
-    // This used to pick an index version from 7.0.0 to 8.9.0. The minimum now is 8.0.0 but it's not clear what the upper range should be
     protected void maybeInitWithOldSnapshotVersion(String repoName, Path repoPath) throws Exception {
         if (randomBoolean() && randomBoolean()) {
             initWithSnapshotVersion(
                 repoName,
                 repoPath,
-                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.V_8_9_0)
+                IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_8_9_0)
             );
         }
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
index 20cb66affddee..d239c6453a7fe 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
@@ -241,6 +241,10 @@ protected static IndexSettings indexSettings() {
         return serviceHolder.idxSettings;
     }
 
+    protected static MapperService mapperService() {
+        return serviceHolder.mapperService;
+    }
+
     protected static String expectedFieldName(String builderFieldName) {
         return ALIAS_TO_CONCRETE_FIELD_NAME.getOrDefault(builderFieldName, builderFieldName);
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index b9a097b4e76f3..c70664940bac6 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -759,24 +759,10 @@ public Settings indexSettings() {
     }
 
     /**
-     * Creates one or more indices and asserts that the indices are acknowledged. If one of the indices
-     * already exists this method will fail and wipe all the indices created so far.
+     * Creates one or more indices and asserts that the indices are acknowledged.
      */
     public final void createIndex(String... names) {
-
-        List<String> created = new ArrayList<>();
-        for (String name : names) {
-            boolean success = false;
-            try {
-                assertAcked(prepareCreate(name));
-                created.add(name);
-                success = true;
-            } finally {
-                if (success == false && created.isEmpty() == false) {
-                    cluster().wipeIndices(created.toArray(new String[created.size()]));
-                }
-            }
-        }
+        assertAcked(Arrays.stream(names).map(this::prepareCreate).toArray(CreateIndexRequestBuilder[]::new));
     }
 
     /**
diff --git a/test/framework/src/main/java/org/elasticsearch/test/FailingFieldPlugin.java b/test/framework/src/main/java/org/elasticsearch/test/FailingFieldPlugin.java
new file mode 100644
index 0000000000000..64c90826fda85
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/FailingFieldPlugin.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.test;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.OnScriptError;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
+import org.elasticsearch.script.LongFieldScript;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.ScriptEngine;
+import org.elasticsearch.search.lookup.SearchLookup;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+
+public class FailingFieldPlugin extends Plugin implements ScriptPlugin {
+
+    @Override
+    public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<?>> contexts) {
+        return new ScriptEngine() {
+            @Override
+            public String getType() {
+                return "failing_field";
+            }
+
+            @Override
+            @SuppressWarnings("unchecked")
+            public <FactoryType> FactoryType compile(
+                String name,
+                String code,
+                ScriptContext<FactoryType> context,
+                Map<String, String> params
+            ) {
+                return (FactoryType) new LongFieldScript.Factory() {
+                    @Override
+                    public LongFieldScript.LeafFactory newFactory(
+                        String fieldName,
+                        Map<String, Object> params,
+                        SearchLookup searchLookup,
+                        OnScriptError onScriptError
+                    ) {
+                        return ctx -> new LongFieldScript(fieldName, params, searchLookup, onScriptError, ctx) {
+                            @Override
+                            public void execute() {
+                                throw new IllegalStateException("Accessing failing field");
+                            }
+                        };
+                    }
+                };
+            }
+
+            @Override
+            public Set<ScriptContext<?>> getSupportedContexts() {
+                return Set.of(LongFieldScript.CONTEXT);
+            }
+        };
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/SkipUnavailableRule.java b/test/framework/src/main/java/org/elasticsearch/test/SkipUnavailableRule.java
new file mode 100644
index 0000000000000..d5ce943b4d8fe
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/SkipUnavailableRule.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.test;
+
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Test rule to process skip_unavailable override annotations
+ */
+public class SkipUnavailableRule implements TestRule {
+    private final Map<String, Boolean> skipMap;
+
+    public SkipUnavailableRule(String... clusterAliases) {
+        this.skipMap = Arrays.stream(clusterAliases).collect(Collectors.toMap(Function.identity(), alias -> true));
+    }
+
+    public Map<String, Boolean> getMap() {
+        return skipMap;
+    }
+
+    @Override
+    public Statement apply(Statement base, Description description) {
+        // Check for annotation named "SkipOverride" and set the overrides accordingly
+        var aliases = description.getAnnotation(NotSkipped.class);
+        if (aliases != null) {
+            for (String alias : aliases.aliases()) {
+                skipMap.put(alias, false);
+            }
+        }
+        return base;
+    }
+
+    /**
+     * Annotation to mark specific cluster in a test as not to be skipped when unavailable
+     */
+    @Retention(RetentionPolicy.RUNTIME)
+    @Target(ElementType.METHOD)
+    public @interface NotSkipped {
+        String[] aliases();
+    }
+
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java
index 75e0c0f459a2a..c79939a4184be 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java
@@ -116,11 +116,15 @@ public void wipe(Set<String> excludeTemplates) {
 
     private void deleteTemplates(Set<String> excludeTemplates, ActionListener<Void> listener) {
         final SubscribableListener<GetComposableIndexTemplateAction.Response> getComposableTemplates = SubscribableListener.newForked(
-            l -> client().execute(GetComposableIndexTemplateAction.INSTANCE, new GetComposableIndexTemplateAction.Request("*"), l)
+            l -> client().execute(
+                GetComposableIndexTemplateAction.INSTANCE,
+                new GetComposableIndexTemplateAction.Request(TEST_REQUEST_TIMEOUT, "*"),
+                l
+            )
         );
 
         final SubscribableListener<GetComponentTemplateAction.Response> getComponentTemplates = SubscribableListener.newForked(
-            l -> client().execute(GetComponentTemplateAction.INSTANCE, new GetComponentTemplateAction.Request("*"), l)
+            l -> client().execute(GetComponentTemplateAction.INSTANCE, new GetComponentTemplateAction.Request(TEST_REQUEST_TIMEOUT, "*"), l)
         );
 
         SubscribableListener
@@ -288,7 +292,7 @@ private void handleWipeIndicesFailure(Exception exception, boolean wipingAllIndi
     private void wipeAllTemplates(Set<String> exclude, RefCountingListener listeners) {
         SubscribableListener
 
-            .<GetIndexTemplatesResponse>newForked(l -> client().admin().indices().prepareGetTemplates().execute(l))
+            .<GetIndexTemplatesResponse>newForked(l -> client().admin().indices().prepareGetTemplates(TEST_REQUEST_TIMEOUT).execute(l))
             .andThenAccept(response -> {
                 for (IndexTemplateMetadata indexTemplate : response.getIndexTemplates()) {
                     if (exclude.contains(indexTemplate.getName())) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetadata.java b/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetadata.java
index 3a24b7ae0f14f..9954739e4a9c3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetadata.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetadata.java
@@ -13,9 +13,9 @@
 import org.elasticsearch.cluster.AbstractNamedDiffable;
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 
@@ -94,8 +94,8 @@ public static <T extends Metadata.Custom> T fromXContent(Function<String, T> sup
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).field("data", getData());
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.single((builder, params) -> builder.field("data", getData()));
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisruption.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisruption.java
index 2558edea9c165..5e503d0fd8b5f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisruption.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisruption.java
@@ -451,8 +451,8 @@ public String toString() {
      */
     public static class NetworkDelay extends NetworkLinkDisruptionType {
 
-        public static TimeValue DEFAULT_DELAY_MIN = TimeValue.timeValueSeconds(10);
-        public static TimeValue DEFAULT_DELAY_MAX = TimeValue.timeValueSeconds(90);
+        public static final TimeValue DEFAULT_DELAY_MIN = TimeValue.timeValueSeconds(10);
+        public static final TimeValue DEFAULT_DELAY_MAX = TimeValue.timeValueSeconds(90);
 
         private final TimeValue delay;
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/fixture/HttpHeaderParser.java b/test/framework/src/main/java/org/elasticsearch/test/fixture/HttpHeaderParser.java
index 7018e5e259584..ec822c6bc42bf 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/fixture/HttpHeaderParser.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/fixture/HttpHeaderParser.java
@@ -16,6 +16,7 @@ public enum HttpHeaderParser {
     ;
 
     private static final Pattern RANGE_HEADER_PATTERN = Pattern.compile("bytes=([0-9]+)-([0-9]+)");
+    private static final Pattern CONTENT_RANGE_HEADER_PATTERN = Pattern.compile("bytes (?:(\\d+)-(\\d+)|\\*)/(?:(\\d+)|\\*)");
 
     /**
      * Parse a "Range" header
@@ -38,5 +39,75 @@ public static Range parseRangeHeader(String rangeHeaderValue) {
         return null;
     }
 
-    public record Range(long start, long end) {}
+    public record Range(long start, long end) {
+
+        public String headerString() {
+            return "bytes=" + start + "-" + end;
+        }
+    }
+
+    /**
+     * Parse a "Content-Range" header
+     *
+     * @see <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range">MDN: Content-Range header</a>
+     *
+     * @param contentRangeHeaderValue The header value as a string
+     * @return a {@link ContentRange} instance representing the parsed value, or null if the header is malformed
+     */
+    public static ContentRange parseContentRangeHeader(String contentRangeHeaderValue) {
+        final Matcher matcher = CONTENT_RANGE_HEADER_PATTERN.matcher(contentRangeHeaderValue);
+        if (matcher.matches()) {
+            try {
+                if (matcher.groupCount() == 3) {
+                    final Long start = parseOptionalLongValue(matcher.group(1));
+                    final Long end = parseOptionalLongValue(matcher.group(2));
+                    final Long size = parseOptionalLongValue(matcher.group(3));
+                    return new ContentRange(start, end, size);
+                }
+            } catch (NumberFormatException e) {
+                return null;
+            }
+        }
+        return null;
+    }
+
+    private static Long parseOptionalLongValue(String value) {
+        return value == null ? null : Long.parseLong(value);
+    }
+
+    /**
+     * A HTTP "Content Range"
+     * <p>
+     * This will always contain one of the following combinations of values:
+     * <ul>
+     *     <li>start, end, size</li>
+     *     <li>start, end</li>
+     *     <li>size</li>
+     *     <li>nothing</li>
+     * </ul>
+     *
+     * @param start The start of the range
+     * @param end The end of the range
+     * @param size The total size
+     */
+    public record ContentRange(Long start, Long end, Long size) {
+
+        public ContentRange {
+            assert (start == null) == (end == null) : "Must have either start and end or neither";
+        }
+
+        public boolean hasRange() {
+            return start != null && end != null;
+        }
+
+        public boolean hasSize() {
+            return size != null;
+        }
+
+        public String headerString() {
+            final String rangeString = hasRange() ? start + "-" + end : "*";
+            final String sizeString = hasSize() ? String.valueOf(size) : "*";
+            return "bytes " + rangeString + "/" + sizeString;
+        }
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
index 6c501898d5fe1..40aee8eed4235 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.action.RequestBuilder;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.bulk.BulkResponse;
@@ -107,8 +108,42 @@ public static void assertAcked(RequestBuilder<?, ? extends IsAcknowledgedSupplie
         assertAcked(builder, TimeValue.timeValueSeconds(30));
     }
 
-    public static void assertAcked(ActionFuture<? extends IsAcknowledgedSupplier> future) {
-        assertAcked(future.actionGet());
+    @SafeVarargs
+    public static void assertAcked(RequestBuilder<?, ? extends IsAcknowledgedSupplier>... builders) {
+        final List<ActionFuture<? extends IsAcknowledgedSupplier>> futures = new ArrayList<>(builders.length);
+        for (RequestBuilder<?, ? extends IsAcknowledgedSupplier> response : builders) {
+            futures.add(response.execute());
+        }
+        Throwable tr = null;
+        for (var future : futures) {
+            try {
+                assertAcked(future.get());
+            } catch (Throwable t) {
+                tr = ExceptionsHelper.useOrSuppress(tr, t);
+            }
+        }
+        if (tr != null) {
+            throw new AssertionError(tr);
+        }
+    }
+
+    @SafeVarargs
+    public static void assertAcked(ActionFuture<? extends IsAcknowledgedSupplier>... futures) {
+        if (futures.length == 1) {
+            assertAcked(futures[0].actionGet());
+            return;
+        }
+        Throwable tr = null;
+        for (var future : futures) {
+            try {
+                assertAcked(future.get());
+            } catch (Throwable t) {
+                tr = ExceptionsHelper.useOrSuppress(tr, t);
+            }
+        }
+        if (tr != null) {
+            throw new AssertionError(tr);
+        }
     }
 
     public static void assertAcked(RequestBuilder<?, ? extends IsAcknowledgedSupplier> builder, TimeValue timeValue) {
@@ -131,6 +166,28 @@ public static void assertAcked(IsAcknowledgedSupplier response) {
      * Assert that an index creation was fully acknowledged, meaning that both the index creation cluster
      * state update was successful and that the requisite number of shard copies were started before returning.
      */
+    public static void assertAcked(CreateIndexRequestBuilder... requests) {
+        if (requests.length == 1) {
+            assertAcked(requests[0]);
+            return;
+        }
+        final List<Future<CreateIndexResponse>> futures = new ArrayList<>(requests.length);
+        for (CreateIndexRequestBuilder response : requests) {
+            futures.add(response.execute());
+        }
+        Throwable tr = null;
+        for (Future<CreateIndexResponse> future : futures) {
+            try {
+                assertAcked(future.get());
+            } catch (Throwable t) {
+                tr = ExceptionsHelper.useOrSuppress(tr, t);
+            }
+        }
+        if (tr != null) {
+            throw new AssertionError(tr);
+        }
+    }
+
     public static void assertAcked(CreateIndexResponse response) {
         assertThat(response.getClass().getSimpleName() + " failed - not acked", response.isAcknowledged(), is(true));
         assertThat(
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/RectangleMatcher.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/RectangleMatcher.java
similarity index 60%
rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/RectangleMatcher.java
rename to test/framework/src/main/java/org/elasticsearch/test/hamcrest/RectangleMatcher.java
index 48fbc9c8e0378..2d55b439bd1b7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/RectangleMatcher.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/RectangleMatcher.java
@@ -1,14 +1,16 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.xpack.esql.expression;
+package org.elasticsearch.test.hamcrest;
 
-import org.elasticsearch.compute.aggregation.spatial.PointType;
 import org.elasticsearch.geometry.Rectangle;
+import org.elasticsearch.lucene.spatial.CoordinateEncoder;
 import org.hamcrest.Description;
 import org.hamcrest.Matchers;
 import org.hamcrest.TypeSafeMatcher;
@@ -19,23 +21,31 @@
  */
 public class RectangleMatcher extends TypeSafeMatcher<Rectangle> {
     private final Rectangle r;
-    private final PointType pointType;
+    private final CoordinateEncoder coordinateEncoder;
     private final double error;
 
-    public static TypeSafeMatcher<Rectangle> closeTo(Rectangle r, double error, PointType pointType) {
-        return new RectangleMatcher(r, error, pointType);
+    public static TypeSafeMatcher<Rectangle> closeTo(Rectangle r, double error, CoordinateEncoder coordinateEncoder) {
+        return new RectangleMatcher(r, error, coordinateEncoder);
     }
 
-    private RectangleMatcher(Rectangle r, double error, PointType pointType) {
+    private RectangleMatcher(Rectangle r, double error, CoordinateEncoder coordinateEncoder) {
         this.r = r;
-        this.pointType = pointType;
+        this.coordinateEncoder = coordinateEncoder;
         this.error = error;
     }
 
+    /**
+     * Casts the rectangle coordinates to floats before comparing. Useful when working with extents which hold the coordinate data as ints.
+     */
+    public static TypeSafeMatcher<Rectangle> closeToFloat(Rectangle r, double v, CoordinateEncoder encoder) {
+        var normalized = new Rectangle((float) r.getMinX(), (float) r.getMaxX(), (float) r.getMaxY(), (float) r.getMinY());
+        return closeTo(normalized, v, encoder);
+    }
+
     @Override
     protected boolean matchesSafely(Rectangle other) {
         // For geo bounds, longitude of (-180, 180) and (epsilon, -epsilon) are actually very close, since both encompass the entire globe.
-        boolean wrapAroundWorkAround = pointType == PointType.GEO && r.getMinX() >= r.getMaxX();
+        boolean wrapAroundWorkAround = coordinateEncoder == CoordinateEncoder.GEO && r.getMinX() >= r.getMaxX();
         boolean matchMinX = Matchers.closeTo(r.getMinX(), error).matches(other.getMinX())
             || (wrapAroundWorkAround && Matchers.closeTo(r.getMinX() - 180, error).matches(other.getMinX()))
             || (wrapAroundWorkAround && Matchers.closeTo(r.getMinX(), error).matches(other.getMinX() - 180));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/WellKnownBinaryBytesRefMatcher.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/WellKnownBinaryBytesRefMatcher.java
similarity index 69%
rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/WellKnownBinaryBytesRefMatcher.java
rename to test/framework/src/main/java/org/elasticsearch/test/hamcrest/WellKnownBinaryBytesRefMatcher.java
index 535bb820458cd..809f2862c208c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/WellKnownBinaryBytesRefMatcher.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/WellKnownBinaryBytesRefMatcher.java
@@ -1,11 +1,13 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.xpack.esql.expression;
+package org.elasticsearch.test.hamcrest;
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.geometry.Geometry;
@@ -23,6 +25,10 @@ public WellKnownBinaryBytesRefMatcher(Matcher<G> matcher) {
         this.matcher = matcher;
     }
 
+    public static <G extends Geometry> Matcher<BytesRef> encodes(TypeSafeMatcher<G> matcher) {
+        return new WellKnownBinaryBytesRefMatcher<G>(matcher);
+    }
+
     @Override
     public boolean matchesSafely(BytesRef bytesRef) {
         return matcher.matches(fromBytesRef(bytesRef));
diff --git a/test/framework/src/main/java/org/elasticsearch/test/index/IndexVersionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/index/IndexVersionUtils.java
index f83e7e17f9aaa..667149e4bdd3e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/index/IndexVersionUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/index/IndexVersionUtils.java
@@ -24,32 +24,38 @@
 public class IndexVersionUtils {
 
     private static final List<IndexVersion> ALL_VERSIONS = KnownIndexVersions.ALL_VERSIONS;
+    private static final List<IndexVersion> ALL_WRITE_VERSIONS = KnownIndexVersions.ALL_WRITE_VERSIONS;
 
     /** Returns all released versions */
     public static List<IndexVersion> allReleasedVersions() {
         return ALL_VERSIONS;
     }
 
-    /** Returns the oldest known {@link IndexVersion} */
-    public static IndexVersion getFirstVersion() {
+    /** Returns the oldest known {@link IndexVersion}. This version can only be read from and not written to */
+    public static IndexVersion getLowestReadCompatibleVersion() {
         return ALL_VERSIONS.get(0);
     }
 
+    /** Returns the oldest known {@link IndexVersion} that can be written to */
+    public static IndexVersion getLowestWriteCompatibleVersion() {
+        return ALL_WRITE_VERSIONS.get(0);
+    }
+
     /** Returns a random {@link IndexVersion} from all available versions. */
     public static IndexVersion randomVersion() {
         return ESTestCase.randomFrom(ALL_VERSIONS);
     }
 
+    /** Returns a random {@link IndexVersion} from all versions that can be written to. */
+    public static IndexVersion randomWriteVersion() {
+        return ESTestCase.randomFrom(ALL_WRITE_VERSIONS);
+    }
+
     /** Returns a random {@link IndexVersion} from all available versions without the ignore set */
     public static IndexVersion randomVersion(Set<IndexVersion> ignore) {
         return ESTestCase.randomFrom(ALL_VERSIONS.stream().filter(v -> ignore.contains(v) == false).collect(Collectors.toList()));
     }
 
-    /** Returns a random {@link IndexVersion} from all available versions. */
-    public static IndexVersion randomVersion(Random random) {
-        return ALL_VERSIONS.get(random.nextInt(ALL_VERSIONS.size()));
-    }
-
     /** Returns a random {@link IndexVersion} between <code>minVersion</code> and <code>maxVersion</code> (inclusive). */
     public static IndexVersion randomVersionBetween(Random random, @Nullable IndexVersion minVersion, @Nullable IndexVersion maxVersion) {
         if (minVersion != null && maxVersion != null && maxVersion.before(minVersion)) {
@@ -116,11 +122,21 @@ public static IndexVersion getNextVersion(IndexVersion version) {
 
     /** Returns a random {@code IndexVersion} that is compatible with {@link IndexVersion#current()} */
     public static IndexVersion randomCompatibleVersion(Random random) {
+        return randomVersionBetween(random, IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersion.current());
+    }
+
+    /** Returns a random {@code IndexVersion} that is compatible with {@link IndexVersion#current()} and can be written to */
+    public static IndexVersion randomCompatibleWriteVersion(Random random) {
         return randomVersionBetween(random, IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current());
     }
 
     /** Returns a random {@code IndexVersion} that is compatible with the previous version to {@code version} */
     public static IndexVersion randomPreviousCompatibleVersion(Random random, IndexVersion version) {
+        return randomVersionBetween(random, IndexVersions.MINIMUM_READONLY_COMPATIBLE, getPreviousVersion(version));
+    }
+
+    /** Returns a random {@code IndexVersion} that is compatible with the previous version to {@code version} and can be written to */
+    public static IndexVersion randomPreviousCompatibleWriteVersion(Random random, IndexVersion version) {
         return randomVersionBetween(random, IndexVersions.MINIMUM_COMPATIBLE, getPreviousVersion(version));
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/jar/JarUtils.java b/test/framework/src/main/java/org/elasticsearch/test/jar/JarUtils.java
index e5bdd66e949f7..0da392cb7fb01 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/jar/JarUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/jar/JarUtils.java
@@ -9,13 +9,24 @@
 
 package org.elasticsearch.test.jar;
 
+import org.elasticsearch.test.PrivilegedOperations.ClosableURLClassLoader;
+
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.lang.module.Configuration;
+import java.lang.module.ModuleFinder;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.jar.JarEntry;
 import java.util.jar.JarOutputStream;
 import java.util.jar.Manifest;
@@ -85,6 +96,28 @@ public static void createJarWithEntriesUTF(Path jarfile, Map<String, String> ent
         createJarWithEntries(jarfile, map);
     }
 
+    /**
+     * Creates a class loader for the given jar file.
+     * @param path Path to the jar file to load
+     * @return A URLClassLoader that will load classes from the jar. It should be closed when no longer needed.
+     */
+    public static ClosableURLClassLoader loadJar(Path path) {
+        try {
+            URL[] urls = new URL[] { path.toUri().toURL() };
+            return new ClosableURLClassLoader(URLClassLoader.newInstance(urls, JarUtils.class.getClassLoader()));
+        } catch (MalformedURLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static ModuleLayer.Controller loadModule(Path path, ClassLoader loader, String name) {
+        var finder = ModuleFinder.of(path.getParent());
+        var cf = Configuration.resolveAndBind(finder, List.of(ModuleLayer.boot().configuration()), ModuleFinder.of(), Set.of(name));
+        return AccessController.doPrivileged(
+            (PrivilegedAction<ModuleLayer.Controller>) () -> ModuleLayer.defineModulesWithOneLoader(cf, List.of(ModuleLayer.boot()), loader)
+        );
+    }
+
     @FunctionalInterface
     interface UncheckedIOFunction<T, R> {
         R apply(T t) throws IOException;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index 4428afaaeabe5..b23ad1e9c548f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -781,7 +781,9 @@ protected Set<String> preserveILMPolicyIds() {
             "profiling-60-days",
             "profiling-60-days@lifecycle",
             "synthetics",
+            "agentless",
             "synthetics@lifecycle",
+            "traces@lifecycle",
             "7-days-default",
             "7-days@lifecycle",
             "30-days-default",
@@ -852,8 +854,7 @@ protected boolean preserveSearchableSnapshotsIndicesUponCompletion() {
     }
 
     private void wipeCluster() throws Exception {
-        logger.info("Waiting for all cluster updates up to this moment to be processed");
-        assertOK(adminClient().performRequest(new Request("GET", "_cluster/health?wait_for_events=languid")));
+        waitForClusterUpdates();
 
         // Cleanup rollup before deleting indices. A rollup job might have bulks in-flight,
         // so we need to fully shut them down first otherwise a job might stall waiting
@@ -991,6 +992,38 @@ private void wipeCluster() throws Exception {
         deleteAllNodeShutdownMetadata();
     }
 
+    private void waitForClusterUpdates() throws Exception {
+        logger.info("Waiting for all cluster updates up to this moment to be processed");
+        try {
+            assertOK(adminClient().performRequest(new Request("GET", "_cluster/health?wait_for_events=languid")));
+        } catch (ResponseException e) {
+            if (e.getResponse().getStatusLine().getStatusCode() == HttpStatus.SC_REQUEST_TIMEOUT) {
+                final var pendingTasks = getPendingClusterStateTasks();
+                if (pendingTasks != null) {
+                    logger.error("Timed out waiting for cluster updates to be processed, {}", pendingTasks);
+                }
+            }
+            throw e;
+        }
+    }
+
+    private static String getPendingClusterStateTasks() {
+        try {
+            Response response = adminClient().performRequest(new Request("GET", "/_cluster/pending_tasks"));
+            List<?> tasks = (List<?>) entityAsMap(response).get("tasks");
+            if (false == tasks.isEmpty()) {
+                StringBuilder message = new StringBuilder("there are still running tasks:");
+                for (Object task : tasks) {
+                    message.append('\n').append(task.toString());
+                }
+                return message.toString();
+            }
+        } catch (IOException e) {
+            fail(e, "Failed to retrieve pending tasks in the cluster during cleanup");
+        }
+        return null;
+    }
+
     /**
      * This method checks whether ILM policies or templates get recreated after they have been deleted. If so, we are probably deleting
      * them unnecessarily, potentially causing test performance problems. This could happen for example if someone adds a new standard ILM
@@ -1138,7 +1171,7 @@ protected static void wipeAllIndices(boolean preserveSecurityIndices) throws IOE
         }
     }
 
-    private static boolean ignoreSystemIndexAccessWarnings(List<String> warnings) {
+    protected static boolean ignoreSystemIndexAccessWarnings(List<String> warnings) {
         for (String warning : warnings) {
             if (warning.startsWith("this request accesses system indices:")) {
                 SUITE_LOGGER.warn("Ignoring system index access warning during test cleanup: {}", warning);
@@ -1461,18 +1494,9 @@ private void logIfThereAreRunningTasks() throws IOException {
      */
     private static void waitForClusterStateUpdatesToFinish() throws Exception {
         assertBusy(() -> {
-            try {
-                Response response = adminClient().performRequest(new Request("GET", "/_cluster/pending_tasks"));
-                List<?> tasks = (List<?>) entityAsMap(response).get("tasks");
-                if (false == tasks.isEmpty()) {
-                    StringBuilder message = new StringBuilder("there are still running tasks:");
-                    for (Object task : tasks) {
-                        message.append('\n').append(task.toString());
-                    }
-                    fail(message.toString());
-                }
-            } catch (IOException e) {
-                fail("cannot get cluster's pending tasks: " + e.getMessage());
+            final var pendingTasks = getPendingClusterStateTasks();
+            if (pendingTasks != null) {
+                fail(pendingTasks);
             }
         }, 30, TimeUnit.SECONDS);
     }
@@ -2192,6 +2216,7 @@ protected static boolean isXPackTemplate(String name) {
             case "metrics-tsdb-settings":
             case "metrics-mappings":
             case "synthetics":
+            case "agentless":
             case "synthetics-settings":
             case "synthetics-mappings":
             case ".snapshot-blob-cache":
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestFeatureService.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestFeatureService.java
index 9054dc6f94182..e2a24eac6f8ab 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestFeatureService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestFeatureService.java
@@ -91,6 +91,12 @@ public boolean clusterHasFeature(String featureId, boolean any) {
         }
 
         if (hasFeatureMetadata()) {
+            if (isRestApiCompatibilityTest()) {
+                // assume this is a feature that has been assumed, then removed in this version
+                // the feature is therefore logically present, but not specified by the cluster
+                return true;
+            }
+
             throw new IllegalArgumentException(
                 Strings.format(
                     "Unknown feature %s: check the respective FeatureSpecification is provided both in module-info.java "
@@ -102,6 +108,10 @@ public boolean clusterHasFeature(String featureId, boolean any) {
         return false;
     }
 
+    private static boolean isRestApiCompatibilityTest() {
+        return Boolean.parseBoolean(System.getProperty("tests.restCompat", "false"));
+    }
+
     public static boolean hasFeatureMetadata() {
         return MetadataHolder.FEATURE_NAMES.isEmpty() == false;
     }
diff --git a/test/framework/src/test/java/org/elasticsearch/http/HttpHeaderParserTests.java b/test/framework/src/test/java/org/elasticsearch/http/HttpHeaderParserTests.java
index e025e7770ea4c..5fb2c528482c2 100644
--- a/test/framework/src/test/java/org/elasticsearch/http/HttpHeaderParserTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/http/HttpHeaderParserTests.java
@@ -50,4 +50,48 @@ public void testParseRangeHeaderEndlessRangeNotMatched() {
     public void testParseRangeHeaderSuffixLengthNotMatched() {
         assertNull(HttpHeaderParser.parseRangeHeader(Strings.format("bytes=-%d", randomLongBetween(0, Long.MAX_VALUE))));
     }
+
+    public void testRangeHeaderString() {
+        final long start = randomLongBetween(0, 10_000);
+        final long end = randomLongBetween(start, start + 10_000);
+        assertEquals("bytes=" + start + "-" + end, new HttpHeaderParser.Range(start, end).headerString());
+    }
+
+    public void testParseContentRangeHeaderFull() {
+        final long start = randomLongBetween(0, 10_000);
+        final long end = randomLongBetween(start, start + 10_000);
+        final long size = randomLongBetween(end, Long.MAX_VALUE);
+        assertEquals(
+            new HttpHeaderParser.ContentRange(start, end, size),
+            HttpHeaderParser.parseContentRangeHeader("bytes " + start + "-" + end + "/" + size)
+        );
+    }
+
+    public void testParseContentRangeHeaderNoSize() {
+        final long start = randomLongBetween(0, 10_000);
+        final long end = randomLongBetween(start, start + 10_000);
+        assertEquals(
+            new HttpHeaderParser.ContentRange(start, end, null),
+            HttpHeaderParser.parseContentRangeHeader("bytes " + start + "-" + end + "/*")
+        );
+    }
+
+    public void testParseContentRangeHeaderNoRange() {
+        final long size = randomNonNegativeLong();
+        assertEquals(new HttpHeaderParser.ContentRange(null, null, size), HttpHeaderParser.parseContentRangeHeader("bytes */" + size));
+    }
+
+    public void testParseNoRangeOrSize() {
+        assertEquals(new HttpHeaderParser.ContentRange(null, null, null), HttpHeaderParser.parseContentRangeHeader("bytes */*"));
+    }
+
+    public void testContentRangeHeaderString() {
+        final long start = randomLongBetween(0, 10_000);
+        final long end = randomLongBetween(start, start + 10_000);
+        final long size = randomLongBetween(end, Long.MAX_VALUE);
+        assertEquals("bytes */*", new HttpHeaderParser.ContentRange(null, null, null).headerString());
+        assertEquals("bytes */" + size, new HttpHeaderParser.ContentRange(null, null, size).headerString());
+        assertEquals("bytes " + start + "-" + end + "/*", new HttpHeaderParser.ContentRange(start, end, null).headerString());
+        assertEquals("bytes " + start + "-" + end + "/" + size, new HttpHeaderParser.ContentRange(start, end, size).headerString());
+    }
 }
diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
index 357c0fe74b5ce..031f8f908847c 100644
--- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
+++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.rest.action.admin.cluster.RestNodesCapabilitiesAction;
 import org.elasticsearch.test.rest.Stash;
 import org.elasticsearch.test.rest.TestFeatureService;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi;
@@ -300,41 +299,24 @@ public Optional<Boolean> clusterHasCapabilities(
             params.put("capabilities", capabilitiesString);
         }
         params.put("error_trace", "false"); // disable error trace
-
-        if (clusterHasFeature(RestNodesCapabilitiesAction.LOCAL_ONLY_CAPABILITIES.id(), false) == false) {
-            // can only check the whole cluster
-            if (any) {
-                logger.warn(
-                    "Cluster does not support checking individual nodes for capabilities,"
-                        + "check for [{} {}?{} {}] may be incorrect in mixed-version clusters",
-                    method,
-                    path,
-                    parametersString,
-                    capabilitiesString
-                );
-            }
-            return checkCapability(NodeSelector.ANY, params);
-        } else {
-            // check each node individually - we can actually check any here
-            params.put("local_only", "true");   // we're calling each node individually
-
-            // individually call each node, so we can control whether we do an 'any' or 'all' check
-            List<Node> nodes = clientYamlTestClient.getRestClient(NodeSelector.ANY).getNodes();
-
-            for (Node n : nodes) {
-                Optional<Boolean> nodeResult = checkCapability(new SpecificNodeSelector(n), params);
-                if (nodeResult.isEmpty()) {
-                    return Optional.empty();
-                } else if (any == nodeResult.get()) {
-                    // either any == true and node has cap,
-                    // or any == false (ie all) and this node does not have cap
-                    return nodeResult;
-                }
+        params.put("local_only", "true");   // we're calling each node individually
+
+        // individually call each node, so we can control whether we do an 'any' or 'all' check
+        List<Node> nodes = clientYamlTestClient.getRestClient(NodeSelector.ANY).getNodes();
+
+        for (Node n : nodes) {
+            Optional<Boolean> nodeResult = checkCapability(new SpecificNodeSelector(n), params);
+            if (nodeResult.isEmpty()) {
+                return Optional.empty();
+            } else if (any == nodeResult.get()) {
+                // either any == true and node has cap,
+                // or any == false (ie all) and this node does not have cap
+                return nodeResult;
             }
-
-            // if we got here, either any is true and no node has it, or any == false and all nodes have it
-            return Optional.of(any == false);
         }
+
+        // if we got here, either any is true and no node has it, or any == false and all nodes have it
+        return Optional.of(any == false);
     }
 
     private Optional<Boolean> checkCapability(NodeSelector nodeSelector, Map<String, String> params) {
diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
index 54602090050ab..15ebcf3d1feb7 100644
--- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
+++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.IOUtils;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.test.ClasspathUtils;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.TestFeatureService;
@@ -33,10 +32,8 @@
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
 import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
 import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
 import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
-import org.elasticsearch.xcontent.ParseField;
 import org.junit.AfterClass;
 import org.junit.Before;
 
@@ -58,7 +55,6 @@
 import java.util.SortedSet;
 import java.util.TreeSet;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 /**
  * Runs a suite of yaml tests shared with all the official Elasticsearch
@@ -217,28 +213,6 @@ public static void closeClient() throws IOException {
         }
     }
 
-    /**
-     * Create parameters for this parameterized test.
-     * Enables support for parsing the legacy version-based node_selector format.
-     */
-    @Deprecated
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
-    public static Iterable<Object[]> createParametersWithLegacyNodeSelectorSupport() throws Exception {
-        var executableSectionRegistry = new NamedXContentRegistry(
-            Stream.concat(
-                ExecutableSection.DEFAULT_EXECUTABLE_CONTEXTS.stream().filter(entry -> entry.name.getPreferredName().equals("do") == false),
-                Stream.of(
-                    new NamedXContentRegistry.Entry(
-                        ExecutableSection.class,
-                        new ParseField("do"),
-                        DoSection::parseWithLegacyNodeSelectorSupport
-                    )
-                )
-            ).toList()
-        );
-        return createParameters(executableSectionRegistry, null);
-    }
-
     /**
      * Create parameters for this parameterized test. Uses the
      * {@link ExecutableSection#XCONTENT_REGISTRY list} of executable sections
diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
index 627554f6b261d..5a212e5b1ec58 100644
--- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
+++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
@@ -84,16 +83,6 @@
  */
 public class DoSection implements ExecutableSection {
     public static DoSection parse(XContentParser parser) throws IOException {
-        return parse(parser, false);
-    }
-
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
-    @Deprecated
-    public static DoSection parseWithLegacyNodeSelectorSupport(XContentParser parser) throws IOException {
-        return parse(parser, true);
-    }
-
-    private static DoSection parse(XContentParser parser, boolean enableLegacyNodeSelectorSupport) throws IOException {
         String currentFieldName = null;
         XContentParser.Token token;
 
@@ -183,7 +172,7 @@ private static DoSection parse(XContentParser parser, boolean enableLegacyNodeSe
                         if (token == XContentParser.Token.FIELD_NAME) {
                             selectorName = parser.currentName();
                         } else {
-                            NodeSelector newSelector = buildNodeSelector(selectorName, parser, enableLegacyNodeSelectorSupport);
+                            NodeSelector newSelector = buildNodeSelector(selectorName, parser);
                             nodeSelector = nodeSelector == NodeSelector.ANY
                                 ? newSelector
                                 : new ComposeNodeSelector(nodeSelector, newSelector);
@@ -604,11 +593,10 @@ private String formatStatusCodeMessage(ClientYamlTestResponse restTestResponse,
         )
     );
 
-    private static NodeSelector buildNodeSelector(String name, XContentParser parser, boolean enableLegacyVersionSupport)
-        throws IOException {
+    private static NodeSelector buildNodeSelector(String name, XContentParser parser) throws IOException {
         return switch (name) {
             case "attribute" -> parseAttributeValuesSelector(parser);
-            case "version" -> parseVersionSelector(parser, enableLegacyVersionSupport);
+            case "version" -> parseVersionSelector(parser);
             default -> throw new XContentParseException(parser.getTokenLocation(), "unknown node_selector [" + name + "]");
         };
     }
@@ -673,7 +661,7 @@ private static boolean matchWithRange(
         }
     }
 
-    private static NodeSelector parseVersionSelector(XContentParser parser, boolean enableLegacyVersionSupport) throws IOException {
+    private static NodeSelector parseVersionSelector(XContentParser parser) throws IOException {
         if (false == parser.currentToken().isValue()) {
             throw new XContentParseException(parser.getTokenLocation(), "expected [version] to be a value");
         }
@@ -687,16 +675,10 @@ private static NodeSelector parseVersionSelector(XContentParser parser, boolean
             nodeMatcher = nodeVersion -> Build.current().version().equals(nodeVersion) == false;
             versionSelectorString = "version is not current (original)";
         } else {
-            if (enableLegacyVersionSupport) {
-                var acceptedVersionRange = VersionRange.parseVersionRanges(parser.text());
-                nodeMatcher = nodeVersion -> matchWithRange(nodeVersion, acceptedVersionRange, parser.getTokenLocation());
-                versionSelectorString = "version ranges " + acceptedVersionRange;
-            } else {
-                throw new XContentParseException(
-                    parser.getTokenLocation(),
-                    "unknown version selector [" + parser.text() + "]. Only [current] and [original] are allowed."
-                );
-            }
+            throw new XContentParseException(
+                parser.getTokenLocation(),
+                "unknown version selector [" + parser.text() + "]. Only [current] and [original] are allowed."
+            );
         }
 
         return new NodeSelector() {
diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSection.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSection.java
index bbbf73d74e4ca..ef18a7852840b 100644
--- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSection.java
+++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSection.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.CheckedFunction;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
 import org.elasticsearch.test.rest.yaml.Features;
 import org.elasticsearch.xcontent.XContentLocation;
@@ -305,7 +304,6 @@ static void parseSkipSection(XContentParser parser, PrerequisiteSectionBuilder b
             boolean valid = false;
             if (parser.currentToken().isValue()) {
                 valid = switch (parser.currentName()) {
-                    case "version" -> parseRestCompatVersion(parser, builder);
                     case "reason" -> parseString(parser, builder::setSkipReason);
                     case "features" -> parseString(parser, f -> parseFeatureField(f, builder));
                     case "os" -> parseString(parser, builder::skipIfOs);
@@ -328,17 +326,6 @@ static void parseSkipSection(XContentParser parser, PrerequisiteSectionBuilder b
         parser.nextToken();
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
-    private static boolean parseRestCompatVersion(XContentParser parser, PrerequisiteSectionBuilder builder) throws IOException {
-        // allow skip version only for v7 REST compatibility tests, to be removed for V9
-        if ("true".equals(System.getProperty("tests.restCompat"))) return parseString(parser, builder::skipIfVersion);
-        throw new IllegalArgumentException(
-            "Skipping by version is no longer supported, please skip based on cluster features. Please check the docs: \n"
-                + "https://github.com/elastic/elasticsearch/tree/main"
-                + "/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test#skipping-tests"
-        );
-    }
-
     private static void throwUnexpectedField(String section, XContentParser parser) throws IOException {
         throw new ParsingException(
             parser.getTokenLocation(),
diff --git a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
index ee8d75b4c034b..465ff7c73e74b 100644
--- a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
+++ b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
@@ -16,9 +16,6 @@
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.core.Strings;
-import org.elasticsearch.core.UpdateForV9;
-import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
-import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
 import org.elasticsearch.xcontent.XContentLocation;
 import org.elasticsearch.xcontent.XContentParseException;
 import org.elasticsearch.xcontent.XContentParser;
@@ -31,11 +28,9 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.regex.Pattern;
 
 import static java.util.Collections.emptyList;
-import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -43,9 +38,6 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
 
 public class DoSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
 
@@ -580,57 +572,6 @@ public void testParseDoSectionAllowedWarnings() throws Exception {
         assertThat(e.getMessage(), equalTo("the warning [foo] was both allowed and expected"));
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // remove
-    public void testLegacyNodeSelectorByVersionRange() throws IOException {
-        parser = createParser(YamlXContent.yamlXContent, """
-            node_selector:
-                version: 5.2.0-6.0.0
-            indices.get_field_mapping:
-                index: test_index""");
-
-        DoSection doSection = DoSection.parseWithLegacyNodeSelectorSupport(parser);
-        assertNotSame(NodeSelector.ANY, doSection.getApiCallSection().getNodeSelector());
-        Node v170 = nodeWithVersion("1.7.0");
-        Node v521 = nodeWithVersion("5.2.1");
-        Node v550 = nodeWithVersion("5.5.0");
-        Node v612 = nodeWithVersion("6.1.2");
-        List<Node> nodes = new ArrayList<>();
-        nodes.add(v170);
-        nodes.add(v521);
-        nodes.add(v550);
-        nodes.add(v612);
-        doSection.getApiCallSection().getNodeSelector().select(nodes);
-        assertEquals(Arrays.asList(v521, v550), nodes);
-        ClientYamlTestExecutionContext context = mock(ClientYamlTestExecutionContext.class);
-        ClientYamlTestResponse mockResponse = mock(ClientYamlTestResponse.class);
-        when(
-            context.callApi(
-                "indices.get_field_mapping",
-                singletonMap("index", "test_index"),
-                emptyList(),
-                emptyMap(),
-                doSection.getApiCallSection().getNodeSelector()
-            )
-        ).thenReturn(mockResponse);
-        when(context.nodesVersions()).thenReturn(Set.of(randomAlphaOfLength(10)));
-        when(mockResponse.getHeaders("X-elastic-product")).thenReturn(List.of("Elasticsearch"));
-        doSection.execute(context);
-        verify(context).callApi(
-            "indices.get_field_mapping",
-            singletonMap("index", "test_index"),
-            emptyList(),
-            emptyMap(),
-            doSection.getApiCallSection().getNodeSelector()
-        );
-
-        {
-            List<Node> badNodes = new ArrayList<>();
-            badNodes.add(new Node(new HttpHost("dummy")));
-            Exception e = expectThrows(IllegalStateException.class, () -> doSection.getApiCallSection().getNodeSelector().select(badNodes));
-            assertEquals("expected [version] metadata to be set but got [host=http://dummy]", e.getMessage());
-        }
-    }
-
     public void testNodeSelectorByVersionRangeFails() throws IOException {
         parser = createParser(YamlXContent.yamlXContent, """
             node_selector:
diff --git a/updatecli-compose.yaml b/updatecli-compose.yaml
index 1893c3abca3ec..19bcac28ab1a9 100644
--- a/updatecli-compose.yaml
+++ b/updatecli-compose.yaml
@@ -7,7 +7,7 @@ policies:
       - .github/updatecli/values.d/scm.yml
       - .github/updatecli/values.d/ironbank.yml
   - name: Update Updatecli policies
-    policy: ghcr.io/updatecli/policies/autodiscovery/updatecli:0.6.0@sha256:6bd6999620674b2fbb1d374f7a1a5e9740d042667f0592900b44259f3e1ae98f
+    policy: ghcr.io/updatecli/policies/autodiscovery/updatecli:0.8.0@sha256:99e9e61b501575c2c176c39f2275998d198b590a3f6b1fe829f7315f8d457e7f
     values:
       - .github/updatecli/values.d/scm.yml
       - .github/updatecli/values.d/updatecli-compose.yml
diff --git a/x-pack/build.gradle b/x-pack/build.gradle
index e79d2fe0e9899..da21ffc829d03 100644
--- a/x-pack/build.gradle
+++ b/x-pack/build.gradle
@@ -10,8 +10,8 @@ subprojects {
   // see: https://wiki.shibboleth.net/confluence/display/DEV/Use+of+Maven+Central
   repositories {
     maven {
-      name "opensaml"
-      url "https://artifactory.elstc.co/artifactory/shibboleth-releases/"
+      name = "opensaml"
+      url = "https://artifactory.elstc.co/artifactory/shibboleth-releases/"
       content {
         includeGroup "org.opensaml"
         includeGroup "net.shibboleth.utilities"
@@ -36,7 +36,7 @@ subprojects {
   }
 
   project.pluginManager.withPlugin("elasticsearch.licensing") {
-    ext.projectLicenses.set(['Elastic License 2.0': ext.elasticLicenseUrl.get()])
+    ext.projectLicenses.set(['Elastic License 2.0': ext.elasticLicenseUrl])
   }
 
   project.pluginManager.withPlugin("elasticsearch.build") {
diff --git a/x-pack/plugin/analytics/build.gradle b/x-pack/plugin/analytics/build.gradle
index 00f28b4badc3d..9a21f40a4c4a9 100644
--- a/x-pack/plugin/analytics/build.gradle
+++ b/x-pack/plugin/analytics/build.gradle
@@ -9,9 +9,9 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  name 'x-pack-analytics'
-  description 'Elasticsearch Expanded Pack Plugin - Analytics'
-  classname 'org.elasticsearch.xpack.analytics.AnalyticsPlugin'
+  name = 'x-pack-analytics'
+  description = 'Elasticsearch Expanded Pack Plugin - Analytics'
+  classname ='org.elasticsearch.xpack.analytics.AnalyticsPlugin'
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/apm-data/build.gradle b/x-pack/plugin/apm-data/build.gradle
index fbd732e18aae1..3c79f3742dc8c 100644
--- a/x-pack/plugin/apm-data/build.gradle
+++ b/x-pack/plugin/apm-data/build.gradle
@@ -9,9 +9,9 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  name 'x-pack-apm-data'
-  description 'The APM plugin defines APM data streams and ingest pipelines.'
-  classname 'org.elasticsearch.xpack.apmdata.APMPlugin'
+  name = 'x-pack-apm-data'
+  description = 'The APM plugin defines APM data streams and ingest pipelines.'
+  classname = 'org.elasticsearch.xpack.apmdata.APMPlugin'
   extendedPlugins = ['x-pack-core']
 }
 
diff --git a/x-pack/plugin/async-search/build.gradle b/x-pack/plugin/async-search/build.gradle
index f22ff7fe3d638..b013d1df46b97 100644
--- a/x-pack/plugin/async-search/build.gradle
+++ b/x-pack/plugin/async-search/build.gradle
@@ -1,9 +1,9 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 esplugin {
-    name 'x-pack-async-search'
-    description 'A module which allows to track the progress of a search asynchronously.'
-    classname 'org.elasticsearch.xpack.search.AsyncSearch'
+    name = 'x-pack-async-search'
+    description = 'A module which allows to track the progress of a search asynchronously.'
+    classname ='org.elasticsearch.xpack.search.AsyncSearch'
     extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/async-search/qa/rest/build.gradle b/x-pack/plugin/async-search/qa/rest/build.gradle
index eb758c2c0ef5e..7363aed336aaf 100644
--- a/x-pack/plugin/async-search/qa/rest/build.gradle
+++ b/x-pack/plugin/async-search/qa/rest/build.gradle
@@ -11,9 +11,9 @@ apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
-  name 'x-pack-test-deprecated-query'
-  description 'Deprecated query plugin'
-  classname 'org.elasticsearch.query.DeprecatedQueryPlugin'
+  name = 'x-pack-test-deprecated-query'
+  description = 'Deprecated query plugin'
+  classname ='org.elasticsearch.query.DeprecatedQueryPlugin'
 }
 
 dependencies {
diff --git a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchErrorTraceIT.java b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchErrorTraceIT.java
new file mode 100644
index 0000000000000..39a6fa1e4b34f
--- /dev/null
+++ b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchErrorTraceIT.java
@@ -0,0 +1,222 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.transport.TransportMessageListener;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xcontent.XContentType;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class AsyncSearchErrorTraceIT extends ESIntegTestCase {
+
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false; // enable http
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(AsyncSearch.class);
+    }
+
+    private AtomicBoolean transportMessageHasStackTrace;
+
+    @Before
+    private void setupMessageListener() {
+        internalCluster().getDataNodeInstances(TransportService.class).forEach(ts -> {
+            ts.addMessageListener(new TransportMessageListener() {
+                @Override
+                public void onResponseSent(long requestId, String action, Exception error) {
+                    TransportMessageListener.super.onResponseSent(requestId, action, error);
+                    if (action.startsWith("indices:data/read/search")) {
+                        Optional<Throwable> throwable = ExceptionsHelper.unwrapCausesAndSuppressed(
+                            error,
+                            t -> t.getStackTrace().length > 0
+                        );
+                        transportMessageHasStackTrace.set(throwable.isPresent());
+                    }
+                }
+            });
+        });
+    }
+
+    private void setupIndexWithDocs() {
+        createIndex("test1", "test2");
+        indexRandom(
+            true,
+            prepareIndex("test1").setId("1").setSource("field", "foo"),
+            prepareIndex("test2").setId("10").setSource("field", 5)
+        );
+        refresh();
+    }
+
+    public void testAsyncSearchFailingQueryErrorTraceDefault() throws IOException, InterruptedException {
+        transportMessageHasStackTrace = new AtomicBoolean();
+        setupIndexWithDocs();
+
+        Request searchRequest = new Request("POST", "/_async_search");
+        searchRequest.setJsonEntity("""
+            {
+                "query": {
+                    "simple_query_string" : {
+                        "query": "foo",
+                        "fields": ["field"]
+                    }
+                }
+            }
+            """);
+        searchRequest.addParameter("keep_on_completion", "true");
+        searchRequest.addParameter("wait_for_completion_timeout", "0ms");
+        Map<String, Object> responseEntity = performRequestAndGetResponseEntityAfterDelay(searchRequest, TimeValue.ZERO);
+        String asyncExecutionId = (String) responseEntity.get("id");
+        Request request = new Request("GET", "/_async_search/" + asyncExecutionId);
+        while (responseEntity.get("is_running") instanceof Boolean isRunning && isRunning) {
+            responseEntity = performRequestAndGetResponseEntityAfterDelay(request, TimeValue.timeValueSeconds(1L));
+        }
+        // check that the stack trace was not sent from the data node to the coordinating node
+        assertFalse(transportMessageHasStackTrace.get());
+    }
+
+    public void testAsyncSearchFailingQueryErrorTraceTrue() throws IOException, InterruptedException {
+        transportMessageHasStackTrace = new AtomicBoolean();
+        setupIndexWithDocs();
+
+        Request searchRequest = new Request("POST", "/_async_search");
+        searchRequest.setJsonEntity("""
+            {
+                "query": {
+                    "simple_query_string" : {
+                        "query": "foo",
+                        "fields": ["field"]
+                    }
+                }
+            }
+            """);
+        searchRequest.addParameter("error_trace", "true");
+        searchRequest.addParameter("keep_on_completion", "true");
+        searchRequest.addParameter("wait_for_completion_timeout", "0ms");
+        Map<String, Object> responseEntity = performRequestAndGetResponseEntityAfterDelay(searchRequest, TimeValue.ZERO);
+        String asyncExecutionId = (String) responseEntity.get("id");
+        Request request = new Request("GET", "/_async_search/" + asyncExecutionId);
+        request.addParameter("error_trace", "true");
+        while (responseEntity.get("is_running") instanceof Boolean isRunning && isRunning) {
+            responseEntity = performRequestAndGetResponseEntityAfterDelay(request, TimeValue.timeValueSeconds(1L));
+        }
+        // check that the stack trace was sent from the data node to the coordinating node
+        assertTrue(transportMessageHasStackTrace.get());
+    }
+
+    public void testAsyncSearchFailingQueryErrorTraceFalse() throws IOException, InterruptedException {
+        transportMessageHasStackTrace = new AtomicBoolean();
+        setupIndexWithDocs();
+
+        Request searchRequest = new Request("POST", "/_async_search");
+        searchRequest.setJsonEntity("""
+            {
+                "query": {
+                    "simple_query_string" : {
+                        "query": "foo",
+                        "fields": ["field"]
+                    }
+                }
+            }
+            """);
+        searchRequest.addParameter("error_trace", "false");
+        searchRequest.addParameter("keep_on_completion", "true");
+        searchRequest.addParameter("wait_for_completion_timeout", "0ms");
+        Map<String, Object> responseEntity = performRequestAndGetResponseEntityAfterDelay(searchRequest, TimeValue.ZERO);
+        String asyncExecutionId = (String) responseEntity.get("id");
+        Request request = new Request("GET", "/_async_search/" + asyncExecutionId);
+        request.addParameter("error_trace", "false");
+        while (responseEntity.get("is_running") instanceof Boolean isRunning && isRunning) {
+            responseEntity = performRequestAndGetResponseEntityAfterDelay(request, TimeValue.timeValueSeconds(1L));
+        }
+        // check that the stack trace was not sent from the data node to the coordinating node
+        assertFalse(transportMessageHasStackTrace.get());
+    }
+
+    public void testAsyncSearchFailingQueryErrorTraceFalseOnSubmitAndTrueOnGet() throws IOException, InterruptedException {
+        transportMessageHasStackTrace = new AtomicBoolean();
+        setupIndexWithDocs();
+
+        Request searchRequest = new Request("POST", "/_async_search");
+        searchRequest.setJsonEntity("""
+            {
+                "query": {
+                    "simple_query_string" : {
+                        "query": "foo",
+                        "fields": ["field"]
+                    }
+                }
+            }
+            """);
+        searchRequest.addParameter("error_trace", "false");
+        searchRequest.addParameter("keep_on_completion", "true");
+        searchRequest.addParameter("wait_for_completion_timeout", "0ms");
+        Map<String, Object> responseEntity = performRequestAndGetResponseEntityAfterDelay(searchRequest, TimeValue.ZERO);
+        String asyncExecutionId = (String) responseEntity.get("id");
+        Request request = new Request("GET", "/_async_search/" + asyncExecutionId);
+        request.addParameter("error_trace", "true");
+        while (responseEntity.get("is_running") instanceof Boolean isRunning && isRunning) {
+            responseEntity = performRequestAndGetResponseEntityAfterDelay(request, TimeValue.timeValueSeconds(1L));
+        }
+        // check that the stack trace was not sent from the data node to the coordinating node
+        assertFalse(transportMessageHasStackTrace.get());
+    }
+
+    public void testAsyncSearchFailingQueryErrorTraceTrueOnSubmitAndFalseOnGet() throws IOException, InterruptedException {
+        transportMessageHasStackTrace = new AtomicBoolean();
+        setupIndexWithDocs();
+
+        Request searchRequest = new Request("POST", "/_async_search");
+        searchRequest.setJsonEntity("""
+            {
+                "query": {
+                    "simple_query_string" : {
+                        "query": "foo",
+                        "fields": ["field"]
+                    }
+                }
+            }
+            """);
+        searchRequest.addParameter("error_trace", "true");
+        searchRequest.addParameter("keep_on_completion", "true");
+        searchRequest.addParameter("wait_for_completion_timeout", "0ms");
+        Map<String, Object> responseEntity = performRequestAndGetResponseEntityAfterDelay(searchRequest, TimeValue.ZERO);
+        String asyncExecutionId = (String) responseEntity.get("id");
+        Request request = new Request("GET", "/_async_search/" + asyncExecutionId);
+        request.addParameter("error_trace", "false");
+        while (responseEntity.get("is_running") instanceof Boolean isRunning && isRunning) {
+            responseEntity = performRequestAndGetResponseEntityAfterDelay(request, TimeValue.timeValueSeconds(1L));
+        }
+        // check that the stack trace was sent from the data node to the coordinating node
+        assertTrue(transportMessageHasStackTrace.get());
+    }
+
+    private Map<String, Object> performRequestAndGetResponseEntityAfterDelay(Request r, TimeValue sleep) throws IOException,
+        InterruptedException {
+        Thread.sleep(sleep.millis());
+        Response response = getRestClient().performRequest(r);
+        XContentType entityContentType = XContentType.fromMediaType(response.getEntity().getContentType().getValue());
+        return XContentHelper.convertToMap(entityContentType.xContent(), response.getEntity().getContent(), false);
+    }
+}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java
index bd09d8f7740a1..952febd46c34c 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java
@@ -55,6 +55,9 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        if (client.threadPool() != null && client.threadPool().getThreadContext() != null) {
+            client.threadPool().getThreadContext().setErrorTraceTransportHeader(request);
+        }
         SubmitAsyncSearchRequest submit = new SubmitAsyncSearchRequest();
         IntConsumer setSize = size -> submit.getSearchRequest().source().size(size);
         // for simplicity, we share parsing with ordinary search. That means a couple of unsupported parameters, like scroll
diff --git a/x-pack/plugin/async/build.gradle b/x-pack/plugin/async/build.gradle
index 36f1c6dea9b98..5ce5d07439cb7 100644
--- a/x-pack/plugin/async/build.gradle
+++ b/x-pack/plugin/async/build.gradle
@@ -1,9 +1,9 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 
 esplugin {
-    name 'x-pack-async'
-    description 'A module which handles common async operations'
-    classname 'org.elasticsearch.xpack.async.AsyncResultsIndexPlugin'
+    name = 'x-pack-async'
+    description = 'A module which handles common async operations'
+    classname ='org.elasticsearch.xpack.async.AsyncResultsIndexPlugin'
     extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/autoscaling/build.gradle b/x-pack/plugin/autoscaling/build.gradle
index edca88ff6bf28..24400a0fc418e 100644
--- a/x-pack/plugin/autoscaling/build.gradle
+++ b/x-pack/plugin/autoscaling/build.gradle
@@ -2,12 +2,12 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  name 'x-pack-autoscaling'
-  description 'Elasticsearch Expanded Pack Plugin - Autoscaling'
-  classname 'org.elasticsearch.xpack.autoscaling.Autoscaling'
+  name = 'x-pack-autoscaling'
+  description = 'Elasticsearch Expanded Pack Plugin - Autoscaling'
+  classname ='org.elasticsearch.xpack.autoscaling.Autoscaling'
   extendedPlugins = ['x-pack-core']
-  hasNativeController false
-  requiresKeystore true
+  hasNativeController =false
+  requiresKeystore =true
 }
 base {
   archivesName = 'x-pack-autoscaling'
diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java
index df1022e45c5a5..2357b80a402b3 100644
--- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java
+++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java
@@ -123,7 +123,7 @@ public void testZeroToOne() throws Exception {
         assertBusy(() -> {
             ExplainLifecycleResponse response = client().execute(
                 ExplainLifecycleAction.INSTANCE,
-                new ExplainLifecycleRequest().indices(INDEX_NAME)
+                new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(INDEX_NAME)
             ).actionGet();
             IndexLifecycleExplainResponse indexResponse = response.getIndexResponses().get(INDEX_NAME);
             assertNotNull(indexResponse);
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java
index b8911d24e1a28..07fceffe11daa 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
@@ -118,8 +118,8 @@ public TransportVersion getMinimalSupportedVersion() {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).xContentObjectFields(POLICIES_FIELD.getPreferredName(), policies);
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.xContentObjectFields(POLICIES_FIELD.getPreferredName(), policies);
     }
 
     @Override
diff --git a/x-pack/plugin/blob-cache/build.gradle b/x-pack/plugin/blob-cache/build.gradle
index c5c91a5ef87e3..6fa4d2c6da723 100644
--- a/x-pack/plugin/blob-cache/build.gradle
+++ b/x-pack/plugin/blob-cache/build.gradle
@@ -9,9 +9,9 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-test-artifact'
 
 esplugin {
-    name 'blob-cache'
-    description 'Provides a blob store cache'
-    classname 'org.elasticsearch.blobcache.BlobCachePlugin'
+    name = 'blob-cache'
+    description = 'Provides a blob store cache'
+    classname = 'org.elasticsearch.blobcache.BlobCachePlugin'
 }
 
 dependencies {
diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java
index 0fb4267745cb8..218569c825258 100644
--- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java
@@ -96,7 +96,7 @@ public BlobCacheMetrics(MeterRegistry meterRegistry) {
         this.cachePopulationTime = cachePopulationTime;
     }
 
-    public static BlobCacheMetrics NOOP = new BlobCacheMetrics(TelemetryProvider.NOOP.getMeterRegistry());
+    public static final BlobCacheMetrics NOOP = new BlobCacheMetrics(TelemetryProvider.NOOP.getMeterRegistry());
 
     public LongCounter getCacheMissCounter() {
         return cacheMissCounter;
diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBytes.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBytes.java
index ad0d99104e8a4..7e1d732befe34 100644
--- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBytes.java
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBytes.java
@@ -44,7 +44,7 @@ public class SharedBytes extends AbstractRefCounted {
     );
     private static final Logger logger = LogManager.getLogger(SharedBytes.class);
 
-    public static int PAGE_SIZE = 4096;
+    public static final int PAGE_SIZE = 4096;
 
     private static final String CACHE_FILE_NAME = "shared_snapshot_cache";
 
diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle
index fb37fb3575551..5ab0112d822ce 100644
--- a/x-pack/plugin/build.gradle
+++ b/x-pack/plugin/build.gradle
@@ -94,5 +94,7 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
   task.skipTest("privileges/11_builtin/Test get builtin privileges" ,"unnecessary to test compatibility")
   task.skipTest("esql/61_enrich_ip/Invalid IP strings", "We switched from exceptions to null+warnings for ENRICH runtime errors")
   task.skipTest("esql/180_match_operator/match with non text field", "Match operator can now be used on non-text fields")
+  task.skipTest("esql/180_match_operator/match with functions", "Error message changed")
+  task.skipTest("esql/40_unsupported_types/semantic_text declared in mapping", "The semantic text field format changed")
 })
 
diff --git a/x-pack/plugin/ccr/build.gradle b/x-pack/plugin/ccr/build.gradle
index b5e96ac2a8b34..765c70abee378 100644
--- a/x-pack/plugin/ccr/build.gradle
+++ b/x-pack/plugin/ccr/build.gradle
@@ -2,11 +2,11 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-java-rest-test'
 esplugin {
-  name 'x-pack-ccr'
-  description 'Elasticsearch Expanded Pack Plugin - CCR'
-  classname 'org.elasticsearch.xpack.ccr.Ccr'
-  hasNativeController false
-  requiresKeystore true
+  name = 'x-pack-ccr'
+  description = 'Elasticsearch Expanded Pack Plugin - CCR'
+  classname ='org.elasticsearch.xpack.ccr.Ccr'
+  hasNativeController =false
+  requiresKeystore =true
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java
index 0bb4afe51b85a..e8e19bad2a7ef 100644
--- a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java
+++ b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java
@@ -368,7 +368,7 @@ public void testSyntheticSource() throws Exception {
         if ("leader".equals(targetCluster)) {
             logger.info("Running against leader cluster");
             Settings settings = Settings.builder()
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
+                .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
                 .build();
             createIndex(adminClient(), leaderIndexName, settings, """
                 "properties": {"kwd": {"type": "keyword"}}}""", null);
diff --git a/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/follow_and_unfollow.yml b/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/follow_and_unfollow.yml
index ac3ed826973b0..2db18f6cf1aa9 100644
--- a/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/follow_and_unfollow.yml
+++ b/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/follow_and_unfollow.yml
@@ -66,8 +66,6 @@
   - do:
       indices.close:
         index: bar
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
 
   - do:
diff --git a/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/forget_follower.yml b/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/forget_follower.yml
index b4ed59a355edc..eb9278f59d0ba 100644
--- a/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/forget_follower.yml
+++ b/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/forget_follower.yml
@@ -75,8 +75,6 @@
   - do:
       indices.close:
         index: follower_index
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
 
   - do:
diff --git a/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/index_directly_into_follower_index.yml b/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/index_directly_into_follower_index.yml
index 7beca43f02f8a..e1269e9885bfa 100644
--- a/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/index_directly_into_follower_index.yml
+++ b/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/index_directly_into_follower_index.yml
@@ -61,8 +61,6 @@
   - do:
       indices.close:
         index: bar
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
 
   - do:
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrAliasesIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrAliasesIT.java
index 45aac762c616a..c389e84ed592a 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrAliasesIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrAliasesIT.java
@@ -306,7 +306,10 @@ public void testStress() throws Exception {
     }
 
     private String[] getAliasesOnLeader() throws InterruptedException, ExecutionException {
-        final GetAliasesResponse response = leaderClient().admin().indices().getAliases(new GetAliasesRequest().indices("leader")).get();
+        final GetAliasesResponse response = leaderClient().admin()
+            .indices()
+            .getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT).indices("leader"))
+            .get();
         return response.getAliases().get("leader").stream().map(AliasMetadata::alias).toArray(String[]::new);
     }
 
@@ -328,7 +331,7 @@ private <E extends Exception> void assertAliasesExist(
         assertBusy(() -> {
             final GetAliasesResponse followerResponse = followerClient().admin()
                 .indices()
-                .getAliases(new GetAliasesRequest().indices(followerIndex))
+                .getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT).indices(followerIndex))
                 .get();
             assertThat(
                 "expected follower to have [" + aliases.length + "] aliases, but was " + followerResponse.getAliases().toString(),
@@ -340,7 +343,7 @@ private <E extends Exception> void assertAliasesExist(
 
                 final GetAliasesResponse leaderResponse = leaderClient().admin()
                     .indices()
-                    .getAliases(new GetAliasesRequest().indices(leaderIndex).aliases(alias))
+                    .getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT).indices(leaderIndex).aliases(alias))
                     .get();
                 final AliasMetadata leaderAliasMetadata = getAliasMetadata(leaderResponse, leaderIndex, alias);
 
@@ -374,7 +377,7 @@ private void assertAliasExistence(final String alias, final boolean exists) thro
             // we must check serially because aliases exist will return true if any but not necessarily all of the requested aliases exist
             final GetAliasesResponse response = followerClient().admin()
                 .indices()
-                .getAliases(new GetAliasesRequest().indices("follower").aliases(alias))
+                .getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT).indices("follower").aliases(alias))
                 .get();
             if (exists) {
                 assertFalse("alias [" + alias + "] did not exist", response.getAliases().isEmpty());
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java
index 7ce55313aa771..cca8228557aba 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java
@@ -488,7 +488,7 @@ public void testFollowerMappingIsUpdated() throws IOException {
             clusterStateRequest.indices(followerIndex);
             MappingMetadata mappingMetadata = followerClient().admin()
                 .indices()
-                .prepareGetMappings("index2")
+                .prepareGetMappings(TEST_REQUEST_TIMEOUT, "index2")
                 .get()
                 .getMappings()
                 .get("index2");
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
index e8940b2f41756..e50627822e315 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
@@ -371,7 +371,12 @@ public void testSyncMappings() throws Exception {
         }
 
         assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), firstBatchNumDocs));
-        MappingMetadata mappingMetadata = followerClient().admin().indices().prepareGetMappings("index2").get().getMappings().get("index2");
+        MappingMetadata mappingMetadata = followerClient().admin()
+            .indices()
+            .prepareGetMappings(TEST_REQUEST_TIMEOUT, "index2")
+            .get()
+            .getMappings()
+            .get("index2");
         assertThat(XContentMapValues.extractValue("properties.f.type", mappingMetadata.sourceAsMap()), equalTo("integer"));
         assertThat(XContentMapValues.extractValue("properties.k", mappingMetadata.sourceAsMap()), nullValue());
 
@@ -382,7 +387,12 @@ public void testSyncMappings() throws Exception {
         }
 
         assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), firstBatchNumDocs + secondBatchNumDocs));
-        mappingMetadata = followerClient().admin().indices().prepareGetMappings("index2").get().getMappings().get("index2");
+        mappingMetadata = followerClient().admin()
+            .indices()
+            .prepareGetMappings(TEST_REQUEST_TIMEOUT, "index2")
+            .get()
+            .getMappings()
+            .get("index2");
         assertThat(XContentMapValues.extractValue("properties.f.type", mappingMetadata.sourceAsMap()), equalTo("integer"));
         assertThat(XContentMapValues.extractValue("properties.k.type", mappingMetadata.sourceAsMap()), equalTo("long"));
         pauseFollow("index2");
@@ -410,7 +420,12 @@ public void testNoMappingDefined() throws Exception {
         assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), 1));
         pauseFollow("index2");
 
-        MappingMetadata mappingMetadata = followerClient().admin().indices().prepareGetMappings("index2").get().getMappings().get("index2");
+        MappingMetadata mappingMetadata = followerClient().admin()
+            .indices()
+            .prepareGetMappings(TEST_REQUEST_TIMEOUT, "index2")
+            .get()
+            .getMappings()
+            .get("index2");
         assertThat(XContentMapValues.extractValue("properties.f.type", mappingMetadata.sourceAsMap()), equalTo("long"));
         assertThat(XContentMapValues.extractValue("properties.k", mappingMetadata.sourceAsMap()), nullValue());
     }
@@ -485,7 +500,7 @@ public void testAddAliasAfterUnfollow() throws Exception {
         assertAcked(followerClient().admin().indices().aliases(request).actionGet());
         final GetAliasesResponse response = followerClient().admin()
             .indices()
-            .getAliases(new GetAliasesRequest("follower_alias"))
+            .getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "follower_alias"))
             .actionGet();
         assertThat(response.getAliases().keySet().size(), equalTo(1));
         assertThat(response.getAliases().keySet().iterator().next(), equalTo("follower"));
@@ -605,8 +620,8 @@ public void testUnfollowNonExistingIndex() {
 
     public void testFollowNonExistentIndex() throws Exception {
         String indexSettings = getIndexSettings(1, 0);
-        assertAcked(leaderClient().admin().indices().prepareCreate("test-leader").setSource(indexSettings, XContentType.JSON).get());
         assertAcked(
+            leaderClient().admin().indices().prepareCreate("test-leader").setSource(indexSettings, XContentType.JSON),
             followerClient().admin()
                 .indices()
                 .prepareCreate("test-follower")
@@ -1188,7 +1203,7 @@ public void testUpdateAnalysisLeaderIndexSettings() throws Exception {
             assertThat(getSettingsResponse.getSetting("follower", "index.analysis.analyzer.my_analyzer.type"), equalTo("custom"));
             assertThat(getSettingsResponse.getSetting("follower", "index.analysis.analyzer.my_analyzer.tokenizer"), equalTo("keyword"));
 
-            GetMappingsRequest getMappingsRequest = new GetMappingsRequest();
+            GetMappingsRequest getMappingsRequest = new GetMappingsRequest(TEST_REQUEST_TIMEOUT);
             getMappingsRequest.indices("follower");
             GetMappingsResponse getMappingsResponse = followerClient().admin().indices().getMappings(getMappingsRequest).actionGet();
             MappingMetadata mappingMetadata = getMappingsResponse.getMappings().get("follower");
diff --git a/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/rest/ShardChangesRestIT.java b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/rest/ShardChangesRestIT.java
index e5dfea7b772f2..4c61904475093 100644
--- a/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/rest/ShardChangesRestIT.java
+++ b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/rest/ShardChangesRestIT.java
@@ -26,6 +26,9 @@
 import org.junit.ClassRule;
 
 import java.io.IOException;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -33,11 +36,14 @@
 public class ShardChangesRestIT extends ESRestTestCase {
     private static final String CCR_SHARD_CHANGES_ENDPOINT = "/%s/ccr/shard_changes";
     private static final String BULK_INDEX_ENDPOINT = "/%s/_bulk";
+    private static final String DATA_STREAM_ENDPOINT = "/_data_stream/%s";
+    private static final String INDEX_TEMPLATE_ENDPOINT = "/_index_template/%s";
 
     private static final String[] SHARD_RESPONSE_FIELDS = new String[] {
         "took_in_millis",
         "operations",
         "shard_id",
+        "index_abstraction",
         "index",
         "settings_version",
         "max_seq_no_of_updates_or_deletes",
@@ -46,6 +52,11 @@ public class ShardChangesRestIT extends ESRestTestCase {
         "aliases_version",
         "max_seq_no",
         "global_checkpoint" };
+
+    private static final String BULK_INDEX_TEMPLATE = """
+        { "index": { "op_type": "create" } }
+        { "@timestamp": "%s", "name": "%s" }
+        """;;
     private static final String[] NAMES = { "skywalker", "leia", "obi-wan", "yoda", "chewbacca", "r2-d2", "c-3po", "darth-vader" };
     @ClassRule
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
@@ -99,13 +110,86 @@ public void testShardChangesDefaultParams() throws IOException {
         createIndex(indexName, settings, mappings);
         assertTrue(indexExists(indexName));
 
-        assertOK(client().performRequest(bulkRequest(indexName, randomIntBetween(10, 20))));
+        assertOK(bulkIndex(indexName, randomIntBetween(10, 20)));
 
         final Request shardChangesRequest = new Request("GET", shardChangesEndpoint(indexName));
         final Response response = client().performRequest(shardChangesRequest);
         assertOK(response);
         assertShardChangesResponse(
-            XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false)
+            XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false),
+            indexName
+        );
+    }
+
+    public void testDataStreamShardChangesDefaultParams() throws IOException {
+        final String templateName = randomAlphanumericOfLength(8).toLowerCase(Locale.ROOT);
+        assertOK(createIndexTemplate(templateName, """
+            {
+              "index_patterns": [ "test-*-*" ],
+              "data_stream": {},
+              "priority": 100,
+              "template": {
+                "mappings": {
+                  "properties": {
+                    "@timestamp": {
+                      "type": "date"
+                    },
+                    "name": {
+                      "type": "keyword"
+                    }
+                  }
+                }
+              }
+            }"""));
+
+        final String dataStreamName = "test-"
+            + randomAlphanumericOfLength(5).toLowerCase(Locale.ROOT)
+            + "-"
+            + randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
+        assertOK(createDataStream(dataStreamName));
+
+        assertOK(bulkIndex(dataStreamName, randomIntBetween(10, 20)));
+
+        final Request shardChangesRequest = new Request("GET", shardChangesEndpoint(dataStreamName));
+        final Response response = client().performRequest(shardChangesRequest);
+        assertOK(response);
+        assertShardChangesResponse(
+            XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false),
+            dataStreamName
+        );
+    }
+
+    public void testIndexAliasShardChangesDefaultParams() throws IOException {
+        final String indexName = randomAlphanumericOfLength(10).toLowerCase(Locale.ROOT);
+        final String aliasName = randomAlphanumericOfLength(8).toLowerCase(Locale.ROOT);
+        final Settings settings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.getKey(), "1s")
+            .build();
+        final String mappings = """
+               {
+                 "properties": {
+                   "name": {
+                     "type": "keyword"
+                   }
+                 }
+               }
+            """;
+        createIndex(indexName, settings, mappings);
+        assertTrue(indexExists(indexName));
+
+        final Request putAliasRequest = new Request("PUT", "/" + indexName + "/_alias/" + aliasName);
+        assertOK(client().performRequest(putAliasRequest));
+
+        assertOK(bulkIndex(aliasName, randomIntBetween(10, 20)));
+
+        final Request shardChangesRequest = new Request("GET", shardChangesEndpoint(aliasName));
+        final Response response = client().performRequest(shardChangesRequest);
+        assertOK(response);
+        assertShardChangesResponse(
+            XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false),
+            aliasName
         );
     }
 
@@ -121,7 +205,7 @@ public void testShardChangesWithAllParameters() throws IOException {
         );
         assertTrue(indexExists(indexName));
 
-        assertOK(client().performRequest(bulkRequest(indexName, randomIntBetween(100, 200))));
+        assertOK(bulkIndex(indexName, randomIntBetween(100, 200)));
 
         final Request shardChangesRequest = new Request("GET", shardChangesEndpoint(indexName));
         shardChangesRequest.addParameter("from_seq_no", "0");
@@ -132,7 +216,8 @@ public void testShardChangesWithAllParameters() throws IOException {
         final Response response = client().performRequest(shardChangesRequest);
         assertOK(response);
         assertShardChangesResponse(
-            XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false)
+            XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false),
+            indexName
         );
     }
 
@@ -148,7 +233,7 @@ public void testShardChangesMultipleRequests() throws IOException {
         );
         assertTrue(indexExists(indexName));
 
-        assertOK(client().performRequest(bulkRequest(indexName, randomIntBetween(100, 200))));
+        assertOK(bulkIndex(indexName, randomIntBetween(100, 200)));
 
         final Request firstRequest = new Request("GET", shardChangesEndpoint(indexName));
         firstRequest.addParameter("from_seq_no", "0");
@@ -159,7 +244,8 @@ public void testShardChangesMultipleRequests() throws IOException {
         final Response firstResponse = client().performRequest(firstRequest);
         assertOK(firstResponse);
         assertShardChangesResponse(
-            XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(firstResponse.getEntity()), false)
+            XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(firstResponse.getEntity()), false),
+            indexName
         );
 
         final Request secondRequest = new Request("GET", shardChangesEndpoint(indexName));
@@ -171,7 +257,8 @@ public void testShardChangesMultipleRequests() throws IOException {
         final Response secondResponse = client().performRequest(secondRequest);
         assertOK(secondResponse);
         assertShardChangesResponse(
-            XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(secondResponse.getEntity()), false)
+            XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(secondResponse.getEntity()), false),
+            indexName
         );
     }
 
@@ -231,17 +318,36 @@ public void testShardChangesMissingIndex() throws IOException {
         assertResponseException(ex, RestStatus.BAD_REQUEST, "Failed to process shard changes for index [" + indexName + "]");
     }
 
-    private static Request bulkRequest(final String indexName, int numberOfDocuments) {
+    private static Response bulkIndex(final String indexName, int numberOfDocuments) throws IOException {
         final StringBuilder sb = new StringBuilder();
 
+        long timestamp = System.currentTimeMillis();
         for (int i = 0; i < numberOfDocuments; i++) {
-            sb.append(String.format(Locale.ROOT, "{ \"index\": { \"_id\": \"%d\" } }\n{ \"name\": \"%s\" }\n", i + 1, randomFrom(NAMES)));
+            sb.append(
+                String.format(
+                    Locale.ROOT,
+                    BULK_INDEX_TEMPLATE,
+                    Instant.ofEpochMilli(timestamp).atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME),
+                    randomFrom(NAMES)
+                )
+            );
+            timestamp += 1000; // 1 second
         }
 
         final Request request = new Request("POST", bulkEndpoint(indexName));
         request.setJsonEntity(sb.toString());
         request.addParameter("refresh", "true");
-        return request;
+        return client().performRequest(request);
+    }
+
+    private Response createDataStream(final String dataStreamName) throws IOException {
+        return client().performRequest(new Request("PUT", dataStreamEndpoint(dataStreamName)));
+    }
+
+    private static Response createIndexTemplate(final String templateName, final String mappings) throws IOException {
+        final Request request = new Request("PUT", indexTemplateEndpoint(templateName));
+        request.setJsonEntity(mappings);
+        return client().performRequest(request);
     }
 
     private static String shardChangesEndpoint(final String indexName) {
@@ -252,16 +358,28 @@ private static String bulkEndpoint(final String indexName) {
         return String.format(Locale.ROOT, BULK_INDEX_ENDPOINT, indexName);
     }
 
+    private static String dataStreamEndpoint(final String dataStreamName) {
+        return String.format(Locale.ROOT, DATA_STREAM_ENDPOINT, dataStreamName);
+    }
+
+    private static String indexTemplateEndpoint(final String templateName) {
+        return String.format(Locale.ROOT, INDEX_TEMPLATE_ENDPOINT, templateName);
+    }
+
     private void assertResponseException(final ResponseException ex, final RestStatus restStatus, final String error) {
         assertEquals(restStatus.getStatus(), ex.getResponse().getStatusLine().getStatusCode());
         assertThat(ex.getMessage(), Matchers.containsString(error));
     }
 
-    private void assertShardChangesResponse(final Map<String, Object> shardChangesResponseBody) {
+    private void assertShardChangesResponse(final Map<String, Object> shardChangesResponseBody, final String indexAbstractionName) {
         for (final String fieldName : SHARD_RESPONSE_FIELDS) {
             final Object fieldValue = shardChangesResponseBody.get(fieldName);
             assertNotNull("Field " + fieldName + " is missing or has a null value.", fieldValue);
 
+            if ("index_abstraction".equals(fieldName)) {
+                assertEquals(indexAbstractionName, fieldValue);
+            }
+
             if ("operations".equals(fieldName)) {
                 if (fieldValue instanceof List<?> operationsList) {
                     assertFalse("Field 'operations' is empty.", operationsList.isEmpty());
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestShardChangesAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestShardChangesAction.java
index 84171ebce162f..4a1d26d05a980 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestShardChangesAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestShardChangesAction.java
@@ -10,6 +10,8 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -32,6 +34,7 @@
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Locale;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
@@ -42,10 +45,14 @@
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 /**
- * A REST handler that retrieves shard changes in a specific index whose name is provided as a parameter.
- * It handles GET requests to the "/{index}/ccr/shard_changes" endpoint retrieving shard-level changes,
- * such as translog operations, mapping version, settings version, aliases version, the global checkpoint,
- * maximum sequence number and maximum sequence number of updates or deletes.
+ * A REST handler that retrieves shard changes in a specific index, data stream or alias whose name is
+ * provided as a parameter. It handles GET requests to the "/{index}/ccr/shard_changes" endpoint retrieving
+ * shard-level changes, such as Translog operations, mapping version, settings version, aliases version,
+ * the global checkpoint, maximum sequence number and maximum sequence number of updates or deletes.
+ * <p>
+ * In the case of a data stream, the first backing index is considered the target for retrieving shard changes.
+ * In the case of an alias, the first index that the alias points to is considered the target for retrieving
+ * shard changes.
  * <p>
  * Note: This handler is only available for snapshot builds.
  */
@@ -84,32 +91,36 @@ public List<Route> routes() {
     */
     @Override
     protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) throws IOException {
-        final var indexName = restRequest.param(INDEX_PARAM_NAME);
+        final var indexAbstractionName = restRequest.param(INDEX_PARAM_NAME);
         final var fromSeqNo = restRequest.paramAsLong(FROM_SEQ_NO_PARAM_NAME, DEFAULT_FROM_SEQ_NO);
         final var maxBatchSize = restRequest.paramAsSize(MAX_BATCH_SIZE_PARAM_NAME, DEFAULT_MAX_BATCH_SIZE);
         final var pollTimeout = restRequest.paramAsTime(POLL_TIMEOUT_PARAM_NAME, DEFAULT_POLL_TIMEOUT);
         final var maxOperationsCount = restRequest.paramAsInt(MAX_OPERATIONS_COUNT_PARAM_NAME, DEFAULT_MAX_OPERATIONS_COUNT);
 
-        final CompletableFuture<String> indexUUIDCompletableFuture = asyncGetIndexUUID(
+        // NOTE: we first retrieve the concrete index name in case we are dealing with an alias or data stream.
+        // Then we use the concrete index name to retrieve the index UUID and shard stats.
+        final CompletableFuture<String> indexNameCompletableFuture = asyncGetIndexName(
             client,
-            indexName,
+            indexAbstractionName,
             client.threadPool().executor(Ccr.CCR_THREAD_POOL_NAME)
         );
-        final CompletableFuture<ShardStats> shardStatsCompletableFuture = asyncShardStats(
-            client,
-            indexName,
-            client.threadPool().executor(Ccr.CCR_THREAD_POOL_NAME)
+        final CompletableFuture<String> indexUUIDCompletableFuture = indexNameCompletableFuture.thenCompose(
+            concreteIndexName -> asyncGetIndexUUID(client, concreteIndexName, client.threadPool().executor(Ccr.CCR_THREAD_POOL_NAME))
+        );
+        final CompletableFuture<ShardStats> shardStatsCompletableFuture = indexNameCompletableFuture.thenCompose(
+            concreteIndexName -> asyncShardStats(client, concreteIndexName, client.threadPool().executor(Ccr.CCR_THREAD_POOL_NAME))
         );
 
         return channel -> CompletableFuture.allOf(indexUUIDCompletableFuture, shardStatsCompletableFuture).thenRun(() -> {
             try {
+                final String concreteIndexName = indexNameCompletableFuture.get(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
                 final String indexUUID = indexUUIDCompletableFuture.get(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
                 final ShardStats shardStats = shardStatsCompletableFuture.get(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
                 final ShardId shardId = shardStats.getShardRouting().shardId();
                 final String expectedHistoryUUID = shardStats.getCommitStats().getUserData().get(Engine.HISTORY_UUID_KEY);
 
                 final ShardChangesAction.Request shardChangesRequest = shardChangesRequest(
-                    indexName,
+                    concreteIndexName,
                     indexUUID,
                     shardId,
                     expectedHistoryUUID,
@@ -121,7 +132,12 @@ protected RestChannelConsumer prepareRequest(final RestRequest restRequest, fina
                 client.execute(ShardChangesAction.INSTANCE, shardChangesRequest, new RestActionListener<>(channel) {
                     @Override
                     protected void processResponse(final ShardChangesAction.Response response) {
-                        channel.sendResponse(new RestResponse(RestStatus.OK, shardChangesResponseToXContent(response, indexName, shardId)));
+                        channel.sendResponse(
+                            new RestResponse(
+                                RestStatus.OK,
+                                shardChangesResponseToXContent(response, indexAbstractionName, concreteIndexName, shardId)
+                            )
+                        );
                     }
                 });
 
@@ -132,7 +148,12 @@ protected void processResponse(final ShardChangesAction.Response response) {
                 throw new IllegalStateException("Timeout while waiting for shard stats or index UUID", te);
             }
         }).exceptionally(ex -> {
-            channel.sendResponse(new RestResponse(RestStatus.BAD_REQUEST, "Failed to process shard changes for index [" + indexName + "]"));
+            channel.sendResponse(
+                new RestResponse(
+                    RestStatus.BAD_REQUEST,
+                    "Failed to process shard changes for index [" + indexAbstractionName + "] " + ex.getMessage()
+                )
+            );
             return null;
         });
     }
@@ -175,17 +196,20 @@ private static ShardChangesAction.Request shardChangesRequest(
      * Converts the response to XContent JSOn format.
      *
      * @param response The ShardChangesAction response.
-     * @param indexName The name of the index.
+     * @param indexAbstractionName The name of the index abstraction.
+     * @param concreteIndexName The name of the index.
      * @param shardId The ShardId.
      */
     private static XContentBuilder shardChangesResponseToXContent(
         final ShardChangesAction.Response response,
-        final String indexName,
+        final String indexAbstractionName,
+        final String concreteIndexName,
         final ShardId shardId
     ) {
         try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
             builder.startObject();
-            builder.field("index", indexName);
+            builder.field("index_abstraction", indexAbstractionName);
+            builder.field("index", concreteIndexName);
             builder.field("shard_id", shardId);
             builder.field("mapping_version", response.getMappingVersion());
             builder.field("settings_version", response.getSettingsVersion());
@@ -249,26 +273,60 @@ private static <T> CompletableFuture<T> supplyAsyncTask(
         }, executorService);
     }
 
+    /**
+     * Asynchronously retrieves the index name for a given index, alias or data stream.
+     * If the name represents a data stream, the name of the first backing index is returned.
+     * If the name represents an alias, the name of the first index that the alias points to is returned.
+     *
+     * @param client The NodeClient for executing the asynchronous request.
+     * @param indexAbstractionName The name of the index, alias or data stream.
+     * @return A CompletableFuture that completes with the retrieved index name.
+     */
+    private static CompletableFuture<String> asyncGetIndexName(
+        final NodeClient client,
+        final String indexAbstractionName,
+        final ExecutorService executorService
+    ) {
+        return supplyAsyncTask(() -> {
+            final ClusterState clusterState = client.admin()
+                .cluster()
+                .prepareState(new TimeValue(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+                .get(GET_INDEX_UUID_TIMEOUT)
+                .getState();
+            final IndexAbstraction indexAbstraction = clusterState.metadata().getIndicesLookup().get(indexAbstractionName);
+            if (indexAbstraction == null) {
+                throw new IllegalArgumentException(
+                    String.format(Locale.ROOT, "Invalid index or data stream name [%s]", indexAbstractionName)
+                );
+            }
+            if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM
+                || indexAbstraction.getType() == IndexAbstraction.Type.ALIAS) {
+                return indexAbstraction.getIndices().getFirst().getName();
+            }
+            return indexAbstractionName;
+        }, executorService, "Error while retrieving index name for index or data stream [" + indexAbstractionName + "]");
+    }
+
     /**
      * Asynchronously retrieves the shard stats for a given index using an executor service.
      *
      * @param client The NodeClient for executing the asynchronous request.
-     * @param indexName The name of the index for which to retrieve shard statistics.
+     * @param concreteIndexName The name of the index for which to retrieve shard statistics.
      * @param executorService The executorService service for executing the asynchronous task.
      * @return A CompletableFuture that completes with the retrieved ShardStats.
      * @throws ElasticsearchException If an error occurs while retrieving shard statistics.
      */
     private static CompletableFuture<ShardStats> asyncShardStats(
         final NodeClient client,
-        final String indexName,
+        final String concreteIndexName,
         final ExecutorService executorService
     ) {
         return supplyAsyncTask(
-            () -> Arrays.stream(client.admin().indices().prepareStats(indexName).clear().get(SHARD_STATS_TIMEOUT).getShards())
+            () -> Arrays.stream(client.admin().indices().prepareStats(concreteIndexName).clear().get(SHARD_STATS_TIMEOUT).getShards())
                 .max(Comparator.comparingLong(shardStats -> shardStats.getCommitStats().getGeneration()))
-                .orElseThrow(() -> new ElasticsearchException("Unable to retrieve shard stats for index: " + indexName)),
+                .orElseThrow(() -> new ElasticsearchException("Unable to retrieve shard stats for index: " + concreteIndexName)),
             executorService,
-            "Error while retrieving shard stats for index [" + indexName + "]"
+            "Error while retrieving shard stats for index [" + concreteIndexName + "]"
         );
     }
 
@@ -276,25 +334,25 @@ private static CompletableFuture<ShardStats> asyncShardStats(
      * Asynchronously retrieves the index UUID for a given index using an executor service.
      *
      * @param client The NodeClient for executing the asynchronous request.
-     * @param indexName The name of the index for which to retrieve the index UUID.
+     * @param concreteIndexName The name of the index for which to retrieve the index UUID.
      * @param executorService The executorService service for executing the asynchronous task.
      * @return A CompletableFuture that completes with the retrieved index UUID.
      * @throws ElasticsearchException If an error occurs while retrieving the index UUID.
      */
     private static CompletableFuture<String> asyncGetIndexUUID(
         final NodeClient client,
-        final String indexName,
+        final String concreteIndexName,
         final ExecutorService executorService
     ) {
         return supplyAsyncTask(
             () -> client.admin()
                 .indices()
                 .prepareGetIndex()
-                .setIndices(indexName)
+                .setIndices(concreteIndexName)
                 .get(GET_INDEX_UUID_TIMEOUT)
-                .getSetting(indexName, IndexMetadata.SETTING_INDEX_UUID),
+                .getSetting(concreteIndexName, IndexMetadata.SETTING_INDEX_UUID),
             executorService,
-            "Error while retrieving index UUID for index [" + indexName + "]"
+            "Error while retrieving index UUID for index [" + concreteIndexName + "]"
         );
     }
 }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java
index 357e1bca38e8f..6bc25215855ee 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.ccr.Ccr;
 import org.elasticsearch.xpack.ccr.CcrSettings;
@@ -335,7 +334,7 @@ public void testDynamicIndexSettingsAreClassified() {
         replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING);
         replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING);
         replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING);
-        replicatedSettings.add(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING);
+        replicatedSettings.add(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING);
 
         for (Setting<?> setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) {
             // removed settings have no effect, they are only there for BWC
diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle
index 51d770936e64e..04458c5ecaa57 100644
--- a/x-pack/plugin/core/build.gradle
+++ b/x-pack/plugin/core/build.gradle
@@ -22,11 +22,11 @@ base {
 }
 
 esplugin {
-  name 'x-pack-core'
-  description 'Elasticsearch Expanded Pack Plugin - Core'
-  classname 'org.elasticsearch.xpack.core.XPackPlugin'
-  hasNativeController false
-  requiresKeystore false
+  name = 'x-pack-core'
+  description = 'Elasticsearch Expanded Pack Plugin - Core'
+  classname ='org.elasticsearch.xpack.core.XPackPlugin'
+  hasNativeController =false
+  requiresKeystore =false
 }
 
 tasks.named("dependencyLicenses").configure {
diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java
index 1bf52b663b30f..af7e223e1c5b9 100644
--- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java
+++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java
@@ -197,7 +197,7 @@ public void testSnapshotWithDanglingLocalSegment() throws Exception {
     }
 
     private static void assertMappings(String sourceIdx, boolean requireRouting, boolean useNested) throws IOException {
-        GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(sourceIdx).get();
+        GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, sourceIdx).get();
         MappingMetadata mapping = getMappingsResponse.getMappings().get(sourceIdx);
         String nested = useNested ? """
             ,"incorrect":{"type":"object"},"nested":{"type":"nested","properties":{"value":{"type":"long"}}}""" : "";
diff --git a/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java b/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java
index 0b6e187930c01..600a41ed01beb 100644
--- a/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java
+++ b/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java
@@ -9,6 +9,7 @@
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -51,7 +52,8 @@ public void testDSXpackUsage() throws Exception {
         assertThat(dataStreams.get("data_streams"), equalTo(0));
         assertThat(dataStreams, hasKey("failure_store"));
         Map<String, Integer> failureStoreStats = (Map<String, Integer>) dataStreams.get("failure_store");
-        assertThat(failureStoreStats.get("enabled_count"), equalTo(0));
+        assertThat(failureStoreStats.get("explicitly_enabled_count"), equalTo(0));
+        assertThat(failureStoreStats.get("effectively_enabled_count"), equalTo(0));
         assertThat(failureStoreStats.get("failure_indices_count"), equalTo(0));
         assertBusy(() -> {
             Map<?, ?> logsTemplate = (Map<?, ?>) ((List<?>) getLocation("/_index_template/logs").get("index_templates")).get(0);
@@ -75,7 +77,7 @@ public void testDSXpackUsage() throws Exception {
         indexRequest.setJsonEntity("{\"@timestamp\": \"2020-01-01\"}");
         client().performRequest(indexRequest);
         // Initialize the failure store
-        rollover = new Request("POST", "/fs/_rollover?target_failure_store=true");
+        rollover = new Request("POST", "/fs::failures/_rollover");
         client().performRequest(rollover);
 
         dataStreams = (Map<?, ?>) getLocation("/_xpack/usage").get("data_streams");
@@ -85,8 +87,21 @@ public void testDSXpackUsage() throws Exception {
         assertThat("got: " + dataStreams, dataStreams.get("data_streams"), equalTo(2));
         assertThat("got: " + dataStreams, dataStreams.get("indices_count"), equalTo(3));
         failureStoreStats = (Map<String, Integer>) dataStreams.get("failure_store");
-        assertThat(failureStoreStats.get("enabled_count"), equalTo(1));
+        assertThat(failureStoreStats.get("explicitly_enabled_count"), equalTo(1));
+        assertThat(failureStoreStats.get("effectively_enabled_count"), equalTo(1));
         assertThat(failureStoreStats.get("failure_indices_count"), equalTo(1));
+
+        // Enable the failure store for logs-mysql-default using the cluster setting...
+        updateClusterSettings(
+            Settings.builder()
+                .put(DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), "logs-mysql-default")
+                .build()
+        );
+        // ...and assert that it counts towards effectively_enabled_count but not explicitly_enabled_count:
+        dataStreams = (Map<?, ?>) getLocation("/_xpack/usage").get("data_streams");
+        failureStoreStats = (Map<String, Integer>) dataStreams.get("failure_store");
+        assertThat(failureStoreStats.get("explicitly_enabled_count"), equalTo(1));
+        assertThat(failureStoreStats.get("effectively_enabled_count"), equalTo(2));
     }
 
     Map<String, Object> getLocation(String path) {
diff --git a/x-pack/plugin/core/src/main/java/module-info.java b/x-pack/plugin/core/src/main/java/module-info.java
index 72436bb9d5171..55b3428907c40 100644
--- a/x-pack/plugin/core/src/main/java/module-info.java
+++ b/x-pack/plugin/core/src/main/java/module-info.java
@@ -126,6 +126,7 @@
     exports org.elasticsearch.xpack.core.ml.stats;
     exports org.elasticsearch.xpack.core.ml.utils.time;
     exports org.elasticsearch.xpack.core.ml.utils;
+    exports org.elasticsearch.xpack.core.ml.vectors;
     exports org.elasticsearch.xpack.core.ml;
     exports org.elasticsearch.xpack.core.monitoring.action;
     exports org.elasticsearch.xpack.core.monitoring.exporter;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/ClusterStateLicenseService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/ClusterStateLicenseService.java
index a38170d87f9a1..873a5b4587e0e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/ClusterStateLicenseService.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/ClusterStateLicenseService.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.license.internal.MutableLicenseService;
 import org.elasticsearch.license.internal.TrialLicenseVersion;
@@ -65,7 +64,6 @@ public class ClusterStateLicenseService extends AbstractLifecycleComponent
     private final Settings settings;
 
     private final ClusterService clusterService;
-    private final FeatureService featureService;
 
     /**
      * The xpack feature state to update when license changes are made.
@@ -104,12 +102,10 @@ public ClusterStateLicenseService(
         ThreadPool threadPool,
         ClusterService clusterService,
         Clock clock,
-        XPackLicenseState xPacklicenseState,
-        FeatureService featureService
+        XPackLicenseState xPacklicenseState
     ) {
         this.settings = settings;
         this.clusterService = clusterService;
-        this.featureService = featureService;
         this.startTrialTaskQueue = clusterService.createTaskQueue(
             "license-service-start-trial",
             Priority.NORMAL,
@@ -344,7 +340,7 @@ public void startTrialLicense(PostStartTrialRequest request, final ActionListene
         }
         startTrialTaskQueue.submitTask(
             StartTrialClusterTask.TASK_SOURCE,
-            new StartTrialClusterTask(logger, clusterService.getClusterName().value(), clock, featureService, request, listener),
+            new StartTrialClusterTask(logger, clusterService.getClusterName().value(), clock, request, listener),
             null             // TODO should pass in request.masterNodeTimeout() here
         );
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java
index 19790e61b6102..d3347f3432a47 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.UpdateForV9;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
@@ -39,8 +38,6 @@
  */
 public class License implements ToXContentObject {
 
-    public static final NodeFeature INDEPENDENT_TRIAL_VERSION_FEATURE = new NodeFeature("license-trial-independent-version");
-
     public enum LicenseType {
         BASIC,
         STANDARD,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/StartTrialClusterTask.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/StartTrialClusterTask.java
index 22f4de105cb2d..bec2ee98b1757 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/StartTrialClusterTask.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/StartTrialClusterTask.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.license.internal.TrialLicenseVersion;
 import org.elasticsearch.xpack.core.XPackPlugin;
 
@@ -41,13 +40,11 @@ public class StartTrialClusterTask implements ClusterStateTaskListener {
     private final PostStartTrialRequest request;
     private final ActionListener<PostStartTrialResponse> listener;
     private final Clock clock;
-    private final FeatureService featureService;
 
     StartTrialClusterTask(
         Logger logger,
         String clusterName,
         Clock clock,
-        FeatureService featureService,
         PostStartTrialRequest request,
         ActionListener<PostStartTrialResponse> listener
     ) {
@@ -56,7 +53,6 @@ public class StartTrialClusterTask implements ClusterStateTaskListener {
         this.request = request;
         this.listener = listener;
         this.clock = clock;
-        this.featureService = featureService;
     }
 
     private LicensesMetadata execute(
@@ -65,9 +61,6 @@ private LicensesMetadata execute(
         ClusterStateTaskExecutor.TaskContext<StartTrialClusterTask> taskContext
     ) {
         assert taskContext.getTask() == this;
-        if (featureService.clusterHasFeature(state, License.INDEPENDENT_TRIAL_VERSION_FEATURE) == false) {
-            throw new IllegalStateException("Please ensure all nodes are up to date before starting your trial");
-        }
         final var listener = ActionListener.runBefore(this.listener, () -> {
             logger.debug("started self generated trial license: {}", currentLicensesMetadata);
         });
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
index f5923a4942634..23f9e91dc32da 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
@@ -395,8 +395,6 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
         );
     }
 
-    // TODO: The WeightedTokensBuilder is slated for removal after the SparseVectorQueryBuilder is available.
-    // The logic to create a Boolean query based on weighted tokens will remain and/or be moved to server.
     @Override
     public List<SearchPlugin.QuerySpec<?>> getQueries() {
         return List.of(
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackFeatures.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackFeatures.java
index f966bf97f4764..42824a553d2bd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackFeatures.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackFeatures.java
@@ -9,8 +9,6 @@
 
 import org.elasticsearch.features.FeatureSpecification;
 import org.elasticsearch.features.NodeFeature;
-import org.elasticsearch.license.License;
-import org.elasticsearch.xpack.core.datatiers.NodesDataTiersUsageTransportAction;
 
 import java.util.Set;
 
@@ -18,16 +16,8 @@
  * Provides the XPack features that this version of the code supports
  */
 public class XPackFeatures implements FeatureSpecification {
-    public static final NodeFeature LOGSDB_TELEMETRY = new NodeFeature("logsdb_telemetry");
-    public static final NodeFeature LOGSDB_TELMETRY_STATS = new NodeFeature("logsdb_telemetry_stats");
-
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(
-            NodesDataTiersUsageTransportAction.LOCALLY_PRECALCULATED_STATS_FEATURE, // Added in 8.12
-            License.INDEPENDENT_TRIAL_VERSION_FEATURE, // 8.14.0
-            LOGSDB_TELEMETRY,
-            LOGSDB_TELMETRY_STATS
-        );
+        return Set.of();
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java
index f79a3fbf124b1..cf5ebc8adc56d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java
@@ -331,8 +331,7 @@ public Collection<?> createComponents(PluginServices services) {
                 services.threadPool(),
                 services.clusterService(),
                 getClock(),
-                getLicenseState(),
-                services.featureService()
+                getLicenseState()
             );
             setLicenseService(licenseService);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamUsageTransportAction.java
index 26f3bdd7654ca..7eece9177cf2b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamUsageTransportAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamUsageTransportAction.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.injection.guice.Inject;
@@ -24,13 +25,16 @@
 
 public class DataStreamUsageTransportAction extends XPackUsageFeatureTransportAction {
 
+    private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings;
+
     @Inject
     public DataStreamUsageTransportAction(
         TransportService transportService,
         ClusterService clusterService,
         ThreadPool threadPool,
         ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        DataStreamFailureStoreSettings dataStreamFailureStoreSettings
     ) {
         super(
             XPackUsageFeatureAction.DATA_STREAMS.name(),
@@ -40,6 +44,7 @@ public DataStreamUsageTransportAction(
             actionFilters,
             indexNameExpressionResolver
         );
+        this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings;
     }
 
     @Override
@@ -51,13 +56,17 @@ protected void masterOperation(
     ) {
         final Map<String, DataStream> dataStreams = state.metadata().dataStreams();
         long backingIndicesCounter = 0;
-        long failureStoreEnabledCounter = 0;
+        long failureStoreExplicitlyEnabledCounter = 0;
+        long failureStoreEffectivelyEnabledCounter = 0;
         long failureIndicesCounter = 0;
         for (DataStream ds : dataStreams.values()) {
             backingIndicesCounter += ds.getIndices().size();
             if (DataStream.isFailureStoreFeatureFlagEnabled()) {
-                if (ds.isFailureStoreEnabled()) {
-                    failureStoreEnabledCounter++;
+                if (ds.isFailureStoreExplicitlyEnabled()) {
+                    failureStoreExplicitlyEnabledCounter++;
+                }
+                if (ds.isFailureStoreEffectivelyEnabled(dataStreamFailureStoreSettings)) {
+                    failureStoreEffectivelyEnabledCounter++;
                 }
                 if (ds.getFailureIndices().getIndices().isEmpty() == false) {
                     failureIndicesCounter += ds.getFailureIndices().getIndices().size();
@@ -67,7 +76,8 @@ protected void masterOperation(
         final DataStreamFeatureSetUsage.DataStreamStats stats = new DataStreamFeatureSetUsage.DataStreamStats(
             dataStreams.size(),
             backingIndicesCounter,
-            failureStoreEnabledCounter,
+            failureStoreExplicitlyEnabledCounter,
+            failureStoreEffectivelyEnabledCounter,
             failureIndicesCounter
         );
         final DataStreamFeatureSetUsage usage = new DataStreamFeatureSetUsage(stats);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/util/ExpandedIdsMatcher.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/util/ExpandedIdsMatcher.java
index 2e0a799ea99a5..ac071fab1d69a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/util/ExpandedIdsMatcher.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/util/ExpandedIdsMatcher.java
@@ -34,8 +34,6 @@
  */
 public final class ExpandedIdsMatcher {
 
-    public static String ALL = "_all";
-
     /**
      * Split {@code expression} into tokens separated by a ','
      *
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncResultsService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncResultsService.java
index 217274f963aec..2236304d8c4d0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncResultsService.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncResultsService.java
@@ -121,7 +121,7 @@ private void getSearchResponseFromTask(
     ) {
         try {
             final Task task = store.getTaskAndCheckAuthentication(taskManager, searchId, asyncTaskClass);
-            if (task == null || task.isCancelled()) {
+            if (task == null || (updateInitialResultsInStore && task.isCancelled())) {
                 getSearchResponseFromIndex(searchId, request, nowInMillis, listener);
                 return;
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java
index a53600348797a..3aeefa3afb796 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java
@@ -13,12 +13,13 @@
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -149,10 +150,11 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params)
-            .xContentObjectFieldObjects(PATTERNS_FIELD.getPreferredName(), patterns)
-            .object(FOLLOWED_LEADER_INDICES_FIELD.getPreferredName(), followedLeaderIndexUUIDs)
-            .object(HEADERS.getPreferredName(), headers);
+        return Iterators.concat(
+            ChunkedToXContentHelper.xContentObjectFieldObjects(PATTERNS_FIELD.getPreferredName(), patterns),
+            ChunkedToXContentHelper.object(FOLLOWED_LEADER_INDICES_FIELD.getPreferredName(), followedLeaderIndexUUIDs),
+            ChunkedToXContentHelper.object(HEADERS.getPreferredName(), headers)
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamFeatureSetUsage.java
index 7379af113fda0..379b07d9b9a77 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamFeatureSetUsage.java
@@ -52,7 +52,8 @@ protected void innerXContent(XContentBuilder builder, Params params) throws IOEx
         builder.field("indices_count", streamStats.indicesBehindDataStream);
         if (DataStream.isFailureStoreFeatureFlagEnabled()) {
             builder.startObject("failure_store");
-            builder.field("enabled_count", streamStats.failureStoreEnabledDataStreamCount);
+            builder.field("explicitly_enabled_count", streamStats.failureStoreExplicitlyEnabledDataStreamCount);
+            builder.field("effectively_enabled_count", streamStats.failureStoreEffectivelyEnabledDataStreamCount);
             builder.field("failure_indices_count", streamStats.failureStoreIndicesCount);
             builder.endObject();
         }
@@ -83,7 +84,8 @@ public boolean equals(Object obj) {
     public record DataStreamStats(
         long totalDataStreamCount,
         long indicesBehindDataStream,
-        long failureStoreEnabledDataStreamCount,
+        long failureStoreExplicitlyEnabledDataStreamCount,
+        long failureStoreEffectivelyEnabledDataStreamCount,
         long failureStoreIndicesCount
     ) implements Writeable {
 
@@ -92,6 +94,7 @@ public DataStreamStats(StreamInput in) throws IOException {
                 in.readVLong(),
                 in.readVLong(),
                 in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0) ? in.readVLong() : 0,
+                in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_ENABLED_BY_CLUSTER_SETTING) ? in.readVLong() : 0,
                 in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0) ? in.readVLong() : 0
             );
         }
@@ -101,7 +104,10 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeVLong(this.totalDataStreamCount);
             out.writeVLong(this.indicesBehindDataStream);
             if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) {
-                out.writeVLong(this.failureStoreEnabledDataStreamCount);
+                out.writeVLong(this.failureStoreExplicitlyEnabledDataStreamCount);
+                if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_ENABLED_BY_CLUSTER_SETTING)) {
+                    out.writeVLong(failureStoreEffectivelyEnabledDataStreamCount);
+                }
                 out.writeVLong(this.failureStoreIndicesCount);
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java
index 9f265f86fae21..e859d81036669 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java
@@ -9,7 +9,6 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
-import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.ParentTaskAssigningClient;
@@ -22,7 +21,6 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.indices.NodeIndicesStats;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
@@ -46,7 +44,6 @@
 public class DataTiersUsageTransportAction extends XPackUsageFeatureTransportAction {
 
     private final Client client;
-    private final FeatureService featureService;
 
     @Inject
     public DataTiersUsageTransportAction(
@@ -55,8 +52,7 @@ public DataTiersUsageTransportAction(
         ThreadPool threadPool,
         ActionFilters actionFilters,
         IndexNameExpressionResolver indexNameExpressionResolver,
-        Client client,
-        FeatureService featureService
+        Client client
     ) {
         super(
             XPackUsageFeatureAction.DATA_TIERS.name(),
@@ -67,7 +63,6 @@ public DataTiersUsageTransportAction(
             indexNameExpressionResolver
         );
         this.client = client;
-        this.featureService = featureService;
     }
 
     @Override
@@ -77,42 +72,22 @@ protected void masterOperation(
         ClusterState state,
         ActionListener<XPackUsageFeatureResponse> listener
     ) {
-        if (featureService.clusterHasFeature(state, NodesDataTiersUsageTransportAction.LOCALLY_PRECALCULATED_STATS_FEATURE)) {
-            new ParentTaskAssigningClient(client, clusterService.localNode(), task).admin()
-                .cluster()
-                .execute(
-                    NodesDataTiersUsageTransportAction.TYPE,
-                    new NodesDataTiersUsageTransportAction.NodesRequest(),
-                    listener.delegateFailureAndWrap((delegate, response) -> {
-                        // Generate tier specific stats for the nodes and indices
-                        delegate.onResponse(
-                            new XPackUsageFeatureResponse(
-                                new DataTiersFeatureSetUsage(
-                                    aggregateStats(response.getNodes(), getIndicesGroupedByTier(state, response.getNodes()))
-                                )
-                            )
-                        );
-                    })
-                );
-        } else {
-            new ParentTaskAssigningClient(client, clusterService.localNode(), task).admin()
-                .cluster()
-                .prepareNodesStats()
-                .setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Docs, CommonStatsFlags.Flag.Store))
-                .execute(listener.delegateFailureAndWrap((delegate, nodesStatsResponse) -> {
-                    List<NodeDataTiersUsage> response = nodesStatsResponse.getNodes()
-                        .stream()
-                        .map(
-                            nodeStats -> new NodeDataTiersUsage(nodeStats.getNode(), precalculateLocalStatsFromNodeStats(nodeStats, state))
-                        )
-                        .toList();
+        new ParentTaskAssigningClient(client, clusterService.localNode(), task).admin()
+            .cluster()
+            .execute(
+                NodesDataTiersUsageTransportAction.TYPE,
+                new NodesDataTiersUsageTransportAction.NodesRequest(),
+                listener.delegateFailureAndWrap((delegate, response) -> {
+                    // Generate tier specific stats for the nodes and indices
                     delegate.onResponse(
                         new XPackUsageFeatureResponse(
-                            new DataTiersFeatureSetUsage(aggregateStats(response, getIndicesGroupedByTier(state, response)))
+                            new DataTiersFeatureSetUsage(
+                                aggregateStats(response.getNodes(), getIndicesGroupedByTier(state, response.getNodes()))
+                            )
                         )
                     );
-                }));
-        }
+                })
+            );
     }
 
     // Visible for testing
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java
index 335a4eaaf5847..99bb926b0a3c7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.store.StoreStats;
 import org.elasticsearch.indices.IndicesService;
@@ -58,7 +57,6 @@ public class NodesDataTiersUsageTransportAction extends TransportNodesAction<
     Void> {
 
     public static final ActionType<NodesResponse> TYPE = new ActionType<>("cluster:monitor/nodes/data_tier_usage");
-    public static final NodeFeature LOCALLY_PRECALCULATED_STATS_FEATURE = new NodeFeature("usage.data_tiers.precalculate_stats");
 
     private static final CommonStatsFlags STATS_FLAGS = new CommonStatsFlags().clear()
         .set(CommonStatsFlags.Flag.Docs, true)
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java
new file mode 100644
index 0000000000000..024d24fdf5151
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.deprecation;
+
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
+
+import java.util.function.Predicate;
+
+public class DeprecatedIndexPredicate {
+
+    public static final IndexVersion MINIMUM_WRITEABLE_VERSION_AFTER_UPGRADE = IndexVersions.UPGRADE_TO_LUCENE_10_0_0;
+
+    /*
+     * This predicate allows through only indices that were created with a previous lucene version, meaning that they need to be reindexed
+     * in order to be writable in the _next_ lucene version.
+     *
+     * It ignores searchable snapshots as they are not writable.
+     */
+    public static Predicate<Index> getReindexRequiredPredicate(Metadata metadata) {
+        return index -> {
+            IndexMetadata indexMetadata = metadata.index(index);
+            return reindexRequired(indexMetadata);
+        };
+    }
+
+    public static boolean reindexRequired(IndexMetadata indexMetadata) {
+        return creationVersionBeforeMinimumWritableVersion(indexMetadata) && isNotSearchableSnapshot(indexMetadata);
+    }
+
+    private static boolean isNotSearchableSnapshot(IndexMetadata indexMetadata) {
+        return indexMetadata.isSearchableSnapshot() == false;
+    }
+
+    private static boolean creationVersionBeforeMinimumWritableVersion(IndexMetadata metadata) {
+        return metadata.getCreationVersion().before(MINIMUM_WRITEABLE_VERSION_AFTER_UPGRADE);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichMetadata.java
index 30ecaf2ff680f..e433093cdfb9a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichMetadata.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
@@ -98,8 +98,8 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).xContentObjectFieldObjects(POLICIES.getPreferredName(), policies);
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.xContentObjectFieldObjects(POLICIES.getPreferredName(), policies);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequest.java
index 5c607335bff39..beff1e6ab39de 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequest.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.support.master.info.ClusterInfoRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -27,8 +28,8 @@ public class ExplainLifecycleRequest extends ClusterInfoRequest<ExplainLifecycle
     private boolean onlyErrors = false;
     private boolean onlyManaged = false;
 
-    public ExplainLifecycleRequest() {
-        super();
+    public ExplainLifecycleRequest(TimeValue masterTimeout) {
+        super(masterTimeout);
     }
 
     public ExplainLifecycleRequest(StreamInput in) throws IOException {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java
index 09e625b96135c..fceeae07ce75b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java
@@ -6,12 +6,10 @@
  */
 package org.elasticsearch.xpack.core.ilm;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
@@ -21,10 +19,14 @@
 import java.util.List;
 
 /**
- * A {@link LifecycleAction} which freezes the index.
+ * A noop {@link LifecycleAction} that replaces the removed freeze action. We keep it for backwards compatibility purposes in case we
+ * encounter a policy or an index that refers to this action and its steps in the lifecycle state.
+ * At 10.x we would like to sanitize the freeze action from input and expunge from the lifecycle execution
+ * state of indices.
  */
+@Deprecated
+@UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
 public class FreezeAction implements LifecycleAction {
-    private static final Logger logger = LogManager.getLogger(FreezeAction.class);
 
     public static final String NAME = "freeze";
     public static final String CONDITIONAL_SKIP_FREEZE_STEP = BranchingStep.NAME + "-freeze-check-prerequisites";
@@ -34,6 +36,7 @@ public class FreezeAction implements LifecycleAction {
     private static final ObjectParser<FreezeAction, Void> PARSER = new ObjectParser<>(NAME, () -> INSTANCE);
 
     public static FreezeAction parse(XContentParser parser) {
+
         return PARSER.apply(parser, null);
     }
 
@@ -63,40 +66,10 @@ public boolean isSafeAction() {
     public List<Step> toSteps(Client client, String phase, StepKey nextStepKey) {
         StepKey preFreezeMergeBranchingKey = new StepKey(phase, NAME, CONDITIONAL_SKIP_FREEZE_STEP);
         StepKey checkNotWriteIndex = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME);
-        StepKey freezeStepKey = new StepKey(phase, NAME, FreezeStep.NAME);
-
-        BranchingStep conditionalSkipFreezeStep = new BranchingStep(
-            preFreezeMergeBranchingKey,
-            checkNotWriteIndex,
-            nextStepKey,
-            (index, clusterState) -> {
-                IndexMetadata indexMetadata = clusterState.getMetadata().index(index);
-                assert indexMetadata != null : "index " + index.getName() + " must exist in the cluster state";
-                String policyName = indexMetadata.getLifecyclePolicyName();
-                if (indexMetadata.getSettings().get(LifecycleSettings.SNAPSHOT_INDEX_NAME) != null) {
-                    logger.warn(
-                        "[{}] action is configured for index [{}] in policy [{}] which is mounted as searchable snapshot. "
-                            + "Skipping this action",
-                        FreezeAction.NAME,
-                        index.getName(),
-                        policyName
-                    );
-                    return true;
-                }
-                if (indexMetadata.getSettings().getAsBoolean("index.frozen", false)) {
-                    logger.debug(
-                        "skipping [{}] action for index [{}] in policy [{}] as the index is already frozen",
-                        FreezeAction.NAME,
-                        index.getName(),
-                        policyName
-                    );
-                    return true;
-                }
-                return false;
-            }
-        );
-        CheckNotDataStreamWriteIndexStep checkNoWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, freezeStepKey);
-        FreezeStep freezeStep = new FreezeStep(freezeStepKey, nextStepKey, client);
+        StepKey freezeStepKey = new StepKey(phase, NAME, NAME);
+        NoopStep conditionalSkipFreezeStep = new NoopStep(preFreezeMergeBranchingKey, nextStepKey);
+        NoopStep checkNoWriteIndexStep = new NoopStep(checkNotWriteIndex, nextStepKey);
+        NoopStep freezeStep = new NoopStep(freezeStepKey, nextStepKey);
         return List.of(conditionalSkipFreezeStep, checkNoWriteIndexStep, freezeStep);
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeStep.java
deleted file mode 100644
index e36548035f939..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeStep.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.core.ilm;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
-
-/**
- * Freezes an index.
- */
-@Deprecated // To be removed in 9.0
-public class FreezeStep extends AsyncRetryDuringSnapshotActionStep {
-    public static final String NAME = "freeze";
-
-    public FreezeStep(StepKey key, StepKey nextStepKey, Client client) {
-        super(key, nextStepKey, client);
-    }
-
-    @Override
-    public void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentState, ActionListener<Void> listener) {
-        // Deprecated in 7.x, the freeze action is a noop in 8.x, so immediately return here
-        listener.onResponse(null);
-    }
-
-    @Override
-    public boolean isRetryable() {
-        return true;
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java
index 26f4f5c92073c..a0e733107e0fb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java
@@ -15,9 +15,10 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.Metadata.Custom;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
@@ -115,10 +116,11 @@ public Diff<Custom> diff(Custom previousState) {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params)
-            .xContentObjectFields(POLICIES_FIELD.getPreferredName(), policyMetadatas)
-            .field(OPERATION_MODE_FIELD.getPreferredName(), operationMode);
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.concat(
+            ChunkedToXContentHelper.xContentObjectFields(POLICIES_FIELD.getPreferredName(), policyMetadatas),
+            Iterators.single((builder, params) -> builder.field(OPERATION_MODE_FIELD.getPreferredName(), operationMode))
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicy.java
index e7021d22de47e..9129e4f029dd6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicy.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicy.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -33,6 +34,8 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.COLD_PHASE;
+
 /**
  * Represents the lifecycle of an index from creation to deletion. A
  * {@link LifecyclePolicy} is made up of a set of {@link Phase}s which it will
@@ -365,4 +368,13 @@ public boolean equals(Object obj) {
     public String toString() {
         return Strings.toString(this, true, true);
     }
+
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
+    public boolean maybeAddDeprecationWarningForFreezeAction(String policyName) {
+        Phase coldPhase = phases.get(COLD_PHASE);
+        if (coldPhase != null) {
+            return coldPhase.maybeAddDeprecationWarningForFreezeAction(policyName);
+        }
+        return false;
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java
index 6a272b0d2271e..18a7fd624bfc1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java
@@ -124,7 +124,7 @@ public static ItemUsage calculateUsage(
 
         final List<String> allDataStreams = indexNameExpressionResolver.dataStreamNames(
             state,
-            IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN
+            IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR
         );
 
         final List<String> dataStreams = allDataStreams.stream().filter(dsName -> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/Phase.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/Phase.java
index b2ace46e3708b..8608a85ddc9af 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/Phase.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/Phase.java
@@ -6,15 +6,15 @@
  */
 package org.elasticsearch.xpack.core.ilm;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.logging.DeprecationCategory;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.core.UpdateForV9;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ContextParser;
 import org.elasticsearch.xcontent.ObjectParser.ValueType;
@@ -34,7 +34,7 @@
  * particular point in the lifecycle of an index.
  */
 public class Phase implements ToXContentObject, Writeable {
-    private static final Logger logger = LogManager.getLogger(Phase.class);
+    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(Phase.class);
 
     public static final ParseField MIN_AGE = new ParseField("min_age");
     public static final ParseField ACTIONS_FIELD = new ParseField("actions");
@@ -51,19 +51,12 @@ public class Phase implements ToXContentObject, Writeable {
         return new Phase(name, (TimeValue) a[0], map);
     });
     static {
-        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), (ContextParser<String, Object>) (p, c) -> {
-            // In earlier versions it was possible to create a Phase with a negative `min_age` which would then cause errors
-            // when the phase is read from the cluster state during startup (even before negative timevalues were strictly
-            // disallowed) so this is a hack to treat negative `min_age`s as 0 to prevent those errors.
-            // They will be saved as `0` so this hack can be removed once we no longer have to read cluster states from 7.x.
-            @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT) // remove this hack now that we don't have to read 7.x cluster states
-            final String timeValueString = p.text();
-            if (timeValueString.startsWith("-")) {
-                logger.warn("phase has negative min_age value of [{}] - this will be treated as a min_age of 0", timeValueString);
-                return TimeValue.ZERO;
-            }
-            return TimeValue.parseTimeValue(timeValueString, MIN_AGE.getPreferredName());
-        }, MIN_AGE, ValueType.VALUE);
+        PARSER.declareField(
+            ConstructingObjectParser.optionalConstructorArg(),
+            (ContextParser<String, Object>) (p, c) -> TimeValue.parseTimeValue(p.text(), MIN_AGE.getPreferredName()),
+            MIN_AGE,
+            ValueType.VALUE
+        );
         PARSER.declareNamedObjects(
             ConstructingObjectParser.constructorArg(),
             (p, c, n) -> p.namedObject(LifecycleAction.class, n, null),
@@ -183,4 +176,20 @@ public String toString() {
         return Strings.toString(this, true, true);
     }
 
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
+    public boolean maybeAddDeprecationWarningForFreezeAction(String policyName) {
+        if (getActions().containsKey(FreezeAction.NAME)) {
+            deprecationLogger.warn(
+                DeprecationCategory.OTHER,
+                "ilm_freeze_action_deprecation",
+                "The freeze action in ILM is deprecated and will be removed in a future version;"
+                    + " this action is already a noop so it can be safely removed. Please remove the freeze action from the '"
+                    + policyName
+                    + "' policy."
+            );
+            return true;
+        }
+        return false;
+    }
+
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java
index b2ca71936ca33..4a8788a849b90 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java
@@ -11,13 +11,14 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
-import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.TimeValue;
 
@@ -122,9 +123,7 @@ public void performAction(
         // Calling rollover with no conditions will always roll over the index
         RolloverRequest rolloverRequest = new RolloverRequest(rolloverTarget, null).masterNodeTimeout(TimeValue.MAX_VALUE);
         if (targetFailureStore) {
-            rolloverRequest.setIndicesOptions(
-                IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build()
-            );
+            rolloverRequest.setRolloverTarget(IndexNameExpressionResolver.combineSelector(rolloverTarget, IndexComponentSelector.FAILURES));
         }
         // We don't wait for active shards when we perform the rollover because the
         // {@link org.elasticsearch.xpack.core.ilm.WaitForActiveShardsStep} step will do so
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java
index 7e074e8caea5b..be1a9d14cfdd1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java
@@ -12,11 +12,12 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.rollover.RolloverConditions;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
-import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -242,9 +243,7 @@ RolloverRequest createRolloverRequest(
         rolloverRequest.dryRun(true);
         rolloverRequest.setConditions(applyDefaultConditions(conditions, rolloverOnlyIfHasDocuments));
         if (targetFailureStore) {
-            rolloverRequest.setIndicesOptions(
-                IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build()
-            );
+            rolloverRequest.setRolloverTarget(IndexNameExpressionResolver.combineSelector(rolloverTarget, IndexComponentSelector.FAILURES));
         }
         return rolloverRequest;
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChatCompletionResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChatCompletionResults.java
index 5c63a60103139..902c69cef558e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChatCompletionResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChatCompletionResults.java
@@ -10,7 +10,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.TaskType;
@@ -50,7 +50,7 @@ public ChatCompletionResults(StreamInput in) throws IOException {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).array(COMPLETION, results.iterator());
+        return ChunkedToXContentHelper.array(COMPLETION, results.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java
index c1be1ce265f6b..16dca7b04d526 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.xcontent.ToXContent;
@@ -62,7 +62,7 @@ public int getFirstEmbeddingSize() {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).array(TEXT_EMBEDDING_BYTES, embeddings.iterator());
+        return ChunkedToXContentHelper.array(TEXT_EMBEDDING_BYTES, embeddings.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java
index 8551250348928..9f9bdfec7cfae 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.TaskType;
@@ -103,7 +103,7 @@ public int getFirstEmbeddingSize() {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).array(TEXT_EMBEDDING, embeddings.iterator());
+        return ChunkedToXContentHelper.array(TEXT_EMBEDDING, embeddings.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java
index a5f72bd51c6c6..b71722d0b9678 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
@@ -178,7 +178,7 @@ public List<RankedDoc> getRankedDocs() {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).array(RERANK, rankedDocs.iterator());
+        return ChunkedToXContentHelper.array(RERANK, rankedDocs.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java
index 318a292b47730..dd8229c604ecb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.TaskType;
@@ -72,7 +72,7 @@ public static SparseEmbeddingResults of(List<? extends InferenceResults> results
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).array(SPARSE_EMBEDDING, embeddings.iterator());
+        return ChunkedToXContentHelper.array(SPARSE_EMBEDDING, embeddings.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobModelSnapshotsUpgradeStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobModelSnapshotsUpgradeStatsAction.java
index ec49603c89cb8..a6df290be67bd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobModelSnapshotsUpgradeStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobModelSnapshotsUpgradeStatsAction.java
@@ -46,7 +46,6 @@ public class GetJobModelSnapshotsUpgradeStatsAction extends ActionType<GetJobMod
 
     // Used for QueryPage
     public static final ParseField RESULTS_FIELD = new ParseField("model_snapshot_upgrades");
-    public static String TYPE = "model_snapshot_upgrade";
 
     private GetJobModelSnapshotsUpgradeStatsAction() {
         super(NAME);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java
index 695b3ffca120f..742b538b6ce4b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java
@@ -102,7 +102,7 @@ public class DatafeedConfig implements SimpleDiffable<DatafeedConfig>, ToXConten
 
     // Used for QueryPage
     public static final ParseField RESULTS_FIELD = new ParseField("datafeeds");
-    public static String TYPE = "datafeed";
+    public static final String TYPE = "datafeed";
 
     /**
      * The field name used to specify document counts in Elasticsearch
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/ModelAliasMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/ModelAliasMetadata.java
index fd07688b1578d..bdf060bf6d7ae 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/ModelAliasMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/ModelAliasMetadata.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
@@ -93,7 +93,7 @@ public Map<String, ModelAliasEntry> modelAliases() {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).xContentObjectFields(MODEL_ALIASES.getPreferredName(), modelAliases);
+        return ChunkedToXContentHelper.xContentObjectFields(MODEL_ALIASES.getPreferredName(), modelAliases);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AllocationStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AllocationStatus.java
index 9216fc89a810d..d27e64752d7d8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AllocationStatus.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AllocationStatus.java
@@ -42,9 +42,9 @@ public String toString() {
         }
     }
 
-    public static ParseField ALLOCATION_COUNT = new ParseField("allocation_count");
-    public static ParseField TARGET_ALLOCATION_COUNT = new ParseField("target_allocation_count");
-    public static ParseField STATE = new ParseField("state");
+    public static final ParseField ALLOCATION_COUNT = new ParseField("allocation_count");
+    public static final ParseField TARGET_ALLOCATION_COUNT = new ParseField("target_allocation_count");
+    public static final ParseField STATE = new ParseField("state");
 
     private static final ConstructingObjectParser<AllocationStatus, Void> PARSER = new ConstructingObjectParser<>(
         "allocation_health",
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedNlpInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedNlpInferenceResults.java
index 9c1316d4d094f..177b9f61c0fd4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedNlpInferenceResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedNlpInferenceResults.java
@@ -13,8 +13,8 @@
 
 public abstract class ChunkedNlpInferenceResults extends NlpInferenceResults {
 
-    public static String TEXT = "text";
-    public static String INFERENCE = "inference";
+    public static final String TEXT = "text";
+    public static final String INFERENCE = "inference";
 
     ChunkedNlpInferenceResults(boolean isTruncated) {
         super(isTruncated);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResults.java
index 12921166b1489..3881625c05aba 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResults.java
@@ -22,7 +22,7 @@
 import java.util.stream.Collectors;
 
 public class ClassificationInferenceResults extends SingleValueInferenceResults {
-    public static String PREDICTION_PROBABILITY = "prediction_probability";
+    public static final String PREDICTION_PROBABILITY = "prediction_probability";
 
     public static final String NAME = "classification";
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertJapaneseTokenizationUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertJapaneseTokenizationUpdate.java
index a833048c76a55..7f4f63dee718a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertJapaneseTokenizationUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertJapaneseTokenizationUpdate.java
@@ -21,7 +21,7 @@ public class BertJapaneseTokenizationUpdate extends AbstractTokenizationUpdate {
 
     public static final ParseField NAME = BertJapaneseTokenization.NAME;
 
-    public static ConstructingObjectParser<BertJapaneseTokenizationUpdate, Void> PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<BertJapaneseTokenizationUpdate, Void> PARSER = new ConstructingObjectParser<>(
         "bert_japanese_tokenization_update",
         a -> new BertJapaneseTokenizationUpdate(a[0] == null ? null : Tokenization.Truncate.fromString((String) a[0]), (Integer) a[1])
     );
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationUpdate.java
index 3bda6f0070f03..065acdfd50307 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationUpdate.java
@@ -21,7 +21,7 @@ public class BertTokenizationUpdate extends AbstractTokenizationUpdate {
 
     public static final ParseField NAME = BertTokenization.NAME;
 
-    public static ConstructingObjectParser<BertTokenizationUpdate, Void> PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<BertTokenizationUpdate, Void> PARSER = new ConstructingObjectParser<>(
         "bert_tokenization_update",
         a -> new BertTokenizationUpdate(a[0] == null ? null : Tokenization.Truncate.fromString((String) a[0]), (Integer) a[1])
     );
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfig.java
index 6985769f08cb6..b04b49e645bcb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfig.java
@@ -29,7 +29,7 @@ public class ClassificationConfig implements LenientlyParsedInferenceConfig, Str
     public static final ParseField PREDICTION_FIELD_TYPE = new ParseField("prediction_field_type");
     private static final MlConfigVersion MIN_SUPPORTED_VERSION = MlConfigVersion.V_7_6_0;
 
-    public static ClassificationConfig EMPTY_PARAMS = new ClassificationConfig(
+    public static final ClassificationConfig EMPTY_PARAMS = new ClassificationConfig(
         0,
         DEFAULT_RESULTS_FIELD,
         DEFAULT_TOP_CLASSES_RESULTS_FIELD,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdate.java
index 1cf1c0ddeb570..9ee97029541f1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdate.java
@@ -32,7 +32,7 @@ public class ClassificationConfigUpdate implements InferenceConfigUpdate, NamedX
 
     public static final ParseField NAME = ClassificationConfig.NAME;
 
-    public static ClassificationConfigUpdate EMPTY_PARAMS = new ClassificationConfigUpdate(null, null, null, null, null);
+    public static final ClassificationConfigUpdate EMPTY_PARAMS = new ClassificationConfigUpdate(null, null, null, null, null);
 
     private final Integer numTopClasses;
     private final String topClassesResultsField;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/DebertaV2TokenizationUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/DebertaV2TokenizationUpdate.java
index 683b27793402d..8f9743231ba60 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/DebertaV2TokenizationUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/DebertaV2TokenizationUpdate.java
@@ -20,7 +20,7 @@
 public class DebertaV2TokenizationUpdate extends AbstractTokenizationUpdate {
     public static final ParseField NAME = new ParseField(DebertaV2Tokenization.NAME);
 
-    public static ConstructingObjectParser<DebertaV2TokenizationUpdate, Void> PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<DebertaV2TokenizationUpdate, Void> PARSER = new ConstructingObjectParser<>(
         "deberta_v2_tokenization_update",
         a -> new DebertaV2TokenizationUpdate(a[0] == null ? null : Tokenization.Truncate.fromString((String) a[0]), (Integer) a[1])
     );
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearningToRankConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearningToRankConfig.java
index a4d7c9c7fa08f..31ebcdf2d1e60 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearningToRankConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearningToRankConfig.java
@@ -45,7 +45,7 @@ public class LearningToRankConfig extends RegressionConfig implements Rewriteabl
     public static final ParseField FEATURE_EXTRACTORS = new ParseField("feature_extractors");
     public static final ParseField DEFAULT_PARAMS = new ParseField("default_params");
 
-    public static LearningToRankConfig EMPTY_PARAMS = new LearningToRankConfig(null, null, null);
+    public static final LearningToRankConfig EMPTY_PARAMS = new LearningToRankConfig(null, null, null);
 
     private static final ObjectParser<LearningToRankConfig.Builder, Boolean> LENIENT_PARSER = createParser(true);
     private static final ObjectParser<LearningToRankConfig.Builder, Boolean> STRICT_PARSER = createParser(false);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfig.java
index 403a747d6801f..13c63b9cc7ba0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfig.java
@@ -25,7 +25,7 @@ public class RegressionConfig implements LenientlyParsedInferenceConfig, Strictl
     private static final MlConfigVersion MIN_SUPPORTED_VERSION = MlConfigVersion.V_7_6_0;
     public static final ParseField NUM_TOP_FEATURE_IMPORTANCE_VALUES = new ParseField("num_top_feature_importance_values");
 
-    public static RegressionConfig EMPTY_PARAMS = new RegressionConfig(DEFAULT_RESULTS_FIELD, null);
+    public static final RegressionConfig EMPTY_PARAMS = new RegressionConfig(DEFAULT_RESULTS_FIELD, null);
 
     private static final ObjectParser<RegressionConfig.Builder, Void> LENIENT_PARSER = createParser(true);
     private static final ObjectParser<RegressionConfig.Builder, Void> STRICT_PARSER = createParser(false);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdate.java
index ceebe3105eb63..c7b66ef95943e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdate.java
@@ -29,7 +29,7 @@ public class RegressionConfigUpdate implements InferenceConfigUpdate, NamedXCont
 
     public static final ParseField NAME = RegressionConfig.NAME;
 
-    public static RegressionConfigUpdate EMPTY_PARAMS = new RegressionConfigUpdate(null, null);
+    public static final RegressionConfigUpdate EMPTY_PARAMS = new RegressionConfigUpdate(null, null);
 
     public static RegressionConfigUpdate fromMap(Map<String, Object> map) {
         Map<String, Object> options = new HashMap<>(map);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RobertaTokenizationUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RobertaTokenizationUpdate.java
index 3763a2350dadc..90e55e8b8ba37 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RobertaTokenizationUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RobertaTokenizationUpdate.java
@@ -20,7 +20,7 @@
 public class RobertaTokenizationUpdate extends AbstractTokenizationUpdate {
     public static final ParseField NAME = new ParseField(RobertaTokenization.NAME);
 
-    public static ConstructingObjectParser<RobertaTokenizationUpdate, Void> PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<RobertaTokenizationUpdate, Void> PARSER = new ConstructingObjectParser<>(
         "roberta_tokenization_update",
         a -> new RobertaTokenizationUpdate(a[0] == null ? null : Tokenization.Truncate.fromString((String) a[0]), (Integer) a[1])
     );
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfig.java
index 57493c7d34d72..6be122c0ccfa1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfig.java
@@ -29,7 +29,7 @@ public class TextEmbeddingConfig implements NlpConfig {
 
     public static final String NAME = "text_embedding";
 
-    public static ParseField EMBEDDING_SIZE = new ParseField("embedding_size");
+    public static final ParseField EMBEDDING_SIZE = new ParseField("embedding_size");
 
     public static TextEmbeddingConfig fromXContentStrict(XContentParser parser) {
         return STRICT_PARSER.apply(parser, null);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigUpdate.java
index e89281a59f7d2..67b54b3642860 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigUpdate.java
@@ -29,7 +29,7 @@ public class TextEmbeddingConfigUpdate extends NlpConfigUpdate implements NamedX
 
     public static final String NAME = TextEmbeddingConfig.NAME;
 
-    public static TextEmbeddingConfigUpdate EMPTY_INSTANCE = new TextEmbeddingConfigUpdate(null, null);
+    public static final TextEmbeddingConfigUpdate EMPTY_INSTANCE = new TextEmbeddingConfigUpdate(null, null);
 
     public static TextEmbeddingConfigUpdate fromMap(Map<String, Object> map) {
         Map<String, Object> options = new HashMap<>(map);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/XLMRobertaTokenizationUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/XLMRobertaTokenizationUpdate.java
index 5ddfdff4a5a41..58df6a8454a43 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/XLMRobertaTokenizationUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/XLMRobertaTokenizationUpdate.java
@@ -20,7 +20,7 @@
 public class XLMRobertaTokenizationUpdate extends AbstractTokenizationUpdate {
     public static final ParseField NAME = new ParseField(XLMRobertaTokenization.NAME);
 
-    public static ConstructingObjectParser<XLMRobertaTokenizationUpdate, Void> PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<XLMRobertaTokenizationUpdate, Void> PARSER = new ConstructingObjectParser<>(
         "xlm_roberta_tokenization_update",
         a -> new XLMRobertaTokenizationUpdate(a[0] == null ? null : Tokenization.Truncate.fromString((String) a[0]), (Integer) a[1])
     );
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ltr/QueryExtractorBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ltr/QueryExtractorBuilder.java
index d9e90b92382e3..4fb79c5b2919b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ltr/QueryExtractorBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ltr/QueryExtractorBuilder.java
@@ -34,7 +34,7 @@ public record QueryExtractorBuilder(String featureName, QueryProvider query, flo
     public static final ParseField QUERY = new ParseField("query");
     public static final ParseField DEFAULT_SCORE = new ParseField("default_score");
 
-    public static float DEFAULT_SCORE_DEFAULT = 0f;
+    public static final float DEFAULT_SCORE_DEFAULT = 0f;
 
     private static final ConstructingObjectParser<QueryExtractorBuilder, Void> PARSER = new ConstructingObjectParser<>(
         NAME.getPreferredName(),
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
index 9f9def6a0678d..f01ac08f922c4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
@@ -281,6 +281,8 @@ public final class Messages {
     public static final String FIELD_CANNOT_BE_NULL = "Field [{0}] cannot be null";
     public static final String MODEL_ID_MATCHES_EXISTING_MODEL_IDS_BUT_MUST_NOT =
         "Model IDs must be unique. Requested model ID [{}] matches existing model IDs but must not.";
+    public static final String INFERENCE_ID_MATCHES_EXISTING_MODEL_IDS_BUT_MUST_NOT =
+        "Inference endpoint IDs must be unique. Requested inference endpoint ID [{}] matches existing trained model ID(s) but must not.";
     public static final String MODEL_ID_DOES_NOT_MATCH_EXISTING_MODEL_IDS_BUT_MUST_FOR_IN_CLUSTER_SERVICE =
         "Requested model ID [{}] does not have a matching trained model and thus cannot be updated.";
     public static final String INFERENCE_ENTITY_NON_EXISTANT_NO_UPDATE = "The inference endpoint [{}] does not exist and cannot be updated";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java
index 3114c03879eb7..66ce4a135e50f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java
@@ -94,7 +94,7 @@ private static ObjectParser<Builder, Void> createParser(boolean ignoreUnknownFie
         return parser;
     }
 
-    public static String EMPTY_SNAPSHOT_ID = "empty";
+    public static final String EMPTY_SNAPSHOT_ID = "empty";
 
     private final String jobId;
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/search/SparseVectorQueryBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/search/SparseVectorQueryBuilder.java
index e9e4e90421adc..35cba890e5e0c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/search/SparseVectorQueryBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/search/SparseVectorQueryBuilder.java
@@ -90,7 +90,8 @@ public SparseVectorQueryBuilder(
             : (this.shouldPruneTokens ? new TokenPruningConfig() : null));
         this.weightedTokensSupplier = null;
 
-        if (queryVectors == null ^ inferenceId == null == false) {
+        // Preserve BWC error messaging
+        if (queryVectors != null && inferenceId != null) {
             throw new IllegalArgumentException(
                 "["
                     + NAME
@@ -98,18 +99,24 @@ public SparseVectorQueryBuilder(
                     + QUERY_VECTOR_FIELD.getPreferredName()
                     + "] or ["
                     + INFERENCE_ID_FIELD.getPreferredName()
-                    + "]"
+                    + "] for "
+                    + ALLOWED_FIELD_TYPE
+                    + " fields"
             );
         }
-        if (inferenceId != null && query == null) {
+
+        // Preserve BWC error messaging
+        if ((queryVectors == null) == (query == null)) {
             throw new IllegalArgumentException(
                 "["
                     + NAME
-                    + "] requires ["
-                    + QUERY_FIELD.getPreferredName()
-                    + "] when ["
+                    + "] requires one of ["
+                    + QUERY_VECTOR_FIELD.getPreferredName()
+                    + "] or ["
                     + INFERENCE_ID_FIELD.getPreferredName()
-                    + "] is specified"
+                    + "] for "
+                    + ALLOWED_FIELD_TYPE
+                    + " fields"
             );
         }
     }
@@ -143,6 +150,14 @@ public List<WeightedToken> getQueryVectors() {
         return queryVectors;
     }
 
+    public String getInferenceId() {
+        return inferenceId;
+    }
+
+    public String getQuery() {
+        return query;
+    }
+
     public boolean shouldPruneTokens() {
         return shouldPruneTokens;
     }
@@ -176,7 +191,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
             }
             builder.endObject();
         } else {
-            builder.field(INFERENCE_ID_FIELD.getPreferredName(), inferenceId);
+            if (inferenceId != null) {
+                builder.field(INFERENCE_ID_FIELD.getPreferredName(), inferenceId);
+            }
             builder.field(QUERY_FIELD.getPreferredName(), query);
         }
         builder.field(PRUNE_FIELD.getPreferredName(), shouldPruneTokens);
@@ -228,6 +245,11 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) {
                 shouldPruneTokens,
                 tokenPruningConfig
             );
+        } else if (inferenceId == null) {
+            // Edge case, where inference_id was not specified in the request,
+            // but we did not intercept this and rewrite to a query o field with
+            // pre-configured inference. So we trap here and output a nicer error message.
+            throw new IllegalArgumentException("inference_id required to perform vector search on query string");
         }
 
         // TODO move this to xpack core and use inference APIs
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/vectors/TextEmbeddingQueryVectorBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/vectors/TextEmbeddingQueryVectorBuilder.java
similarity index 84%
rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/vectors/TextEmbeddingQueryVectorBuilder.java
rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/vectors/TextEmbeddingQueryVectorBuilder.java
index 6c7d9ef1b8a1c..b859f59fc3e0e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/vectors/TextEmbeddingQueryVectorBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/vectors/TextEmbeddingQueryVectorBuilder.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.ml.vectors;
+package org.elasticsearch.xpack.core.ml.vectors;
 
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
@@ -30,7 +30,9 @@
 import java.util.List;
 import java.util.Objects;
 
+import static org.elasticsearch.TransportVersions.TEXT_EMBEDDING_QUERY_VECTOR_BUILDER_INFER_MODEL_ID;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
 
@@ -46,7 +48,7 @@ public class TextEmbeddingQueryVectorBuilder implements QueryVectorBuilder {
     );
 
     static {
-        PARSER.declareString(constructorArg(), TrainedModelConfig.MODEL_ID);
+        PARSER.declareString(optionalConstructorArg(), TrainedModelConfig.MODEL_ID);
         PARSER.declareString(constructorArg(), MODEL_TEXT);
     }
 
@@ -63,7 +65,11 @@ public TextEmbeddingQueryVectorBuilder(String modelId, String modelText) {
     }
 
     public TextEmbeddingQueryVectorBuilder(StreamInput in) throws IOException {
-        this.modelId = in.readString();
+        if (in.getTransportVersion().onOrAfter(TEXT_EMBEDDING_QUERY_VECTOR_BUILDER_INFER_MODEL_ID)) {
+            this.modelId = in.readOptionalString();
+        } else {
+            this.modelId = in.readString();
+        }
         this.modelText = in.readString();
     }
 
@@ -79,14 +85,20 @@ public TransportVersion getMinimalSupportedVersion() {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(modelId);
+        if (out.getTransportVersion().onOrAfter(TEXT_EMBEDDING_QUERY_VECTOR_BUILDER_INFER_MODEL_ID)) {
+            out.writeOptionalString(modelId);
+        } else {
+            out.writeString(modelId);
+        }
         out.writeString(modelText);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(TrainedModelConfig.MODEL_ID.getPreferredName(), modelId);
+        if (modelId != null) {
+            builder.field(TrainedModelConfig.MODEL_ID.getPreferredName(), modelId);
+        }
         builder.field(MODEL_TEXT.getPreferredName(), modelText);
         builder.endObject();
         return builder;
@@ -94,6 +106,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public void buildVector(Client client, ActionListener<float[]> listener) {
+
+        if (modelId == null) {
+            throw new IllegalArgumentException("[model_id] must not be null.");
+        }
+
         CoordinatedInferenceAction.Request inferRequest = CoordinatedInferenceAction.Request.forTextInput(
             modelId,
             List.of(modelText),
@@ -101,6 +118,7 @@ public void buildVector(Client client, ActionListener<float[]> listener) {
             false,
             InferModelAction.Request.DEFAULT_TIMEOUT_FOR_API
         );
+
         inferRequest.setHighPriority(true);
         inferRequest.setPrefixType(TrainedModelPrefixStrings.PrefixType.SEARCH);
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
index 32b401ebfb32d..1d63d2710bd6a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
@@ -10,9 +10,10 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.AbstractRefCounted;
 import org.elasticsearch.core.Nullable;
@@ -24,6 +25,7 @@
 import org.elasticsearch.xpack.core.async.AsyncResponse;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Iterator;
 
 import static org.elasticsearch.rest.RestStatus.OK;
@@ -232,7 +234,8 @@ public RestStatus status() {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).object(ob -> ob.append((builder, p) -> {
+        return Iterators.concat(ChunkedToXContentHelper.chunk((builder, p) -> {
+            builder.startObject();
             if (id != null) {
                 builder.field("id", id);
             }
@@ -252,14 +255,18 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
                 builder.field("response");
             }
             return builder;
-        }).appendIfPresent(searchResponse).append((builder, p) -> {
-            if (error != null) {
-                builder.startObject("error");
-                ElasticsearchException.generateThrowableXContent(builder, p, error);
+        }),
+            searchResponse == null ? Collections.emptyIterator() : searchResponse.toXContentChunked(params),
+            ChunkedToXContentHelper.chunk((builder, p) -> {
+                if (error != null) {
+                    builder.startObject("error");
+                    ElasticsearchException.generateThrowableXContent(builder, params, error);
+                    builder.endObject();
+                }
                 builder.endObject();
-            }
-            return builder;
-        }));
+                return builder;
+            })
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java
index 4b9700a37ad71..e68a1d73a14d8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java
@@ -29,7 +29,7 @@
  * @see AsyncSearchResponse
  */
 public class SubmitAsyncSearchRequest extends ActionRequest {
-    public static long MIN_KEEP_ALIVE = TimeValue.timeValueSeconds(1).millis();
+    public static final long MIN_KEEP_ALIVE = TimeValue.timeValueSeconds(1).millis();
 
     private TimeValue waitForCompletionTimeout = TimeValue.timeValueSeconds(1);
     private boolean keepOnCompletion = false;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/ScrollHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/ScrollHelper.java
index 129619f6976e2..0962b4c84e2ba 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/ScrollHelper.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/ScrollHelper.java
@@ -97,7 +97,7 @@ public void onResponse(SearchResponse resp) {
                                     listener.onResponse(Collections.unmodifiableList(results));
                                 } else {
                                     SearchScrollRequest scrollRequest = new SearchScrollRequest(resp.getScrollId());
-                                    scrollRequest.scroll(request.scroll().keepAlive());
+                                    scrollRequest.scroll(request.scroll());
                                     client.searchScroll(scrollRequest, this);
                                 }
                             } else {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java
index f44409daa37f8..3ebfad04a0f13 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java
@@ -55,8 +55,10 @@ public SecurityFeatureSetUsage(StreamInput in) throws IOException {
         realmsUsage = in.readGenericMap();
         rolesStoreUsage = in.readGenericMap();
         sslUsage = in.readGenericMap();
-        tokenServiceUsage = in.readGenericMap();
-        apiKeyServiceUsage = in.readGenericMap();
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) {
+            tokenServiceUsage = in.readGenericMap();
+            apiKeyServiceUsage = in.readGenericMap();
+        }
         auditUsage = in.readGenericMap();
         ipFilterUsage = in.readGenericMap();
         anonymousUsage = in.readGenericMap();
@@ -121,8 +123,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeGenericMap(realmsUsage);
         out.writeGenericMap(rolesStoreUsage);
         out.writeGenericMap(sslUsage);
-        out.writeGenericMap(tokenServiceUsage);
-        out.writeGenericMap(apiKeyServiceUsage);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) {
+            out.writeGenericMap(tokenServiceUsage);
+            out.writeGenericMap(apiKeyServiceUsage);
+        }
         out.writeGenericMap(auditUsage);
         out.writeGenericMap(ipFilterUsage);
         out.writeGenericMap(anonymousUsage);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
index c2f40a3e393b9..561a94dd3cc05 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
@@ -55,6 +55,7 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.elasticsearch.common.Strings.EMPTY_ARRAY;
 import static org.elasticsearch.transport.RemoteClusterPortSettings.TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
@@ -170,21 +171,49 @@ public Authentication(StreamInput in) throws IOException {
             type = AuthenticationType.REALM;
             metadata = Map.of();
         }
+
         if (innerUser != null) {
-            authenticatingSubject = new Subject(innerUser, authenticatedBy, version, metadata);
+            authenticatingSubject = new Subject(
+                copyUserWithRolesRemovedForLegacyApiKeys(version, innerUser),
+                authenticatedBy,
+                version,
+                metadata
+            );
             // The lookup user for run-as currently doesn't have authentication metadata associated with them because
             // lookupUser only returns the User object. The lookup user for authorization delegation does have
             // authentication metadata, but the realm does not expose this difference between authenticatingUser and
             // delegateUser so effectively this is handled together with the authenticatingSubject not effectiveSubject.
+            // Note: we do not call copyUserWithRolesRemovedForLegacyApiKeys here because an API key is never the target of run-as
             effectiveSubject = new Subject(outerUser, lookedUpBy, version, Map.of());
         } else {
-            authenticatingSubject = effectiveSubject = new Subject(outerUser, authenticatedBy, version, metadata);
+            authenticatingSubject = effectiveSubject = new Subject(
+                copyUserWithRolesRemovedForLegacyApiKeys(version, outerUser),
+                authenticatedBy,
+                version,
+                metadata
+            );
         }
+
         if (Assertions.ENABLED) {
             checkConsistency();
         }
     }
 
+    private User copyUserWithRolesRemovedForLegacyApiKeys(TransportVersion version, User user) {
+        // API keys prior to 7.8 had synthetic role names. Strip these out to maintain the invariant that API keys don't have role names
+        if (type == AuthenticationType.API_KEY && version.onOrBefore(TransportVersions.V_7_8_0) && user.roles().length > 0) {
+            logger.debug(
+                "Stripping [{}] roles from API key user [{}] for legacy version [{}]",
+                user.roles().length,
+                user.principal(),
+                version
+            );
+            return new User(user.principal(), EMPTY_ARRAY, user.fullName(), user.email(), user.metadata(), user.enabled());
+        } else {
+            return user;
+        }
+    }
+
     /**
      * Get the {@link Subject} that performs the actual authentication. This normally means it provides a credentials.
      */
@@ -1198,7 +1227,7 @@ public static boolean isFileOrNativeRealm(String realmType) {
         return FileRealmSettings.TYPE.equals(realmType) || NativeRealmSettings.TYPE.equals(realmType);
     }
 
-    public static ConstructingObjectParser<RealmRef, Void> REALM_REF_PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<RealmRef, Void> REALM_REF_PARSER = new ConstructingObjectParser<>(
         "realm_ref",
         false,
         (args, v) -> new RealmRef((String) args[0], (String) args[1], (String) args[2], (RealmDomain) args[3])
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationResult.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationResult.java
index 5d144931a906d..30d76da35370c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationResult.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationResult.java
@@ -26,7 +26,7 @@
 public final class AuthenticationResult<T> {
     private static final AuthenticationResult<?> NOT_HANDLED = new AuthenticationResult<>(Status.CONTINUE, null, null, null, null);
 
-    public static String THREAD_CONTEXT_KEY = "_xpack_security_auth_result";
+    public static final String THREAD_CONTEXT_KEY = "_xpack_security_auth_result";
 
     public enum Status {
         /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmConfig.java
index c2c64b45941cd..0e1080ba9fe69 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmConfig.java
@@ -261,7 +261,7 @@ public int compareTo(RealmIdentifier other) {
         }
     }
 
-    public static ConstructingObjectParser<RealmIdentifier, Void> REALM_IDENTIFIER_PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<RealmIdentifier, Void> REALM_IDENTIFIER_PARSER = new ConstructingObjectParser<>(
         "realm_identifier",
         false,
         (args, v) -> new RealmIdentifier((String) args[0], (String) args[1])
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmDomain.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmDomain.java
index 573023739db11..c458a3ba3981a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmDomain.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmDomain.java
@@ -59,7 +59,7 @@ public String toString() {
     }
 
     @SuppressWarnings("unchecked")
-    public static ConstructingObjectParser<RealmDomain, Void> REALM_DOMAIN_PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<RealmDomain, Void> REALM_DOMAIN_PARSER = new ConstructingObjectParser<>(
         "realm_domain",
         false,
         (args, v) -> new RealmDomain((String) args[0], Set.copyOf((List<RealmConfig.RealmIdentifier>) args[1]))
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/TokensInvalidationResult.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/TokensInvalidationResult.java
index 59c16fc8a7a72..8fe018a825468 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/TokensInvalidationResult.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/TokensInvalidationResult.java
@@ -59,6 +59,9 @@ public TokensInvalidationResult(StreamInput in) throws IOException {
         this.invalidatedTokens = in.readStringCollectionAsList();
         this.previouslyInvalidatedTokens = in.readStringCollectionAsList();
         this.errors = in.readCollectionAsList(StreamInput::readException);
+        if (in.getTransportVersion().before(TransportVersions.V_7_2_0)) {
+            in.readVInt();
+        }
         if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_0_0)) {
             this.restStatus = RestStatus.readFrom(in);
         }
@@ -108,6 +111,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeStringCollection(invalidatedTokens);
         out.writeStringCollection(previouslyInvalidatedTokens);
         out.writeCollection(errors, StreamOutput::writeException);
+        if (out.getTransportVersion().before(TransportVersions.V_7_2_0)) {
+            out.writeVInt(5);
+        }
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_0_0)) {
             RestStatus.writeTo(out, restStatus);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorsIntersection.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorsIntersection.java
index 38aa1bc106e99..4d98767444f4b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorsIntersection.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorsIntersection.java
@@ -24,7 +24,7 @@
 
 public record RoleDescriptorsIntersection(Collection<Set<RoleDescriptor>> roleDescriptorsList) implements ToXContentObject, Writeable {
 
-    public static RoleDescriptorsIntersection EMPTY = new RoleDescriptorsIntersection(Collections.emptyList());
+    public static final RoleDescriptorsIntersection EMPTY = new RoleDescriptorsIntersection(Collections.emptyList());
 
     private static final RoleDescriptor.Parser ROLE_DESCRIPTOR_PARSER = RoleDescriptor.parserBuilder()
         .allowRestriction(true)
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java
index 6d693b8ad8fdb..3f0fb18e825c0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java
@@ -6,11 +6,13 @@
  */
 package org.elasticsearch.xpack.core.security.authz.accesscontrol;
 
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.CachedSupplier;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField;
 import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
@@ -59,7 +61,8 @@ protected IndicesAccessControl(IndicesAccessControl copy) {
      */
     @Nullable
     public IndexAccessControl getIndexPermissions(String index) {
-        return this.getAllIndexPermissions().get(index);
+        Tuple<String, String> indexAndSelector = IndexNameExpressionResolver.splitSelectorExpression(index);
+        return this.getAllIndexPermissions().get(indexAndSelector.v1());
     }
 
     public boolean hasIndexPermissions(String index) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java
index 23c93226d5494..5ba5c1fd1218a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java
@@ -53,7 +53,7 @@ public final class ApplicationPermission {
                 return new PermissionEntry(
                     appPriv,
                     Sets.union(existing.resourceNames, resourceNames),
-                    Automatons.unionAndDeterminize(Arrays.asList(existing.resourceAutomaton, patterns))
+                    Automatons.unionAndMinimize(Arrays.asList(existing.resourceAutomaton, patterns))
                 );
             }
         }));
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java
index 5f3da8f73a708..4e608281a7858 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java
@@ -137,7 +137,7 @@ public ClusterPermission build() {
             }
             List<PermissionCheck> checks = this.permissionChecks;
             if (false == actionAutomatons.isEmpty()) {
-                final Automaton mergedAutomaton = Automatons.unionAndDeterminize(this.actionAutomatons);
+                final Automaton mergedAutomaton = Automatons.unionAndMinimize(this.actionAutomatons);
                 checks = new ArrayList<>(this.permissionChecks.size() + 1);
                 checks.add(new AutomatonPermissionCheck(mergedAutomaton));
                 checks.addAll(this.permissionChecks);
@@ -156,7 +156,7 @@ private static Automaton createAutomaton(Set<String> allowedActionPatterns, Set<
             } else {
                 final Automaton allowedAutomaton = Automatons.patterns(allowedActionPatterns);
                 final Automaton excludedAutomaton = Automatons.patterns(excludeActionPatterns);
-                return Automatons.minusAndDeterminize(allowedAutomaton, excludedAutomaton);
+                return Automatons.minusAndMinimize(allowedAutomaton, excludedAutomaton);
             }
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java
index 24f0a52436203..92bb037888495 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java
@@ -159,13 +159,15 @@ private static void buildRoleQuery(
             if (queryBuilder != null) {
                 failIfQueryUsesClient(queryBuilder, context);
                 Query roleQuery = context.toQuery(queryBuilder).query();
-                filter.add(roleQuery, SHOULD);
-                if (context.nestedLookup() != NestedLookup.EMPTY) {
+                if (context.nestedLookup() == NestedLookup.EMPTY) {
+                    filter.add(roleQuery, SHOULD);
+                } else {
                     if (NestedHelper.mightMatchNestedDocs(roleQuery, context)) {
                         roleQuery = new BooleanQuery.Builder().add(roleQuery, FILTER)
                             .add(Queries.newNonNestedFilter(context.indexVersionCreated()), FILTER)
                             .build();
                     }
+                    filter.add(roleQuery, SHOULD);
                     // If access is allowed on root doc then also access is allowed on all nested docs of that root document:
                     BitSetProducer rootDocs = context.bitsetFilter(Queries.newNonNestedFilter(context.indexVersionCreated()));
                     ToChildBlockJoinQuery includeNestedDocs = new ToChildBlockJoinQuery(roleQuery, rootDocs);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java
index ed7bbf9158278..c46f1350776b1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.lucene.util.automaton.MinimizationOperations;
 import org.elasticsearch.plugins.FieldPredicate;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.FieldSubsetReader;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition.FieldGrantExcludeGroup;
@@ -147,7 +148,7 @@ public static Automaton initializePermittedFieldsAutomaton(FieldPermissionsDefin
         List<Automaton> automatonList = groups.stream()
             .map(g -> FieldPermissions.buildPermittedFieldsAutomaton(g.getGrantedFields(), g.getExcludedFields()))
             .collect(Collectors.toList());
-        return Automatons.unionAndDeterminize(automatonList);
+        return Automatons.unionAndMinimize(automatonList);
     }
 
     /**
@@ -172,12 +173,8 @@ public static Automaton buildPermittedFieldsAutomaton(final String[] grantedFiel
             deniedFieldsAutomaton = Automatons.patterns(deniedFields);
         }
 
-        grantedFieldsAutomaton = Operations.removeDeadStates(
-            Operations.determinize(grantedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT)
-        );
-        deniedFieldsAutomaton = Operations.removeDeadStates(
-            Operations.determinize(deniedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT)
-        );
+        grantedFieldsAutomaton = MinimizationOperations.minimize(grantedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+        deniedFieldsAutomaton = MinimizationOperations.minimize(deniedFieldsAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
 
         if (Automatons.subsetOf(deniedFieldsAutomaton, grantedFieldsAutomaton) == false) {
             throw new ElasticsearchSecurityException(
@@ -189,7 +186,7 @@ public static Automaton buildPermittedFieldsAutomaton(final String[] grantedFiel
             );
         }
 
-        grantedFieldsAutomaton = Automatons.minusAndDeterminize(grantedFieldsAutomaton, deniedFieldsAutomaton);
+        grantedFieldsAutomaton = Automatons.minusAndMinimize(grantedFieldsAutomaton, deniedFieldsAutomaton);
         return grantedFieldsAutomaton;
     }
 
@@ -206,10 +203,7 @@ public static Automaton buildPermittedFieldsAutomaton(final String[] grantedFiel
     public FieldPermissions limitFieldPermissions(FieldPermissions limitedBy) {
         if (hasFieldLevelSecurity() && limitedBy != null && limitedBy.hasFieldLevelSecurity()) {
             // TODO: cache the automaton computation with FieldPermissionsCache
-            Automaton _permittedFieldsAutomaton = Automatons.intersectAndDeterminize(
-                getIncludeAutomaton(),
-                limitedBy.getIncludeAutomaton()
-            );
+            Automaton _permittedFieldsAutomaton = Automatons.intersectAndMinimize(getIncludeAutomaton(), limitedBy.getIncludeAutomaton());
             return new FieldPermissions(
                 CollectionUtils.concatLists(fieldPermissionsDefinitions, limitedBy.fieldPermissionsDefinitions),
                 _permittedFieldsAutomaton
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissionsCache.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissionsCache.java
index a1e14bfde8aa5..46261937a0228 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissionsCache.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissionsCache.java
@@ -107,7 +107,7 @@ FieldPermissions union(Collection<FieldPermissions> fieldPermissionsCollection)
                     List<Automaton> automatonList = fieldPermissionsCollection.stream()
                         .map(FieldPermissions::getIncludeAutomaton)
                         .collect(Collectors.toList());
-                    return new FieldPermissions(key, Automatons.unionAndDeterminize(automatonList));
+                    return new FieldPermissions(key, Automatons.unionAndMinimize(automatonList));
                 });
             } catch (ExecutionException e) {
                 throw new ElasticsearchException("unable to compute field permissions", e);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java
index cdd5a6f6ff72d..4ea590afff864 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java
@@ -10,8 +10,10 @@
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.action.admin.indices.mapping.put.TransportAutoPutMappingAction;
 import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.logging.DeprecationCategory;
@@ -39,7 +41,6 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiPredicate;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
@@ -283,14 +284,14 @@ public boolean checkResourcePrivileges(
         for (String forIndexPattern : checkForIndexPatterns) {
             Automaton checkIndexAutomaton = Automatons.patterns(forIndexPattern);
             if (false == allowRestrictedIndices && false == isConcreteRestrictedIndex(forIndexPattern)) {
-                checkIndexAutomaton = Automatons.minusAndDeterminize(checkIndexAutomaton, restrictedIndices.getAutomaton());
+                checkIndexAutomaton = Automatons.minusAndMinimize(checkIndexAutomaton, restrictedIndices.getAutomaton());
             }
             if (false == Operations.isEmpty(checkIndexAutomaton)) {
                 Automaton allowedIndexPrivilegesAutomaton = null;
                 for (var indexAndPrivilegeAutomaton : indexGroupAutomatons.entrySet()) {
                     if (Automatons.subsetOf(checkIndexAutomaton, indexAndPrivilegeAutomaton.getValue())) {
                         if (allowedIndexPrivilegesAutomaton != null) {
-                            allowedIndexPrivilegesAutomaton = Automatons.unionAndDeterminize(
+                            allowedIndexPrivilegesAutomaton = Automatons.unionAndMinimize(
                                 Arrays.asList(allowedIndexPrivilegesAutomaton, indexAndPrivilegeAutomaton.getKey())
                             );
                         } else {
@@ -342,7 +343,7 @@ public Automaton allowedActionsMatcher(String index) {
                 automatonList.add(group.privilege.getAutomaton());
             }
         }
-        return automatonList.isEmpty() ? Automatons.EMPTY : Automatons.unionAndDeterminize(automatonList);
+        return automatonList.isEmpty() ? Automatons.EMPTY : Automatons.unionAndMinimize(automatonList);
     }
 
     /**
@@ -355,18 +356,34 @@ private static class IndexResource {
          */
         private final String name;
 
+        /**
+         * The selector to be applied to the IndexAbstraction which selects which indices to return when resolving
+         */
+        @Nullable
+        private final IndexComponentSelector selector;
+
         /**
          * The IndexAbstraction on which authorization is being performed, or {@code null} if nothing in the cluster matches the name
          */
         @Nullable
         private final IndexAbstraction indexAbstraction;
 
-        private IndexResource(String name, @Nullable IndexAbstraction abstraction) {
+        private IndexResource(String name, @Nullable IndexAbstraction abstraction, @Nullable IndexComponentSelector selector) {
             assert name != null : "Resource name cannot be null";
             assert abstraction == null || abstraction.getName().equals(name)
                 : "Index abstraction has unexpected name [" + abstraction.getName() + "] vs [" + name + "]";
+            assert abstraction == null
+                || selector == null
+                || IndexComponentSelector.FAILURES.equals(selector) == false
+                || abstraction.isDataStreamRelated()
+                : "Invalid index component selector ["
+                    + selector.getKey()
+                    + "] applied to abstraction of type ["
+                    + abstraction.getType()
+                    + "]";
             this.name = name;
             this.indexAbstraction = abstraction;
+            this.selector = selector;
         }
 
         /**
@@ -403,21 +420,69 @@ public boolean checkIndex(Group group) {
         /**
          * @return the number of distinct objects to which this expansion refers.
          */
-        public int size() {
+        public int size(Map<String, IndexAbstraction> lookup) {
             if (indexAbstraction == null) {
                 return 1;
             } else if (indexAbstraction.getType() == IndexAbstraction.Type.CONCRETE_INDEX) {
                 return 1;
+            } else if (selector != null) {
+                int size = 1;
+                if (selector.shouldIncludeData()) {
+                    size += indexAbstraction.getIndices().size();
+                }
+                if (selector.shouldIncludeFailures()) {
+                    if (IndexAbstraction.Type.ALIAS.equals(indexAbstraction.getType())) {
+                        Set<DataStream> aliasDataStreams = new HashSet<>();
+                        int failureIndices = 0;
+                        for (Index index : indexAbstraction.getIndices()) {
+                            DataStream parentDataStream = lookup.get(index.getName()).getParentDataStream();
+                            if (parentDataStream != null && aliasDataStreams.add(parentDataStream)) {
+                                failureIndices += parentDataStream.getFailureIndices().getIndices().size();
+                            }
+                        }
+                        size += failureIndices;
+                    } else {
+                        DataStream parentDataStream = (DataStream) indexAbstraction;
+                        size += parentDataStream.getFailureIndices().getIndices().size();
+                    }
+                }
+                return size;
             } else {
                 return 1 + indexAbstraction.getIndices().size();
             }
         }
 
-        public Collection<String> resolveConcreteIndices() {
+        public Collection<String> resolveConcreteIndices(Map<String, IndexAbstraction> lookup) {
             if (indexAbstraction == null) {
                 return List.of();
             } else if (indexAbstraction.getType() == IndexAbstraction.Type.CONCRETE_INDEX) {
                 return List.of(indexAbstraction.getName());
+            } else if (IndexComponentSelector.FAILURES.equals(selector)) {
+                if (IndexAbstraction.Type.ALIAS.equals(indexAbstraction.getType())) {
+                    Set<DataStream> aliasDataStreams = new HashSet<>();
+                    for (Index index : indexAbstraction.getIndices()) {
+                        DataStream parentDataStream = lookup.get(index.getName()).getParentDataStream();
+                        if (parentDataStream != null) {
+                            aliasDataStreams.add(parentDataStream);
+                        }
+                    }
+                    List<String> concreteIndexNames = new ArrayList<>(aliasDataStreams.size());
+                    for (DataStream aliasDataStream : aliasDataStreams) {
+                        DataStream.DataStreamIndices failureIndices = aliasDataStream.getFailureIndices();
+                        for (Index index : failureIndices.getIndices()) {
+                            concreteIndexNames.add(index.getName());
+                        }
+                    }
+                    return concreteIndexNames;
+                } else {
+                    DataStream parentDataStream = (DataStream) indexAbstraction;
+                    DataStream.DataStreamIndices failureIndices = parentDataStream.getFailureIndices();
+                    List<String> concreteIndexNames = new ArrayList<>(failureIndices.getIndices().size());
+                    for (Index index : failureIndices.getIndices()) {
+                        concreteIndexNames.add(index.getName());
+                    }
+                    return concreteIndexNames;
+                }
             } else {
                 final List<Index> indices = indexAbstraction.getIndices();
                 final List<String> concreteIndexNames = new ArrayList<>(indices.size());
@@ -443,26 +508,35 @@ public IndicesAccessControl authorize(
         FieldPermissionsCache fieldPermissionsCache
     ) {
         // Short circuit if the indicesPermission allows all access to every index
-        if (Arrays.stream(groups).anyMatch(Group::isTotal)) {
-            return IndicesAccessControl.allowAll();
+        for (Group group : groups) {
+            if (group.isTotal()) {
+                return IndicesAccessControl.allowAll();
+            }
         }
 
         final Map<String, IndexResource> resources = Maps.newMapWithExpectedSize(requestedIndicesOrAliases.size());
-        final AtomicInteger totalResourceCountHolder = new AtomicInteger(0);
+        int totalResourceCount = 0;
 
         for (String indexOrAlias : requestedIndicesOrAliases) {
-            final IndexResource resource = new IndexResource(indexOrAlias, lookup.get(indexOrAlias));
+            // Remove any selectors from abstraction name. Discard them for this check as we do not have access control for them (yet)
+            Tuple<String, String> expressionAndSelector = IndexNameExpressionResolver.splitSelectorExpression(indexOrAlias);
+            indexOrAlias = expressionAndSelector.v1();
+            IndexComponentSelector selector = expressionAndSelector.v2() == null
+                ? null
+                : IndexComponentSelector.getByKey(expressionAndSelector.v2());
+            final IndexResource resource = new IndexResource(indexOrAlias, lookup.get(indexOrAlias), selector);
             resources.put(resource.name, resource);
-            totalResourceCountHolder.getAndAdd(resource.size());
+            totalResourceCount += resource.size(lookup);
         }
 
         final boolean overallGranted = isActionGranted(action, resources);
-
+        final int finalTotalResourceCount = totalResourceCount;
         final Supplier<Map<String, IndicesAccessControl.IndexAccessControl>> indexPermissions = () -> buildIndicesAccessControl(
             action,
             resources,
-            totalResourceCountHolder.get(),
-            fieldPermissionsCache
+            finalTotalResourceCount,
+            fieldPermissionsCache,
+            lookup
         );
 
         return new IndicesAccessControl(overallGranted, indexPermissions);
@@ -472,7 +546,8 @@ private Map<String, IndicesAccessControl.IndexAccessControl> buildIndicesAccessC
         final String action,
         final Map<String, IndexResource> requestedResources,
         final int totalResourceCount,
-        final FieldPermissionsCache fieldPermissionsCache
+        final FieldPermissionsCache fieldPermissionsCache,
+        final Map<String, IndexAbstraction> lookup
     ) {
 
         // now... every index that is associated with the request, must be granted
@@ -487,7 +562,7 @@ private Map<String, IndicesAccessControl.IndexAccessControl> buildIndicesAccessC
             // true if ANY group covers the given index AND the given action
             boolean granted = false;
 
-            final Collection<String> concreteIndices = resource.resolveConcreteIndices();
+            final Collection<String> concreteIndices = resource.resolveConcreteIndices(lookup);
             for (Group group : groups) {
                 // the group covers the given index OR the given index is a backing index and the group covers the parent data stream
                 if (resource.checkIndex(group)) {
@@ -704,7 +779,7 @@ private Map<Automaton, Automaton> indexGroupAutomatons(boolean combine) {
             Automaton indexAutomaton = group.getIndexMatcherAutomaton();
             allAutomatons.compute(
                 group.privilege().getAutomaton(),
-                (key, value) -> value == null ? indexAutomaton : Automatons.unionAndDeterminize(List.of(value, indexAutomaton))
+                (key, value) -> value == null ? indexAutomaton : Automatons.unionAndMinimize(List.of(value, indexAutomaton))
             );
             if (combine) {
                 List<Tuple<Automaton, Automaton>> combinedAutomatons = new ArrayList<>();
@@ -714,7 +789,7 @@ private Map<Automaton, Automaton> indexGroupAutomatons(boolean combine) {
                         group.privilege().getAutomaton()
                     );
                     if (Operations.isEmpty(intersectingPrivileges) == false) {
-                        Automaton indexPatternAutomaton = Automatons.unionAndDeterminize(
+                        Automaton indexPatternAutomaton = Automatons.unionAndMinimize(
                             List.of(indexAndPrivilegeAutomatons.getValue(), indexAutomaton)
                         );
                         combinedAutomatons.add(new Tuple<>(intersectingPrivileges, indexPatternAutomaton));
@@ -723,7 +798,7 @@ private Map<Automaton, Automaton> indexGroupAutomatons(boolean combine) {
                 combinedAutomatons.forEach(
                     automatons -> allAutomatons.compute(
                         automatons.v1(),
-                        (key, value) -> value == null ? automatons.v2() : Automatons.unionAndDeterminize(List.of(value, automatons.v2()))
+                        (key, value) -> value == null ? automatons.v2() : Automatons.unionAndMinimize(List.of(value, automatons.v2()))
                     )
                 );
             }
@@ -768,7 +843,7 @@ public Group(
                 this.indexNameMatcher = StringMatcher.of(indices).and(name -> restrictedIndices.isRestricted(name) == false);
                 this.indexNameAutomaton = () -> indexNameAutomatonMemo.computeIfAbsent(
                     indices,
-                    k -> Automatons.minusAndDeterminize(Automatons.patterns(indices), restrictedIndices.getAutomaton())
+                    k -> Automatons.minusAndMinimize(Automatons.patterns(indices), restrictedIndices.getAutomaton())
                 );
             }
             this.fieldPermissions = Objects.requireNonNull(fieldPermissions);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java
index e4d283aba75a3..ea32ba13ae576 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java
@@ -212,7 +212,7 @@ public IsResourceAuthorizedPredicate allowedIndicesMatcher(String action) {
     public Automaton allowedActionsMatcher(String index) {
         final Automaton allowedMatcher = baseRole.allowedActionsMatcher(index);
         final Automaton limitedByMatcher = limitedByRole.allowedActionsMatcher(index);
-        return Automatons.intersectAndDeterminize(allowedMatcher, limitedByMatcher);
+        return Automatons.intersectAndMinimize(allowedMatcher, limitedByMatcher);
     }
 
     /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java
index f4df99dcefea4..7174b2f616c2a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java
@@ -57,7 +57,7 @@
 
 import static java.util.Map.entry;
 import static org.elasticsearch.xpack.core.security.support.Automatons.patterns;
-import static org.elasticsearch.xpack.core.security.support.Automatons.unionAndDeterminize;
+import static org.elasticsearch.xpack.core.security.support.Automatons.unionAndMinimize;
 
 /**
  * The name of an index related action always being with `indices:` followed by a sequence of slash-separated terms
@@ -110,7 +110,7 @@ public final class IndexPrivilege extends Privilege {
     private static final Automaton DELETE_AUTOMATON = patterns("indices:data/write/delete*", "indices:data/write/bulk*");
     private static final Automaton WRITE_AUTOMATON = patterns("indices:data/write/*", TransportAutoPutMappingAction.TYPE.name());
     private static final Automaton MONITOR_AUTOMATON = patterns("indices:monitor/*");
-    private static final Automaton MANAGE_AUTOMATON = unionAndDeterminize(
+    private static final Automaton MANAGE_AUTOMATON = unionAndMinimize(
         Arrays.asList(
             MONITOR_AUTOMATON,
             patterns("indices:admin/*", TransportFieldCapabilitiesAction.NAME + "*", GetRollupIndexCapsAction.NAME + "*")
@@ -303,7 +303,7 @@ private static IndexPrivilege resolve(Set<String> name) {
         if (actions.isEmpty() == false) {
             automata.add(patterns(actions));
         }
-        return new IndexPrivilege(name, unionAndDeterminize(automata));
+        return new IndexPrivilege(name, unionAndMinimize(automata));
     }
 
     static Map<String, IndexPrivilege> values() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java
index 9233841891461..68baab4469e48 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java
@@ -20,7 +20,7 @@
 
 public final class SystemPrivilege extends Privilege {
 
-    public static SystemPrivilege INSTANCE = new SystemPrivilege();
+    public static final SystemPrivilege INSTANCE = new SystemPrivilege();
 
     private static final Predicate<String> ALLOWED_ACTIONS = StringMatcher.of(
         "internal:*",
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java
index 5e19b26b8f4de..fcd1ba54a8de2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.action.admin.indices.alias.TransportIndicesAliasesAction;
 import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction;
+import org.elasticsearch.action.admin.indices.mapping.put.TransportAutoPutMappingAction;
 import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction;
 import org.elasticsearch.action.admin.indices.rollover.RolloverAction;
 import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction;
@@ -225,18 +226,29 @@ static RoleDescriptor kibanaSystem(String name) {
                 RoleDescriptor.IndicesPrivileges.builder().indices("logs-fleet_server*").privileges("read", "delete_index").build(),
                 // Legacy "Alerts as data" used in Security Solution.
                 // Kibana user creates these indices; reads / writes to them.
-                RoleDescriptor.IndicesPrivileges.builder().indices(ReservedRolesStore.ALERTS_LEGACY_INDEX).privileges("all").build(),
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(ReservedRolesStore.ALERTS_LEGACY_INDEX, ReservedRolesStore.ALERTS_LEGACY_INDEX_REINDEXED_V8)
+                    .privileges("all")
+                    .build(),
                 // Used in Security Solution for value lists.
                 // Kibana user creates these indices; reads / writes to them.
                 RoleDescriptor.IndicesPrivileges.builder()
-                    .indices(ReservedRolesStore.LISTS_INDEX, ReservedRolesStore.LISTS_ITEMS_INDEX)
+                    .indices(
+                        ReservedRolesStore.LISTS_INDEX,
+                        ReservedRolesStore.LISTS_ITEMS_INDEX,
+                        ReservedRolesStore.LISTS_INDEX_REINDEXED_V8,
+                        ReservedRolesStore.LISTS_ITEMS_INDEX_REINDEXED_V8
+                    )
                     .privileges("all")
                     .build(),
                 // "Alerts as data" internal backing indices used in Security Solution,
                 // Observability, etc.
                 // Kibana system user creates these indices; reads / writes to them via the
                 // aliases (see below).
-                RoleDescriptor.IndicesPrivileges.builder().indices(ReservedRolesStore.ALERTS_BACKING_INDEX).privileges("all").build(),
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(ReservedRolesStore.ALERTS_BACKING_INDEX, ReservedRolesStore.ALERTS_BACKING_INDEX_REINDEXED)
+                    .privileges("all")
+                    .build(),
                 // "Alerts as data" public index aliases used in Security Solution,
                 // Observability, etc.
                 // Kibana system user uses them to read / write alerts.
@@ -248,7 +260,7 @@ static RoleDescriptor kibanaSystem(String name) {
                 // Kibana system user creates these indices; reads / writes to them via the
                 // aliases (see below).
                 RoleDescriptor.IndicesPrivileges.builder()
-                    .indices(ReservedRolesStore.PREVIEW_ALERTS_BACKING_INDEX_ALIAS)
+                    .indices(ReservedRolesStore.PREVIEW_ALERTS_BACKING_INDEX, ReservedRolesStore.PREVIEW_ALERTS_BACKING_INDEX_REINDEXED)
                     .privileges("all")
                     .build(),
                 // Endpoint / Fleet policy responses. Kibana requires read access to send
@@ -417,7 +429,6 @@ static RoleDescriptor kibanaSystem(String name) {
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices(
                         "logs-cloud_security_posture.findings_latest-default*",
-                        "logs-cloud_security_posture.scores-default*",
                         "logs-cloud_security_posture.vulnerabilities_latest-default*"
                     )
                     .privileges(
@@ -429,6 +440,20 @@ static RoleDescriptor kibanaSystem(String name) {
                         TransportUpdateSettingsAction.TYPE.name()
                     )
                     .build(),
+                // For destination indices of the Cloud Security Posture packages that ships a
+                // transform (specific for scores indexes, as of 9.0.0 score indices will need to have auto_put priviliges)
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices("logs-cloud_security_posture.scores-default*")
+                    .privileges(
+                        "create_index",
+                        "read",
+                        "index",
+                        "delete",
+                        TransportIndicesAliasesAction.NAME,
+                        TransportUpdateSettingsAction.TYPE.name(),
+                        TransportAutoPutMappingAction.TYPE.name()
+                    )
+                    .build(),
                 // For source indices of the Cloud Detection & Response (CDR) packages that ships a
                 // transform
                 RoleDescriptor.IndicesPrivileges.builder()
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
index bdaf75203ee5d..3648d8a0c7daa 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
@@ -43,9 +43,11 @@
 public class ReservedRolesStore implements BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>> {
     /** "Security Solutions" only legacy signals index */
     public static final String ALERTS_LEGACY_INDEX = ".siem-signals*";
+    public static final String ALERTS_LEGACY_INDEX_REINDEXED_V8 = ".reindexed-v8-siem-signals*";
 
     /** Alerts, Rules, Cases (RAC) index used by multiple solutions */
     public static final String ALERTS_BACKING_INDEX = ".internal.alerts*";
+    public static final String ALERTS_BACKING_INDEX_REINDEXED = ".reindexed-v8-internal.alerts*";
 
     /** Alerts, Rules, Cases (RAC) index used by multiple solutions */
     public static final String ALERTS_INDEX_ALIAS = ".alerts*";
@@ -54,13 +56,16 @@ public class ReservedRolesStore implements BiConsumer<Set<String>, ActionListene
     public static final String PREVIEW_ALERTS_INDEX_ALIAS = ".preview.alerts*";
 
     /** Alerts, Rules, Cases (RAC) preview index used by multiple solutions */
-    public static final String PREVIEW_ALERTS_BACKING_INDEX_ALIAS = ".internal.preview.alerts*";
+    public static final String PREVIEW_ALERTS_BACKING_INDEX = ".internal.preview.alerts*";
+    public static final String PREVIEW_ALERTS_BACKING_INDEX_REINDEXED = ".reindexed-v8-internal.preview.alerts*";
 
     /** "Security Solutions" only lists index for value lists for detections */
     public static final String LISTS_INDEX = ".lists-*";
+    public static final String LISTS_INDEX_REINDEXED_V8 = ".reindexed-v8-lists-*";
 
     /** "Security Solutions" only lists index for value list items for detections */
     public static final String LISTS_ITEMS_INDEX = ".items-*";
+    public static final String LISTS_ITEMS_INDEX_REINDEXED_V8 = ".reindexed-v8-items-*";
 
     /** Index pattern for Universal Profiling */
     public static final String UNIVERSAL_PROFILING_ALIASES = "profiling-*";
@@ -534,70 +539,6 @@ private static Map<String, RoleDescriptor> initializeReservedRoles() {
                         + "and roles that grant access to Kibana."
                 )
             ),
-            // DEPRECATED: to be removed in 9.0.0
-            entry(
-                "data_frame_transforms_admin",
-                new RoleDescriptor(
-                    "data_frame_transforms_admin",
-                    new String[] { "manage_data_frame_transforms" },
-                    new RoleDescriptor.IndicesPrivileges[] {
-                        RoleDescriptor.IndicesPrivileges.builder()
-                            .indices(
-                                TransformInternalIndexConstants.AUDIT_INDEX_PATTERN,
-                                TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED,
-                                TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS
-                            )
-                            .privileges("view_index_metadata", "read")
-                            .build() },
-                    new RoleDescriptor.ApplicationResourcePrivileges[] {
-                        RoleDescriptor.ApplicationResourcePrivileges.builder()
-                            .application("kibana-*")
-                            .resources("*")
-                            .privileges("reserved_ml_user")
-                            .build() },
-                    null,
-                    null,
-                    MetadataUtils.getDeprecatedReservedMetadata("Please use the [transform_admin] role instead"),
-                    null,
-                    null,
-                    null,
-                    null,
-                    "Grants manage_data_frame_transforms cluster privileges, which enable you to manage transforms. "
-                        + "This role also includes all Kibana privileges for the machine learning features."
-                )
-            ),
-            // DEPRECATED: to be removed in 9.0.0
-            entry(
-                "data_frame_transforms_user",
-                new RoleDescriptor(
-                    "data_frame_transforms_user",
-                    new String[] { "monitor_data_frame_transforms" },
-                    new RoleDescriptor.IndicesPrivileges[] {
-                        RoleDescriptor.IndicesPrivileges.builder()
-                            .indices(
-                                TransformInternalIndexConstants.AUDIT_INDEX_PATTERN,
-                                TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED,
-                                TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS
-                            )
-                            .privileges("view_index_metadata", "read")
-                            .build() },
-                    new RoleDescriptor.ApplicationResourcePrivileges[] {
-                        RoleDescriptor.ApplicationResourcePrivileges.builder()
-                            .application("kibana-*")
-                            .resources("*")
-                            .privileges("reserved_ml_user")
-                            .build() },
-                    null,
-                    null,
-                    MetadataUtils.getDeprecatedReservedMetadata("Please use the [transform_user] role instead"),
-                    null,
-                    null,
-                    null,
-                    null,
-                    "Grants monitor_data_frame_transforms cluster privileges, which enable you to use transforms. "
-                        + "This role also includes all Kibana privileges for the machine learning features. "
-                )
-            ),
             entry(
                 "transform_admin",
                 new RoleDescriptor(
@@ -827,7 +768,14 @@ private static RoleDescriptor buildViewerRoleDescriptor() {
                     .build(),
                 // Security
                 RoleDescriptor.IndicesPrivileges.builder()
-                    .indices(ReservedRolesStore.ALERTS_LEGACY_INDEX, ReservedRolesStore.LISTS_INDEX, ReservedRolesStore.LISTS_ITEMS_INDEX)
+                    .indices(
+                        ReservedRolesStore.ALERTS_LEGACY_INDEX,
+                        ReservedRolesStore.LISTS_INDEX,
+                        ReservedRolesStore.LISTS_ITEMS_INDEX,
+                        ReservedRolesStore.ALERTS_LEGACY_INDEX_REINDEXED_V8,
+                        ReservedRolesStore.LISTS_INDEX_REINDEXED_V8,
+                        ReservedRolesStore.LISTS_ITEMS_INDEX_REINDEXED_V8
+                    )
                     .privileges("read", "view_index_metadata")
                     .build(),
                 // Alerts-as-data
@@ -878,15 +826,24 @@ private static RoleDescriptor buildEditorRoleDescriptor() {
                     .build(),
                 // Security
                 RoleDescriptor.IndicesPrivileges.builder()
-                    .indices(ReservedRolesStore.ALERTS_LEGACY_INDEX, ReservedRolesStore.LISTS_INDEX, ReservedRolesStore.LISTS_ITEMS_INDEX)
+                    .indices(
+                        ReservedRolesStore.ALERTS_LEGACY_INDEX,
+                        ReservedRolesStore.LISTS_INDEX,
+                        ReservedRolesStore.LISTS_ITEMS_INDEX,
+                        ReservedRolesStore.ALERTS_LEGACY_INDEX_REINDEXED_V8,
+                        ReservedRolesStore.LISTS_INDEX_REINDEXED_V8,
+                        ReservedRolesStore.LISTS_ITEMS_INDEX_REINDEXED_V8
+                    )
                     .privileges("read", "view_index_metadata", "write", "maintenance")
                     .build(),
                 // Alerts-as-data
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices(
                         ReservedRolesStore.ALERTS_BACKING_INDEX,
+                        ReservedRolesStore.ALERTS_BACKING_INDEX_REINDEXED,
                         ReservedRolesStore.ALERTS_INDEX_ALIAS,
-                        ReservedRolesStore.PREVIEW_ALERTS_BACKING_INDEX_ALIAS,
+                        ReservedRolesStore.PREVIEW_ALERTS_BACKING_INDEX,
+                        ReservedRolesStore.PREVIEW_ALERTS_BACKING_INDEX_REINDEXED,
                         ReservedRolesStore.PREVIEW_ALERTS_INDEX_ALIAS
                     )
                     .privileges("read", "view_index_metadata", "write", "maintenance")
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java
index d3790ea64ba4b..52a3650159a08 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.Predicates;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.lucene.util.automaton.MinimizationOperations;
 
 import java.util.ArrayDeque;
 import java.util.ArrayList;
@@ -76,7 +77,7 @@ public final class Automatons {
     static final char WILDCARD_ESCAPE = '\\';    // Escape character
 
     // for testing only -Dtests.jvm.argline="-Dtests.automaton.record.patterns=true"
-    public static boolean recordPatterns = System.getProperty("tests.automaton.record.patterns", "false").equals("true");
+    public static final boolean recordPatterns = System.getProperty("tests.automaton.record.patterns", "false").equals("true");
     private static final Map<Automaton, List<String>> patternsMap = new HashMap<>();
 
     private Automatons() {}
@@ -112,7 +113,7 @@ public static Automaton patterns(Collection<String> patterns) {
 
     private static Automaton buildAutomaton(Collection<String> patterns) {
         if (patterns.size() == 1) {
-            return determinize(pattern(patterns.iterator().next()));
+            return minimize(pattern(patterns.iterator().next()));
         }
 
         final Function<Collection<String>, Automaton> build = strings -> {
@@ -121,7 +122,7 @@ private static Automaton buildAutomaton(Collection<String> patterns) {
                 final Automaton patternAutomaton = pattern(pattern);
                 automata.add(patternAutomaton);
             }
-            return unionAndDeterminize(automata);
+            return unionAndMinimize(automata);
         };
 
         // We originally just compiled each automaton separately and then unioned them all.
@@ -188,7 +189,7 @@ private static Automaton buildAutomaton(Collection<String> patterns) {
         if (misc.isEmpty() == false) {
             automata.add(build.apply(misc));
         }
-        return unionAndDeterminize(automata);
+        return unionAndMinimize(automata);
     }
 
     /**
@@ -277,23 +278,23 @@ static Automaton wildcard(String text) {
         return Operations.determinize(concatenate(automata), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
     }
 
-    public static Automaton unionAndDeterminize(Collection<Automaton> automata) {
+    public static Automaton unionAndMinimize(Collection<Automaton> automata) {
         Automaton res = automata.size() == 1 ? automata.iterator().next() : union(automata);
-        return determinize(res);
+        return minimize(res);
     }
 
-    public static Automaton minusAndDeterminize(Automaton a1, Automaton a2) {
+    public static Automaton minusAndMinimize(Automaton a1, Automaton a2) {
         Automaton res = minus(a1, a2, maxDeterminizedStates);
-        return determinize(res);
+        return minimize(res);
     }
 
-    public static Automaton intersectAndDeterminize(Automaton a1, Automaton a2) {
+    public static Automaton intersectAndMinimize(Automaton a1, Automaton a2) {
         Automaton res = intersection(a1, a2);
-        return determinize(res);
+        return minimize(res);
     }
 
-    private static Automaton determinize(Automaton automaton) {
-        return Operations.determinize(automaton, maxDeterminizedStates);
+    private static Automaton minimize(Automaton automaton) {
+        return MinimizationOperations.minimize(automaton, maxDeterminizedStates);
     }
 
     public static Predicate<String> predicate(String... patterns) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/NoOpLogger.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/NoOpLogger.java
index af1a2726b4f55..8844fec09cf53 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/NoOpLogger.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/NoOpLogger.java
@@ -20,7 +20,7 @@
  */
 public class NoOpLogger implements Logger {
 
-    public static NoOpLogger INSTANCE = new NoOpLogger();
+    public static final NoOpLogger INSTANCE = new NoOpLogger();
 
     private NoOpLogger() {
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java
index 7a5bf9f9e1aea..136070429c36d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java
@@ -15,9 +15,10 @@
 import org.elasticsearch.cluster.SimpleDiffable;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
@@ -143,10 +144,13 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params)
-            .xContentObjectFields(POLICIES_FIELD.getPreferredName(), snapshotConfigurations)
-            .field(OPERATION_MODE_FIELD.getPreferredName(), operationMode)
-            .field(STATS_FIELD.getPreferredName(), slmStats);
+        return Iterators.concat(
+            ChunkedToXContentHelper.xContentObjectFields(POLICIES_FIELD.getPreferredName(), this.snapshotConfigurations),
+            Iterators.single(
+                (builder, p) -> builder.field(OPERATION_MODE_FIELD.getPreferredName(), operationMode)
+                    .field(STATS_FIELD.getPreferredName(), this.slmStats)
+            )
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequest.java
index 95edd3fc3bb92..1c787389602a8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequest.java
@@ -33,8 +33,8 @@
 public final class TermsEnumRequest extends BroadcastRequest<TermsEnumRequest> implements ToXContentObject {
 
     public static final IndicesOptions DEFAULT_INDICES_OPTIONS = SearchRequest.DEFAULT_INDICES_OPTIONS;
-    public static int DEFAULT_SIZE = 10;
-    public static TimeValue DEFAULT_TIMEOUT = new TimeValue(1000);
+    public static final int DEFAULT_SIZE = 10;
+    public static final TimeValue DEFAULT_TIMEOUT = new TimeValue(1000);
 
     private String field;
     private String string = null;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java
index 1de584d5593f1..79a679441de3a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java
@@ -27,12 +27,5 @@ public class TransformDeprecations {
 
     public static final String MAX_PAGE_SEARCH_SIZE_BREAKING_CHANGES_URL = "https://ela.st/es-deprecation-7-transform-max-page-search-size";
 
-    public static final String DATA_FRAME_TRANSFORMS_ROLES_BREAKING_CHANGES_URL =
-        "https://ela.st/es-deprecation-9-data-frame-transforms-roles";
-
-    public static final String DATA_FRAME_TRANSFORMS_ROLES_IS_DEPRECATED = "This transform configuration uses one or more obsolete roles "
-        + "prefixed with [data_frame_transformers_] which will be unsupported after the next upgrade. Switch to a user with the equivalent "
-        + "roles prefixed with [transform_] and use [/_transform/_upgrade] to upgrade all transforms to the latest roles.";;
-
     private TransformDeprecations() {}
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java
index 204b81914816a..b0ba81d8eeb37 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.core.transform;
 
+import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.xcontent.ParseField;
 
 /*
@@ -99,7 +100,16 @@ public final class TransformField {
     public static final String EXCLUDE_GENERATED = "exclude_generated";
 
     // internal document id
-    public static String DOCUMENT_ID_FIELD = "_id";
+    public static final String DOCUMENT_ID_FIELD = "_id";
+
+    public static final PersistentTasksCustomMetadata.Assignment AWAITING_UPGRADE = new PersistentTasksCustomMetadata.Assignment(
+        null,
+        "Transform task will not be assigned while upgrade mode is enabled."
+    );
+    public static final PersistentTasksCustomMetadata.Assignment RESET_IN_PROGRESS = new PersistentTasksCustomMetadata.Assignment(
+        null,
+        "Transform task will not be assigned as a feature reset is in progress."
+    );
 
     private TransformField() {}
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMessages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMessages.java
index 8c3b133d580a5..9cb6a86f433d1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMessages.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMessages.java
@@ -89,6 +89,9 @@ public class TransformMessages {
     public static final String INVALID_ID = "Invalid {0}; ''{1}'' can contain lowercase alphanumeric (a-z and 0-9), hyphens or "
         + "underscores; must start and end with alphanumeric";
 
+    public static final String MAX_PAGE_SEARCH_SIZE_MIGRATION =
+        "Automatically migrating [max_page_search_size] from the pivot to the transform setting.";
+
     private TransformMessages() {}
 
     /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMetadata.java
index e54cf2edb2690..a489a4f088862 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMetadata.java
@@ -14,9 +14,9 @@
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -30,8 +30,9 @@
 public class TransformMetadata implements Metadata.Custom {
     public static final String TYPE = "transform";
     public static final ParseField RESET_MODE = new ParseField("reset_mode");
+    public static final ParseField UPGRADE_MODE = new ParseField("upgrade_mode");
 
-    public static final TransformMetadata EMPTY_METADATA = new TransformMetadata(false);
+    public static final TransformMetadata EMPTY_METADATA = new TransformMetadata(false, false);
     // This parser follows the pattern that metadata is parsed leniently (to allow for enhancements)
     public static final ObjectParser<TransformMetadata.Builder, Void> LENIENT_PARSER = new ObjectParser<>(
         "" + "transform_metadata",
@@ -40,19 +41,26 @@ public class TransformMetadata implements Metadata.Custom {
     );
 
     static {
-        LENIENT_PARSER.declareBoolean(TransformMetadata.Builder::isResetMode, RESET_MODE);
+        LENIENT_PARSER.declareBoolean(TransformMetadata.Builder::resetMode, RESET_MODE);
+        LENIENT_PARSER.declareBoolean(TransformMetadata.Builder::upgradeMode, UPGRADE_MODE);
     }
 
     private final boolean resetMode;
+    private final boolean upgradeMode;
 
-    private TransformMetadata(boolean resetMode) {
+    private TransformMetadata(boolean resetMode, boolean upgradeMode) {
         this.resetMode = resetMode;
+        this.upgradeMode = upgradeMode;
     }
 
-    public boolean isResetMode() {
+    public boolean resetMode() {
         return resetMode;
     }
 
+    public boolean upgradeMode() {
+        return upgradeMode;
+    }
+
     @Override
     public TransportVersion getMinimalSupportedVersion() {
         return TransportVersions.MINIMUM_COMPATIBLE;
@@ -75,28 +83,46 @@ public Diff<Metadata.Custom> diff(Metadata.Custom previousState) {
 
     public TransformMetadata(StreamInput in) throws IOException {
         this.resetMode = in.readBoolean();
+        if (in.getTransportVersion().onOrAfter(TransportVersions.TRANSFORMS_UPGRADE_MODE)) {
+            this.upgradeMode = in.readBoolean();
+        } else {
+            this.upgradeMode = false;
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(resetMode);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.TRANSFORMS_UPGRADE_MODE)) {
+            out.writeBoolean(upgradeMode);
+        }
     }
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
-        return ChunkedToXContentHelper.field(RESET_MODE.getPreferredName(), resetMode);
+        return Iterators.single(
+            ((builder, params) -> builder.field(UPGRADE_MODE.getPreferredName(), upgradeMode)
+                .field(RESET_MODE.getPreferredName(), resetMode))
+        );
     }
 
     public static class TransformMetadataDiff implements NamedDiff<Metadata.Custom> {
 
         final boolean resetMode;
+        final boolean upgradeMode;
 
         TransformMetadataDiff(TransformMetadata before, TransformMetadata after) {
             this.resetMode = after.resetMode;
+            this.upgradeMode = after.upgradeMode;
         }
 
         public TransformMetadataDiff(StreamInput in) throws IOException {
             resetMode = in.readBoolean();
+            if (in.getTransportVersion().onOrAfter(TransportVersions.TRANSFORMS_UPGRADE_MODE)) {
+                this.upgradeMode = in.readBoolean();
+            } else {
+                this.upgradeMode = false;
+            }
         }
 
         /**
@@ -106,12 +132,15 @@ public TransformMetadataDiff(StreamInput in) throws IOException {
          */
         @Override
         public Metadata.Custom apply(Metadata.Custom part) {
-            return new TransformMetadata(resetMode);
+            return new TransformMetadata(resetMode, upgradeMode);
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(resetMode);
+            if (out.getTransportVersion().onOrAfter(TransportVersions.TRANSFORMS_UPGRADE_MODE)) {
+                out.writeBoolean(upgradeMode);
+            }
         }
 
         @Override
@@ -130,7 +159,7 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         TransformMetadata that = (TransformMetadata) o;
-        return resetMode == that.resetMode;
+        return resetMode == that.resetMode && upgradeMode == that.upgradeMode;
     }
 
     @Override
@@ -140,12 +169,17 @@ public final String toString() {
 
     @Override
     public int hashCode() {
-        return Objects.hash(resetMode);
+        return Objects.hash(resetMode, upgradeMode);
+    }
+
+    public Builder builder() {
+        return new TransformMetadata.Builder(this);
     }
 
     public static class Builder {
 
         private boolean resetMode;
+        private boolean upgradeMode;
 
         public static TransformMetadata.Builder from(@Nullable TransformMetadata previous) {
             return new TransformMetadata.Builder(previous);
@@ -156,16 +190,22 @@ public Builder() {}
         public Builder(@Nullable TransformMetadata previous) {
             if (previous != null) {
                 resetMode = previous.resetMode;
+                upgradeMode = previous.upgradeMode;
             }
         }
 
-        public TransformMetadata.Builder isResetMode(boolean isResetMode) {
+        public TransformMetadata.Builder resetMode(boolean isResetMode) {
             this.resetMode = isResetMode;
             return this;
         }
 
+        public TransformMetadata.Builder upgradeMode(boolean upgradeMode) {
+            this.upgradeMode = upgradeMode;
+            return this;
+        }
+
         public TransformMetadata build() {
-            return new TransformMetadata(resetMode);
+            return new TransformMetadata(resetMode, upgradeMode);
         }
     }
 
@@ -176,4 +216,8 @@ public static TransformMetadata getTransformMetadata(ClusterState state) {
         }
         return TransformMetadata;
     }
+
+    public static boolean upgradeMode(ClusterState state) {
+        return getTransformMetadata(state).upgradeMode();
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/SetTransformUpgradeModeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/SetTransformUpgradeModeAction.java
new file mode 100644
index 0000000000000..b93df8125708f
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/SetTransformUpgradeModeAction.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.transform.action;
+
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+
+public class SetTransformUpgradeModeAction extends ActionType<AcknowledgedResponse> {
+    public static final SetTransformUpgradeModeAction INSTANCE = new SetTransformUpgradeModeAction();
+    public static final String NAME = "cluster:admin/transform/upgrade_mode";
+
+    private SetTransformUpgradeModeAction() {
+        super(NAME);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpoint.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpoint.java
index fea6299c73e62..e4d8f8790c42d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpoint.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpoint.java
@@ -46,8 +46,8 @@
  */
 public class TransformCheckpoint implements Writeable, ToXContentObject {
 
-    public static String EMPTY_NAME = "_empty";
-    public static TransformCheckpoint EMPTY = createEmpty(0);
+    public static final String EMPTY_NAME = "_empty";
+    public static final TransformCheckpoint EMPTY = createEmpty(0);
 
     public static TransformCheckpoint createEmpty(long timestampMillis) {
         return new TransformCheckpoint(EMPTY_NAME, timestampMillis, -1L, Collections.emptyMap(), timestampMillis);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java
index 745da71539992..41279d83fdae1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java
@@ -24,13 +24,11 @@
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.common.time.TimeUtils;
 import org.elasticsearch.xpack.core.common.validation.SourceDestValidator;
 import org.elasticsearch.xpack.core.common.validation.SourceDestValidator.SourceDestValidation;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level;
-import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer;
 import org.elasticsearch.xpack.core.security.xcontent.XContentUtils;
 import org.elasticsearch.xpack.core.transform.TransformConfigVersion;
 import org.elasticsearch.xpack.core.transform.TransformDeprecations;
@@ -43,7 +41,6 @@
 import java.io.IOException;
 import java.time.Instant;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
@@ -52,7 +49,6 @@
 
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
-import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY;
 
 /**
  * This class holds the configuration details of a data frame transform
@@ -69,10 +65,6 @@ public final class TransformConfig implements SimpleDiffable<TransformConfig>, W
     public static final ParseField HEADERS = new ParseField("headers");
     /** Version in which {@code FieldCapabilitiesRequest.runtime_fields} field was introduced. */
     private static final TransportVersion FIELD_CAPS_RUNTIME_MAPPINGS_INTRODUCED_TRANSPORT_VERSION = TransportVersions.V_7_12_0;
-    private static final List<String> DEPRECATED_DATA_FRAME_TRANSFORMS_ROLES = List.of(
-        "data_frame_transforms_admin",
-        "data_frame_transforms_user"
-    );
 
     /** Specifies all the possible transform functions. */
     public enum Function {
@@ -413,37 +405,9 @@ public List<DeprecationIssue> checkForDeprecations(NamedXContentRegistry namedXC
             retentionPolicyConfig.checkForDeprecations(getId(), namedXContentRegistry, deprecations::add);
         }
 
-        var deprecatedTransformRoles = getRolesFromHeaders().stream().filter(DEPRECATED_DATA_FRAME_TRANSFORMS_ROLES::contains).toList();
-        if (deprecatedTransformRoles.isEmpty() == false) {
-            deprecations.add(
-                new DeprecationIssue(
-                    Level.CRITICAL,
-                    "Transform [" + id + "] uses deprecated transform roles " + deprecatedTransformRoles,
-                    TransformDeprecations.DATA_FRAME_TRANSFORMS_ROLES_BREAKING_CHANGES_URL,
-                    TransformDeprecations.DATA_FRAME_TRANSFORMS_ROLES_IS_DEPRECATED,
-                    false,
-                    null
-                )
-            );
-        }
-
         return deprecations;
     }
 
-    private List<String> getRolesFromHeaders() throws IOException {
-        if (headers == null) {
-            return Collections.emptyList();
-        }
-
-        var encodedAuthenticationHeader = ClientHelper.filterSecurityHeaders(headers).getOrDefault(AUTHENTICATION_KEY, "");
-        if (encodedAuthenticationHeader.isEmpty()) {
-            return Collections.emptyList();
-        }
-
-        var decodedAuthenticationHeader = AuthenticationContextSerializer.decode(encodedAuthenticationHeader);
-        return Arrays.asList(decodedAuthenticationHeader.getEffectiveSubject().getUser().roles());
-    }
-
     @Override
     public void writeTo(final StreamOutput out) throws IOException {
         out.writeString(id);
@@ -615,24 +579,15 @@ public static TransformConfig rewriteForUpdate(final TransformConfig transformCo
 
     private static TransformConfig applyRewriteForUpdate(Builder builder) {
         // 1. Move pivot.max_page_size_search to settings.max_page_size_search
-        if (builder.getPivotConfig() != null && builder.getPivotConfig().getMaxPageSearchSize() != null) {
+        migrateMaxPageSearchSize(builder);
 
-            // find maxPageSearchSize value
-            Integer maxPageSearchSizeDeprecated = builder.getPivotConfig().getMaxPageSearchSize();
-            Integer maxPageSearchSize = builder.getSettings().getMaxPageSearchSize() != null
-                ? builder.getSettings().getMaxPageSearchSize()
-                : maxPageSearchSizeDeprecated;
-
-            // create a new pivot config but set maxPageSearchSize to null
-            builder.setPivotConfig(
-                new PivotConfig(builder.getPivotConfig().getGroupConfig(), builder.getPivotConfig().getAggregationConfig(), null)
-            );
-            // create new settings with maxPageSearchSize
+        // 2. set dates_as_epoch_millis to true for transforms < 7.11 to keep BWC
+        if (builder.getVersion() != null && builder.getVersion().before(TransformConfigVersion.V_7_11_0)) {
             builder.setSettings(
                 new SettingsConfig(
-                    maxPageSearchSize,
+                    builder.getSettings().getMaxPageSearchSize(),
                     builder.getSettings().getDocsPerSecond(),
-                    builder.getSettings().getDatesAsEpochMillis(),
+                    true,
                     builder.getSettings().getAlignCheckpoints(),
                     builder.getSettings().getUsePit(),
                     builder.getSettings().getDeduceMappings(),
@@ -642,14 +597,14 @@ private static TransformConfig applyRewriteForUpdate(Builder builder) {
             );
         }
 
-        // 2. set dates_as_epoch_millis to true for transforms < 7.11 to keep BWC
-        if (builder.getVersion() != null && builder.getVersion().before(TransformConfigVersion.V_7_11_0)) {
+        // 3. set align_checkpoints to false for transforms < 7.15 to keep BWC
+        if (builder.getVersion() != null && builder.getVersion().before(TransformConfigVersion.V_7_15_0)) {
             builder.setSettings(
                 new SettingsConfig(
                     builder.getSettings().getMaxPageSearchSize(),
                     builder.getSettings().getDocsPerSecond(),
-                    true,
-                    builder.getSettings().getAlignCheckpoints(),
+                    builder.getSettings().getDatesAsEpochMillis(),
+                    false,
                     builder.getSettings().getUsePit(),
                     builder.getSettings().getDeduceMappings(),
                     builder.getSettings().getNumFailureRetries(),
@@ -658,14 +613,33 @@ private static TransformConfig applyRewriteForUpdate(Builder builder) {
             );
         }
 
-        // 3. set align_checkpoints to false for transforms < 7.15 to keep BWC
-        if (builder.getVersion() != null && builder.getVersion().before(TransformConfigVersion.V_7_15_0)) {
+        return builder.setVersion(TransformConfigVersion.CURRENT).build();
+    }
+
+    public boolean shouldAutoMigrateMaxPageSearchSize() {
+        return getPivotConfig() != null && getPivotConfig().getMaxPageSearchSize() != null;
+    }
+
+    public static Builder migrateMaxPageSearchSize(Builder builder) {
+        if (builder.getPivotConfig() != null && builder.getPivotConfig().getMaxPageSearchSize() != null) {
+
+            // find maxPageSearchSize value
+            Integer maxPageSearchSizeDeprecated = builder.getPivotConfig().getMaxPageSearchSize();
+            Integer maxPageSearchSize = builder.getSettings().getMaxPageSearchSize() != null
+                ? builder.getSettings().getMaxPageSearchSize()
+                : maxPageSearchSizeDeprecated;
+
+            // create a new pivot config but set maxPageSearchSize to null
+            builder.setPivotConfig(
+                new PivotConfig(builder.getPivotConfig().getGroupConfig(), builder.getPivotConfig().getAggregationConfig(), null)
+            );
+            // create new settings with maxPageSearchSize
             builder.setSettings(
                 new SettingsConfig(
-                    builder.getSettings().getMaxPageSearchSize(),
+                    maxPageSearchSize,
                     builder.getSettings().getDocsPerSecond(),
                     builder.getSettings().getDatesAsEpochMillis(),
-                    false,
+                    builder.getSettings().getAlignCheckpoints(),
                     builder.getSettings().getUsePit(),
                     builder.getSettings().getDeduceMappings(),
                     builder.getSettings().getNumFailureRetries(),
@@ -673,8 +647,7 @@ private static TransformConfig applyRewriteForUpdate(Builder builder) {
                 )
             );
         }
-
-        return builder.setVersion(TransformConfigVersion.CURRENT).build();
+        return builder;
     }
 
     public static Builder builder() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java
index 7cf84ac3ecea8..f638f7b92cd59 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java
@@ -24,23 +24,23 @@
 public class TransformIndexerStats extends IndexerJobStats {
 
     public static final String NAME = "data_frame_indexer_transform_stats";
-    public static ParseField NUM_PAGES = new ParseField("pages_processed");
-    public static ParseField NUM_INPUT_DOCUMENTS = new ParseField("documents_processed");
-    public static ParseField NUM_OUTPUT_DOCUMENTS = new ParseField("documents_indexed");
-    public static ParseField NUM_DELETED_DOCUMENTS = new ParseField("documents_deleted");
-    public static ParseField NUM_INVOCATIONS = new ParseField("trigger_count");
-    public static ParseField INDEX_TIME_IN_MS = new ParseField("index_time_in_ms");
-    public static ParseField SEARCH_TIME_IN_MS = new ParseField("search_time_in_ms");
-    public static ParseField PROCESSING_TIME_IN_MS = new ParseField("processing_time_in_ms");
-    public static ParseField DELETE_TIME_IN_MS = new ParseField("delete_time_in_ms");
-    public static ParseField INDEX_TOTAL = new ParseField("index_total");
-    public static ParseField SEARCH_TOTAL = new ParseField("search_total");
-    public static ParseField PROCESSING_TOTAL = new ParseField("processing_total");
-    public static ParseField SEARCH_FAILURES = new ParseField("search_failures");
-    public static ParseField INDEX_FAILURES = new ParseField("index_failures");
-    public static ParseField EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS = new ParseField("exponential_avg_checkpoint_duration_ms");
-    public static ParseField EXPONENTIAL_AVG_DOCUMENTS_INDEXED = new ParseField("exponential_avg_documents_indexed");
-    public static ParseField EXPONENTIAL_AVG_DOCUMENTS_PROCESSED = new ParseField("exponential_avg_documents_processed");
+    public static final ParseField NUM_PAGES = new ParseField("pages_processed");
+    public static final ParseField NUM_INPUT_DOCUMENTS = new ParseField("documents_processed");
+    public static final ParseField NUM_OUTPUT_DOCUMENTS = new ParseField("documents_indexed");
+    public static final ParseField NUM_DELETED_DOCUMENTS = new ParseField("documents_deleted");
+    public static final ParseField NUM_INVOCATIONS = new ParseField("trigger_count");
+    public static final ParseField INDEX_TIME_IN_MS = new ParseField("index_time_in_ms");
+    public static final ParseField SEARCH_TIME_IN_MS = new ParseField("search_time_in_ms");
+    public static final ParseField PROCESSING_TIME_IN_MS = new ParseField("processing_time_in_ms");
+    public static final ParseField DELETE_TIME_IN_MS = new ParseField("delete_time_in_ms");
+    public static final ParseField INDEX_TOTAL = new ParseField("index_total");
+    public static final ParseField SEARCH_TOTAL = new ParseField("search_total");
+    public static final ParseField PROCESSING_TOTAL = new ParseField("processing_total");
+    public static final ParseField SEARCH_FAILURES = new ParseField("search_failures");
+    public static final ParseField INDEX_FAILURES = new ParseField("index_failures");
+    public static final ParseField EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS = new ParseField("exponential_avg_checkpoint_duration_ms");
+    public static final ParseField EXPONENTIAL_AVG_DOCUMENTS_INDEXED = new ParseField("exponential_avg_documents_indexed");
+    public static final ParseField EXPONENTIAL_AVG_DOCUMENTS_PROCESSED = new ParseField("exponential_avg_documents_processed");
 
     // This changes how much "weight" past calculations have.
     // The shorter the window, the less "smoothing" will occur.
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/AbstractClusterStateLicenseServiceTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/AbstractClusterStateLicenseServiceTestCase.java
index a3a12792df4aa..a85656d8e32bd 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/AbstractClusterStateLicenseServiceTestCase.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/AbstractClusterStateLicenseServiceTestCase.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -27,7 +26,6 @@
 import org.junit.After;
 import org.junit.Before;
 
-import java.util.List;
 import java.util.stream.Stream;
 
 import static java.util.Collections.emptySet;
@@ -66,14 +64,7 @@ protected void setInitialState(License license, XPackLicenseState licenseState,
     protected void setInitialState(License license, XPackLicenseState licenseState, Settings settings, String selfGeneratedType) {
         licenseType = selfGeneratedType;
         settings = Settings.builder().put(settings).put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), licenseType).build();
-        licenseService = new ClusterStateLicenseService(
-            settings,
-            threadPool,
-            clusterService,
-            clock,
-            licenseState,
-            new FeatureService(List.of())
-        );
+        licenseService = new ClusterStateLicenseService(settings, threadPool, clusterService, clock, licenseState);
         ClusterState state = mock(ClusterState.class);
         final ClusterBlocks noBlock = ClusterBlocks.builder().build();
         when(state.blocks()).thenReturn(noBlock);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/ClusterStateLicenseServiceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/ClusterStateLicenseServiceTests.java
index aaadecef6021c..61530d1bd77cb 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/ClusterStateLicenseServiceTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/ClusterStateLicenseServiceTests.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.license.licensor.LicenseSigner;
 import org.elasticsearch.protocol.xpack.license.LicensesStatus;
 import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
@@ -91,8 +90,7 @@ public void testLogExpirationWarning() {
             mock(ThreadPool.class),
             mockDefaultClusterService(),
             mock(Clock.class),
-            mock(XPackLicenseState.class),
-            new FeatureService(List.of())
+            mock(XPackLicenseState.class)
         );
         final String message = service.buildExpirationMessage(time, expired).toString();
         if (expired) {
@@ -187,8 +185,7 @@ public void testStartBasicStartsNewLicenseIfFieldsDifferent() throws Exception {
             mock(ThreadPool.class),
             clusterService,
             clock,
-            mock(XPackLicenseState.class),
-            new FeatureService(List.of())
+            mock(XPackLicenseState.class)
         );
         verify(clusterService).createTaskQueue(eq("license-service-start-basic"), any(), taskExecutorCaptor.capture());
 
@@ -280,8 +277,7 @@ private void tryRegisterLicense(Settings baseSettings, License license, Consumer
             threadPool,
             clusterService,
             clock,
-            licenseState,
-            new FeatureService(List.of())
+            licenseState
         );
 
         final PutLicenseRequest request = new PutLicenseRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseScheduleTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseScheduleTests.java
index d7b308bc51f46..78bd500b55e24 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseScheduleTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseScheduleTests.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.common.scheduler.SchedulerEngine;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.Before;
@@ -18,7 +17,6 @@
 import java.time.Clock;
 import java.time.Instant;
 import java.time.format.DateTimeFormatter;
-import java.util.List;
 import java.util.Locale;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -37,8 +35,7 @@ public void setup() throws Exception {
             mock(ThreadPool.class),
             mock(ClusterService.class),
             mock(Clock.class),
-            mock(XPackLicenseState.class),
-            new FeatureService(List.of())
+            mock(XPackLicenseState.class)
         );
         schedule = service.nextLicenseCheck(license);
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java
index 81d194ce84131..a2cdfff2e2c80 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java
@@ -46,6 +46,7 @@
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.Engine;
@@ -53,7 +54,6 @@
 import org.elasticsearch.index.engine.InternalEngineFactory;
 import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
-import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.seqno.RetentionLeaseSyncer;
 import org.elasticsearch.index.seqno.SeqNoStats;
@@ -162,7 +162,7 @@ public void testSourceIncompleteSyntheticSourceNoDoc() throws IOException {
             .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic")
+            .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic")
             .build();
         IndexMetadata metadata = IndexMetadata.builder(shardRouting.getIndexName()).settings(settings).primaryTerm(0, primaryTerm).build();
         IndexShard shard = newShard(shardRouting, metadata, null, new InternalEngineFactory());
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java
index c42f0c008548c..692835c9123a6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java
@@ -759,27 +759,15 @@ private SingleNodeShutdownMetadata randomShutdownMetadataRemovingNode(String nod
             SingleNodeShutdownMetadata.Type.REPLACE,
             SingleNodeShutdownMetadata.Type.REMOVE
         );
+        var builder = SingleNodeShutdownMetadata.builder()
+            .setNodeId(nodeId)
+            .setNodeEphemeralId(nodeId)
+            .setType(type)
+            .setReason(this.getTestName());
         return switch (type) {
-            case REMOVE -> SingleNodeShutdownMetadata.builder()
-                .setNodeId(nodeId)
-                .setType(type)
-                .setReason(this.getTestName())
-                .setStartedAtMillis(randomNonNegativeLong())
-                .build();
-            case REPLACE -> SingleNodeShutdownMetadata.builder()
-                .setNodeId(nodeId)
-                .setType(type)
-                .setTargetNodeName(randomAlphaOfLength(10))
-                .setReason(this.getTestName())
-                .setStartedAtMillis(randomNonNegativeLong())
-                .build();
-            case SIGTERM -> SingleNodeShutdownMetadata.builder()
-                .setNodeId(nodeId)
-                .setType(type)
-                .setGracePeriod(randomTimeValue())
-                .setReason(this.getTestName())
-                .setStartedAtMillis(randomNonNegativeLong())
-                .build();
+            case REMOVE -> builder.setStartedAtMillis(randomNonNegativeLong()).build();
+            case REPLACE -> builder.setTargetNodeName(randomAlphaOfLength(10)).setStartedAtMillis(randomNonNegativeLong()).build();
+            case SIGTERM -> builder.setGracePeriod(randomTimeValue()).setStartedAtMillis(randomNonNegativeLong()).build();
             case RESTART -> throw new AssertionError("bad randomization, this method only generates removal type shutdowns");
         };
     }
@@ -1019,6 +1007,7 @@ private NodesShutdownMetadata randomRestartInCluster(Set<String> currentNodes) {
                 nodeId,
                 SingleNodeShutdownMetadata.builder()
                     .setNodeId(nodeId)
+                    .setNodeEphemeralId(nodeId)
                     .setType(SingleNodeShutdownMetadata.Type.RESTART)
                     .setReason(this.getTestName())
                     .setStartedAtMillis(randomNonNegativeLong())
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckShrinkReadyStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckShrinkReadyStepTests.java
index 72bf7cedb2fb9..a427f8c9154cc 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckShrinkReadyStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckShrinkReadyStepTests.java
@@ -465,6 +465,7 @@ public void testStepCompletableIfAllShardsActive() {
                                         .setStartedAtMillis(randomNonNegativeLong())
                                         .setReason("test")
                                         .setNodeId("node1")
+                                        .setNodeEphemeralId("node1")
                                         .setTargetNodeName(targetNodeName)
                                         .setGracePeriod(grace)
                                         .build()
@@ -544,6 +545,7 @@ public void testStepBecomesUncompletable() {
                                         .setStartedAtMillis(randomNonNegativeLong())
                                         .setReason("test")
                                         .setNodeId("node1")
+                                        .setNodeEphemeralId("node1")
                                         .setTargetNodeName(targetNodeName)
                                         .setGracePeriod(grace)
                                         .build()
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequestTests.java
index 25de084a34b04..6cea7100a9da9 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequestTests.java
@@ -17,7 +17,7 @@ public class ExplainLifecycleRequestTests extends AbstractWireSerializingTestCas
 
     @Override
     protected ExplainLifecycleRequest createTestInstance() {
-        ExplainLifecycleRequest request = new ExplainLifecycleRequest();
+        ExplainLifecycleRequest request = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT);
         if (randomBoolean()) {
             request.indices(generateRandomStringArray(20, 20, false, false));
         }
@@ -71,7 +71,7 @@ protected ExplainLifecycleRequest mutateInstance(ExplainLifecycleRequest instanc
             case 3 -> onlyManaged = onlyManaged == false;
             default -> throw new AssertionError("Illegal randomisation branch");
         }
-        ExplainLifecycleRequest newRequest = new ExplainLifecycleRequest();
+        ExplainLifecycleRequest newRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT);
         newRequest.indices(indices);
         newRequest.indicesOptions(indicesOptions);
         newRequest.onlyErrors(onlyErrors);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java
index 858ba31d27a76..efbd15ac2065f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java
@@ -29,7 +29,8 @@ protected FreezeAction createTestInstance() {
 
     @Override
     protected FreezeAction mutateInstance(FreezeAction instance) {
-        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+        // This class is a singleton
+        return null;
     }
 
     @Override
@@ -50,16 +51,15 @@ public void testToSteps() {
         assertEquals(3, steps.size());
         StepKey expectedFirstStepKey = new StepKey(phase, FreezeAction.NAME, FreezeAction.CONDITIONAL_SKIP_FREEZE_STEP);
         StepKey expectedSecondStepKey = new StepKey(phase, FreezeAction.NAME, CheckNotDataStreamWriteIndexStep.NAME);
-        StepKey expectedThirdStepKey = new StepKey(phase, FreezeAction.NAME, FreezeStep.NAME);
-
-        BranchingStep firstStep = (BranchingStep) steps.get(0);
-        CheckNotDataStreamWriteIndexStep secondStep = (CheckNotDataStreamWriteIndexStep) steps.get(1);
-        FreezeStep thirdStep = (FreezeStep) steps.get(2);
+        StepKey expectedThirdStepKey = new StepKey(phase, FreezeAction.NAME, FreezeAction.NAME);
 
+        NoopStep firstStep = (NoopStep) steps.get(0);
         assertThat(firstStep.getKey(), equalTo(expectedFirstStepKey));
-
+        assertThat(firstStep.getNextStepKey(), equalTo(nextStepKey));
+        NoopStep secondStep = (NoopStep) steps.get(1);
         assertEquals(expectedSecondStepKey, secondStep.getKey());
-        assertEquals(expectedThirdStepKey, secondStep.getNextStepKey());
+        assertEquals(nextStepKey, secondStep.getNextStepKey());
+        NoopStep thirdStep = (NoopStep) steps.get(2);
         assertEquals(expectedThirdStepKey, thirdStep.getKey());
         assertEquals(nextStepKey, thirdStep.getNextStepKey());
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeStepTests.java
deleted file mode 100644
index 32386ceb2b882..0000000000000
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeStepTests.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.core.ilm;
-
-import org.elasticsearch.xpack.core.ilm.Step.StepKey;
-
-public class FreezeStepTests extends AbstractStepTestCase<FreezeStep> {
-
-    @Override
-    public FreezeStep createRandomInstance() {
-        StepKey stepKey = randomStepKey();
-        StepKey nextStepKey = randomStepKey();
-
-        return new FreezeStep(stepKey, nextStepKey, client);
-    }
-
-    @Override
-    public FreezeStep mutateInstance(FreezeStep instance) {
-        StepKey key = instance.getKey();
-        StepKey nextKey = instance.getNextStepKey();
-
-        switch (between(0, 1)) {
-            case 0 -> key = new StepKey(key.phase(), key.action(), key.name() + randomAlphaOfLength(5));
-            case 1 -> nextKey = new StepKey(nextKey.phase(), nextKey.action(), nextKey.name() + randomAlphaOfLength(5));
-            default -> throw new AssertionError("Illegal randomisation branch");
-        }
-
-        return new FreezeStep(key, nextKey, instance.getClient());
-    }
-
-    @Override
-    public FreezeStep copyInstance(FreezeStep instance) {
-        return new FreezeStep(instance.getKey(), instance.getNextStepKey(), instance.getClient());
-    }
-
-    public void testIndexSurvives() {
-        assertTrue(createRandomInstance().indexSurvives());
-    }
-}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java
index 1bea0ac6d192c..4d34115919710 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java
@@ -35,6 +35,7 @@
 
 public class LifecyclePolicyTests extends AbstractXContentSerializingTestCase<LifecyclePolicy> {
 
+    // Excluding the deprecated freeze action and test it separately
     private String lifecycleName;
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RolloverStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RolloverStepTests.java
index 3683690763d93..d839d6102b18a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RolloverStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RolloverStepTests.java
@@ -11,11 +11,13 @@
 import org.elasticsearch.action.admin.indices.rollover.RolloverInfo;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.IndexVersion;
@@ -68,11 +70,15 @@ private IndexMetadata getIndexMetadata(String alias) {
             .build();
     }
 
-    private static void assertRolloverIndexRequest(RolloverRequest request, String rolloverTarget) {
+    private static void assertRolloverIndexRequest(RolloverRequest request, String rolloverTarget, boolean targetFailureStores) {
+        String target = targetFailureStores
+            ? IndexNameExpressionResolver.combineSelector(rolloverTarget, IndexComponentSelector.FAILURES)
+            : rolloverTarget;
+
         assertNotNull(request);
         assertEquals(1, request.indices().length);
-        assertEquals(rolloverTarget, request.indices()[0]);
-        assertEquals(rolloverTarget, request.getRolloverTarget());
+        assertEquals(target, request.indices()[0]);
+        assertEquals(target, request.getRolloverTarget());
         assertFalse(request.isDryRun());
         assertEquals(0, request.getConditions().getConditions().size());
     }
@@ -83,7 +89,7 @@ public void testPerformAction() throws Exception {
 
         RolloverStep step = createRandomInstance();
 
-        mockClientRolloverCall(alias);
+        mockClientRolloverCall(alias, false);
 
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexMetadata, true)).build();
         performActionAndWait(step, indexMetadata, clusterState, null);
@@ -109,8 +115,6 @@ public void testPerformActionOnDataStream() throws Exception {
 
         RolloverStep step = createRandomInstance();
 
-        mockClientRolloverCall(dataStreamName);
-
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
             .metadata(
                 Metadata.builder()
@@ -121,6 +125,7 @@ public void testPerformActionOnDataStream() throws Exception {
             .build();
         boolean useFailureStore = randomBoolean();
         IndexMetadata indexToOperateOn = useFailureStore ? failureIndexMetadata : indexMetadata;
+        mockClientRolloverCall(dataStreamName, useFailureStore);
         performActionAndWait(step, indexToOperateOn, clusterState, null);
 
         Mockito.verify(client, Mockito.only()).admin();
@@ -179,12 +184,12 @@ public void testSkipRolloverIfDataStreamIsAlreadyRolledOver() throws Exception {
         verifyNoMoreInteractions(indicesClient);
     }
 
-    private void mockClientRolloverCall(String rolloverTarget) {
+    private void mockClientRolloverCall(String rolloverTarget, boolean targetFailureStore) {
         Mockito.doAnswer(invocation -> {
             RolloverRequest request = (RolloverRequest) invocation.getArguments()[0];
             @SuppressWarnings("unchecked")
             ActionListener<RolloverResponse> listener = (ActionListener<RolloverResponse>) invocation.getArguments()[1];
-            assertRolloverIndexRequest(request, rolloverTarget);
+            assertRolloverIndexRequest(request, rolloverTarget, targetFailureStore);
             listener.onResponse(new RolloverResponse(null, null, Map.of(), request.isDryRun(), true, true, true, false));
             return null;
         }).when(indicesClient).rolloverIndex(Mockito.any(), Mockito.any());
@@ -237,7 +242,7 @@ public void testPerformActionFailure() {
             RolloverRequest request = (RolloverRequest) invocation.getArguments()[0];
             @SuppressWarnings("unchecked")
             ActionListener<RolloverResponse> listener = (ActionListener<RolloverResponse>) invocation.getArguments()[1];
-            assertRolloverIndexRequest(request, alias);
+            assertRolloverIndexRequest(request, alias, false);
             listener.onFailure(exception);
             return null;
         }).when(indicesClient).rolloverIndex(Mockito.any(), Mockito.any());
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStepTests.java
index db0c2957b3ccb..453404d096c30 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStepTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.action.admin.indices.rollover.RolloverInfo;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
@@ -45,9 +46,9 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SelectorResolver.SELECTOR_SEPARATOR;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.not;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
@@ -270,7 +271,11 @@ public void testEvaluateConditionOnDataStreamTarget() {
 
         WaitForRolloverReadyStep step = createRandomInstance();
 
-        mockRolloverIndexCall(dataStreamName, step, true);
+        mockRolloverIndexCall(
+            failureStoreIndex ? dataStreamName + SELECTOR_SEPARATOR + IndexComponentSelector.FAILURES.getKey() : dataStreamName,
+            step,
+            true
+        );
 
         SetOnce<Boolean> conditionsMet = new SetOnce<>();
         Metadata metadata = Metadata.builder()
@@ -307,8 +312,14 @@ public void onFailure(Exception e) {
         verify(indicesClient, Mockito.only()).rolloverIndex(requestCaptor.capture(), Mockito.any());
 
         RolloverRequest request = requestCaptor.getValue();
-        assertThat(request.indicesOptions().includeFailureIndices(), equalTo(failureStoreIndex));
-        assertThat(request.indicesOptions().includeRegularIndices(), not(equalTo(failureStoreIndex)));
+        if (failureStoreIndex == false) {
+            assertThat(request.getRolloverTarget(), equalTo(dataStreamName));
+        } else {
+            assertThat(
+                request.getRolloverTarget(),
+                equalTo(dataStreamName + SELECTOR_SEPARATOR + IndexComponentSelector.FAILURES.getKey())
+            );
+        }
     }
 
     public void testSkipRolloverIfDataStreamIsAlreadyRolledOver() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionRequestTests.java
index 47a0814a584b7..120c2a6dbc5e7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionRequestTests.java
@@ -35,7 +35,6 @@ public void testParseAllFields() throws IOException {
                           "type": "string"
                         }
                     ],
-                    "name": "a name",
                     "tool_call_id": "100",
                     "tool_calls": [
                         {
@@ -83,7 +82,6 @@ public void testParseAllFields() throws IOException {
                             List.of(new UnifiedCompletionRequest.ContentObject("some text", "string"))
                         ),
                         "user",
-                        "a name",
                         "100",
                         List.of(
                             new UnifiedCompletionRequest.ToolCall(
@@ -155,7 +153,6 @@ public void testParsing() throws IOException {
                         new UnifiedCompletionRequest.ContentString("What is the weather like in Boston today?"),
                         "user",
                         null,
-                        null,
                         null
                     )
                 ),
@@ -200,7 +197,6 @@ public static UnifiedCompletionRequest.Message randomMessage() {
             randomContent(),
             randomAlphaOfLength(10),
             randomAlphaOfLengthOrNull(10),
-            randomAlphaOfLengthOrNull(10),
             randomToolCallListOrNull()
         );
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/search/SparseVectorQueryBuilderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/search/SparseVectorQueryBuilderTests.java
index a5c1ba45d90b7..af557ed6b7f82 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/search/SparseVectorQueryBuilderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/search/SparseVectorQueryBuilderTests.java
@@ -260,16 +260,16 @@ public void testIllegalValues() {
         {
             IllegalArgumentException e = expectThrows(
                 IllegalArgumentException.class,
-                () -> new SparseVectorQueryBuilder("field name", null, "model id")
+                () -> new SparseVectorQueryBuilder("field name", null, null)
             );
-            assertEquals("[sparse_vector] requires one of [query_vector] or [inference_id]", e.getMessage());
+            assertEquals("[sparse_vector] requires one of [query_vector] or [inference_id] for sparse_vector fields", e.getMessage());
         }
         {
             IllegalArgumentException e = expectThrows(
                 IllegalArgumentException.class,
                 () -> new SparseVectorQueryBuilder("field name", "model text", null)
             );
-            assertEquals("[sparse_vector] requires [query] when [inference_id] is specified", e.getMessage());
+            assertEquals("[sparse_vector] requires one of [query_vector] or [inference_id] for sparse_vector fields", e.getMessage());
         }
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/DataStreamFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/DataStreamFeatureSetUsageTests.java
index 3ff36c52229e7..5f879d508a52b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/DataStreamFeatureSetUsageTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/DataStreamFeatureSetUsageTests.java
@@ -20,6 +20,7 @@ protected DataStreamFeatureSetUsage createTestInstance() {
                 randomNonNegativeLong(),
                 randomNonNegativeLong(),
                 randomNonNegativeLong(),
+                randomNonNegativeLong(),
                 randomNonNegativeLong()
             )
         );
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java
index 11a8f894d7cf7..d9d235c8ce41b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java
@@ -13,16 +13,21 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.transport.RemoteClusterPortSettings;
+import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer;
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
 import org.elasticsearch.xpack.core.security.user.InternalUsers;
 import org.elasticsearch.xpack.core.security.user.KibanaSystemUser;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
 import org.elasticsearch.xpack.core.security.user.User;
 
+import java.io.IOException;
 import java.util.Arrays;
+import java.util.Map;
 
 import static org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationSerializationHelper;
+import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.emptyArray;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
@@ -171,4 +176,47 @@ public void testReservedUserSerialization() throws Exception {
 
         assertEquals(kibanaSystemUser, readFrom);
     }
+
+    public void testRolesRemovedFromUserForLegacyApiKeys() throws IOException {
+        TransportVersion transportVersion = TransportVersionUtils.randomVersionBetween(
+            random(),
+            TransportVersions.V_7_0_0,
+            TransportVersions.V_7_8_0
+        );
+        Subject authenticatingSubject = new Subject(
+            new User("foo", "role"),
+            new Authentication.RealmRef(AuthenticationField.API_KEY_REALM_NAME, AuthenticationField.API_KEY_REALM_TYPE, "node"),
+            transportVersion,
+            Map.of(AuthenticationField.API_KEY_ID_KEY, "abc")
+        );
+        Subject effectiveSubject = new Subject(
+            new User("bar", "role"),
+            new Authentication.RealmRef("native", "native", "node"),
+            transportVersion,
+            Map.of()
+        );
+
+        {
+            Authentication actual = AuthenticationContextSerializer.decode(
+                Authentication.doEncode(authenticatingSubject, authenticatingSubject, Authentication.AuthenticationType.API_KEY)
+            );
+            assertThat(actual.getAuthenticatingSubject().getUser().roles(), is(emptyArray()));
+        }
+
+        {
+            Authentication actual = AuthenticationContextSerializer.decode(
+                Authentication.doEncode(effectiveSubject, authenticatingSubject, Authentication.AuthenticationType.API_KEY)
+            );
+            assertThat(actual.getAuthenticatingSubject().getUser().roles(), is(emptyArray()));
+            assertThat(actual.getEffectiveSubject().getUser().roles(), is(arrayContaining("role")));
+        }
+
+        {
+            // do not strip roles for authentication methods other than API key
+            Authentication actual = AuthenticationContextSerializer.decode(
+                Authentication.doEncode(effectiveSubject, effectiveSubject, Authentication.AuthenticationType.REALM)
+            );
+            assertThat(actual.getAuthenticatingSubject().getUser().roles(), is(arrayContaining("role")));
+        }
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java
index 4751f66cf548e..89b42228d8918 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java
@@ -23,9 +23,12 @@
 import org.apache.lucene.search.TotalHitCountCollectorManager;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
+import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.IndexSettings;
@@ -33,9 +36,11 @@
 import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperMetrics;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.Mapping;
 import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.MockFieldMapper;
+import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.query.TermsQueryBuilder;
@@ -45,6 +50,9 @@
 import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.test.AbstractBuilderTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper;
@@ -52,6 +60,8 @@
 import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
 
+import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -340,6 +350,176 @@ protected IndicesAccessControl getIndicesAccessControl() {
         directory.close();
     }
 
+    @Override
+    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("properties")
+            .startObject("f1")
+            .field("type", "keyword")
+            .endObject()
+            .startObject("nested1")
+            .field("type", "nested")
+            .startObject("properties")
+            .startObject("field")
+            .field("type", "keyword")
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject();
+        mapperService.merge(
+            MapperService.SINGLE_MAPPING_NAME,
+            new CompressedXContent(Strings.toString(builder)),
+            MapperService.MergeReason.MAPPING_UPDATE
+        );
+    }
+
+    public void testDLSWithNestedDocs() throws Exception {
+        Directory directory = newDirectory();
+        try (
+            IndexWriter iw = new IndexWriter(
+                directory,
+                new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(NoMergePolicy.INSTANCE)
+            )
+        ) {
+            var parser = mapperService().documentParser();
+            String doc = """
+                        {
+                            "f1": "value",
+                            "nested1": [
+                                {
+                                    "field": "0"
+                                },
+                                {
+                                    "field": "1"
+                                },
+                                {}
+                            ]
+                        }
+                """;
+            var parsedDoc = parser.parseDocument(
+                new SourceToParse("0", new BytesArray(doc), XContentType.JSON),
+                mapperService().mappingLookup()
+            );
+            iw.addDocuments(parsedDoc.docs());
+
+            doc = """
+                        {
+                            "nested1": [
+                                {
+                                    "field": "12"
+                                },
+                                {
+                                    "field": "13"
+                                },
+                                {}
+                            ]
+                        }
+                """;
+            parsedDoc = parser.parseDocument(
+                new SourceToParse("1", new BytesArray(doc), XContentType.JSON),
+                mapperService().mappingLookup()
+            );
+            iw.addDocuments(parsedDoc.docs());
+
+            doc = """
+                       {
+                            "f1": "value",
+                            "nested1": [
+                                {
+                                    "field": "12"
+                                },
+                                {}
+                            ]
+                       }
+                """;
+            parsedDoc = parser.parseDocument(
+                new SourceToParse("2", new BytesArray(doc), XContentType.JSON),
+                mapperService().mappingLookup()
+            );
+            iw.addDocuments(parsedDoc.docs());
+
+            doc = """
+                        {
+                            "nested1": [
+                                {
+                                    "field": "12"
+                                },
+                                {}
+                            ]
+                        }
+                """;
+            parsedDoc = parser.parseDocument(
+                new SourceToParse("3", new BytesArray(doc), XContentType.JSON),
+                mapperService().mappingLookup()
+            );
+            iw.addDocuments(parsedDoc.docs());
+
+            iw.commit();
+        }
+
+        DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(
+            DirectoryReader.open(directory),
+            new ShardId(indexSettings().getIndex(), 0)
+        );
+        SearchExecutionContext context = createSearchExecutionContext(new IndexSearcher(directoryReader));
+
+        final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        final SecurityContext securityContext = new SecurityContext(Settings.EMPTY, threadContext);
+        final Authentication authentication = AuthenticationTestHelper.builder().build();
+        new AuthenticationContextSerializer().writeToContext(authentication, threadContext);
+
+        Set<BytesReference> queries = new HashSet<>();
+        queries.add(new BytesArray("{\"bool\": { \"must_not\": { \"exists\": { \"field\": \"f1\" } } } }"));
+        IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(
+            FieldPermissions.DEFAULT,
+            DocumentPermissions.filteredBy(queries)
+        );
+
+        DocumentSubsetBitsetCache bitsetCache = new DocumentSubsetBitsetCache(Settings.EMPTY, Executors.newSingleThreadExecutor());
+
+        final MockLicenseState licenseState = mock(MockLicenseState.class);
+        when(licenseState.isAllowed(DOCUMENT_LEVEL_SECURITY_FEATURE)).thenReturn(true);
+        ScriptService scriptService = mock(ScriptService.class);
+        SecurityIndexReaderWrapper wrapper = new SecurityIndexReaderWrapper(
+            s -> context,
+            bitsetCache,
+            securityContext,
+            licenseState,
+            scriptService
+        ) {
+
+            @Override
+            protected IndicesAccessControl getIndicesAccessControl() {
+                IndicesAccessControl indicesAccessControl = new IndicesAccessControl(
+                    true,
+                    singletonMap(indexSettings().getIndex().getName(), indexAccessControl)
+                );
+                return indicesAccessControl;
+            }
+        };
+
+        DirectoryReader wrappedDirectoryReader = wrapper.apply(directoryReader);
+        IndexSearcher indexSearcher = new ContextIndexSearcher(
+            wrappedDirectoryReader,
+            IndexSearcher.getDefaultSimilarity(),
+            IndexSearcher.getDefaultQueryCache(),
+            IndexSearcher.getDefaultQueryCachingPolicy(),
+            true
+        );
+
+        ScoreDoc[] hits = indexSearcher.search(new MatchAllDocsQuery(), 1000).scoreDocs;
+        assertThat(Arrays.stream(hits).map(h -> h.doc).collect(Collectors.toSet()), containsInAnyOrder(4, 5, 6, 7, 11, 12, 13));
+
+        hits = indexSearcher.search(Queries.newNonNestedFilter(context.indexVersionCreated()), 1000).scoreDocs;
+        assertThat(Arrays.stream(hits).map(h -> h.doc).collect(Collectors.toSet()), containsInAnyOrder(7, 13));
+
+        bitsetCache.close();
+        directoryReader.close();
+        directory.close();
+    }
+
     private static MappingLookup createMappingLookup(List<MappedFieldType> concreteFields) {
         List<FieldMapper> mappers = concreteFields.stream().map(MockFieldMapper::new).collect(Collectors.toList());
         return MappingLookup.fromMappers(Mapping.EMPTY, mappers, emptyList());
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
index 1b9a65d12d8d9..b08dd90ae9065 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
@@ -197,6 +197,7 @@
 import org.elasticsearch.xpack.core.transform.action.GetTransformStatsAction;
 import org.elasticsearch.xpack.core.transform.action.PreviewTransformAction;
 import org.elasticsearch.xpack.core.transform.action.PutTransformAction;
+import org.elasticsearch.xpack.core.transform.action.SetTransformUpgradeModeAction;
 import org.elasticsearch.xpack.core.transform.action.StartTransformAction;
 import org.elasticsearch.xpack.core.transform.action.StopTransformAction;
 import org.elasticsearch.xpack.core.transform.action.UpdateTransformAction;
@@ -276,8 +277,6 @@ public void testIsReserved() {
         assertThat(ReservedRolesStore.isReserved("reporting_user"), is(true));
         assertThat(ReservedRolesStore.isReserved("machine_learning_user"), is(true));
         assertThat(ReservedRolesStore.isReserved("machine_learning_admin"), is(true));
-        assertThat(ReservedRolesStore.isReserved("data_frame_transforms_user"), is(true));
-        assertThat(ReservedRolesStore.isReserved("data_frame_transforms_admin"), is(true));
         assertThat(ReservedRolesStore.isReserved("transform_user"), is(true));
         assertThat(ReservedRolesStore.isReserved("transform_admin"), is(true));
         assertThat(ReservedRolesStore.isReserved("watcher_user"), is(true));
@@ -612,12 +611,17 @@ public void testKibanaSystemRole() {
             ".apm-custom-link",
             ".apm-source-map",
             ReservedRolesStore.ALERTS_LEGACY_INDEX + randomAlphaOfLength(randomIntBetween(0, 13)),
+            ReservedRolesStore.ALERTS_LEGACY_INDEX_REINDEXED_V8 + randomAlphaOfLength(randomIntBetween(0, 13)),
             ReservedRolesStore.ALERTS_BACKING_INDEX + randomAlphaOfLength(randomIntBetween(0, 13)),
+            ReservedRolesStore.ALERTS_BACKING_INDEX_REINDEXED + randomAlphaOfLength(randomIntBetween(0, 13)),
             ReservedRolesStore.ALERTS_INDEX_ALIAS + randomAlphaOfLength(randomIntBetween(0, 13)),
             ReservedRolesStore.PREVIEW_ALERTS_INDEX_ALIAS + randomAlphaOfLength(randomIntBetween(0, 13)),
-            ReservedRolesStore.PREVIEW_ALERTS_BACKING_INDEX_ALIAS + randomAlphaOfLength(randomIntBetween(0, 13)),
+            ReservedRolesStore.PREVIEW_ALERTS_BACKING_INDEX + randomAlphaOfLength(randomIntBetween(0, 13)),
+            ReservedRolesStore.PREVIEW_ALERTS_BACKING_INDEX_REINDEXED + randomAlphaOfLength(randomIntBetween(0, 13)),
             ReservedRolesStore.LISTS_INDEX + randomAlphaOfLength(randomIntBetween(0, 13)),
+            ReservedRolesStore.LISTS_INDEX_REINDEXED_V8 + randomAlphaOfLength(randomIntBetween(0, 13)),
             ReservedRolesStore.LISTS_ITEMS_INDEX + randomAlphaOfLength(randomIntBetween(0, 13)),
+            ReservedRolesStore.LISTS_ITEMS_INDEX_REINDEXED_V8 + randomAlphaOfLength(randomIntBetween(0, 13)),
             ".slo-observability." + randomAlphaOfLength(randomIntBetween(0, 13))
         ).forEach(index -> assertAllIndicesAccessAllowed(kibanaRole, index));
 
@@ -3418,183 +3422,99 @@ public void testTransformAdminRole() {
         final TransportRequest request = mock(TransportRequest.class);
         final Authentication authentication = AuthenticationTestHelper.builder().build();
 
-        RoleDescriptor[] roleDescriptors = {
-            ReservedRolesStore.roleDescriptor("data_frame_transforms_admin"),
-            ReservedRolesStore.roleDescriptor("transform_admin") };
+        RoleDescriptor roleDescriptor = ReservedRolesStore.roleDescriptor("transform_admin");
 
-        for (RoleDescriptor roleDescriptor : roleDescriptors) {
-            assertNotNull(roleDescriptor);
-            assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
-            if (roleDescriptor.getName().equals("data_frame_transforms_admin")) {
-                assertThat(roleDescriptor.getMetadata(), hasEntry("_deprecated", true));
-            } else {
-                assertThat(roleDescriptor.getMetadata(), not(hasEntry("_deprecated", true)));
-            }
+        assertNotNull(roleDescriptor);
+        assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
+        assertThat(roleDescriptor.getMetadata(), not(hasEntry("_deprecated", true)));
 
-            final String allowedApplicationActionPattern = "example/custom/action/*";
-            final String kibanaApplicationWithRandomIndex = "kibana-" + randomFrom(randomAlphaOfLengthBetween(8, 24), ".kibana");
-            List<ApplicationPrivilegeDescriptor> lookup = roleDescriptor.getName().equals("data_frame_transforms_admin")
-                ? List.of(
-                    new ApplicationPrivilegeDescriptor(
-                        kibanaApplicationWithRandomIndex,
-                        "reserved_ml_user",
-                        Set.of(allowedApplicationActionPattern),
-                        Map.of()
-                    )
-                )
-                : List.of();
-            Role role = Role.buildFromRoleDescriptor(roleDescriptor, new FieldPermissionsCache(Settings.EMPTY), RESTRICTED_INDICES, lookup);
-            assertThat(role.cluster().check(DeleteTransformAction.NAME, request, authentication), is(true));
-            assertThat(role.cluster().check(GetTransformAction.NAME, request, authentication), is(true));
-            assertThat(role.cluster().check(GetTransformStatsAction.NAME, request, authentication), is(true));
-            assertThat(role.cluster().check(PreviewTransformAction.NAME, request, authentication), is(true));
-            assertThat(role.cluster().check(PutTransformAction.NAME, request, authentication), is(true));
-            assertThat(role.cluster().check(StartTransformAction.NAME, request, authentication), is(true));
-            assertThat(role.cluster().check(StopTransformAction.NAME, request, authentication), is(true));
-            assertThat(role.cluster().check(DelegatePkiAuthenticationAction.NAME, request, authentication), is(false));
-
-            assertThat(role.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false));
-
-            assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS);
-            assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_PATTERN);
-            assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED);
-            assertNoAccessAllowed(role, "foo");
-            assertNoAccessAllowed(role, TransformInternalIndexConstants.LATEST_INDEX_NAME); // internal use only
-
-            assertNoAccessAllowed(role, TestRestrictedIndices.SAMPLE_RESTRICTED_NAMES);
-            assertNoAccessAllowed(role, XPackPlugin.ASYNC_RESULTS_INDEX + randomAlphaOfLengthBetween(0, 2));
+        final String allowedApplicationActionPattern = "example/custom/action/*";
+        final String kibanaApplicationWithRandomIndex = "kibana-" + randomFrom(randomAlphaOfLengthBetween(8, 24), ".kibana");
+        List<ApplicationPrivilegeDescriptor> lookup = List.of();
+        Role role = Role.buildFromRoleDescriptor(roleDescriptor, new FieldPermissionsCache(Settings.EMPTY), RESTRICTED_INDICES, lookup);
+        assertThat(role.cluster().check(DeleteTransformAction.NAME, request, authentication), is(true));
+        assertThat(role.cluster().check(GetTransformAction.NAME, request, authentication), is(true));
+        assertThat(role.cluster().check(GetTransformStatsAction.NAME, request, authentication), is(true));
+        assertThat(role.cluster().check(PreviewTransformAction.NAME, request, authentication), is(true));
+        assertThat(role.cluster().check(PutTransformAction.NAME, request, authentication), is(true));
+        assertThat(role.cluster().check(StartTransformAction.NAME, request, authentication), is(true));
+        assertThat(role.cluster().check(StopTransformAction.NAME, request, authentication), is(true));
+        assertThat(role.cluster().check(SetTransformUpgradeModeAction.NAME, request, authentication), is(true));
+        assertThat(role.cluster().check(DelegatePkiAuthenticationAction.NAME, request, authentication), is(false));
 
-            assertThat(
-                role.application()
-                    .grants(ApplicationPrivilegeTests.createPrivilege(kibanaApplicationWithRandomIndex, "app-foo", "foo"), "*"),
-                is(false)
-            );
+        assertThat(role.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false));
 
-            if (roleDescriptor.getName().equals("data_frame_transforms_admin")) {
-                assertThat(
-                    role.application()
-                        .grants(
-                            ApplicationPrivilegeTests.createPrivilege(
-                                kibanaApplicationWithRandomIndex,
-                                "app-reserved_ml",
-                                allowedApplicationActionPattern
-                            ),
-                            "*"
-                        ),
-                    is(true)
-                );
-            }
+        assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS);
+        assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_PATTERN);
+        assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED);
+        assertNoAccessAllowed(role, "foo");
+        assertNoAccessAllowed(role, TransformInternalIndexConstants.LATEST_INDEX_NAME); // internal use only
 
-            final String otherApplication = "logstash-" + randomAlphaOfLengthBetween(8, 24);
-            assertThat(
-                role.application().grants(ApplicationPrivilegeTests.createPrivilege(otherApplication, "app-foo", "foo"), "*"),
-                is(false)
-            );
-            if (roleDescriptor.getName().equals("data_frame_transforms_admin")) {
-                assertThat(
-                    role.application()
-                        .grants(
-                            ApplicationPrivilegeTests.createPrivilege(otherApplication, "app-reserved_ml", allowedApplicationActionPattern),
-                            "*"
-                        ),
-                    is(false)
-                );
-            }
-        }
+        assertNoAccessAllowed(role, TestRestrictedIndices.SAMPLE_RESTRICTED_NAMES);
+        assertNoAccessAllowed(role, XPackPlugin.ASYNC_RESULTS_INDEX + randomAlphaOfLengthBetween(0, 2));
+
+        assertThat(
+            role.application().grants(ApplicationPrivilegeTests.createPrivilege(kibanaApplicationWithRandomIndex, "app-foo", "foo"), "*"),
+            is(false)
+        );
+
+        final String otherApplication = "logstash-" + randomAlphaOfLengthBetween(8, 24);
+        assertThat(
+            role.application().grants(ApplicationPrivilegeTests.createPrivilege(otherApplication, "app-foo", "foo"), "*"),
+            is(false)
+        );
     }
 
     public void testTransformUserRole() {
         final TransportRequest request = mock(TransportRequest.class);
         final Authentication authentication = AuthenticationTestHelper.builder().build();
 
-        RoleDescriptor[] roleDescriptors = {
-            ReservedRolesStore.roleDescriptor("data_frame_transforms_user"),
-            ReservedRolesStore.roleDescriptor("transform_user") };
+        RoleDescriptor roleDescriptor = ReservedRolesStore.roleDescriptor("transform_user");
 
-        for (RoleDescriptor roleDescriptor : roleDescriptors) {
-            assertNotNull(roleDescriptor);
-            assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
-            if (roleDescriptor.getName().equals("data_frame_transforms_user")) {
-                assertThat(roleDescriptor.getMetadata(), hasEntry("_deprecated", true));
-            } else {
-                assertThat(roleDescriptor.getMetadata(), not(hasEntry("_deprecated", true)));
-            }
+        assertNotNull(roleDescriptor);
+        assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
+        assertThat(roleDescriptor.getMetadata(), not(hasEntry("_deprecated", true)));
 
-            final String allowedApplicationActionPattern = "example/custom/action/*";
-            final String kibanaApplicationWithRandomIndex = "kibana-" + randomFrom(randomAlphaOfLengthBetween(8, 24), ".kibana");
-            List<ApplicationPrivilegeDescriptor> lookup = roleDescriptor.getName().equals("data_frame_transforms_user")
-                ? List.of(
-                    new ApplicationPrivilegeDescriptor(
-                        kibanaApplicationWithRandomIndex,
-                        "reserved_ml_user",
-                        Set.of(allowedApplicationActionPattern),
-                        Map.of()
-                    )
-                )
-                : List.of();
-            Role role = Role.buildFromRoleDescriptor(roleDescriptor, new FieldPermissionsCache(Settings.EMPTY), RESTRICTED_INDICES, lookup);
-            assertThat(role.cluster().check(DeleteTransformAction.NAME, request, authentication), is(false));
-            assertThat(role.cluster().check(GetTransformAction.NAME, request, authentication), is(true));
-            assertThat(role.cluster().check(GetTransformStatsAction.NAME, request, authentication), is(true));
-            assertThat(role.cluster().check(PreviewTransformAction.NAME, request, authentication), is(false));
-            assertThat(role.cluster().check(PutTransformAction.NAME, request, authentication), is(false));
-            assertThat(role.cluster().check(StartTransformAction.NAME, request, authentication), is(false));
-            assertThat(role.cluster().check(StopTransformAction.NAME, request, authentication), is(false));
-            assertThat(role.cluster().check(DelegatePkiAuthenticationAction.NAME, request, authentication), is(false));
-            assertThat(role.cluster().check(ActivateProfileAction.NAME, request, authentication), is(false));
-            assertThat(role.cluster().check(SuggestProfilesAction.NAME, request, authentication), is(false));
-            assertThat(role.cluster().check(UpdateProfileDataAction.NAME, request, authentication), is(false));
-            assertThat(role.cluster().check(GetProfilesAction.NAME, request, authentication), is(false));
-            assertThat(role.cluster().check(ProfileHasPrivilegesAction.NAME, request, authentication), is(false));
-
-            assertThat(role.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false));
-
-            assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS);
-            assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_PATTERN);
-            assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED);
-            assertNoAccessAllowed(role, "foo");
-            assertNoAccessAllowed(role, TransformInternalIndexConstants.LATEST_INDEX_NAME);
-
-            assertNoAccessAllowed(role, TestRestrictedIndices.SAMPLE_RESTRICTED_NAMES);
-            assertNoAccessAllowed(role, XPackPlugin.ASYNC_RESULTS_INDEX + randomAlphaOfLengthBetween(0, 2));
+        final String allowedApplicationActionPattern = "example/custom/action/*";
+        final String kibanaApplicationWithRandomIndex = "kibana-" + randomFrom(randomAlphaOfLengthBetween(8, 24), ".kibana");
+        List<ApplicationPrivilegeDescriptor> lookup = List.of();
+        Role role = Role.buildFromRoleDescriptor(roleDescriptor, new FieldPermissionsCache(Settings.EMPTY), RESTRICTED_INDICES, lookup);
+        assertThat(role.cluster().check(DeleteTransformAction.NAME, request, authentication), is(false));
+        assertThat(role.cluster().check(GetTransformAction.NAME, request, authentication), is(true));
+        assertThat(role.cluster().check(GetTransformStatsAction.NAME, request, authentication), is(true));
+        assertThat(role.cluster().check(PreviewTransformAction.NAME, request, authentication), is(false));
+        assertThat(role.cluster().check(PutTransformAction.NAME, request, authentication), is(false));
+        assertThat(role.cluster().check(StartTransformAction.NAME, request, authentication), is(false));
+        assertThat(role.cluster().check(StopTransformAction.NAME, request, authentication), is(false));
+        assertThat(role.cluster().check(SetTransformUpgradeModeAction.NAME, request, authentication), is(false));
+        assertThat(role.cluster().check(DelegatePkiAuthenticationAction.NAME, request, authentication), is(false));
+        assertThat(role.cluster().check(ActivateProfileAction.NAME, request, authentication), is(false));
+        assertThat(role.cluster().check(SuggestProfilesAction.NAME, request, authentication), is(false));
+        assertThat(role.cluster().check(UpdateProfileDataAction.NAME, request, authentication), is(false));
+        assertThat(role.cluster().check(GetProfilesAction.NAME, request, authentication), is(false));
+        assertThat(role.cluster().check(ProfileHasPrivilegesAction.NAME, request, authentication), is(false));
 
-            assertThat(
-                role.application()
-                    .grants(ApplicationPrivilegeTests.createPrivilege(kibanaApplicationWithRandomIndex, "app-foo", "foo"), "*"),
-                is(false)
-            );
+        assertThat(role.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false));
 
-            if (roleDescriptor.getName().equals("data_frame_transforms_user")) {
-                assertThat(
-                    role.application()
-                        .grants(
-                            ApplicationPrivilegeTests.createPrivilege(
-                                kibanaApplicationWithRandomIndex,
-                                "app-reserved_ml",
-                                allowedApplicationActionPattern
-                            ),
-                            "*"
-                        ),
-                    is(true)
-                );
-            }
+        assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS);
+        assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_PATTERN);
+        assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED);
+        assertNoAccessAllowed(role, "foo");
+        assertNoAccessAllowed(role, TransformInternalIndexConstants.LATEST_INDEX_NAME);
 
-            final String otherApplication = "logstash-" + randomAlphaOfLengthBetween(8, 24);
-            assertThat(
-                role.application().grants(ApplicationPrivilegeTests.createPrivilege(otherApplication, "app-foo", "foo"), "*"),
-                is(false)
-            );
-            if (roleDescriptor.getName().equals("data_frame_transforms_user")) {
-                assertThat(
-                    role.application()
-                        .grants(
-                            ApplicationPrivilegeTests.createPrivilege(otherApplication, "app-reserved_ml", allowedApplicationActionPattern),
-                            "*"
-                        ),
-                    is(false)
-                );
-            }
-        }
+        assertNoAccessAllowed(role, TestRestrictedIndices.SAMPLE_RESTRICTED_NAMES);
+        assertNoAccessAllowed(role, XPackPlugin.ASYNC_RESULTS_INDEX + randomAlphaOfLengthBetween(0, 2));
+
+        assertThat(
+            role.application().grants(ApplicationPrivilegeTests.createPrivilege(kibanaApplicationWithRandomIndex, "app-foo", "foo"), "*"),
+            is(false)
+        );
+
+        final String otherApplication = "logstash-" + randomAlphaOfLengthBetween(8, 24);
+        assertThat(
+            role.application().grants(ApplicationPrivilegeTests.createPrivilege(otherApplication, "app-foo", "foo"), "*"),
+            is(false)
+        );
     }
 
     public void testWatcherAdminRole() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
index 4d1ebf6cbaabc..68987bbd19815 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
@@ -170,7 +170,6 @@ public void testReloadingKeyStore() throws Exception {
      * Tests the reloading of SSLContext when a PEM key and certificate are used.
      */
     public void testPEMKeyConfigReloading() throws Exception {
-        assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm());
         Path tempDir = createTempDir();
         Path keyPath = tempDir.resolve("testnode.pem");
         Path certPath = tempDir.resolve("testnode.crt");
@@ -221,11 +220,19 @@ public void testPEMKeyConfigReloading() throws Exception {
                 try (MockWebServer server = new MockWebServer(updatedContext, false)) {
                     server.enqueue(new MockResponse().setResponseCode(200).setBody("body"));
                     server.start();
-                    SSLHandshakeException sslException = expectThrows(
-                        SSLHandshakeException.class,
-                        () -> privilegedConnect(() -> client.execute(new HttpGet("https://localhost:" + server.getPort())).close())
-                    );
-                    assertThat(sslException.getCause().getMessage(), containsString("PKIX path validation failed"));
+                    if (inFipsJvm()) {
+                        Exception sslException = expectThrows(
+                            IOException.class,
+                            () -> privilegedConnect(() -> client.execute(new HttpGet("https://localhost:" + server.getPort())).close())
+                        );
+                        assertThat(sslException.getCause().getMessage(), containsString("Unable to construct a valid chain"));
+                    } else {
+                        SSLHandshakeException sslException = expectThrows(
+                            SSLHandshakeException.class,
+                            () -> privilegedConnect(() -> client.execute(new HttpGet("https://localhost:" + server.getPort())).close())
+                        );
+                        assertThat(sslException.getCause().getMessage(), containsString("PKIX path validation failed"));
+                    }
                 } catch (Exception e) {
                     throw new RuntimeException("Exception starting or connecting to the mock server", e);
                 }
@@ -290,7 +297,6 @@ public void testReloadingTrustStore() throws Exception {
      * Test the reloading of SSLContext whose trust config is backed by PEM certificate files.
      */
     public void testReloadingPEMTrustConfig() throws Exception {
-        assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm());
         Path tempDir = createTempDir();
         Path serverCertPath = tempDir.resolve("testnode.crt");
         Path serverKeyPath = tempDir.resolve("testnode.pem");
@@ -324,11 +330,19 @@ public void testReloadingPEMTrustConfig() throws Exception {
             // Client doesn't trust the Server certificate anymore so SSLHandshake should fail
             final Consumer<SSLContext> trustMaterialPostChecks = (updatedContext) -> {
                 try (CloseableHttpClient client = createHttpClient(updatedContext)) {
-                    SSLHandshakeException sslException = expectThrows(
-                        SSLHandshakeException.class,
-                        () -> privilegedConnect(() -> client.execute(new HttpGet("https://localhost:" + server.getPort())).close())
-                    );
-                    assertThat(sslException.getCause().getMessage(), containsString("PKIX path validation failed"));
+                    if (inFipsJvm()) {
+                        Exception sslException = expectThrows(
+                            IOException.class,
+                            () -> privilegedConnect(() -> client.execute(new HttpGet("https://localhost:" + server.getPort())).close())
+                        );
+                        assertThat(sslException.getCause().getMessage(), containsString("Unable to construct a valid chain"));
+                    } else {
+                        SSLHandshakeException sslException = expectThrows(
+                            SSLHandshakeException.class,
+                            () -> privilegedConnect(() -> client.execute(new HttpGet("https://localhost:" + server.getPort())).close())
+                        );
+                        assertThat(sslException.getCause().getMessage(), containsString("PKIX path validation failed"));
+                    }
                 } catch (Exception e) {
                     throw new RuntimeException("Error closing CloseableHttpClient", e);
                 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java
index 2e7e5293c835f..180c9ca1f674d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java
@@ -27,8 +27,6 @@
 import org.elasticsearch.xpack.core.common.validation.SourceDestValidator.SourceDestValidation;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level;
-import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper;
-import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase;
 import org.elasticsearch.xpack.core.transform.TransformConfigVersion;
 import org.elasticsearch.xpack.core.transform.TransformDeprecations;
@@ -46,7 +44,6 @@
 import java.util.Map;
 
 import static org.elasticsearch.test.TestMatchers.matchesPattern;
-import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY;
 import static org.elasticsearch.xpack.core.transform.transforms.DestConfigTests.randomDestConfig;
 import static org.elasticsearch.xpack.core.transform.transforms.SourceConfigTests.randomInvalidSourceConfig;
 import static org.elasticsearch.xpack.core.transform.transforms.SourceConfigTests.randomSourceConfig;
@@ -61,8 +58,6 @@ public class TransformConfigTests extends AbstractSerializingTransformTestCase<T
 
     private String transformId;
     private boolean runWithHeaders;
-    private static final String DATA_FRAME_TRANSFORMS_ADMIN_ROLE = "data_frame_transforms_admin";
-    private static final String DATA_FRAME_TRANSFORMS_USER_ROLE = "data_frame_transforms_user";
 
     public static TransformConfig randomTransformConfigWithoutHeaders() {
         return randomTransformConfigWithoutHeaders(randomAlphaOfLengthBetween(1, 10));
@@ -152,6 +147,14 @@ public static TransformConfig randomTransformConfigWithSettings(SettingsConfig s
             latestConfig = LatestConfigTests.randomLatestConfig();
         }
 
+        return randomTransformConfigWithSettings(settingsConfig, pivotConfig, latestConfig);
+    }
+
+    public static TransformConfig randomTransformConfigWithSettings(
+        SettingsConfig settingsConfig,
+        PivotConfig pivotConfig,
+        LatestConfig latestConfig
+    ) {
         return new TransformConfig(
             randomAlphaOfLengthBetween(1, 10),
             randomSourceConfig(),
@@ -1027,44 +1030,6 @@ public void testCheckForDeprecations_WithDeprecatedFields_Version_7_4() throws I
         );
     }
 
-    public void testCheckForDeprecations_WithDeprecatedTransformUserAdmin() throws IOException {
-        testCheckForDeprecations_WithDeprecatedRoles(List.of(DATA_FRAME_TRANSFORMS_ADMIN_ROLE));
-    }
-
-    public void testCheckForDeprecations_WithDeprecatedTransformUserRole() throws IOException {
-        testCheckForDeprecations_WithDeprecatedRoles(List.of(DATA_FRAME_TRANSFORMS_USER_ROLE));
-    }
-
-    public void testCheckForDeprecations_WithDeprecatedTransformRoles() throws IOException {
-        testCheckForDeprecations_WithDeprecatedRoles(List.of(DATA_FRAME_TRANSFORMS_ADMIN_ROLE, DATA_FRAME_TRANSFORMS_USER_ROLE));
-    }
-
-    private void testCheckForDeprecations_WithDeprecatedRoles(List<String> roles) throws IOException {
-        var authentication = AuthenticationTestHelper.builder()
-            .realm()
-            .user(new User(randomAlphaOfLength(10), roles.toArray(String[]::new)))
-            .build();
-        Map<String, String> headers = Map.of(AUTHENTICATION_KEY, authentication.encode());
-        TransformConfig deprecatedConfig = randomTransformConfigWithHeaders(headers);
-
-        // important: checkForDeprecations does _not_ create new deprecation warnings
-        assertThat(
-            deprecatedConfig.checkForDeprecations(xContentRegistry()),
-            equalTo(
-                List.of(
-                    new DeprecationIssue(
-                        Level.CRITICAL,
-                        "Transform [" + deprecatedConfig.getId() + "] uses deprecated transform roles " + roles,
-                        TransformDeprecations.DATA_FRAME_TRANSFORMS_ROLES_BREAKING_CHANGES_URL,
-                        TransformDeprecations.DATA_FRAME_TRANSFORMS_ROLES_IS_DEPRECATED,
-                        false,
-                        null
-                    )
-                )
-            )
-        );
-    }
-
     public void testSerializingMetadataPreservesOrder() throws IOException {
         String json = Strings.format("""
             {
diff --git a/x-pack/plugin/core/template-resources/src/main/resources/agentless@mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/agentless@mappings.json
new file mode 100644
index 0000000000000..5afb66de923db
--- /dev/null
+++ b/x-pack/plugin/core/template-resources/src/main/resources/agentless@mappings.json
@@ -0,0 +1,22 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "v": {
+          "type": "object",
+          "enabled": false
+        },
+        "updated_at": {
+          "type": "date",
+          "format": "strict_date_optional_time||epoch_millis"
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "default mappings for the agentless index template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version},
+  "deprecated": ${xpack.stack.template.deprecated}
+}
diff --git a/x-pack/plugin/core/template-resources/src/main/resources/agentless@settings.json b/x-pack/plugin/core/template-resources/src/main/resources/agentless@settings.json
new file mode 100644
index 0000000000000..23f7c1ef88296
--- /dev/null
+++ b/x-pack/plugin/core/template-resources/src/main/resources/agentless@settings.json
@@ -0,0 +1,15 @@
+{
+  "template": {
+    "settings": {
+      "index": {
+        "hidden": true
+      }
+    }
+  },
+  "_meta": {
+    "description": "default settings for the agentless index template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version},
+  "deprecated": ${xpack.stack.template.deprecated}
+}
diff --git a/x-pack/plugin/core/template-resources/src/main/resources/agentless@template.json b/x-pack/plugin/core/template-resources/src/main/resources/agentless@template.json
new file mode 100644
index 0000000000000..35c5f5581b1e8
--- /dev/null
+++ b/x-pack/plugin/core/template-resources/src/main/resources/agentless@template.json
@@ -0,0 +1,15 @@
+{
+  "index_patterns": ["agentless-*-*"],
+  "priority": 100,
+  "composed_of": [
+    "agentless@mappings",
+    "agentless@settings"
+  ],
+  "allow_auto_create": true,
+  "_meta": {
+    "description": "default agentless template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version},
+  "deprecated": ${xpack.stack.template.deprecated}
+}
diff --git a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json
index 651e1c84da73a..25409dbf8460e 100644
--- a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json
+++ b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json
@@ -7,7 +7,7 @@
       "dynamic": "false",
       "_meta": {
         "pipeline": {
-          "default_name": "ent-search-generic-ingestion",
+          "default_name": "search-default-ingestion",
           "default_extract_binary_content": true,
           "default_run_ml_inference": true,
           "default_reduce_whitespace": true
@@ -302,6 +302,9 @@
         },
         "sync_now": {
           "type": "boolean"
+        },
+        "deleted": {
+          "type": "boolean"
         }
       }
     }
diff --git a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/generic_ingestion_pipeline.json b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/generic_ingestion_pipeline.json
deleted file mode 100644
index e2a2cbd460117..0000000000000
--- a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/generic_ingestion_pipeline.json
+++ /dev/null
@@ -1,130 +0,0 @@
-{
-  "version": ${xpack.application.connector.template.version},
-  "description": "Generic Enterprise Search ingest pipeline",
-  "_meta": {
-    "managed_by": "Enterprise Search",
-    "managed": true
-  },
-  "processors": [
-    {
-      "attachment": {
-        "description": "Extract text from binary attachments",
-        "field": "_attachment",
-        "target_field": "_extracted_attachment",
-        "ignore_missing": true,
-        "indexed_chars_field": "_attachment_indexed_chars",
-        "if": "ctx?._extract_binary_content == true",
-        "on_failure": [
-          {
-            "append": {
-              "description": "Record error information",
-              "field": "_ingestion_errors",
-              "value": "Processor 'attachment' in pipeline '{{ _ingest.on_failure_pipeline }}' failed with message '{{ _ingest.on_failure_message }}'"
-            }
-          }
-        ],
-        "remove_binary": false
-      }
-    },
-    {
-      "set": {
-        "tag": "set_body",
-        "description": "Set any extracted text on the 'body' field",
-        "field": "body",
-        "copy_from": "_extracted_attachment.content",
-        "ignore_empty_value": true,
-        "if": "ctx?._extract_binary_content == true",
-        "on_failure": [
-          {
-            "append": {
-              "description": "Record error information",
-              "field": "_ingestion_errors",
-              "value": "Processor 'set' with tag 'set_body' in pipeline '{{ _ingest.on_failure_pipeline }}' failed with message '{{ _ingest.on_failure_message }}'"
-            }
-          }
-        ]
-      }
-    },
-    {
-      "gsub": {
-        "tag": "remove_replacement_chars",
-        "description": "Remove unicode 'replacement' characters",
-        "field": "body",
-        "pattern": "�",
-        "replacement": "",
-        "ignore_missing": true,
-        "if": "ctx?._extract_binary_content == true",
-        "on_failure": [
-          {
-            "append": {
-              "description": "Record error information",
-              "field": "_ingestion_errors",
-              "value": "Processor 'gsub' with tag 'remove_replacement_chars' in pipeline '{{ _ingest.on_failure_pipeline }}' failed with message '{{ _ingest.on_failure_message }}'"
-            }
-          }
-        ]
-      }
-    },
-    {
-      "gsub": {
-        "tag": "remove_extra_whitespace",
-        "description": "Squish whitespace",
-        "field": "body",
-        "pattern": "\\s+",
-        "replacement": " ",
-        "ignore_missing": true,
-        "if": "ctx?._reduce_whitespace == true",
-        "on_failure": [
-          {
-            "append": {
-              "description": "Record error information",
-              "field": "_ingestion_errors",
-              "value": "Processor 'gsub' with tag 'remove_extra_whitespace' in pipeline '{{ _ingest.on_failure_pipeline }}' failed with message '{{ _ingest.on_failure_message }}'"
-            }
-          }
-        ]
-      }
-    },
-    {
-      "trim" : {
-        "description": "Trim leading and trailing whitespace",
-        "field": "body",
-        "ignore_missing": true,
-        "if": "ctx?._reduce_whitespace == true",
-        "on_failure": [
-          {
-            "append": {
-              "description": "Record error information",
-              "field": "_ingestion_errors",
-              "value": "Processor 'trim' in pipeline '{{ _ingest.on_failure_pipeline }}' failed with message '{{ _ingest.on_failure_message }}'"
-            }
-          }
-        ]
-      }
-    },
-    {
-      "remove": {
-        "tag": "remove_meta_fields",
-        "description": "Remove meta fields",
-        "field": [
-          "_attachment",
-          "_attachment_indexed_chars",
-          "_extracted_attachment",
-          "_extract_binary_content",
-          "_reduce_whitespace",
-          "_run_ml_inference"
-        ],
-        "ignore_missing": true,
-        "on_failure": [
-          {
-            "append": {
-              "description": "Record error information",
-              "field": "_ingestion_errors",
-              "value": "Processor 'remove' with tag 'remove_meta_fields' in pipeline '{{ _ingest.on_failure_pipeline }}' failed with message '{{ _ingest.on_failure_message }}'"
-            }
-          }
-        ]
-      }
-    }
-  ]
-}
diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/results_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/results_index_mappings.json
index 4415afe50a998..e0bde4715839e 100644
--- a/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/results_index_mappings.json
+++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/results_index_mappings.json
@@ -5,7 +5,15 @@
   },
   "dynamic_templates" : [
     {
-      "strings_as_keywords" : {
+      "map_objects": {
+        "match_mapping_type": "object",
+        "mapping": {
+          "type": "object"
+        }
+      }
+    },
+    {
+      "non_objects_as_keywords" : {
         "match" : "*",
         "mapping" : {
           "type" : "keyword"
diff --git a/x-pack/plugin/core/template-resources/src/main/resources/traces@lifecycle.json b/x-pack/plugin/core/template-resources/src/main/resources/traces@lifecycle.json
new file mode 100644
index 0000000000000..95e6e964b485c
--- /dev/null
+++ b/x-pack/plugin/core/template-resources/src/main/resources/traces@lifecycle.json
@@ -0,0 +1,17 @@
+{
+  "phases": {
+    "hot": {
+      "actions": {
+        "rollover": {
+          "max_primary_shard_size": "50gb",
+          "max_age": "30d"
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "default policy for the traces index template installed by x-pack",
+    "managed": true
+  },
+  "deprecated": ${xpack.stack.template.deprecated}
+}
diff --git a/x-pack/plugin/core/template-resources/src/main/resources/traces@settings.json b/x-pack/plugin/core/template-resources/src/main/resources/traces@settings.json
index 3f4fdba6f4f46..e33f8f13a7ce9 100644
--- a/x-pack/plugin/core/template-resources/src/main/resources/traces@settings.json
+++ b/x-pack/plugin/core/template-resources/src/main/resources/traces@settings.json
@@ -2,6 +2,9 @@
   "template": {
     "settings": {
       "index": {
+        "lifecycle": {
+          "name": "traces@lifecycle"
+        },
         "codec": "best_compression",
         "mapping": {
           "ignore_malformed": true
diff --git a/x-pack/plugin/deprecation/build.gradle b/x-pack/plugin/deprecation/build.gradle
index e62a3441af65a..57124cda9a791 100644
--- a/x-pack/plugin/deprecation/build.gradle
+++ b/x-pack/plugin/deprecation/build.gradle
@@ -1,9 +1,9 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 
 esplugin {
-  name 'x-pack-deprecation'
-  description 'Elasticsearch Expanded Pack Plugin - Deprecation'
-  classname 'org.elasticsearch.xpack.deprecation.Deprecation'
+  name = 'x-pack-deprecation'
+  description = 'Elasticsearch Expanded Pack Plugin - Deprecation'
+  classname ='org.elasticsearch.xpack.deprecation.Deprecation'
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/deprecation/qa/early-deprecation-rest/build.gradle b/x-pack/plugin/deprecation/qa/early-deprecation-rest/build.gradle
index 7e61533c818ec..2fe46efca8a1c 100644
--- a/x-pack/plugin/deprecation/qa/early-deprecation-rest/build.gradle
+++ b/x-pack/plugin/deprecation/qa/early-deprecation-rest/build.gradle
@@ -11,8 +11,8 @@ apply plugin: 'elasticsearch.base-internal-es-plugin'
 apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 esplugin {
-  description 'Deprecated query plugin'
-  classname 'org.elasticsearch.xpack.deprecation.EarlyDeprecationTestPlugin'
+  description = 'Deprecated query plugin'
+  classname ='org.elasticsearch.xpack.deprecation.EarlyDeprecationTestPlugin'
 }
 
 dependencies {
diff --git a/x-pack/plugin/deprecation/qa/rest/build.gradle b/x-pack/plugin/deprecation/qa/rest/build.gradle
index 45b543d910a75..e3ad5799aaf75 100644
--- a/x-pack/plugin/deprecation/qa/rest/build.gradle
+++ b/x-pack/plugin/deprecation/qa/rest/build.gradle
@@ -11,8 +11,8 @@ apply plugin: 'elasticsearch.base-internal-es-plugin'
 apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 esplugin {
-  description 'Deprecated query plugin'
-  classname 'org.elasticsearch.xpack.deprecation.TestDeprecationPlugin'
+  description = 'Deprecated query plugin'
+  classname ='org.elasticsearch.xpack.deprecation.TestDeprecationPlugin'
 }
 
 dependencies {
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecks.java
index ee029d01427aa..65f2659fda04a 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecks.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecks.java
@@ -10,10 +10,12 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexVersions;
+import org.elasticsearch.xpack.core.deprecation.DeprecatedIndexPredicate;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
 
 import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import static java.util.Map.entry;
 import static java.util.Map.ofEntries;
@@ -21,54 +23,28 @@
 public class DataStreamDeprecationChecks {
     static DeprecationIssue oldIndicesCheck(DataStream dataStream, ClusterState clusterState) {
         List<Index> backingIndices = dataStream.getIndices();
-        boolean hasOldIndices = backingIndices.stream()
-            .anyMatch(index -> clusterState.metadata().index(index).getCompatibilityVersion().before(IndexVersions.V_8_0_0));
-        if (hasOldIndices) {
-            long totalIndices = backingIndices.size();
-            List<Index> oldIndices = backingIndices.stream()
-                .filter(index -> clusterState.metadata().index(index).getCompatibilityVersion().before(IndexVersions.V_8_0_0))
-                .toList();
-            long totalOldIndices = oldIndices.size();
-            long totalOldSearchableSnapshots = oldIndices.stream()
-                .filter(index -> clusterState.metadata().index(index).isSearchableSnapshot())
-                .count();
-            long totalOldPartiallyMountedSearchableSnapshots = oldIndices.stream()
-                .filter(index -> clusterState.metadata().index(index).isPartialSearchableSnapshot())
-                .count();
-            long totalOldFullyMountedSearchableSnapshots = totalOldSearchableSnapshots - totalOldPartiallyMountedSearchableSnapshots;
+
+        Set<String> indicesNeedingUpgrade = backingIndices.stream()
+            .filter(DeprecatedIndexPredicate.getReindexRequiredPredicate(clusterState.metadata()))
+            .map(Index::getName)
+            .collect(Collectors.toUnmodifiableSet());
+
+        if (indicesNeedingUpgrade.isEmpty() == false) {
             return new DeprecationIssue(
                 DeprecationIssue.Level.CRITICAL,
-                "Old data stream with a compatibility version < 8.0",
+                "Old data stream with a compatibility version < 9.0",
                 "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html",
-                "This data stream has backing indices that were created before Elasticsearch 8.0.0",
+                "This data stream has backing indices that were created before Elasticsearch 9.0.0",
                 false,
                 ofEntries(
-                    entry(
-                        "backing_indices",
-                        ofEntries(
-                            entry("count", totalIndices),
-                            entry(
-                                "need_upgrading",
-                                ofEntries(
-                                    entry("count", totalOldIndices),
-                                    entry(
-                                        "searchable_snapshots",
-                                        ofEntries(
-                                            entry("count", totalOldSearchableSnapshots),
-                                            entry("fully_mounted", ofEntries(entry("count", totalOldFullyMountedSearchableSnapshots))),
-                                            entry(
-                                                "partially_mounted",
-                                                ofEntries(entry("count", totalOldPartiallyMountedSearchableSnapshots))
-                                            )
-                                        )
-                                    )
-                                )
-                            )
-                        )
-                    )
+                    entry("reindex_required", true),
+                    entry("total_backing_indices", backingIndices.size()),
+                    entry("indices_requiring_upgrade_count", indicesNeedingUpgrade.size()),
+                    entry("indices_requiring_upgrade", indicesNeedingUpgrade)
                 )
             );
         }
+
         return null;
     }
 }
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java
index f9b2cc5afe3a5..0b9a538d505c9 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java
@@ -87,7 +87,8 @@ private DeprecationChecks() {}
                 NodeDeprecationChecks::checkLifecyleStepMasterTimeoutSetting,
                 NodeDeprecationChecks::checkEqlEnabledSetting,
                 NodeDeprecationChecks::checkNodeAttrData,
-                NodeDeprecationChecks::checkWatcherBulkConcurrentRequestsSetting
+                NodeDeprecationChecks::checkWatcherBulkConcurrentRequestsSetting,
+                NodeDeprecationChecks::checkTracingApmSettings
             );
 
     static List<BiFunction<IndexMetadata, ClusterState, DeprecationIssue>> INDEX_SETTINGS_CHECKS = List.of(
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java
index 7ad0758d99832..809870730292a 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java
@@ -304,7 +304,7 @@ public static DeprecationInfoAction.Response from(
 
             List<String> dataStreamNames = indexNameExpressionResolver.dataStreamNames(
                 state,
-                IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN
+                IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR
             );
             Map<String, List<DeprecationIssue>> dataStreamIssues = new HashMap<>();
             for (String dataStreamName : dataStreamNames) {
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java
index aaf58a44a6565..de06e270a867e 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java
@@ -14,12 +14,13 @@
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
-import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.engine.frozen.FrozenEngine;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.xpack.core.deprecation.DeprecatedIndexPredicate;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -36,14 +37,14 @@ static DeprecationIssue oldIndicesCheck(IndexMetadata indexMetadata, ClusterStat
         // TODO: this check needs to be revised. It's trivially true right now.
         IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion();
         // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks
-        if (currentCompatibilityVersion.before(IndexVersions.V_8_0_0) && isNotDataStreamIndex(indexMetadata, clusterState)) {
+        if (DeprecatedIndexPredicate.reindexRequired(indexMetadata) && isNotDataStreamIndex(indexMetadata, clusterState)) {
             return new DeprecationIssue(
                 DeprecationIssue.Level.CRITICAL,
-                "Old index with a compatibility version < 8.0",
+                "Old index with a compatibility version < 9.0",
                 "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html",
                 "This index has version: " + currentCompatibilityVersion.toReleaseVersion(),
                 false,
-                null
+                Collections.singletonMap("reindex_required", true)
             );
         }
         return null;
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java
index 349762456cd3b..b6fff5a82f0cd 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java
@@ -34,6 +34,9 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static org.elasticsearch.common.settings.Setting.Property.Deprecated;
+import static org.elasticsearch.common.settings.Setting.Property.NodeScope;
+import static org.elasticsearch.common.settings.Setting.Property.OperatorDynamic;
 import static org.elasticsearch.xpack.core.security.authc.RealmSettings.RESERVED_REALM_AND_DOMAIN_NAME_PREFIX;
 
 public class NodeDeprecationChecks {
@@ -95,15 +98,13 @@ static DeprecationIssue checkRemovedSetting(
             return null;
         }
         final String removedSettingKey = removedSetting.getKey();
-        Object removedSettingValue = removedSetting.exists(clusterSettings)
-            ? removedSetting.get(clusterSettings)
-            : removedSetting.get(nodeSettings);
-        String value;
-        if (removedSettingValue instanceof TimeValue) {
-            value = ((TimeValue) removedSettingValue).getStringRep();
+        // read setting to force the deprecation warning
+        if (removedSetting.exists(clusterSettings)) {
+            removedSetting.get(clusterSettings);
         } else {
-            value = removedSettingValue.toString();
+            removedSetting.get(nodeSettings);
         }
+
         final String message = String.format(Locale.ROOT, "Setting [%s] is deprecated", removedSettingKey);
         final String details = additionalDetailMessage == null
             ? String.format(Locale.ROOT, "Remove the [%s] setting.", removedSettingKey)
@@ -113,6 +114,45 @@ static DeprecationIssue checkRemovedSetting(
         return new DeprecationIssue(deprecationLevel, message, url, details, false, meta);
     }
 
+    static DeprecationIssue checkMultipleRemovedSettings(
+        final Settings clusterSettings,
+        final Settings nodeSettings,
+        final List<Setting<?>> removedSettings,
+        final String url,
+        String additionalDetailMessage,
+        DeprecationIssue.Level deprecationLevel
+    ) {
+
+        var removedSettingsRemaining = removedSettings.stream().filter(s -> s.exists(clusterSettings) || s.exists(nodeSettings)).toList();
+        if (removedSettingsRemaining.isEmpty()) {
+            return null;
+        }
+        if (removedSettingsRemaining.size() == 1) {
+            Setting<?> removedSetting = removedSettingsRemaining.get(0);
+            return checkRemovedSetting(clusterSettings, nodeSettings, removedSetting, url, additionalDetailMessage, deprecationLevel);
+        }
+
+        // read settings to force the deprecation warning
+        removedSettingsRemaining.forEach(s -> {
+            if (s.exists(clusterSettings)) {
+                s.get(clusterSettings);
+            } else {
+                s.get(nodeSettings);
+            }
+        });
+
+        var removedSettingKeysRemaining = removedSettingsRemaining.stream().map(Setting::getKey).sorted().toList();
+        final String message = String.format(Locale.ROOT, "Settings %s are deprecated", removedSettingKeysRemaining);
+        final String details = additionalDetailMessage == null
+            ? String.format(Locale.ROOT, "Remove each setting in %s.", removedSettingKeysRemaining)
+            : String.format(Locale.ROOT, "Remove each setting in %s. %s", removedSettingKeysRemaining, additionalDetailMessage);
+
+        var canAutoRemoveSettings = removedSettingsRemaining.stream()
+            .allMatch(s -> s.exists(clusterSettings) && s.exists(nodeSettings) == false);
+        var meta = createMetaMapForRemovableSettings(canAutoRemoveSettings, removedSettingKeysRemaining);
+        return new DeprecationIssue(deprecationLevel, message, url, details, false, meta);
+    }
+
     static DeprecationIssue checkMultipleDataPaths(
         Settings nodeSettings,
         PluginsAndModules plugins,
@@ -944,4 +984,32 @@ static DeprecationIssue checkWatcherBulkConcurrentRequestsSetting(
             DeprecationIssue.Level.WARNING
         );
     }
+
+    static DeprecationIssue checkTracingApmSettings(
+        final Settings settings,
+        final PluginsAndModules pluginsAndModules,
+        final ClusterState clusterState,
+        final XPackLicenseState licenseState
+    ) {
+        String url = "https://ela.st/es-deprecation-9-tracing-apm-settings";
+        Setting.Property[] properties = { NodeScope, OperatorDynamic, Deprecated };
+        List<Setting<?>> tracingApmSettings = List.of(
+            Setting.prefixKeySetting("tracing.apm.agent.", key -> Setting.simpleString(key, properties)),
+            Setting.stringListSetting("tracing.apm.names.include", properties),
+            Setting.stringListSetting("tracing.apm.names.exclude", properties),
+            Setting.stringListSetting("tracing.apm.sanitize_field_names", properties),
+            Setting.boolSetting("tracing.apm.enabled", false, properties),
+            SecureSetting.secureString("tracing.apm.api_key", null, Deprecated),
+            SecureSetting.secureString("tracing.apm.secret_token", null, Deprecated)
+        );
+        return checkMultipleRemovedSettings(
+            clusterState.metadata().settings(),
+            settings,
+            tracingApmSettings,
+            url,
+            "[tracing.apm.*] settings are no longer accepted as of 9.0.0"
+                + " and should be replaced by [telemetry.*] or [telemetry.tracing.*] settings.",
+            DeprecationIssue.Level.CRITICAL
+        );
+    }
 }
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckAction.java
index 2e40481f7c561..48009f8105429 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckAction.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckAction.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
 
@@ -32,7 +31,6 @@ private NodesDeprecationCheckAction() {
         super(NAME);
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // this can be replaced with TransportRequest.Empty in v9
     public static class NodeRequest extends TransportRequest {
 
         public NodeRequest() {}
@@ -40,11 +38,6 @@ public NodeRequest() {}
         public NodeRequest(StreamInput in) throws IOException {
             super(in);
         }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-        }
     }
 
     public static class NodeResponse extends BaseNodeResponse {
diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecksTests.java
index d5325fb0ff3a4..b297cc1a5bdf8 100644
--- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecksTests.java
+++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecksTests.java
@@ -17,41 +17,46 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.snapshots.SearchableSnapshotsSettings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static java.util.Collections.singletonList;
+import static java.util.Map.entry;
+import static java.util.Map.ofEntries;
+import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
 import static org.elasticsearch.xpack.deprecation.DeprecationChecks.DATA_STREAM_CHECKS;
 import static org.hamcrest.Matchers.equalTo;
 
 public class DataStreamDeprecationChecksTests extends ESTestCase {
 
     public void testOldIndicesCheck() {
-        long oldIndexCount = randomIntBetween(1, 100);
-        long newIndexCount = randomIntBetween(1, 100);
-        long oldSearchableSnapshotCount = 0;
-        long oldFullyManagedSearchableSnapshotCount = 0;
-        long oldPartiallyManagedSearchableSnapshotCount = 0;
+        int oldIndexCount = randomIntBetween(1, 100);
+        int newIndexCount = randomIntBetween(1, 100);
+
         List<Index> allIndices = new ArrayList<>();
         Map<String, IndexMetadata> nameToIndexMetadata = new HashMap<>();
+        Set<String> expectedIndices = new HashSet<>();
+
         for (int i = 0; i < oldIndexCount; i++) {
-            Settings.Builder settingsBuilder = settings(IndexVersion.fromId(7170099));
-            if (randomBoolean()) {
-                settingsBuilder.put("index.store.type", "snapshot");
-                if (randomBoolean()) {
-                    oldFullyManagedSearchableSnapshotCount++;
-                } else {
-                    settingsBuilder.put("index.store.snapshot.partial", true);
-                    oldPartiallyManagedSearchableSnapshotCount++;
-                }
-                oldSearchableSnapshotCount++;
+            Settings.Builder settings = settings(IndexVersion.fromId(7170099));
+
+            String indexName = "old-data-stream-index-" + i;
+            if (expectedIndices.isEmpty() == false && randomIntBetween(0, 2) == 0) {
+                settings.put(INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE);
+            } else {
+                expectedIndices.add(indexName);
             }
-            IndexMetadata oldIndexMetadata = IndexMetadata.builder("old-data-stream-index-" + i)
+
+            Settings.Builder settingsBuilder = settings;
+            IndexMetadata oldIndexMetadata = IndexMetadata.builder(indexName)
                 .settings(settingsBuilder)
                 .numberOfShards(1)
                 .numberOfReplicas(0)
@@ -59,11 +64,9 @@ public void testOldIndicesCheck() {
             allIndices.add(oldIndexMetadata.getIndex());
             nameToIndexMetadata.put(oldIndexMetadata.getIndex().getName(), oldIndexMetadata);
         }
+
         for (int i = 0; i < newIndexCount; i++) {
             Settings.Builder settingsBuilder = settings(IndexVersion.current());
-            if (randomBoolean()) {
-                settingsBuilder.put("index.store.type", "snapshot");
-            }
             IndexMetadata newIndexMetadata = IndexMetadata.builder("new-data-stream-index-" + i)
                 .settings(settingsBuilder)
                 .numberOfShards(1)
@@ -72,6 +75,7 @@ public void testOldIndicesCheck() {
             allIndices.add(newIndexMetadata.getIndex());
             nameToIndexMetadata.put(newIndexMetadata.getIndex().getName(), newIndexMetadata);
         }
+
         DataStream dataStream = new DataStream(
             randomAlphaOfLength(10),
             allIndices,
@@ -88,37 +92,27 @@ public void testOldIndicesCheck() {
             randomBoolean(),
             null
         );
+
         Metadata metadata = Metadata.builder().indices(nameToIndexMetadata).build();
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build();
+
         DeprecationIssue expected = new DeprecationIssue(
             DeprecationIssue.Level.CRITICAL,
-            "Old data stream with a compatibility version < 8.0",
+            "Old data stream with a compatibility version < 9.0",
             "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html",
-            "This data stream has backing indices that were created before Elasticsearch 8.0.0",
+            "This data stream has backing indices that were created before Elasticsearch 9.0.0",
             false,
-            Map.of(
-                "backing_indices",
-                Map.of(
-                    "count",
-                    oldIndexCount + newIndexCount,
-                    "need_upgrading",
-                    Map.of(
-                        "count",
-                        oldIndexCount,
-                        "searchable_snapshots",
-                        Map.of(
-                            "count",
-                            oldSearchableSnapshotCount,
-                            "fully_mounted",
-                            Map.of("count", oldFullyManagedSearchableSnapshotCount),
-                            "partially_mounted",
-                            Map.of("count", oldPartiallyManagedSearchableSnapshotCount)
-                        )
-                    )
-                )
+            ofEntries(
+                entry("reindex_required", true),
+                entry("total_backing_indices", oldIndexCount + newIndexCount),
+                entry("indices_requiring_upgrade_count", expectedIndices.size()),
+                entry("indices_requiring_upgrade", expectedIndices)
             )
         );
+
         List<DeprecationIssue> issues = DeprecationChecks.filterChecks(DATA_STREAM_CHECKS, c -> c.apply(dataStream, clusterState));
+
         assertThat(issues, equalTo(singletonList(expected)));
     }
+
 }
diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java
index 48cbef6831a2b..c6f3208a1cfb0 100644
--- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java
+++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java
@@ -19,8 +19,8 @@
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
-import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.engine.frozen.FrozenEngine;
+import org.elasticsearch.snapshots.SearchableSnapshotsSettings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
 
@@ -29,6 +29,8 @@
 import java.util.Map;
 
 import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
 import static org.elasticsearch.xpack.deprecation.DeprecationChecks.INDEX_SETTINGS_CHECKS;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
@@ -48,11 +50,11 @@ public void testOldIndicesCheck() {
             .build();
         DeprecationIssue expected = new DeprecationIssue(
             DeprecationIssue.Level.CRITICAL,
-            "Old index with a compatibility version < 8.0",
+            "Old index with a compatibility version < 9.0",
             "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html",
             "This index has version: " + createdWith.toReleaseVersion(),
             false,
-            null
+            singletonMap("reindex_required", true)
         );
         List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata, clusterState));
         assertEquals(singletonList(expected), issues);
@@ -100,6 +102,20 @@ public void testOldIndicesCheckDataStreamIndex() {
         assertThat(issues.size(), equalTo(0));
     }
 
+    public void testOldIndicesCheckSnapshotIgnored() {
+        IndexVersion createdWith = IndexVersion.fromId(7170099);
+        Settings.Builder settings = settings(createdWith);
+        settings.put(INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE);
+        IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build();
+        ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE)
+            .metadata(Metadata.builder().put(indexMetadata, true))
+            .build();
+
+        List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata, clusterState));
+
+        assertThat(issues, empty());
+    }
+
     public void testTranslogRetentionSettings() {
         Settings.Builder settings = settings(IndexVersion.current());
         settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue());
@@ -229,7 +245,7 @@ public void testCamelCaseDeprecation() throws IOException {
             + "} }";
 
         IndexMetadata simpleIndex = IndexMetadata.builder(randomAlphaOfLengthBetween(5, 10))
-            .settings(settings(IndexVersions.MINIMUM_COMPATIBLE))
+            .settings(settings(IndexVersion.current()))
             .numberOfShards(1)
             .numberOfReplicas(1)
             .putMapping(simpleMapping)
diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java
index 9d89a3cbe328e..3aaee0e5cdb52 100644
--- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java
+++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java
@@ -75,6 +75,29 @@ public void testRemovedSetting() {
         assertThat(issue.getUrl(), equalTo("https://removed-setting.example.com"));
     }
 
+    public void testMultipleRemovedSettings() {
+        final Settings clusterSettings = Settings.EMPTY;
+        final Settings nodeSettings = Settings.builder()
+            .put("node.removed_setting1", "value")
+            .put("node.removed_setting2", "value")
+            .build();
+        final Setting<?> removedSetting1 = Setting.simpleString("node.removed_setting1");
+        final Setting<?> removedSetting2 = Setting.simpleString("node.removed_setting2");
+        final DeprecationIssue issue = NodeDeprecationChecks.checkMultipleRemovedSettings(
+            clusterSettings,
+            nodeSettings,
+            shuffledList(List.of(removedSetting1, removedSetting2)),
+            "https://removed-setting.example.com",
+            "Some detail.",
+            DeprecationIssue.Level.CRITICAL
+        );
+        assertThat(issue, not(nullValue()));
+        assertThat(issue.getLevel(), equalTo(DeprecationIssue.Level.CRITICAL));
+        assertThat(issue.getMessage(), equalTo("Settings [node.removed_setting1, node.removed_setting2] are deprecated"));
+        assertThat(issue.getDetails(), equalTo("Remove each setting in [node.removed_setting1, node.removed_setting2]. Some detail."));
+        assertThat(issue.getUrl(), equalTo("https://removed-setting.example.com"));
+    }
+
     public void testMultipleDataPaths() {
         final Settings settings = Settings.builder().putList("path.data", Arrays.asList("d1", "d2")).build();
         final XPackLicenseState licenseState = new XPackLicenseState(() -> 0);
diff --git a/x-pack/plugin/downsample/build.gradle b/x-pack/plugin/downsample/build.gradle
index 74999a7620e51..2de12b89b5d3d 100644
--- a/x-pack/plugin/downsample/build.gradle
+++ b/x-pack/plugin/downsample/build.gradle
@@ -1,9 +1,9 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 esplugin {
-  name 'x-pack-downsample'
-  description 'Elasticsearch Expanded Pack Plugin - Downsample'
-  classname 'org.elasticsearch.xpack.downsample.Downsample'
+  name = 'x-pack-downsample'
+  description = 'Elasticsearch Expanded Pack Plugin - Downsample'
+  classname ='org.elasticsearch.xpack.downsample.Downsample'
   extendedPlugins = ['x-pack-aggregate-metric']
 }
 
diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/70_flattened_field_type.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/70_flattened_field_type.yml
new file mode 100644
index 0000000000000..0f586ec0ed669
--- /dev/null
+++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/70_flattened_field_type.yml
@@ -0,0 +1,307 @@
+---
+"A flattened label field":
+  - do:
+      indices.create:
+        index: source_index
+        body:
+          settings:
+            number_of_shards: 1
+            index:
+              mode: time_series
+              routing_path: [ metricset, k8s.pod.uid ]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            subobjects: false
+            properties:
+              "@timestamp":
+                type: date
+              metricset:
+                type: keyword
+                time_series_dimension: true
+              k8s:
+                properties:
+                  pod:
+                    properties:
+                      uid:
+                        type: keyword
+                        time_series_dimension: true
+                      name:
+                        type: keyword
+                  agent:
+                    type: flattened
+                  value:
+                    type: long
+                    time_series_metric: gauge
+
+  - do:
+      bulk:
+        refresh: true
+        index: source_index
+        body:
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.4" }, "value": 10 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.5" }, "value": 20 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T20:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.6" }, "value": 12 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T20:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.7" }, "value": 15 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.7" }, "value": 9 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.8" }, "value": 16 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T19:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.9" }, "value": 25 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T19:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.10" }, "value": 17 }}'
+
+  - do:
+      indices.put_settings:
+        index: source_index
+        body:
+          index.blocks.write: true
+
+  - do:
+      indices.downsample:
+        index: source_index
+        target_index: target_index
+        body: >
+          {
+            "fixed_interval": "1h"
+          }
+  - is_true: acknowledged
+
+  - do:
+      search:
+        index: target_index
+        body:
+          sort: [ "_tsid", "@timestamp" ]
+
+  - length: { hits.hits: 4 }
+  - match: { hits.hits.0._source._doc_count: 2 }
+  - match: { hits.hits.0._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
+  - match: { hits.hits.0._source.k8s\.agent: { "id": "second", "version": "2.1.8" } }
+
+  - match: { hits.hits.1._source._doc_count: 2 }
+  - match: { hits.hits.1._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.1._source.@timestamp: 2021-04-28T19:00:00.000Z }
+  - match: { hits.hits.1._source.k8s\.agent: { "id": "second", "version": "2.1.10" } }
+
+  - match: { hits.hits.2._source._doc_count: 2 }
+  - match: { hits.hits.2._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z }
+  - match: { hits.hits.2._source.k8s\.agent: { "id": "first", "version": "2.0.5" } }
+
+  - match: { hits.hits.3._source._doc_count: 2 }
+  - match: { hits.hits.3._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.3._source.@timestamp: 2021-04-28T20:00:00.000Z }
+  - match: { hits.hits.3._source.k8s\.agent: { "id": "first", "version": "2.0.7" } }
+
+---
+"A flattened label field with no doc values":
+  - do:
+      indices.create:
+        index: source_index
+        body:
+          settings:
+            number_of_shards: 1
+            index:
+              mode: time_series
+              routing_path: [ metricset, k8s.pod.uid ]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            subobjects: false
+            properties:
+              "@timestamp":
+                type: date
+              metricset:
+                type: keyword
+                time_series_dimension: true
+              k8s:
+                properties:
+                  pod:
+                    properties:
+                      uid:
+                        type: keyword
+                        time_series_dimension: true
+                      name:
+                        type: keyword
+                  agent:
+                    type: flattened
+                    doc_values: false
+                  value:
+                    type: long
+                    time_series_metric: gauge
+
+  - do:
+      bulk:
+        refresh: true
+        index: source_index
+        body:
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.4" }, "value": 10 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.5" }, "value": 20 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T20:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.6" }, "value": 12 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T20:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.7" }, "value": 15 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.7" }, "value": 9 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.8" }, "value": 16 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T19:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.9" }, "value": 25 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T19:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.10" }, "value": 17 }}'
+
+  - do:
+      indices.put_settings:
+        index: source_index
+        body:
+          index.blocks.write: true
+
+  - do:
+      indices.downsample:
+        index: source_index
+        target_index: target_index
+        body: >
+          {
+            "fixed_interval": "1h"
+          }
+  - is_true: acknowledged
+
+  - do:
+      search:
+        index: target_index
+        body:
+          sort: [ "_tsid", "@timestamp" ]
+
+  - length: { hits.hits: 4 }
+  - match: { hits.hits.0._source._doc_count: 2 }
+  - match: { hits.hits.0._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
+  - is_false: hits.hits.0._source.k8s\.agent
+
+  - match: { hits.hits.1._source._doc_count: 2 }
+  - match: { hits.hits.1._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.1._source.@timestamp: 2021-04-28T19:00:00.000Z }
+  - is_false: hits.hits.1._source.k8s\.agent
+
+  - match: { hits.hits.2._source._doc_count: 2 }
+  - match: { hits.hits.2._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z }
+  - is_false: hits.hits.2._source.k8s\.agent
+
+  - match: { hits.hits.3._source._doc_count: 2 }
+  - match: { hits.hits.3._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.3._source.@timestamp: 2021-04-28T20:00:00.000Z }
+  - is_false: hits.hits.3._source.k8s\.agent
+
+---
+"A flattened label field with mixed content":
+  - do:
+      indices.create:
+        index: source_index
+        body:
+          settings:
+            number_of_shards: 1
+            index:
+              mode: time_series
+              routing_path: [ metricset, k8s.pod.uid ]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            subobjects: false
+            properties:
+              "@timestamp":
+                type: date
+              metricset:
+                type: keyword
+                time_series_dimension: true
+              k8s:
+                properties:
+                  pod:
+                    properties:
+                      uid:
+                        type: keyword
+                        time_series_dimension: true
+                      name:
+                        type: keyword
+                  agent:
+                    type: flattened
+                    null_value: my_null_value
+                  value:
+                    type: long
+                    time_series_metric: gauge
+
+  - do:
+      bulk:
+        refresh: true
+        index: source_index
+        body:
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.4", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11 }, "value": 10 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.5", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 20 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T20:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.6", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 12 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T20:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.7", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 15 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.7", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 9 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.8", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 16 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T19:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.9", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 25 }}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T19:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.10", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 17 }}'
+
+  - do:
+      indices.put_settings:
+        index: source_index
+        body:
+          index.blocks.write: true
+
+  - do:
+      indices.downsample:
+        index: source_index
+        target_index: target_index
+        body: >
+          {
+            "fixed_interval": "1h"
+          }
+  - is_true: acknowledged
+
+  - do:
+      search:
+        index: target_index
+        body:
+          sort: [ "_tsid", "@timestamp" ]
+
+  - length: { hits.hits: 4 }
+  - match: { hits.hits.0._source._doc_count: 2 }
+  - match: { hits.hits.0._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
+  - match: { hits.hits.0._source.k8s\.agent: { "id": "second", "version": "2.1.8", "versions": ["1", "2", "3"], "dotted": {"version": "1.1"}, "numeric_version": "11", optional_version: "my_null_value" } }
+
+  - match: { hits.hits.1._source._doc_count: 2 }
+  - match: { hits.hits.1._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.1._source.@timestamp: 2021-04-28T19:00:00.000Z }
+  - match: { hits.hits.1._source.k8s\.agent: { "id": "second", "version": "2.1.10", "versions": ["1", "2", "3"], "dotted": {"version": "1.1"}, "numeric_version": "11", optional_version: "my_null_value" } }
+
+  - match: { hits.hits.2._source._doc_count: 2 }
+  - match: { hits.hits.2._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z }
+  - match: { hits.hits.2._source.k8s\.agent: { "id": "first", "version": "2.0.5", "versions": ["1", "2", "3"], "dotted": {"version": "1.1"}, "numeric_version": "11", optional_version: "my_null_value" } }
+
+  - match: { hits.hits.3._source._doc_count: 2 }
+  - match: { hits.hits.3._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.3._source.@timestamp: 2021-04-28T20:00:00.000Z }
+  - match: { hits.hits.3._source.k8s\.agent: { "id": "first", "version": "2.0.7", "versions": ["1", "2", "3"], "dotted": {"version": "1.1"}, "numeric_version": "11", optional_version: "my_null_value" } }
diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java
index 5c2a05da1e8d2..5fba98b765a6b 100644
--- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java
+++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java
@@ -10,8 +10,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.DocWriteRequest;
-import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
-import org.elasticsearch.action.admin.cluster.shards.TransportClusterSearchShardsAction;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
@@ -57,9 +55,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
 
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -144,7 +140,7 @@ public void setup(final String sourceIndex, int numOfShards, int numOfReplicas,
 
     public void testILMDownsampleRollingRestart() throws Exception {
         final InternalTestCluster cluster = internalCluster();
-        final List<String> masterNodes = cluster.startMasterOnlyNodes(1);
+        cluster.startMasterOnlyNodes(1);
         cluster.startDataOnlyNodes(3);
         ensureStableCluster(cluster.size());
         ensureGreen();
@@ -169,46 +165,16 @@ public void testILMDownsampleRollingRestart() throws Exception {
                 .endObject();
         };
         int indexedDocs = bulkIndex(sourceIndex, sourceSupplier, DOC_COUNT);
-        final CountDownLatch disruptionStart = new CountDownLatch(1);
-        final CountDownLatch disruptionEnd = new CountDownLatch(1);
 
-        new Thread(new Disruptor(cluster, sourceIndex, new DisruptionListener() {
-            @Override
-            public void disruptionStart() {
-                disruptionStart.countDown();
-            }
-
-            @Override
-            public void disruptionEnd() {
-                disruptionEnd.countDown();
-            }
-        }, masterNodes.get(0), (ignored) -> {
-            try {
-                cluster.rollingRestart(new InternalTestCluster.RestartCallback() {
-                    @Override
-                    public boolean validateClusterForming() {
-                        return true;
-                    }
-                });
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        })).start();
+        cluster.rollingRestart(new InternalTestCluster.RestartCallback());
 
         final String targetIndex = "downsample-1h-" + sourceIndex;
-        startDownsampleTaskViaIlm(sourceIndex, targetIndex, disruptionStart, disruptionEnd);
-        waitUntil(() -> getClusterPendingTasks(cluster.client()).pendingTasks().isEmpty(), 60, TimeUnit.SECONDS);
-        ensureStableCluster(cluster.numDataAndMasterNodes());
-        assertTargetIndex(cluster, targetIndex, indexedDocs);
+        startDownsampleTaskViaIlm(sourceIndex, targetIndex);
+        assertBusy(() -> assertTargetIndex(cluster, targetIndex, indexedDocs));
+        ensureGreen(targetIndex);
     }
 
-    private void startDownsampleTaskViaIlm(
-        String sourceIndex,
-        String targetIndex,
-        CountDownLatch disruptionStart,
-        CountDownLatch disruptionEnd
-    ) throws Exception {
-        disruptionStart.await();
+    private void startDownsampleTaskViaIlm(String sourceIndex, String targetIndex) throws Exception {
         var request = new UpdateSettingsRequest(sourceIndex).settings(
             Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, POLICY_NAME)
         );
@@ -231,7 +197,6 @@ private void startDownsampleTaskViaIlm(
             var getSettingsResponse = client().admin().indices().getSettings(new GetSettingsRequest().indices(targetIndex)).actionGet();
             assertThat(getSettingsResponse.getSetting(targetIndex, IndexMetadata.INDEX_DOWNSAMPLE_STATUS.getKey()), equalTo("success"));
         }, 60, TimeUnit.SECONDS);
-        disruptionEnd.await();
     }
 
     private void assertTargetIndex(final InternalTestCluster cluster, final String targetIndex, int indexedDocs) {
@@ -294,53 +259,4 @@ private String randomDateForRange(long start, long end) {
     public interface SourceSupplier {
         XContentBuilder get() throws IOException;
     }
-
-    interface DisruptionListener {
-        void disruptionStart();
-
-        void disruptionEnd();
-    }
-
-    private class Disruptor implements Runnable {
-        final InternalTestCluster cluster;
-        private final String sourceIndex;
-        private final DisruptionListener listener;
-        private final String clientNode;
-        private final Consumer<String> disruption;
-
-        private Disruptor(
-            final InternalTestCluster cluster,
-            final String sourceIndex,
-            final DisruptionListener listener,
-            final String clientNode,
-            final Consumer<String> disruption
-        ) {
-            this.cluster = cluster;
-            this.sourceIndex = sourceIndex;
-            this.listener = listener;
-            this.clientNode = clientNode;
-            this.disruption = disruption;
-        }
-
-        @Override
-        public void run() {
-            listener.disruptionStart();
-            try {
-                final String candidateNode = safeExecute(
-                    cluster.client(clientNode),
-                    TransportClusterSearchShardsAction.TYPE,
-                    new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, sourceIndex)
-                ).getNodes()[0].getName();
-                logger.info("Candidate node [" + candidateNode + "]");
-                disruption.accept(candidateNode);
-                ensureGreen(sourceIndex);
-                ensureStableCluster(cluster.numDataAndMasterNodes(), clientNode);
-
-            } catch (Exception e) {
-                logger.error("Ignoring Error while injecting disruption [" + e.getMessage() + "]");
-            } finally {
-                listener.disruptionEnd();
-            }
-        }
-    }
 }
diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java
index 74375bbe27939..3657e4989ccbd 100644
--- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java
+++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper;
@@ -65,6 +66,8 @@ private AbstractDownsampleFieldProducer createFieldProducer() {
             // If field is not a metric, we downsample it as a label
             if ("histogram".equals(fieldType.typeName())) {
                 return new LabelFieldProducer.HistogramLastLabelFieldProducer(name());
+            } else if ("flattened".equals(fieldType.typeName())) {
+                return new LabelFieldProducer.FlattenedLastValueFieldProducer(name());
             }
             return new LabelFieldProducer.LabelLastValueFieldProducer(name());
         }
@@ -90,7 +93,13 @@ static List<FieldValueFetcher> create(SearchExecutionContext context, String[] f
                 }
             } else {
                 if (context.fieldExistsInIndex(field)) {
-                    final IndexFieldData<?> fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH);
+                    final IndexFieldData<?> fieldData;
+                    if (fieldType instanceof FlattenedFieldMapper.RootFlattenedFieldType flattenedFieldType) {
+                        var keyedFieldType = flattenedFieldType.getKeyedFieldType();
+                        fieldData = context.getForField(keyedFieldType, MappedFieldType.FielddataOperation.SEARCH);
+                    } else {
+                        fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH);
+                    }
                     final String fieldName = context.isMultiField(field)
                         ? fieldType.name().substring(0, fieldType.name().lastIndexOf('.'))
                         : fieldType.name();
diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LabelFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LabelFieldProducer.java
index 05b4852d0dfd3..b211c5bfb0d12 100644
--- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LabelFieldProducer.java
+++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LabelFieldProducer.java
@@ -7,8 +7,10 @@
 
 package org.elasticsearch.xpack.downsample;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.index.fielddata.FormattedDocValues;
 import org.elasticsearch.index.fielddata.HistogramValue;
+import org.elasticsearch.index.mapper.flattened.FlattenedFieldSyntheticWriterHelper;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric;
 
@@ -141,14 +143,14 @@ public void reset() {
         }
     }
 
-    static class AggregateMetricFieldProducer extends LabelLastValueFieldProducer {
+    static final class AggregateMetricFieldProducer extends LabelLastValueFieldProducer {
 
         AggregateMetricFieldProducer(String name, Metric metric) {
             super(name, new LastValueLabel(metric.name()));
         }
     }
 
-    public static class HistogramLastLabelFieldProducer extends LabelLastValueFieldProducer {
+    static final class HistogramLastLabelFieldProducer extends LabelLastValueFieldProducer {
         HistogramLastLabelFieldProducer(String name) {
             super(name);
         }
@@ -167,4 +169,40 @@ public void write(XContentBuilder builder) throws IOException {
             }
         }
     }
+
+    static final class FlattenedLastValueFieldProducer extends LabelLastValueFieldProducer {
+
+        FlattenedLastValueFieldProducer(String name) {
+            super(name);
+        }
+
+        @Override
+        public void write(XContentBuilder builder) throws IOException {
+            if (isEmpty() == false) {
+                builder.startObject(name());
+
+                var value = label.get();
+                List<BytesRef> list;
+                if (value instanceof Object[] values) {
+                    list = new ArrayList<>(values.length);
+                    for (Object v : values) {
+                        list.add(new BytesRef(v.toString()));
+                    }
+                } else {
+                    list = List.of(new BytesRef(value.toString()));
+                }
+
+                var iterator = list.iterator();
+                var helper = new FlattenedFieldSyntheticWriterHelper(() -> {
+                    if (iterator.hasNext()) {
+                        return iterator.next();
+                    } else {
+                        return null;
+                    }
+                });
+                helper.write(builder);
+                builder.endObject();
+            }
+        }
+    }
 }
diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java
index 58a0370efb50e..41d815c8206a7 100644
--- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java
+++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java
@@ -58,6 +58,7 @@
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
@@ -284,7 +285,7 @@ protected void masterOperation(
         // At any point if there is an issue, delete the downsample index
 
         // 1. Extract source index mappings
-        final GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(sourceIndexName);
+        final GetMappingsRequest getMappingsRequest = new GetMappingsRequest(request.masterNodeTimeout()).indices(sourceIndexName);
         getMappingsRequest.setParentTask(parentTask);
         client.admin().indices().getMappings(getMappingsRequest, listener.delegateFailureAndWrap((delegate, getMappingsResponse) -> {
             final Map<String, Object> sourceIndexMappings = getMappingsResponse.mappings()
@@ -723,6 +724,9 @@ private static void addTimestampField(
                     if (mapping.get("format") != null) {
                         builder.field("format", mapping.get("format"));
                     }
+                    if (mapping.get("ignore_malformed") != null) {
+                        builder.field("ignore_malformed", mapping.get("ignore_malformed"));
+                    }
                 }
             } catch (IOException e) {
                 throw new ElasticsearchException("Unable to create timestamp field mapping for field [" + timestampField + "]", e);
@@ -897,6 +901,12 @@ private void createDownsampleIndex(
                 sourceIndexMetadata.getSettings().get(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())
             );
         }
+        if (sourceIndexMetadata.getSettings().hasValue(FieldMapper.IGNORE_MALFORMED_SETTING.getKey())) {
+            builder.put(
+                FieldMapper.IGNORE_MALFORMED_SETTING.getKey(),
+                sourceIndexMetadata.getSettings().get(FieldMapper.IGNORE_MALFORMED_SETTING.getKey())
+            );
+        }
 
         CreateIndexClusterStateUpdateRequest createIndexClusterStateUpdateRequest = new CreateIndexClusterStateUpdateRequest(
             "downsample",
diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java
index 812b48ee4cae5..31b415270915b 100644
--- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java
+++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java
@@ -49,6 +49,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
@@ -201,14 +202,19 @@ public void setup() throws IOException {
                 IndexSettings.TIME_SERIES_START_TIME.getKey(),
                 DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(Instant.ofEpochMilli(startTime).toEpochMilli())
             )
-            .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2106-01-08T23:40:53.384Z");
+            .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2106-01-08T23:40:53.384Z")
+            .put(FieldMapper.IGNORE_MALFORMED_SETTING.getKey(), randomBoolean());
 
         if (randomBoolean()) {
             settings.put(IndexMetadata.SETTING_INDEX_HIDDEN, randomBoolean());
         }
 
         XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties");
-        mapping.startObject(FIELD_TIMESTAMP).field("type", "date").endObject();
+        mapping.startObject(FIELD_TIMESTAMP).field("type", "date");
+        if (settings.get(FieldMapper.IGNORE_MALFORMED_SETTING.getKey()).equals("true")) {
+            mapping.field("ignore_malformed", false);
+        }
+        mapping.endObject();
 
         // Dimensions
         mapping.startObject(FIELD_DIMENSION_1).field("type", "keyword").field("time_series_dimension", true).endObject();
@@ -1173,7 +1179,7 @@ private InternalAggregations aggregate(final String index, AggregationBuilder ag
     @SuppressWarnings("unchecked")
     private void assertDownsampleIndex(String sourceIndex, String downsampleIndex, DownsampleConfig config) throws Exception {
         // Retrieve field information for the metric fields
-        final GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(sourceIndex).get();
+        final GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, sourceIndex).get();
         final Map<String, Object> sourceIndexMappings = getMappingsResponse.mappings()
             .entrySet()
             .stream()
@@ -1215,8 +1221,9 @@ private void assertDownsampleIndex(String sourceIndex, String downsampleIndex, D
 
         assertFieldMappings(config, metricFields, mappings);
 
-        GetMappingsResponse indexMappings = indicesAdmin().getMappings(new GetMappingsRequest().indices(downsampleIndex, sourceIndex))
-            .actionGet();
+        GetMappingsResponse indexMappings = indicesAdmin().getMappings(
+            new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(downsampleIndex, sourceIndex)
+        ).actionGet();
         Map<String, String> downsampleIndexProperties = (Map<String, String>) indexMappings.mappings()
             .get(downsampleIndex)
             .sourceAsMap()
diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LabelFieldProducerTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LabelFieldProducerTests.java
index 469e00f7af9af..844eb1b8e27d8 100644
--- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LabelFieldProducerTests.java
+++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LabelFieldProducerTests.java
@@ -7,10 +7,18 @@
 
 package org.elasticsearch.xpack.downsample;
 
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.index.fielddata.FormattedDocValues;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentType;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
 
 public class LabelFieldProducerTests extends AggregatorTestCase {
 
@@ -93,4 +101,50 @@ public Object nextValue() {
         assertTrue(producer.isEmpty());
         assertNull(producer.label().get());
     }
+
+    public void testFlattenedLastValueFieldProducer() throws IOException {
+        var producer = new LabelFieldProducer.FlattenedLastValueFieldProducer("dummy");
+        assertTrue(producer.isEmpty());
+        assertEquals("dummy", producer.name());
+        assertEquals("last_value", producer.label().name());
+
+        var bytes = List.of("a\0value_a", "b\0value_b", "c\0value_c", "d\0value_d");
+        var docValues = new FormattedDocValues() {
+
+            Iterator<String> iterator = bytes.iterator();
+
+            @Override
+            public boolean advanceExact(int docId) {
+                return true;
+            }
+
+            @Override
+            public int docValueCount() {
+                return bytes.size();
+            }
+
+            @Override
+            public Object nextValue() {
+                return iterator.next();
+            }
+        };
+
+        producer.collect(docValues, 1);
+        assertFalse(producer.isEmpty());
+        assertEquals("a\0value_a", (((Object[]) producer.label().get())[0]).toString());
+        assertEquals("b\0value_b", (((Object[]) producer.label().get())[1]).toString());
+        assertEquals("c\0value_c", (((Object[]) producer.label().get())[2]).toString());
+        assertEquals("d\0value_d", (((Object[]) producer.label().get())[3]).toString());
+
+        var builder = new XContentBuilder(XContentType.JSON.xContent(), new ByteArrayOutputStream());
+        builder.startObject();
+        producer.write(builder);
+        builder.endObject();
+        var content = Strings.toString(builder);
+        assertThat(content, equalTo("{\"dummy\":{\"a\":\"value_a\",\"b\":\"value_b\",\"c\":\"value_c\",\"d\":\"value_d\"}}"));
+
+        producer.reset();
+        assertTrue(producer.isEmpty());
+        assertNull(producer.label().get());
+    }
 }
diff --git a/x-pack/plugin/enrich/build.gradle b/x-pack/plugin/enrich/build.gradle
index 134008c53fe1f..352b7a3e64171 100644
--- a/x-pack/plugin/enrich/build.gradle
+++ b/x-pack/plugin/enrich/build.gradle
@@ -1,9 +1,9 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 esplugin {
-  name 'x-pack-enrich'
-  description 'Elasticsearch Expanded Pack Plugin - Enrich'
-  classname 'org.elasticsearch.xpack.enrich.EnrichPlugin'
+  name = 'x-pack-enrich'
+  description = 'Elasticsearch Expanded Pack Plugin - Enrich'
+  classname ='org.elasticsearch.xpack.enrich.EnrichPlugin'
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java
index ef455acb645d9..16bc81785f53a 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java
@@ -308,7 +308,8 @@ public List<Setting<?>> getSettings() {
             COORDINATOR_PROXY_MAX_LOOKUPS_PER_REQUEST,
             COORDINATOR_PROXY_QUEUE_CAPACITY,
             ENRICH_MAX_FORCE_MERGE_ATTEMPTS,
-            CACHE_SIZE
+            CACHE_SIZE,
+            CACHE_SIZE_BWC
         );
     }
 
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java
index 69bc54457785c..37498bb9d54ab 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java
@@ -748,7 +748,7 @@ private void validateIndexBeforePromotion(String destinationIndexName, ClusterSt
     private void updateEnrichPolicyAlias(ActionListener<IndicesAliasesResponse> listener) {
         String enrichIndexBase = EnrichPolicy.getBaseName(policyName);
         logger.debug("Policy [{}]: Promoting new enrich index [{}] to alias [{}]", policyName, enrichIndexName, enrichIndexBase);
-        GetAliasesRequest aliasRequest = new GetAliasesRequest(enrichIndexBase);
+        GetAliasesRequest aliasRequest = new GetAliasesRequest(ENRICH_MASTER_REQUEST_TIMEOUT, enrichIndexBase);
         ClusterState clusterState = clusterService.state();
         validateIndexBeforePromotion(enrichIndexName, clusterState);
         String[] concreteIndices = indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(clusterState, aliasRequest);
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyMaintenanceServiceTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyMaintenanceServiceTests.java
index 34df040205496..0c19c2867a489 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyMaintenanceServiceTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyMaintenanceServiceTests.java
@@ -203,7 +203,10 @@ private String fakeRunPolicy(String forPolicy) throws IOException {
 
     private void promoteFakePolicyIndex(String indexName, String forPolicy) {
         String enrichIndexBase = EnrichPolicy.getBaseName(forPolicy);
-        GetAliasesResponse getAliasesResponse = client().admin().indices().getAliases(new GetAliasesRequest(enrichIndexBase)).actionGet();
+        GetAliasesResponse getAliasesResponse = client().admin()
+            .indices()
+            .getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT, enrichIndexBase))
+            .actionGet();
         IndicesAliasesRequest aliasToggleRequest = new IndicesAliasesRequest();
         String[] indices = getAliasesResponse.getAliases().keySet().toArray(new String[0]);
         if (indices.length > 0) {
diff --git a/x-pack/plugin/ent-search/build.gradle b/x-pack/plugin/ent-search/build.gradle
index 66be40152d8ba..52634ad788d97 100644
--- a/x-pack/plugin/ent-search/build.gradle
+++ b/x-pack/plugin/ent-search/build.gradle
@@ -3,9 +3,9 @@ apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 esplugin {
-  name 'x-pack-ent-search'
-  description 'Elasticsearch Expanded Pack Plugin - Enterprise Search'
-  classname 'org.elasticsearch.xpack.application.EnterpriseSearch'
+  name = 'x-pack-ent-search'
+  description = 'Elasticsearch Expanded Pack Plugin - Enterprise Search'
+  classname = 'org.elasticsearch.xpack.application.EnterpriseSearch'
   extendedPlugins = ['x-pack-core']
 }
 
diff --git a/x-pack/plugin/ent-search/qa/rest/roles.yml b/x-pack/plugin/ent-search/qa/rest/roles.yml
index d32f05b7b749e..661ba482c6367 100644
--- a/x-pack/plugin/ent-search/qa/rest/roles.yml
+++ b/x-pack/plugin/ent-search/qa/rest/roles.yml
@@ -4,13 +4,12 @@ admin:
     - manage_behavioral_analytics
     - manage
     - monitor
+    - manage_connector
   indices:
     - names: [
         # indices and search applications
         "test-*",
         "another-test-search-application",
-        ".elastic-connectors-v1",
-        ".elastic-connectors-sync-jobs-v1"
     ]
       privileges: [ "manage", "write", "read" ]
 
@@ -20,6 +19,7 @@ user:
     - manage_api_key
     - read_connector_secrets
     - write_connector_secrets
+    - monitor_connector
   indices:
     - names: [
       "test-index1",
@@ -27,9 +27,7 @@ user:
       "test-search-application-1",
       "test-search-application-with-aggs",
       "test-search-application-with-list",
-      "test-search-application-with-list-invalid",
-      ".elastic-connectors-v1",
-      ".elastic-connectors-sync-jobs-v1"
+      "test-search-application-with-list-invalid"
     ]
       privileges: [ "read" ]
 
diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/10_connector_put.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/10_connector_put.yml
index 094d9cbf43089..a4d9176cf01e3 100644
--- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/10_connector_put.yml
+++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/10_connector_put.yml
@@ -152,6 +152,19 @@ setup:
           service_type: super-connector
 
 
+---
+'Create Connector - Invalid Managed Connector Index Prefix':
+  - do:
+      catch: "bad_request"
+      connector.put:
+        connector_id: test-connector-test-managed
+        body:
+          index_name: wrong-prefix-index
+          name: my-connector
+          language: pl
+          is_native: true
+          service_type: super-connector
+
 ---
 'Create Connector - Id returned as part of response':
   - do:
@@ -205,6 +218,30 @@ setup:
         body:
           index_name: search-test
 
+---
+'Create Connector - Index name used by deleted connector':
+  - do:
+      connector.put:
+        connector_id: test-connector-1
+        body:
+          index_name: search-test
+
+  - match: { result: 'created' }
+
+  - do:
+      connector.delete:
+        connector_id: test-connector-1
+
+  - match: { acknowledged: true }
+
+  - do:
+      connector.put:
+        connector_id: test-connector-2
+        body:
+          index_name: search-test
+
+  - match: { result: 'created' }
+
 ---
 'Create Connector - Without index attached':
   - do:
diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/130_connector_update_index_name.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/130_connector_update_index_name.yml
index f804dc02a9e01..b63bf595af5f4 100644
--- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/130_connector_update_index_name.yml
+++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/130_connector_update_index_name.yml
@@ -151,3 +151,18 @@ setup:
 
   - match: { index_name: content-search-2-test }
 
+---
+"Update Managed Connector Index Name - Bad Prefix":
+  - do:
+      connector.put:
+        connector_id: test-connector-2
+        body:
+          is_native: true
+          service_type: super-connector
+
+  - do:
+      catch: "bad_request"
+      connector.update_index_name:
+        connector_id: test-connector-2
+        body:
+          index_name: wrong-prefix-search-2-test
diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/140_connector_update_native.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/140_connector_update_native.yml
index f8cd24d175312..6811c3340ce42 100644
--- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/140_connector_update_native.yml
+++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/140_connector_update_native.yml
@@ -73,3 +73,43 @@ setup:
             field_1: test
             field_2: something
 
+---
+"Update Connector Native - changing connector to Elastic-managed wrong index name":
+
+  - do:
+      connector.put:
+        connector_id: test-connector-1
+        body:
+          is_native: false
+          index_name: super-connector
+
+  - do:
+      catch: "bad_request"
+      connector.update_native:
+        connector_id: test-connector-1
+        body:
+          is_native: true
+
+---
+"Update Connector Native - changing connector to Elastic-managed correct index name":
+
+  - do:
+      connector.put:
+        connector_id: test-connector-1
+        body:
+          is_native: false
+          index_name: content-super-connector
+
+  - do:
+      connector.update_native:
+        connector_id: test-connector-1
+        body:
+          is_native: true
+
+  - match: { result: updated }
+
+  - do:
+      connector.get:
+        connector_id: test-connector-1
+
+  - match: { is_native: true }
diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/15_connector_post.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/15_connector_post.yml
index 634f99cd53fde..4acca493c42c8 100644
--- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/15_connector_post.yml
+++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/15_connector_post.yml
@@ -103,6 +103,18 @@ setup:
           service_type: super-connector
 
 
+---
+'Create Connector - Invalid Managed Connector Index Prefix':
+  - do:
+      catch: "bad_request"
+      connector.post:
+        body:
+          index_name: wrong-prefix-index
+          name: my-connector
+          language: pl
+          is_native: true
+          service_type: super-connector
+
 ---
 'Create Connector - Index name used by another connector':
   - do:
diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml
index 697b0ee419181..c54d764fc4761 100644
--- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml
+++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml
@@ -71,7 +71,7 @@ setup:
   - match: { results.1.language: "nl" }
 
 ---
-"List Connector- with size":
+"List Connector - with size":
   - do:
       connector.list:
         size: 2
@@ -106,7 +106,6 @@ setup:
 
   - match: { count: 0 }
 
-
 ---
 "List Connector - filter by index names":
   - do:
@@ -279,4 +278,145 @@ setup:
       connector.list: { }
 
 
+---
+"List Connectors - Soft deleted connectors / no deleted":
+  - requires:
+      cluster_features: ["connector_soft_deletes"]
+      reason: Soft deletes were introduced in 9.0 release
+
+  - do:
+      connector.list:
+        include_deleted: true
+
+  - match: { count: 3 }
+
+
+---
+"List Connectors - Single soft deleted connector":
+  - requires:
+      cluster_features: ["connector_soft_deletes"]
+      reason: Soft deletes were introduced in 9.0 release
+
+  - do:
+      connector.delete:
+        connector_id: connector-a
+
+  - do:
+      connector.list: {}
+
+  - match: { count: 2 }
+
+  - do:
+      connector.list:
+        include_deleted: true
+
+  - match: { count: 3 }
+
+---
+"List Connectors - Soft deleted connectors":
+  - requires:
+      cluster_features: ["connector_soft_deletes"]
+      reason: Soft deletes were introduced in 9.0 release
+
+  - do:
+      connector.delete:
+        connector_id: connector-a
+  - do:
+      connector.delete:
+        connector_id: connector-b
+  - do:
+      connector.delete:
+        connector_id: connector-c
+
+  - do:
+      connector.list:
+        include_deleted: true
+
+  - match: { count: 3 }
+
+  # Alphabetical order by index_name for results
+  - match: { results.0.id: "connector-b" }
+  - match: { results.0.index_name: "content-search-2-test" }
+  - match: { results.0.language: "en" }
+  - match: { results.0.deleted: true }
+
+  - match: { results.1.id: "connector-a" }
+  - match: { results.1.index_name: "search-1-test" }
+  - match: { results.1.language: "pl" }
+  - match: { results.1.deleted: true }
+
+  - match: { results.2.id: "connector-c" }
+  - match: { results.2.index_name: "search-3-test" }
+  - match: { results.2.language: "nl" }
+  - match: { results.2.deleted: true }
+
+
+---
+"List Connectors - Soft deleted with from":
+  - requires:
+      cluster_features: ["connector_soft_deletes"]
+      reason: Soft deletes were introduced in 9.0 release
+
+  - do:
+      connector.delete:
+        connector_id: connector-a
+  - do:
+      connector.delete:
+        connector_id: connector-b
+  - do:
+      connector.delete:
+        connector_id: connector-c
+
+  - do:
+      connector.list:
+        from: 1
+        include_deleted: true
+
+  - match: { count: 3 }
+
+  # Alphabetical order by index_name for results
+  - match: { results.0.id: "connector-a" }
+  - match: { results.0.index_name: "search-1-test" }
+  - match: { results.0.language: "pl" }
+  - match: { results.0.deleted: true }
+
+  - match: { results.1.id: "connector-c" }
+  - match: { results.1.index_name: "search-3-test" }
+  - match: { results.1.language: "nl" }
+  - match: { results.0.deleted: true }
+
+---
+"List Connector - Soft deleted with size":
+  - requires:
+      cluster_features: ["connector_soft_deletes"]
+      reason: Soft deletes were introduced in 9.0 release
+
+  - do:
+      connector.delete:
+        connector_id: connector-a
+  - do:
+      connector.delete:
+        connector_id: connector-b
+  - do:
+      connector.delete:
+        connector_id: connector-c
+
+  - do:
+      connector.list:
+        size: 2
+        include_deleted: true
+
+  - match: { count: 3 }
+
+  # Alphabetical order by index_name for results
+  - match: { results.0.id: "connector-b" }
+  - match: { results.0.index_name: "content-search-2-test" }
+  - match: { results.0.language: "en" }
+  - match: { results.0.deleted: true }
+
+  - match: { results.1.id: "connector-a" }
+  - match: { results.1.index_name: "search-1-test" }
+  - match: { results.1.language: "pl" }
+  - match: { results.1.deleted: true }
+
 
diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/30_connector_delete.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/30_connector_delete.yml
index 484c5264227f9..aa820732be2e2 100644
--- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/30_connector_delete.yml
+++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/30_connector_delete.yml
@@ -108,6 +108,33 @@ setup:
         connector_id: test-nonexistent-connector
 
 
+---
+"Delete Connector - Supports soft deletes":
+  - requires:
+      cluster_features: ["connector_soft_deletes"]
+      reason: Soft deletes were introduced in 9.0 release
+
+  - do:
+      connector.delete:
+        connector_id: test-connector-to-delete
+
+  - match: { acknowledged: true }
+
+  - do:
+      catch: "missing"
+      connector.get:
+        connector_id: test-connector-to-delete
+
+  - do:
+      connector.get:
+        connector_id: test-connector-to-delete
+        include_deleted: true
+
+  - match: { id: test-connector-to-delete }
+  - match: { index_name: search-1-test }
+  - match: { service_type: super-connector }
+  - match: { name: my-connector }
+
 ---
 "Delete connector fails for unprivileged user":
   - skip:
diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/20_query_ruleset_list.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/20_query_ruleset_list.yml
index 0b98182b39602..f29deb943d957 100644
--- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/20_query_ruleset_list.yml
+++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/20_query_ruleset_list.yml
@@ -306,10 +306,6 @@ teardown:
 
 ---
 'List query rulesets - include rule types':
-  - requires:
-      cluster_features: [ "query_rule_list_types" ]
-      reason: 'List responses updated in 8.15.5 and 8.16.1'
-
   - do:
       query_rules.put_ruleset:
         ruleset_id: a-test-query-ruleset-with-lots-of-criteria
@@ -389,4 +385,4 @@ teardown:
         suffix: 1
         always: 1
   - match: { results.0.rule_type_counts: { pinned: 4, exclude: 1 } }
-  
+
diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/70_query_rule_test.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/70_query_rule_test.yml
index 016d9f10fe77f..433a0f6705c73 100644
--- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/70_query_rule_test.yml
+++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/70_query_rule_test.yml
@@ -1,8 +1,4 @@
 setup:
-  - requires:
-      cluster_features: [ "query_rules.test" ]
-      reason: Introduced in 8.16.0
-
   - do:
       query_rules.put_ruleset:
         ruleset_id: test-ruleset
diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/80_query_rules_retriever.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/80_query_rules_retriever.yml
index 7967516c6ad5a..089a078c62207 100644
--- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/80_query_rules_retriever.yml
+++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/80_query_rules_retriever.yml
@@ -1,8 +1,4 @@
 setup:
-  - requires:
-      cluster_features: 'query_rule_retriever_supported'
-      reason: 'test requires query rule retriever implementation'
-
   - do:
       indices.create:
         index: test-index1
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java
index ba121f2cf865e..0f5c38e6f75ed 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java
@@ -9,21 +9,13 @@
 
 import org.elasticsearch.features.FeatureSpecification;
 import org.elasticsearch.features.NodeFeature;
-import org.elasticsearch.xpack.application.rules.action.ListQueryRulesetsAction;
-import org.elasticsearch.xpack.application.rules.retriever.QueryRuleRetrieverBuilder;
 
 import java.util.Set;
 
-import static org.elasticsearch.xpack.application.rules.action.TestQueryRulesetAction.QUERY_RULES_TEST_API;
-
 public class EnterpriseSearchFeatures implements FeatureSpecification {
 
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(
-            QUERY_RULES_TEST_API,
-            QueryRuleRetrieverBuilder.QUERY_RULE_RETRIEVERS_SUPPORTED,
-            ListQueryRulesetsAction.QUERY_RULE_LIST_TYPES
-        );
+        return Set.of();
     }
 }
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionResolver.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionResolver.java
index f68feebfb5034..405c29b00586d 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionResolver.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionResolver.java
@@ -82,7 +82,7 @@ public List<AnalyticsCollection> collections(ClusterState state, String... expre
         // Listing data streams that are matching the analytics collection pattern.
         List<String> dataStreams = indexNameExpressionResolver.dataStreamNames(
             state,
-            IndicesOptions.lenientExpandOpen(),
+            IndicesOptions.lenientExpandOpenNoSelectors(),
             EVENT_DATA_STREAM_INDEX_PATTERN
         );
 
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/DeleteAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/DeleteAnalyticsCollectionAction.java
index 0001e59aedd94..f6fd758882930 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/DeleteAnalyticsCollectionAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/DeleteAnalyticsCollectionAction.java
@@ -34,7 +34,7 @@ private DeleteAnalyticsCollectionAction() {/* no instances */}
     public static class Request extends MasterNodeRequest<Request> implements ToXContentObject {
         private final String collectionName;
 
-        public static ParseField COLLECTION_NAME_FIELD = new ParseField("collection_name");
+        public static final ParseField COLLECTION_NAME_FIELD = new ParseField("collection_name");
 
         public Request(StreamInput in) throws IOException {
             super(in);
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/GetAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/GetAnalyticsCollectionAction.java
index dd1e341a8d900..833fc64f03f85 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/GetAnalyticsCollectionAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/GetAnalyticsCollectionAction.java
@@ -35,7 +35,7 @@ private GetAnalyticsCollectionAction() {/* no instances */}
     public static class Request extends MasterNodeReadRequest<Request> implements ToXContentObject {
         private final String[] names;
 
-        public static ParseField NAMES_FIELD = new ParseField("names");
+        public static final ParseField NAMES_FIELD = new ParseField("names");
 
         public Request(TimeValue masterNodeTimeout, String[] names) {
             super(masterNodeTimeout);
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/PostAnalyticsEventAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/PostAnalyticsEventAction.java
index 7e5d05e9e222e..7d93c33208e5c 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/PostAnalyticsEventAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/PostAnalyticsEventAction.java
@@ -311,7 +311,7 @@ public RequestBuilder clientAddress(String clientAddress) {
     }
 
     public static class Response extends ActionResponse implements ToXContentObject {
-        public static Response ACCEPTED = new Response(true);
+        public static final Response ACCEPTED = new Response(true);
 
         public static Response readFromStreamInput(StreamInput in) throws IOException {
             boolean accepted = in.readBoolean();
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/DocumentAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/DocumentAnalyticsEventField.java
index bf73be9a5ddd7..1cab0d1911975 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/DocumentAnalyticsEventField.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/DocumentAnalyticsEventField.java
@@ -20,11 +20,11 @@
 
 public class DocumentAnalyticsEventField {
 
-    public static ParseField DOCUMENT_FIELD = new ParseField("document");
+    public static final ParseField DOCUMENT_FIELD = new ParseField("document");
 
-    public static ParseField DOCUMENT_ID_FIELD = new ParseField("id");
+    public static final ParseField DOCUMENT_ID_FIELD = new ParseField("id");
 
-    public static ParseField DOCUMENT_INDEX_FIELD = new ParseField("index");
+    public static final ParseField DOCUMENT_INDEX_FIELD = new ParseField("index");
 
     private static final ObjectParser<Map<String, String>, AnalyticsEvent.Context> PARSER = new ObjectParser<>(
         DOCUMENT_FIELD.getPreferredName(),
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/PageAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/PageAnalyticsEventField.java
index e450d89eead3b..b2436be319190 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/PageAnalyticsEventField.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/PageAnalyticsEventField.java
@@ -17,13 +17,13 @@
 import java.util.Map;
 
 public class PageAnalyticsEventField {
-    public static ParseField PAGE_FIELD = new ParseField("page");
+    public static final ParseField PAGE_FIELD = new ParseField("page");
 
-    public static ParseField PAGE_URL_FIELD = new ParseField("url");
+    public static final ParseField PAGE_URL_FIELD = new ParseField("url");
 
-    public static ParseField PAGE_TITLE_FIELD = new ParseField("title");
+    public static final ParseField PAGE_TITLE_FIELD = new ParseField("title");
 
-    public static ParseField PAGE_REFERRER_FIELD = new ParseField("referrer");
+    public static final ParseField PAGE_REFERRER_FIELD = new ParseField("referrer");
 
     private static final ObjectParser<Map<String, String>, AnalyticsEvent.Context> PARSER = new ObjectParser<>(
         PAGE_FIELD.getPreferredName(),
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/PaginationAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/PaginationAnalyticsEventField.java
index 78dc75befa924..3dca22d94ad7b 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/PaginationAnalyticsEventField.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/PaginationAnalyticsEventField.java
@@ -19,11 +19,11 @@
 
 public class PaginationAnalyticsEventField {
 
-    public static ParseField PAGINATION_FIELD = new ParseField("page");
+    public static final ParseField PAGINATION_FIELD = new ParseField("page");
 
-    public static ParseField CURRENT_PAGE_FIELD = new ParseField("current");
+    public static final ParseField CURRENT_PAGE_FIELD = new ParseField("current");
 
-    public static ParseField PAGE_SIZE_FIELD = new ParseField("size");
+    public static final ParseField PAGE_SIZE_FIELD = new ParseField("size");
 
     private static final ObjectParser<Map<String, Integer>, AnalyticsEvent.Context> PARSER = new ObjectParser<>(
         PAGINATION_FIELD.getPreferredName(),
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchAnalyticsEventField.java
index 27d63f4dd0a69..438fb20dcf0bf 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchAnalyticsEventField.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchAnalyticsEventField.java
@@ -21,13 +21,13 @@
 import static org.elasticsearch.xpack.application.analytics.event.parser.field.SortOrderAnalyticsEventField.SORT_FIELD;
 
 public class SearchAnalyticsEventField {
-    public static ParseField SEARCH_FIELD = new ParseField("search");
+    public static final ParseField SEARCH_FIELD = new ParseField("search");
 
-    public static ParseField SEARCH_QUERY_FIELD = new ParseField("query");
+    public static final ParseField SEARCH_QUERY_FIELD = new ParseField("query");
 
-    public static ParseField SEARCH_APPLICATION_FIELD = new ParseField("search_application");
+    public static final ParseField SEARCH_APPLICATION_FIELD = new ParseField("search_application");
 
-    public static ParseField SEARCH_RESULTS_FIELD = new ParseField("results");
+    public static final ParseField SEARCH_RESULTS_FIELD = new ParseField("results");
 
     private static final ObjectParser<Map<String, Object>, AnalyticsEvent.Context> PARSER = new ObjectParser<>(
         SEARCH_FIELD.getPreferredName(),
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchFiltersAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchFiltersAnalyticsEventField.java
index dc4ef6f816b9b..261a6c623a7a5 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchFiltersAnalyticsEventField.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchFiltersAnalyticsEventField.java
@@ -19,7 +19,7 @@
 import java.util.Map;
 
 public class SearchFiltersAnalyticsEventField {
-    public static ParseField SEARCH_FILTERS_FIELD = new ParseField("filters");
+    public static final ParseField SEARCH_FILTERS_FIELD = new ParseField("filters");
 
     private static final ObjectParser<Map<String, List<String>>, AnalyticsEvent.Context> PARSER = new ObjectParser<>(
         SEARCH_FILTERS_FIELD.getPreferredName(),
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchResultAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchResultAnalyticsEventField.java
index a680a151142d7..e95e777bb88ec 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchResultAnalyticsEventField.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchResultAnalyticsEventField.java
@@ -20,9 +20,9 @@
 import static org.elasticsearch.xpack.application.analytics.event.parser.field.PageAnalyticsEventField.PAGE_FIELD;
 
 public class SearchResultAnalyticsEventField {
-    public static ParseField SEARCH_RESULTS_TOTAL_FIELD = new ParseField("total_results");
+    public static final ParseField SEARCH_RESULTS_TOTAL_FIELD = new ParseField("total_results");
 
-    public static ParseField SEARCH_RESULT_ITEMS_FIELD = new ParseField("items");
+    public static final ParseField SEARCH_RESULT_ITEMS_FIELD = new ParseField("items");
 
     private static final ObjectParser<Map<String, Object>, AnalyticsEvent.Context> PARSER = new ObjectParser<>(
         "search_results",
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SessionAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SessionAnalyticsEventField.java
index 406d59a7d081e..866984c62b75e 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SessionAnalyticsEventField.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SessionAnalyticsEventField.java
@@ -21,9 +21,9 @@
 import static org.elasticsearch.common.Strings.requireNonBlank;
 
 public class SessionAnalyticsEventField {
-    public static ParseField SESSION_FIELD = new ParseField("session");
+    public static final ParseField SESSION_FIELD = new ParseField("session");
 
-    public static ParseField SESSION_ID_FIELD = new ParseField("id");
+    public static final ParseField SESSION_ID_FIELD = new ParseField("id");
 
     public static final ParseField CLIENT_ADDRESS_FIELD = new ParseField("ip");
 
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SortOrderAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SortOrderAnalyticsEventField.java
index 5b70bd8c91c2c..df591572a7672 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SortOrderAnalyticsEventField.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SortOrderAnalyticsEventField.java
@@ -20,11 +20,11 @@
 
 public class SortOrderAnalyticsEventField {
 
-    public static ParseField SORT_FIELD = new ParseField("sort");
+    public static final ParseField SORT_FIELD = new ParseField("sort");
 
-    public static ParseField SORT_ORDER_NAME_FIELD = new ParseField("name");
+    public static final ParseField SORT_ORDER_NAME_FIELD = new ParseField("name");
 
-    public static ParseField SORT_ORDER_DIRECTION_FIELD = new ParseField("direction");
+    public static final ParseField SORT_ORDER_DIRECTION_FIELD = new ParseField("direction");
 
     private static final ObjectParser<Map<String, String>, AnalyticsEvent.Context> PARSER = new ObjectParser<>(
         SORT_FIELD.getPreferredName(),
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/UserAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/UserAnalyticsEventField.java
index dbcedde825725..186f581d4b6ff 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/UserAnalyticsEventField.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/UserAnalyticsEventField.java
@@ -20,9 +20,9 @@
 import static org.elasticsearch.common.Strings.requireNonBlank;
 
 public class UserAnalyticsEventField {
-    public static ParseField USER_FIELD = new ParseField("user");
+    public static final ParseField USER_FIELD = new ParseField("user");
 
-    public static ParseField USER_ID_FIELD = new ParseField("id");
+    public static final ParseField USER_ID_FIELD = new ParseField("id");
 
     private static final ObjectParser<Map<String, String>, AnalyticsEvent.Context> PARSER = new ObjectParser<>(
         USER_FIELD.getPreferredName(),
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java
index 46275bb623b7a..3683cdce5ad8f 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java
@@ -9,9 +9,6 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.NamedWriteable;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -63,9 +60,10 @@
  *     <li>A {@link ConnectorStatus} indicating the current status of the connector.</li>
  *     <li>A sync cursor, used for incremental syncs.</li>
  *     <li>A boolean flag 'syncNow', which, when set, triggers an immediate synchronization operation.</li>
+ *     <li>A boolean flag 'isDeleted', when set indicates that connector has been soft-deleted. </li>
  * </ul>
  */
-public class Connector implements NamedWriteable, ToXContentObject {
+public class Connector implements ToXContentObject {
 
     public static final String NAME = Connector.class.getName().toUpperCase(Locale.ROOT);
 
@@ -106,6 +104,7 @@ public class Connector implements NamedWriteable, ToXContentObject {
     @Nullable
     private final Object syncCursor;
     private final boolean syncNow;
+    private final boolean isDeleted;
 
     /**
      * Constructor for Connector.
@@ -132,6 +131,7 @@ public class Connector implements NamedWriteable, ToXContentObject {
      * @param status             Current status of the connector.
      * @param syncCursor         Position or state indicating the current point of synchronization.
      * @param syncNow            Flag indicating whether an immediate synchronization is requested.
+     * @param isDeleted          Flag indicating whether connector has been soft-deleted.
      */
     private Connector(
         String connectorId,
@@ -155,7 +155,8 @@ private Connector(
         String serviceType,
         ConnectorStatus status,
         Object syncCursor,
-        boolean syncNow
+        boolean syncNow,
+        Boolean isDeleted
     ) {
         this.connectorId = connectorId;
         this.apiKeyId = apiKeyId;
@@ -179,31 +180,7 @@ private Connector(
         this.status = status;
         this.syncCursor = syncCursor;
         this.syncNow = syncNow;
-    }
-
-    public Connector(StreamInput in) throws IOException {
-        this.connectorId = in.readOptionalString();
-        this.apiKeyId = in.readOptionalString();
-        this.apiKeySecretId = in.readOptionalString();
-        this.configuration = in.readMap(ConnectorConfiguration::new);
-        this.customScheduling = in.readMap(ConnectorCustomSchedule::new);
-        this.description = in.readOptionalString();
-        this.error = in.readOptionalString();
-        this.features = in.readOptionalWriteable(ConnectorFeatures::new);
-        this.filtering = in.readOptionalCollectionAsList(ConnectorFiltering::new);
-        this.syncJobFiltering = in.readOptionalWriteable(FilteringRules::new);
-        this.indexName = in.readOptionalString();
-        this.isNative = in.readBoolean();
-        this.language = in.readOptionalString();
-        this.lastSeen = in.readOptionalInstant();
-        this.syncInfo = in.readOptionalWriteable(ConnectorSyncInfo::new);
-        this.name = in.readOptionalString();
-        this.pipeline = in.readOptionalWriteable(ConnectorIngestPipeline::new);
-        this.scheduling = in.readOptionalWriteable(ConnectorScheduling::new);
-        this.serviceType = in.readOptionalString();
-        this.status = in.readEnum(ConnectorStatus.class);
-        this.syncCursor = in.readGenericValue();
-        this.syncNow = in.readBoolean();
+        this.isDeleted = isDeleted;
     }
 
     public static final ParseField ID_FIELD = new ParseField("id");
@@ -226,6 +203,7 @@ public Connector(StreamInput in) throws IOException {
     public static final ParseField STATUS_FIELD = new ParseField("status");
     public static final ParseField SYNC_CURSOR_FIELD = new ParseField("sync_cursor");
     static final ParseField SYNC_NOW_FIELD = new ParseField("sync_now");
+    public static final ParseField IS_DELETED_FIELD = new ParseField("deleted");
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<Connector, String> PARSER = new ConstructingObjectParser<>(
@@ -265,7 +243,8 @@ public Connector(StreamInput in) throws IOException {
                 .setServiceType((String) args[i++])
                 .setStatus((ConnectorStatus) args[i++])
                 .setSyncCursor(args[i++])
-                .setSyncNow((Boolean) args[i])
+                .setSyncNow((Boolean) args[i++])
+                .setIsDeleted((Boolean) args[i])
                 .build();
         }
     );
@@ -357,6 +336,7 @@ public String getAccessControlIndexName() {
         );
         PARSER.declareObjectOrNull(optionalConstructorArg(), (p, c) -> p.map(), null, SYNC_CURSOR_FIELD);
         PARSER.declareBoolean(optionalConstructorArg(), SYNC_NOW_FIELD);
+        PARSER.declareBoolean(optionalConstructorArg(), IS_DELETED_FIELD);
     }
 
     public static Connector fromXContentBytes(BytesReference source, String docId, XContentType xContentType) {
@@ -433,6 +413,7 @@ public void toInnerXContent(XContentBuilder builder, Params params) throws IOExc
             builder.field(STATUS_FIELD.getPreferredName(), status.toString());
         }
         builder.field(SYNC_NOW_FIELD.getPreferredName(), syncNow);
+        builder.field(IS_DELETED_FIELD.getPreferredName(), isDeleted);
     }
 
     @Override
@@ -445,32 +426,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeOptionalString(connectorId);
-        out.writeOptionalString(apiKeyId);
-        out.writeOptionalString(apiKeySecretId);
-        out.writeMap(configuration, StreamOutput::writeWriteable);
-        out.writeMap(customScheduling, StreamOutput::writeWriteable);
-        out.writeOptionalString(description);
-        out.writeOptionalString(error);
-        out.writeOptionalWriteable(features);
-        out.writeOptionalCollection(filtering);
-        out.writeOptionalWriteable(syncJobFiltering);
-        out.writeOptionalString(indexName);
-        out.writeBoolean(isNative);
-        out.writeOptionalString(language);
-        out.writeOptionalInstant(lastSeen);
-        out.writeOptionalWriteable(syncInfo);
-        out.writeOptionalString(name);
-        out.writeOptionalWriteable(pipeline);
-        out.writeOptionalWriteable(scheduling);
-        out.writeOptionalString(serviceType);
-        out.writeEnum(status);
-        out.writeGenericValue(syncCursor);
-        out.writeBoolean(syncNow);
-    }
-
     public String getConnectorId() {
         return connectorId;
     }
@@ -559,6 +514,10 @@ public boolean isSyncNow() {
         return syncNow;
     }
 
+    public boolean isDeleted() {
+        return isDeleted;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -566,6 +525,7 @@ public boolean equals(Object o) {
         Connector connector = (Connector) o;
         return isNative == connector.isNative
             && syncNow == connector.syncNow
+            && isDeleted == connector.isDeleted
             && Objects.equals(connectorId, connector.connectorId)
             && Objects.equals(apiKeyId, connector.apiKeyId)
             && Objects.equals(apiKeySecretId, connector.apiKeySecretId)
@@ -612,15 +572,11 @@ public int hashCode() {
             serviceType,
             status,
             syncCursor,
-            syncNow
+            syncNow,
+            isDeleted
         );
     }
 
-    @Override
-    public String getWriteableName() {
-        return NAME;
-    }
-
     public static class Builder {
 
         private String connectorId;
@@ -645,6 +601,7 @@ public static class Builder {
         private ConnectorStatus status = ConnectorStatus.CREATED;
         private Object syncCursor;
         private boolean syncNow;
+        private boolean isDeleted;
 
         public Builder setConnectorId(String connectorId) {
             this.connectorId = connectorId;
@@ -756,6 +713,11 @@ public Builder setSyncNow(Boolean syncNow) {
             return this;
         }
 
+        public Builder setIsDeleted(Boolean isDeleted) {
+            this.isDeleted = Objects.requireNonNullElse(isDeleted, false);
+            return this;
+        }
+
         public Connector build() {
             return new Connector(
                 connectorId,
@@ -779,7 +741,8 @@ public Connector build() {
                 serviceType,
                 status,
                 syncCursor,
-                syncNow
+                syncNow,
+                isDeleted
             );
         }
     }
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java
index 5e1fde0dfb942..bb80f5fee4ec9 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java
@@ -13,9 +13,6 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DelegatingActionListener;
 import org.elasticsearch.action.DocWriteRequest;
-import org.elasticsearch.action.DocWriteResponse;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
@@ -24,6 +21,7 @@
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.query.BoolQueryBuilder;
@@ -76,13 +74,16 @@
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.fromXContentBytesConnectorFiltering;
 import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.sortFilteringRulesByOrder;
+import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.MANAGED_CONNECTOR_INDEX_PREFIX;
+import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN;
 
 /**
  * A service that manages persistent {@link Connector} configurations.
  */
 public class ConnectorIndexService {
 
-    private final Client client;
+    // The client to interact with the system index (internal user).
+    private final Client clientWithOrigin;
 
     public static final String CONNECTOR_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN;
 
@@ -90,7 +91,7 @@ public class ConnectorIndexService {
      * @param client A client for executing actions on the connector index
      */
     public ConnectorIndexService(Client client) {
-        this.client = client;
+        this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN);
     }
 
     /**
@@ -136,7 +137,7 @@ public void createConnector(
                         indexRequest = indexRequest.id(connectorId);
                     }
 
-                    client.index(
+                    clientWithOrigin.index(
                         indexRequest,
                         listener.delegateFailureAndWrap(
                             (ll, indexResponse) -> ll.onResponse(
@@ -193,14 +194,17 @@ private Connector createConnectorWithDefaultValues(
     /**
      * Gets the {@link Connector} from the underlying index.
      *
-     * @param connectorId The id of the connector object.
-     * @param listener    The action listener to invoke on response/failure.
+     * @param connectorId     The id of the connector object.
+     * @param includeDeleted  If false, returns only the non-deleted connector with the matching ID;
+     *                        if true, returns the connector with the matching ID.
+     * @param listener        The action listener to invoke on response/failure.
      */
-    public void getConnector(String connectorId, ActionListener<ConnectorSearchResult> listener) {
+    public void getConnector(String connectorId, boolean includeDeleted, ActionListener<ConnectorSearchResult> listener) {
+
         try {
             final GetRequest getRequest = new GetRequest(CONNECTOR_INDEX_NAME).id(connectorId).realtime(true);
 
-            client.get(getRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, getResponse) -> {
+            clientWithOrigin.get(getRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, getResponse) -> {
                 if (getResponse.isExists() == false) {
                     l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
                     return;
@@ -211,6 +215,12 @@ public void getConnector(String connectorId, ActionListener<ConnectorSearchResul
                         .setResultMap(getResponse.getSourceAsMap())
                         .build();
 
+                    boolean connectorIsSoftDeleted = includeDeleted == false && isConnectorDeleted(connector);
+
+                    if (connectorIsSoftDeleted) {
+                        l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                        return;
+                    }
                     l.onResponse(connector);
                 } catch (Exception e) {
                     listener.onFailure(e);
@@ -222,33 +232,43 @@ public void getConnector(String connectorId, ActionListener<ConnectorSearchResul
     }
 
     /**
-     * Deletes the {@link Connector} and the related instances of {@link ConnectorSyncJob} in the underlying index.
+     * Soft deletes the {@link Connector} and optionally removes the related instances of {@link ConnectorSyncJob} in the underlying index.
      *
      * @param connectorId          The id of the {@link Connector}.
      * @param shouldDeleteSyncJobs The flag indicating if {@link ConnectorSyncJob} should also be deleted.
      * @param listener             The action listener to invoke on response/failure.
      */
-    public void deleteConnector(String connectorId, boolean shouldDeleteSyncJobs, ActionListener<DeleteResponse> listener) {
-
-        final DeleteRequest deleteRequest = new DeleteRequest(CONNECTOR_INDEX_NAME).id(connectorId)
-            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+    public void deleteConnector(String connectorId, boolean shouldDeleteSyncJobs, ActionListener<UpdateResponse> listener) {
 
         try {
-            client.delete(deleteRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, deleteResponse) -> {
-                if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) {
-                    l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
-                    return;
-                }
-                if (shouldDeleteSyncJobs) {
-                    new ConnectorSyncJobIndexService(client).deleteAllSyncJobsByConnectorId(connectorId, l.map(r -> deleteResponse));
-                } else {
-                    l.onResponse(deleteResponse);
-                }
+            // ensure that if connector is soft-deleted, deleting it again results in 404
+            getConnector(connectorId, false, listener.delegateFailure((l, connector) -> {
+                final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).setRefreshPolicy(
+                    WriteRequest.RefreshPolicy.IMMEDIATE
+                )
+                    .doc(
+                        new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX)
+                            .id(connectorId)
+                            .source(Map.of(Connector.IS_DELETED_FIELD.getPreferredName(), true))
+                    );
+                clientWithOrigin.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> {
+                    if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+                        ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                        return;
+                    }
+                    if (shouldDeleteSyncJobs) {
+                        new ConnectorSyncJobIndexService(clientWithOrigin).deleteAllSyncJobsByConnectorId(
+                            connectorId,
+                            ll.map(r -> updateResponse)
+                        );
+                    } else {
+                        ll.onResponse(updateResponse);
+                    }
+                }));
             }));
         } catch (Exception e) {
             listener.onFailure(e);
         }
-
     }
 
     /**
@@ -260,6 +280,7 @@ public void deleteConnector(String connectorId, boolean shouldDeleteSyncJobs, Ac
      * @param connectorNames Filter connectors by connector names, if provided.
      * @param serviceTypes Filter connectors by service types, if provided.
      * @param searchQuery Apply a wildcard search on index name, connector name, and description, if provided.
+     * @param includeDeleted  If false, filters to include only non-deleted connectors; if true, no filter is applied.
      * @param listener Invoked with search results or upon failure.
      */
     public void listConnectors(
@@ -269,16 +290,17 @@ public void listConnectors(
         List<String> connectorNames,
         List<String> serviceTypes,
         String searchQuery,
+        boolean includeDeleted,
         ActionListener<ConnectorIndexService.ConnectorResult> listener
     ) {
         try {
             final SearchSourceBuilder source = new SearchSourceBuilder().from(from)
                 .size(size)
-                .query(buildListQuery(indexNames, connectorNames, serviceTypes, searchQuery))
+                .query(buildListQuery(indexNames, connectorNames, serviceTypes, searchQuery, includeDeleted))
                 .fetchSource(true)
                 .sort(Connector.INDEX_NAME_FIELD.getPreferredName(), SortOrder.ASC);
             final SearchRequest req = new SearchRequest(CONNECTOR_INDEX_NAME).source(source);
-            client.search(req, new ActionListener<>() {
+            clientWithOrigin.search(req, new ActionListener<>() {
                 @Override
                 public void onResponse(SearchResponse searchResponse) {
                     try {
@@ -310,13 +332,15 @@ public void onFailure(Exception e) {
      * @param connectorNames List of connector names for filtering, or null/empty to skip.
      * @param serviceTypes List of connector service types for filtering, or null/empty to skip.
      * @param searchQuery Search query for wildcard filtering on index name, connector name, and description, or null/empty to skip.
+     * @param includeDeleted  If false, filters to include only non-deleted connectors; if true, no filter is applied.
      * @return A {@link QueryBuilder} customized based on provided filters.
      */
     private QueryBuilder buildListQuery(
         List<String> indexNames,
         List<String> connectorNames,
         List<String> serviceTypes,
-        String searchQuery
+        String searchQuery,
+        boolean includeDeleted
     ) {
         boolean filterByIndexNames = indexNames != null && indexNames.isEmpty() == false;
         boolean filterByConnectorNames = indexNames != null && connectorNames.isEmpty() == false;
@@ -348,7 +372,12 @@ private QueryBuilder buildListQuery(
                     );
             }
         }
-        return usesFilter ? boolFilterQueryBuilder : new MatchAllQueryBuilder();
+
+        if (includeDeleted == false) {
+            boolFilterQueryBuilder.mustNot(new TermQueryBuilder(Connector.IS_DELETED_FIELD.getPreferredName(), true));
+        }
+
+        return boolFilterQueryBuilder;
     }
 
     /**
@@ -367,7 +396,7 @@ public void updateConnectorConfiguration(UpdateConnectorConfigurationAction.Requ
             Map<String, Object> configurationValues = request.getConfigurationValues();
             String connectorId = request.getConnectorId();
 
-            getConnector(connectorId, listener.delegateFailure((l, connector) -> {
+            getConnector(connectorId, false, listener.delegateFailure((l, connector) -> {
 
                 UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).setRefreshPolicy(
                     WriteRequest.RefreshPolicy.IMMEDIATE
@@ -453,7 +482,7 @@ else if (configurationValues != null) {
                     return;
                 }
 
-                client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> {
+                clientWithOrigin.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> {
                     if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
                         ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
                         return;
@@ -490,13 +519,16 @@ public void updateConnectorError(String connectorId, String error, ActionListene
                         }
                     })
             );
-            client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
-                if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
-                    l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
-                    return;
-                }
-                l.onResponse(updateResponse);
-            }));
+            clientWithOrigin.update(
+                updateRequest,
+                new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
+                    if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+                        l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                        return;
+                    }
+                    l.onResponse(updateResponse);
+                })
+            );
         } catch (Exception e) {
             listener.onFailure(e);
         }
@@ -518,13 +550,16 @@ public void updateConnectorNameOrDescription(UpdateConnectorNameAction.Request r
                     .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                     .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS))
             );
-            client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
-                if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
-                    l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
-                    return;
-                }
-                l.onResponse(updateResponse);
-            }));
+            clientWithOrigin.update(
+                updateRequest,
+                new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
+                    if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+                        l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                        return;
+                    }
+                    l.onResponse(updateResponse);
+                })
+            );
         } catch (Exception e) {
             listener.onFailure(e);
         }
@@ -545,13 +580,16 @@ public void updateConnectorFiltering(String connectorId, List<ConnectorFiltering
                     .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                     .source(Map.of(Connector.FILTERING_FIELD.getPreferredName(), filtering))
             );
-            client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
-                if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
-                    l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
-                    return;
-                }
-                l.onResponse(updateResponse);
-            }));
+            clientWithOrigin.update(
+                updateRequest,
+                new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
+                    if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+                        l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                        return;
+                    }
+                    l.onResponse(updateResponse);
+                })
+            );
         } catch (Exception e) {
             listener.onFailure(e);
         }
@@ -572,13 +610,16 @@ public void updateConnectorFeatures(String connectorId, ConnectorFeatures featur
                     .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                     .source(Map.of(Connector.FEATURES_FIELD.getPreferredName(), features))
             );
-            client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
-                if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
-                    l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
-                    return;
-                }
-                l.onResponse(updateResponse);
-            }));
+            clientWithOrigin.update(
+                updateRequest,
+                new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
+                    if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+                        l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                        return;
+                    }
+                    l.onResponse(updateResponse);
+                })
+            );
         } catch (Exception e) {
             listener.onFailure(e);
         }
@@ -599,7 +640,7 @@ public void updateConnectorFilteringDraft(
         ActionListener<UpdateResponse> listener
     ) {
         try {
-            getConnector(connectorId, listener.delegateFailure((l, connector) -> {
+            getConnector(connectorId, false, listener.delegateFailure((l, connector) -> {
                 List<ConnectorFiltering> connectorFilteringList = fromXContentBytesConnectorFiltering(
                     connector.getSourceRef(),
                     XContentType.JSON
@@ -634,13 +675,16 @@ public void updateConnectorFilteringDraft(
                         .source(Map.of(Connector.FILTERING_FIELD.getPreferredName(), List.of(connectorFilteringWithUpdatedDraft)))
                 );
 
-                client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> {
-                    if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
-                        ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
-                        return;
-                    }
-                    ll.onResponse(updateResponse);
-                }));
+                clientWithOrigin.update(
+                    updateRequest,
+                    new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> {
+                        if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+                            ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                            return;
+                        }
+                        ll.onResponse(updateResponse);
+                    })
+                );
             }));
 
         } catch (Exception e) {
@@ -659,7 +703,7 @@ public void updateConnectorDraftFilteringValidation(
         FilteringValidationInfo validation,
         ActionListener<UpdateResponse> listener
     ) {
-        getConnector(connectorId, listener.delegateFailure((l, connector) -> {
+        getConnector(connectorId, false, listener.delegateFailure((l, connector) -> {
             try {
                 List<ConnectorFiltering> connectorFilteringList = fromXContentBytesConnectorFiltering(
                     connector.getSourceRef(),
@@ -682,7 +726,7 @@ public void updateConnectorDraftFilteringValidation(
                         .source(Map.of(Connector.FILTERING_FIELD.getPreferredName(), List.of(activatedConnectorFiltering)))
                 );
 
-                client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> {
+                clientWithOrigin.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> {
                     if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
                         ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
                         return;
@@ -703,7 +747,7 @@ public void updateConnectorDraftFilteringValidation(
      * @param listener     Listener to respond to a successful response or an error.
      */
     public void activateConnectorDraftFiltering(String connectorId, ActionListener<UpdateResponse> listener) {
-        getConnector(connectorId, listener.delegateFailure((l, connector) -> {
+        getConnector(connectorId, false, listener.delegateFailure((l, connector) -> {
             try {
                 List<ConnectorFiltering> connectorFilteringList = fromXContentBytesConnectorFiltering(
                     connector.getSourceRef(),
@@ -739,7 +783,7 @@ public void activateConnectorDraftFiltering(String connectorId, ActionListener<U
                         .source(Map.of(Connector.FILTERING_FIELD.getPreferredName(), List.of(activatedConnectorFiltering)))
                 );
 
-                client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> {
+                clientWithOrigin.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> {
                     if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
                         ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
                         return;
@@ -767,13 +811,16 @@ public void checkInConnector(String connectorId, ActionListener<UpdateResponse>
                     .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                     .source(Map.of(Connector.LAST_SEEN_FIELD.getPreferredName(), Instant.now()))
             );
-            client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
-                if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
-                    l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
-                    return;
-                }
-                l.onResponse(updateResponse);
-            }));
+            clientWithOrigin.update(
+                updateRequest,
+                new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
+                    if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+                        l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                        return;
+                    }
+                    l.onResponse(updateResponse);
+                })
+            );
         } catch (Exception e) {
             listener.onFailure(e);
         }
@@ -794,21 +841,24 @@ public void updateConnectorLastSyncStats(UpdateConnectorLastSyncStatsAction.Requ
                     .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                     .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS))
             );
-            client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
-                if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
-                    l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
-                    return;
-                }
-                l.onResponse(updateResponse);
-            }));
+            clientWithOrigin.update(
+                updateRequest,
+                new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
+                    if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+                        l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                        return;
+                    }
+                    l.onResponse(updateResponse);
+                })
+            );
         } catch (Exception e) {
             listener.onFailure(e);
         }
     }
 
     /**
-     * Updates the is_native property of a {@link Connector}. It always sets the {@link ConnectorStatus} to
-     * CONFIGURED.
+     * Updates the is_native property of a {@link Connector}. It sets the {@link ConnectorStatus} to
+     * CONFIGURED when connector is in CONNECTED state to indicate that connector needs to reconnect.
      *
      * @param request  The request for updating the connector's is_native property.
      * @param listener The listener for handling responses, including successful updates or errors.
@@ -816,29 +866,65 @@ public void updateConnectorLastSyncStats(UpdateConnectorLastSyncStatsAction.Requ
     public void updateConnectorNative(UpdateConnectorNativeAction.Request request, ActionListener<UpdateResponse> listener) {
         try {
             String connectorId = request.getConnectorId();
+            boolean isNative = request.isNative();
 
-            final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc(
-                new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX)
-                    .id(connectorId)
-                    .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
-                    .source(
-                        Map.of(
-                            Connector.IS_NATIVE_FIELD.getPreferredName(),
-                            request.isNative(),
-                            Connector.STATUS_FIELD.getPreferredName(),
-                            ConnectorStatus.CONFIGURED.toString()
-                        )
-                    )
+            getConnector(connectorId, false, listener.delegateFailure((l, connector) -> {
 
-            );
-            client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
-                if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
-                    l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                String indexName = getConnectorIndexNameFromSearchResult(connector);
+
+                boolean doesNotHaveContentPrefix = indexName != null && isValidManagedConnectorIndexName(indexName) == false;
+                // Ensure attached content index is prefixed correctly
+                if (isNative && doesNotHaveContentPrefix) {
+                    l.onFailure(
+                        new ElasticsearchStatusException(
+                            "The index name ["
+                                + indexName
+                                + "] attached to the connector ["
+                                + connectorId
+                                + "] must start with the required prefix: ["
+                                + MANAGED_CONNECTOR_INDEX_PREFIX
+                                + "] to be Elastic-managed. Please update the attached index first to comply with this requirement.",
+                            RestStatus.BAD_REQUEST
+                        )
+                    );
                     return;
                 }
-                l.onResponse(updateResponse);
-            }));
 
+                ConnectorStatus status = getConnectorStatusFromSearchResult(connector);
+
+                // If connector was connected already, change its status to CONFIGURED as we need to re-connect
+                boolean isConnected = status == ConnectorStatus.CONNECTED;
+                boolean isValidTransitionToConfigured = ConnectorStateMachine.isValidTransition(status, ConnectorStatus.CONFIGURED);
+                if (isConnected && isValidTransitionToConfigured) {
+                    status = ConnectorStatus.CONFIGURED;
+                }
+
+                final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).setRefreshPolicy(
+                    WriteRequest.RefreshPolicy.IMMEDIATE
+                )
+                    .doc(
+                        new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX)
+                            .id(connectorId)
+                            .source(
+                                Map.of(
+                                    Connector.IS_NATIVE_FIELD.getPreferredName(),
+                                    isNative,
+                                    Connector.STATUS_FIELD.getPreferredName(),
+                                    status.toString()
+                                )
+                            )
+                    );
+                clientWithOrigin.update(
+                    updateRequest,
+                    new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> {
+                        if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+                            ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                            return;
+                        }
+                        ll.onResponse(updateResponse);
+                    })
+                );
+            }));
         } catch (Exception e) {
             listener.onFailure(e);
         }
@@ -860,13 +946,16 @@ public void updateConnectorPipeline(UpdateConnectorPipelineAction.Request reques
                     .source(Map.of(Connector.PIPELINE_FIELD.getPreferredName(), request.getPipeline()))
                     .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS))
             );
-            client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
-                if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
-                    l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
-                    return;
-                }
-                l.onResponse(updateResponse);
-            }));
+            clientWithOrigin.update(
+                updateRequest,
+                new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
+                    if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+                        l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                        return;
+                    }
+                    l.onResponse(updateResponse);
+                })
+            );
         } catch (Exception e) {
             listener.onFailure(e);
         }
@@ -896,22 +985,45 @@ public void updateConnectorIndexName(UpdateConnectorIndexNameAction.Request requ
                     return;
                 }
 
-                final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc(
-                    new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX)
-                        .id(connectorId)
-                        .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
-                        .source(new HashMap<>() {
-                            {
-                                put(Connector.INDEX_NAME_FIELD.getPreferredName(), request.getIndexName());
-                            }
-                        })
-                );
-                client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> {
-                    if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
-                        ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                getConnector(connectorId, false, l.delegateFailure((ll, connector) -> {
+
+                    Boolean isNativeConnector = getConnectorIsNativeFlagFromSearchResult(connector);
+                    Boolean doesNotHaveContentPrefix = indexName != null && isValidManagedConnectorIndexName(indexName) == false;
+
+                    if (isNativeConnector && doesNotHaveContentPrefix) {
+                        ll.onFailure(
+                            new ElasticsearchStatusException(
+                                "Index attached to an Elastic-managed connector must start with the prefix: ["
+                                    + MANAGED_CONNECTOR_INDEX_PREFIX
+                                    + "]. The index name in the payload ["
+                                    + indexName
+                                    + "] doesn't comply with this requirement.",
+                                RestStatus.BAD_REQUEST
+                            )
+                        );
                         return;
                     }
-                    ll.onResponse(updateResponse);
+
+                    final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc(
+                        new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX)
+                            .id(connectorId)
+                            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                            .source(new HashMap<>() {
+                                {
+                                    put(Connector.INDEX_NAME_FIELD.getPreferredName(), request.getIndexName());
+                                }
+                            })
+                    );
+                    clientWithOrigin.update(
+                        updateRequest,
+                        new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (lll, updateResponse) -> {
+                            if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+                                lll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                                return;
+                            }
+                            lll.onResponse(updateResponse);
+                        })
+                    );
                 }));
             }));
 
@@ -935,13 +1047,16 @@ public void updateConnectorScheduling(UpdateConnectorSchedulingAction.Request re
                     .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                     .source(Map.of(Connector.SCHEDULING_FIELD.getPreferredName(), request.getScheduling()))
             );
-            client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
-                if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
-                    l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
-                    return;
-                }
-                l.onResponse(updateResponse);
-            }));
+            clientWithOrigin.update(
+                updateRequest,
+                new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
+                    if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+                        l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                        return;
+                    }
+                    l.onResponse(updateResponse);
+                })
+            );
         } catch (Exception e) {
             listener.onFailure(e);
         }
@@ -956,7 +1071,7 @@ public void updateConnectorScheduling(UpdateConnectorSchedulingAction.Request re
     public void updateConnectorServiceType(UpdateConnectorServiceTypeAction.Request request, ActionListener<UpdateResponse> listener) {
         try {
             String connectorId = request.getConnectorId();
-            getConnector(connectorId, listener.delegateFailure((l, connector) -> {
+            getConnector(connectorId, false, listener.delegateFailure((l, connector) -> {
 
                 ConnectorStatus prevStatus = getConnectorStatusFromSearchResult(connector);
                 ConnectorStatus newStatus = prevStatus == ConnectorStatus.CREATED
@@ -977,7 +1092,7 @@ public void updateConnectorServiceType(UpdateConnectorServiceTypeAction.Request
                         )
 
                 );
-                client.update(
+                clientWithOrigin.update(
                     updateRequest,
                     new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (updateListener, updateResponse) -> {
                         if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
@@ -1003,7 +1118,7 @@ public void updateConnectorStatus(UpdateConnectorStatusAction.Request request, A
         try {
             String connectorId = request.getConnectorId();
             ConnectorStatus newStatus = request.getStatus();
-            getConnector(connectorId, listener.delegateFailure((l, connector) -> {
+            getConnector(connectorId, false, listener.delegateFailure((l, connector) -> {
 
                 ConnectorStatus prevStatus = getConnectorStatusFromSearchResult(connector);
 
@@ -1020,7 +1135,7 @@ public void updateConnectorStatus(UpdateConnectorStatusAction.Request request, A
                         .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                         .source(Map.of(Connector.STATUS_FIELD.getPreferredName(), request.getStatus()))
                 );
-                client.update(
+                clientWithOrigin.update(
                     updateRequest,
                     new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (updateListener, updateResponse) -> {
                         if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
@@ -1048,13 +1163,16 @@ public void updateConnectorApiKeyIdOrApiKeySecretId(
                     .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                     .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS))
             );
-            client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
-                if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
-                    l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
-                    return;
-                }
-                l.onResponse(updateResponse);
-            }));
+            clientWithOrigin.update(
+                updateRequest,
+                new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
+                    if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+                        l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                        return;
+                    }
+                    l.onResponse(updateResponse);
+                })
+            );
         } catch (Exception e) {
             listener.onFailure(e);
         }
@@ -1064,11 +1182,28 @@ private ConnectorStatus getConnectorStatusFromSearchResult(ConnectorSearchResult
         return ConnectorStatus.connectorStatus((String) searchResult.getResultMap().get(Connector.STATUS_FIELD.getPreferredName()));
     }
 
+    private Boolean getConnectorIsNativeFlagFromSearchResult(ConnectorSearchResult searchResult) {
+        return (Boolean) searchResult.getResultMap().get(Connector.IS_NATIVE_FIELD.getPreferredName());
+    }
+
+    private String getConnectorIndexNameFromSearchResult(ConnectorSearchResult searchResult) {
+        return (String) searchResult.getResultMap().get(Connector.INDEX_NAME_FIELD.getPreferredName());
+    }
+
+    private boolean isValidManagedConnectorIndexName(String indexName) {
+        return indexName.startsWith(MANAGED_CONNECTOR_INDEX_PREFIX);
+    }
+
     @SuppressWarnings("unchecked")
     private Map<String, Object> getConnectorConfigurationFromSearchResult(ConnectorSearchResult searchResult) {
         return (Map<String, Object>) searchResult.getResultMap().get(Connector.CONFIGURATION_FIELD.getPreferredName());
     }
 
+    private boolean isConnectorDeleted(ConnectorSearchResult searchResult) {
+        Boolean isDeletedFlag = (Boolean) searchResult.getResultMap().get(Connector.IS_DELETED_FIELD.getPreferredName());
+        return Boolean.TRUE.equals(isDeletedFlag);
+    }
+
     private static ConnectorIndexService.ConnectorResult mapSearchResponseToConnectorList(SearchResponse response) {
         final List<ConnectorSearchResult> connectorResults = Arrays.stream(response.getHits().getHits())
             .map(ConnectorIndexService::hitToConnector)
@@ -1088,7 +1223,7 @@ private static ConnectorSearchResult hitToConnector(SearchHit searchHit) {
 
     /**
      * This method determines if any documents in the connector index have the same index name as the one specified,
-     * excluding the document with the given _id if it is provided.
+     * excluding the docs marked as deleted (soft-deleted) and document with the given _id if it is provided.
      *
      * @param indexName    The name of the index to check for existence in the connector index.
      * @param connectorId  The ID of the {@link Connector} to exclude from the search. Can be null if no document should be excluded.
@@ -1104,6 +1239,9 @@ private void isDataIndexNameAlreadyInUse(String indexName, String connectorId, A
 
             boolFilterQueryBuilder.must().add(new TermQueryBuilder(Connector.INDEX_NAME_FIELD.getPreferredName(), indexName));
 
+            // exclude soft-deleted connectors
+            boolFilterQueryBuilder.mustNot(new TermQueryBuilder(Connector.IS_DELETED_FIELD.getPreferredName(), true));
+
             // If we know the connector _id, exclude this from search query
             if (connectorId != null) {
                 boolFilterQueryBuilder.mustNot(new IdsQueryBuilder().addIds(connectorId));
@@ -1112,7 +1250,7 @@ private void isDataIndexNameAlreadyInUse(String indexName, String connectorId, A
             final SearchSourceBuilder searchSource = new SearchSourceBuilder().query(boolFilterQueryBuilder);
 
             final SearchRequest searchRequest = new SearchRequest(CONNECTOR_INDEX_NAME).source(searchSource);
-            client.search(searchRequest, new ActionListener<>() {
+            clientWithOrigin.search(searchRequest, new ActionListener<>() {
                 @Override
                 public void onResponse(SearchResponse searchResponse) {
                     boolean indexNameIsInUse = searchResponse.getHits().getTotalHits().value() > 0L;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java
index 9b8cc7cfdbe4f..fd35acc89db5c 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java
@@ -45,11 +45,9 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry {
     public static final String ACCESS_CONTROL_INDEX_NAME_PATTERN = ".search-acl-filter-*";
     public static final String ACCESS_CONTROL_TEMPLATE_NAME = "search-acl-filter";
 
-    // Pipeline constants
-
-    public static final String ENT_SEARCH_GENERIC_PIPELINE_NAME = "ent-search-generic-ingestion";
-    public static final String ENT_SEARCH_GENERIC_PIPELINE_FILE = "generic_ingestion_pipeline";
+    public static final String MANAGED_CONNECTOR_INDEX_PREFIX = "content-";
 
+    // Pipeline constants
     public static final String SEARCH_DEFAULT_PIPELINE_NAME = "search-default-ingestion";
     public static final String SEARCH_DEFAULT_PIPELINE_FILE = "search_default_pipeline";
 
@@ -109,12 +107,6 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry {
     @Override
     protected List<IngestPipelineConfig> getIngestPipelines() {
         return List.of(
-            new JsonIngestPipelineConfig(
-                ENT_SEARCH_GENERIC_PIPELINE_NAME,
-                ROOT_RESOURCE_PATH + ENT_SEARCH_GENERIC_PIPELINE_FILE + ".json",
-                REGISTRY_VERSION,
-                TEMPLATE_VERSION_VARIABLE
-            ),
             new JsonIngestPipelineConfig(
                 SEARCH_DEFAULT_PIPELINE_NAME,
                 ROOT_RESOURCE_PATH + SEARCH_DEFAULT_PIPELINE_FILE + ".json",
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java
index 1799121505da5..723ff12b3b1e1 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java
@@ -9,22 +9,19 @@
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.IndicesRequest;
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.indices.InvalidIndexNameException;
-import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry;
 
 import java.io.IOException;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.MANAGED_CONNECTOR_INDEX_PREFIX;
 
 /**
- * Abstract base class for action requests targeting the connectors index. Implements {@link org.elasticsearch.action.IndicesRequest}
- * to ensure index-level privilege support. This class defines the connectors index as the target for all derived action requests.
+ * Abstract base class for action requests targeting the connectors index.
  */
-public abstract class ConnectorActionRequest extends ActionRequest implements IndicesRequest {
+public abstract class ConnectorActionRequest extends ActionRequest {
 
     public ConnectorActionRequest() {
         super();
@@ -52,13 +49,29 @@ public ActionRequestValidationException validateIndexName(String indexName, Acti
         return validationException;
     }
 
-    @Override
-    public String[] indices() {
-        return new String[] { ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN };
-    }
-
-    @Override
-    public IndicesOptions indicesOptions() {
-        return IndicesOptions.lenientExpandHidden();
+    /**
+     * Validates that the given index name starts with the required prefix for Elastic-managed connectors.
+     * If the index name does not start with the required prefix, the validation exception is updated with an error message.
+     *
+     * @param indexName The index name to validate. If null, no validation is performed.
+     * @param validationException The exception to accumulate validation errors.
+     * @return The updated or original {@code validationException} with any new validation errors added,
+     *         if the index name does not start with the required prefix.
+     */
+    public ActionRequestValidationException validateManagedConnectorIndexPrefix(
+        String indexName,
+        ActionRequestValidationException validationException
+    ) {
+        if (indexName != null && indexName.startsWith(MANAGED_CONNECTOR_INDEX_PREFIX) == false) {
+            return addValidationError(
+                "Index ["
+                    + indexName
+                    + "] is invalid. Index attached to an Elastic-managed connector must start with the prefix: ["
+                    + MANAGED_CONNECTOR_INDEX_PREFIX
+                    + "]",
+                validationException
+            );
+        }
+        return validationException;
     }
 }
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java
index 930068a2a46ef..5d98f9703ecea 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -28,7 +27,7 @@
 
 public class DeleteConnectorAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/delete";
+    public static final String NAME = "cluster:admin/xpack/connector/delete";
     public static final ActionType<AcknowledgedResponse> INSTANCE = new ActionType<>(NAME);
 
     private DeleteConnectorAction() {/* no instances */}
@@ -71,14 +70,6 @@ public boolean shouldDeleteSyncJobs() {
             return deleteSyncJobs;
         }
 
-        @Override
-        public String[] indices() {
-            // When deleting a connector, corresponding sync jobs can also be deleted
-            return new String[] {
-                ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN,
-                ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN };
-        }
-
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java
index 2edd47b1fce3a..6f3401e6d85b9 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -25,10 +26,11 @@
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 public class GetConnectorAction {
 
-    public static final String NAME = "indices:data/read/xpack/connector/get";
+    public static final String NAME = "cluster:admin/xpack/connector/get";
     public static final ActionType<GetConnectorAction.Response> INSTANCE = new ActionType<>(NAME);
 
     private GetConnectorAction() {/* no instances */}
@@ -36,22 +38,25 @@ private GetConnectorAction() {/* no instances */}
     public static class Request extends ConnectorActionRequest implements ToXContentObject {
 
         private final String connectorId;
+        private final Boolean includeDeleted;
 
         private static final ParseField CONNECTOR_ID_FIELD = new ParseField("connector_id");
 
-        public Request(String connectorId) {
-            this.connectorId = connectorId;
-        }
+        private static final ParseField INCLUDE_DELETED_FIELD = new ParseField("include_deleted");
 
-        public Request(StreamInput in) throws IOException {
-            super(in);
-            this.connectorId = in.readString();
+        public Request(String connectorId, Boolean includeDeleted) {
+            this.connectorId = connectorId;
+            this.includeDeleted = includeDeleted;
         }
 
         public String getConnectorId() {
             return connectorId;
         }
 
+        public Boolean getIncludeDeleted() {
+            return includeDeleted;
+        }
+
         @Override
         public ActionRequestValidationException validate() {
             ActionRequestValidationException validationException = null;
@@ -65,8 +70,7 @@ public ActionRequestValidationException validate() {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeString(connectorId);
+            TransportAction.localOnly();
         }
 
         @Override
@@ -74,12 +78,12 @@ public boolean equals(Object o) {
             if (this == o) return true;
             if (o == null || getClass() != o.getClass()) return false;
             Request request = (Request) o;
-            return Objects.equals(connectorId, request.connectorId);
+            return Objects.equals(connectorId, request.connectorId) && Objects.equals(includeDeleted, request.includeDeleted);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(connectorId);
+            return Objects.hash(connectorId, includeDeleted);
         }
 
         @Override
@@ -87,6 +91,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.startObject();
             {
                 builder.field(CONNECTOR_ID_FIELD.getPreferredName(), connectorId);
+                builder.field(INCLUDE_DELETED_FIELD.getPreferredName(), includeDeleted);
             }
             builder.endObject();
             return builder;
@@ -95,11 +100,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         private static final ConstructingObjectParser<Request, Void> PARSER = new ConstructingObjectParser<>(
             "get_connector_request",
             false,
-            (p) -> new Request((String) p[0])
+            (p) -> new Request((String) p[0], (Boolean) p[1])
 
         );
         static {
             PARSER.declareString(constructorArg(), CONNECTOR_ID_FIELD);
+            PARSER.declareBoolean(optionalConstructorArg(), INCLUDE_DELETED_FIELD);
         }
 
         public static Request parse(XContentParser parser) {
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java
index e543d805b7099..048f651ab2a1c 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -34,7 +35,7 @@
 
 public class ListConnectorAction {
 
-    public static final String NAME = "indices:data/read/xpack/connector/list";
+    public static final String NAME = "cluster:admin/xpack/connector/list";
     public static final ActionType<ListConnectorAction.Response> INSTANCE = new ActionType<>(NAME);
 
     private ListConnectorAction() {/* no instances */}
@@ -46,33 +47,28 @@ public static class Request extends ConnectorActionRequest implements ToXContent
         private final List<String> connectorNames;
         private final List<String> connectorServiceTypes;
         private final String connectorSearchQuery;
+        private final Boolean includeDeleted;
 
         private static final ParseField PAGE_PARAMS_FIELD = new ParseField("pageParams");
         private static final ParseField INDEX_NAMES_FIELD = new ParseField("index_names");
         private static final ParseField NAMES_FIELD = new ParseField("names");
         private static final ParseField SEARCH_QUERY_FIELD = new ParseField("query");
-
-        public Request(StreamInput in) throws IOException {
-            super(in);
-            this.pageParams = new PageParams(in);
-            this.indexNames = in.readOptionalStringCollectionAsList();
-            this.connectorNames = in.readOptionalStringCollectionAsList();
-            this.connectorServiceTypes = in.readOptionalStringCollectionAsList();
-            this.connectorSearchQuery = in.readOptionalString();
-        }
+        private static final ParseField INCLUDE_DELETED_FIELD = new ParseField("include_deleted");
 
         public Request(
             PageParams pageParams,
             List<String> indexNames,
             List<String> connectorNames,
             List<String> serviceTypes,
-            String connectorSearchQuery
+            String connectorSearchQuery,
+            Boolean includeDeleted
         ) {
             this.pageParams = pageParams;
             this.indexNames = indexNames;
             this.connectorNames = connectorNames;
             this.connectorServiceTypes = serviceTypes;
             this.connectorSearchQuery = connectorSearchQuery;
+            this.includeDeleted = includeDeleted;
         }
 
         public PageParams getPageParams() {
@@ -95,6 +91,10 @@ public String getConnectorSearchQuery() {
             return connectorSearchQuery;
         }
 
+        public Boolean getIncludeDeleted() {
+            return includeDeleted;
+        }
+
         @Override
         public ActionRequestValidationException validate() {
             ActionRequestValidationException validationException = null;
@@ -114,12 +114,7 @@ public ActionRequestValidationException validate() {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            pageParams.writeTo(out);
-            out.writeOptionalStringCollection(indexNames);
-            out.writeOptionalStringCollection(connectorNames);
-            out.writeOptionalStringCollection(connectorServiceTypes);
-            out.writeOptionalString(connectorSearchQuery);
+            TransportAction.localOnly();
         }
 
         @Override
@@ -131,7 +126,8 @@ public boolean equals(Object o) {
                 && Objects.equals(indexNames, request.indexNames)
                 && Objects.equals(connectorNames, request.connectorNames)
                 && Objects.equals(connectorServiceTypes, request.connectorServiceTypes)
-                && Objects.equals(connectorSearchQuery, request.connectorSearchQuery);
+                && Objects.equals(connectorSearchQuery, request.connectorSearchQuery)
+                && Objects.equals(includeDeleted, request.includeDeleted);
         }
 
         @Override
@@ -147,7 +143,8 @@ public int hashCode() {
                 (List<String>) p[1],
                 (List<String>) p[2],
                 (List<String>) p[3],
-                (String) p[4]
+                (String) p[4],
+                (Boolean) p[5]
             )
         );
 
@@ -157,6 +154,7 @@ public int hashCode() {
             PARSER.declareStringArray(optionalConstructorArg(), NAMES_FIELD);
             PARSER.declareStringArray(optionalConstructorArg(), Connector.SERVICE_TYPE_FIELD);
             PARSER.declareString(optionalConstructorArg(), SEARCH_QUERY_FIELD);
+            PARSER.declareBoolean(optionalConstructorArg(), INCLUDE_DELETED_FIELD);
         }
 
         public static ListConnectorAction.Request parse(XContentParser parser) {
@@ -172,6 +170,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.field(NAMES_FIELD.getPreferredName(), connectorNames);
                 builder.field(Connector.SERVICE_TYPE_FIELD.getPreferredName(), connectorServiceTypes);
                 builder.field(SEARCH_QUERY_FIELD.getPreferredName(), connectorSearchQuery);
+                builder.field(INCLUDE_DELETED_FIELD.getPreferredName(), includeDeleted);
             }
             builder.endObject();
             return builder;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java
index fad349cd31877..b5087634a77f2 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java
@@ -25,7 +25,7 @@
 
 public class PostConnectorAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/post";
+    public static final String NAME = "cluster:admin/xpack/connector/post";
     public static final ActionType<ConnectorCreateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private PostConnectorAction() {/* no instances */}
@@ -127,6 +127,10 @@ public ActionRequestValidationException validate() {
 
             validationException = validateIndexName(indexName, validationException);
 
+            if (Boolean.TRUE.equals(isNative)) {
+                validationException = validateManagedConnectorIndexPrefix(indexName, validationException);
+            }
+
             return validationException;
         }
 
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java
index 687a801ab8fd6..c5922ebfafe1b 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java
@@ -9,7 +9,6 @@
 
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
-import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -27,12 +26,12 @@
 
 public class PutConnectorAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/put";
+    public static final String NAME = "cluster:admin/xpack/connector/put";
     public static final ActionType<ConnectorCreateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private PutConnectorAction() {/* no instances */}
 
-    public static class Request extends ConnectorActionRequest implements IndicesRequest, ToXContentObject {
+    public static class Request extends ConnectorActionRequest implements ToXContentObject {
 
         @Nullable
         private final String connectorId;
@@ -147,6 +146,10 @@ public ActionRequestValidationException validate() {
 
             validationException = validateIndexName(indexName, validationException);
 
+            if (Boolean.TRUE.equals(isNative)) {
+                validationException = validateManagedConnectorIndexPrefix(indexName, validationException);
+            }
+
             return validationException;
         }
 
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java
index 8d3d5914ca695..81960db46ad62 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java
@@ -36,7 +36,8 @@ public List<Route> routes() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
-        GetConnectorAction.Request request = new GetConnectorAction.Request(restRequest.param(CONNECTOR_ID_PARAM));
+        Boolean includeDeleted = restRequest.paramAsBoolean("include_deleted", false);
+        GetConnectorAction.Request request = new GetConnectorAction.Request(restRequest.param(CONNECTOR_ID_PARAM), includeDeleted);
         return channel -> client.execute(GetConnectorAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java
index 765f6eca12290..df33718462911 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java
@@ -43,13 +43,15 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
         List<String> connectorNames = List.of(restRequest.paramAsStringArray("connector_name", new String[0]));
         List<String> serviceTypes = List.of(restRequest.paramAsStringArray("service_type", new String[0]));
         String searchQuery = restRequest.param("query");
+        Boolean includeDeleted = restRequest.paramAsBoolean("include_deleted", false);
 
         ListConnectorAction.Request request = new ListConnectorAction.Request(
             new PageParams(from, size),
             indexNames,
             connectorNames,
             serviceTypes,
-            searchQuery
+            searchQuery,
+            includeDeleted
         );
 
         return channel -> client.execute(ListConnectorAction.INSTANCE, request, new RestToXContentListener<>(channel));
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java
index e0beeb9b19515..98891427ab848 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java
@@ -9,7 +9,7 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.injection.guice.Inject;
@@ -17,23 +17,21 @@
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
 
-public class TransportGetConnectorAction extends HandledTransportAction<GetConnectorAction.Request, GetConnectorAction.Response> {
+public class TransportGetConnectorAction extends TransportAction<GetConnectorAction.Request, GetConnectorAction.Response> {
     protected final ConnectorIndexService connectorIndexService;
 
     @Inject
     public TransportGetConnectorAction(TransportService transportService, ActionFilters actionFilters, Client client) {
-        super(
-            GetConnectorAction.NAME,
-            transportService,
-            actionFilters,
-            GetConnectorAction.Request::new,
-            EsExecutors.DIRECT_EXECUTOR_SERVICE
-        );
+        super(GetConnectorAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.connectorIndexService = new ConnectorIndexService(client);
     }
 
     @Override
     protected void doExecute(Task task, GetConnectorAction.Request request, ActionListener<GetConnectorAction.Response> listener) {
-        connectorIndexService.getConnector(request.getConnectorId(), listener.map(GetConnectorAction.Response::new));
+        connectorIndexService.getConnector(
+            request.getConnectorId(),
+            request.getIncludeDeleted(),
+            listener.map(GetConnectorAction.Response::new)
+        );
     }
 }
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java
index 8ff180aa6189b..408aa6fc12405 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java
@@ -9,7 +9,7 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.injection.guice.Inject;
@@ -18,18 +18,12 @@
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
 import org.elasticsearch.xpack.core.action.util.PageParams;
 
-public class TransportListConnectorAction extends HandledTransportAction<ListConnectorAction.Request, ListConnectorAction.Response> {
+public class TransportListConnectorAction extends TransportAction<ListConnectorAction.Request, ListConnectorAction.Response> {
     protected final ConnectorIndexService connectorIndexService;
 
     @Inject
     public TransportListConnectorAction(TransportService transportService, ActionFilters actionFilters, Client client) {
-        super(
-            ListConnectorAction.NAME,
-            transportService,
-            actionFilters,
-            ListConnectorAction.Request::new,
-            EsExecutors.DIRECT_EXECUTOR_SERVICE
-        );
+        super(ListConnectorAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.connectorIndexService = new ConnectorIndexService(client);
     }
 
@@ -44,6 +38,8 @@ protected void doExecute(Task task, ListConnectorAction.Request request, ActionL
             request.getConnectorNames(),
             request.getConnectorServiceTypes(),
             request.getConnectorSearchQuery(),
+            request.getIncludeDeleted(),
+
             listener.map(r -> new ListConnectorAction.Response(r.connectors(), r.totalResults()))
         );
     }
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringAction.java
index 7b4ce08ef8320..a7bd9c2382331 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringAction.java
@@ -22,7 +22,7 @@
 
 public class UpdateConnectorActiveFilteringAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/update_filtering/activate";
+    public static final String NAME = "cluster:admin/xpack/connector/update_filtering/activate";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private UpdateConnectorActiveFilteringAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java
index 7f726f21ce225..e76c5191a58a6 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java
@@ -27,7 +27,7 @@
 
 public class UpdateConnectorApiKeyIdAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/update_api_key_id";
+    public static final String NAME = "cluster:admin/xpack/connector/update_api_key_id";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private UpdateConnectorApiKeyIdAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java
index 5d36c5f886ea0..6948667fa7351 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java
@@ -32,7 +32,7 @@
 
 public class UpdateConnectorConfigurationAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/update_configuration";
+    public static final String NAME = "cluster:admin/xpack/connector/update_configuration";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private UpdateConnectorConfigurationAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java
index 3e506fc835f65..a8c2b334cfee3 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java
@@ -27,7 +27,7 @@
 
 public class UpdateConnectorErrorAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/update_error";
+    public static final String NAME = "cluster:admin/xpack/connector/update_error";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private UpdateConnectorErrorAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java
index 56656855583aa..4bd51794c1f9e 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java
@@ -27,7 +27,7 @@
 
 public class UpdateConnectorFeaturesAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/update_features";
+    public static final String NAME = "cluster:admin/xpack/connector/update_features";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private UpdateConnectorFeaturesAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java
index 660956b2e9d7f..e527e79161f2f 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java
@@ -32,7 +32,7 @@
 
 public class UpdateConnectorFilteringAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/update_filtering";
+    public static final String NAME = "cluster:admin/xpack/connector/update_filtering";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private UpdateConnectorFilteringAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationAction.java
index 92291506d0719..a22b08152f509 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationAction.java
@@ -27,7 +27,7 @@
 
 public class UpdateConnectorFilteringValidationAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/update_filtering/draft_validation";
+    public static final String NAME = "cluster:admin/xpack/connector/update_filtering/draft_validation";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private UpdateConnectorFilteringValidationAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java
index e7840e1f84fad..ddc98d4756dc4 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java
@@ -27,7 +27,7 @@
 
 public class UpdateConnectorIndexNameAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/update_index_name";
+    public static final String NAME = "cluster:admin/xpack/connector/update_index_name";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private UpdateConnectorIndexNameAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java
index f72938ec8dba2..deae10d901c13 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java
@@ -23,7 +23,7 @@
 
 public class UpdateConnectorLastSeenAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/update_last_seen";
+    public static final String NAME = "cluster:admin/xpack/connector/update_last_seen";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private UpdateConnectorLastSeenAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java
index ae3be3801786c..029e261e51fab 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java
@@ -32,7 +32,7 @@
 
 public class UpdateConnectorLastSyncStatsAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/update_last_sync_stats";
+    public static final String NAME = "cluster:admin/xpack/connector/update_last_sync_stats";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private UpdateConnectorLastSyncStatsAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java
index bbc1f992b48e2..5a63bb106747f 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java
@@ -27,7 +27,7 @@
 
 public class UpdateConnectorNameAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/update_name";
+    public static final String NAME = "cluster:admin/xpack/connector/update_name";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private UpdateConnectorNameAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeAction.java
index 7b3f2e4577f4e..d20b535d5cc57 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeAction.java
@@ -26,7 +26,7 @@
 
 public class UpdateConnectorNativeAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/update_native";
+    public static final String NAME = "cluster:admin/xpack/connector/update_native";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private UpdateConnectorNativeAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java
index e58d614f4ef21..2e738033c3384 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java
@@ -27,7 +27,7 @@
 
 public class UpdateConnectorPipelineAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/update_pipeline";
+    public static final String NAME = "cluster:admin/xpack/connector/update_pipeline";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private UpdateConnectorPipelineAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java
index 578639f065a0b..65cfd645ea60b 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java
@@ -32,7 +32,7 @@
 
 public class UpdateConnectorSchedulingAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/update_scheduling";
+    public static final String NAME = "cluster:admin/xpack/connector/update_scheduling";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private UpdateConnectorSchedulingAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeAction.java
index de07a6db21bab..1d4df12e10eba 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeAction.java
@@ -26,7 +26,7 @@
 
 public class UpdateConnectorServiceTypeAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/update_service_type";
+    public static final String NAME = "cluster:admin/xpack/connector/update_service_type";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private UpdateConnectorServiceTypeAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java
index aebaa0afb9052..79f097db2dec9 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java
@@ -28,7 +28,7 @@
 
 public class UpdateConnectorStatusAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/update_status";
+    public static final String NAME = "cluster:admin/xpack/connector/update_status";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     public UpdateConnectorStatusAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java
index 4aabb9e1af663..5ba45ea4757bf 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java
@@ -10,9 +10,6 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -65,7 +62,7 @@
  *     <li>The hostname of the worker to run the sync job.</li>
  * </ul>
  */
-public class ConnectorSyncJob implements Writeable, ToXContentObject {
+public class ConnectorSyncJob implements ToXContentObject {
 
     static final ParseField CANCELATION_REQUESTED_AT_FIELD = new ParseField("cancelation_requested_at");
 
@@ -213,27 +210,6 @@ private ConnectorSyncJob(
         this.workerHostname = workerHostname;
     }
 
-    public ConnectorSyncJob(StreamInput in) throws IOException {
-        this.cancelationRequestedAt = in.readOptionalInstant();
-        this.canceledAt = in.readOptionalInstant();
-        this.completedAt = in.readOptionalInstant();
-        this.connector = in.readNamedWriteable(Connector.class);
-        this.createdAt = in.readInstant();
-        this.deletedDocumentCount = in.readLong();
-        this.error = in.readOptionalString();
-        this.id = in.readString();
-        this.indexedDocumentCount = in.readLong();
-        this.indexedDocumentVolume = in.readLong();
-        this.jobType = in.readEnum(ConnectorSyncJobType.class);
-        this.lastSeen = in.readOptionalInstant();
-        this.metadata = in.readMap(StreamInput::readString, StreamInput::readGenericValue);
-        this.startedAt = in.readOptionalInstant();
-        this.status = in.readEnum(ConnectorSyncStatus.class);
-        this.totalDocumentCount = in.readOptionalLong();
-        this.triggerMethod = in.readEnum(ConnectorSyncJobTriggerMethod.class);
-        this.workerHostname = in.readOptionalString();
-    }
-
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<ConnectorSyncJob, String> PARSER = new ConstructingObjectParser<>(
         "connector_sync_job",
@@ -548,28 +524,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeOptionalInstant(cancelationRequestedAt);
-        out.writeOptionalInstant(canceledAt);
-        out.writeOptionalInstant(completedAt);
-        out.writeNamedWriteable(connector);
-        out.writeInstant(createdAt);
-        out.writeLong(deletedDocumentCount);
-        out.writeOptionalString(error);
-        out.writeString(id);
-        out.writeLong(indexedDocumentCount);
-        out.writeLong(indexedDocumentVolume);
-        out.writeEnum(jobType);
-        out.writeOptionalInstant(lastSeen);
-        out.writeMap(metadata, StreamOutput::writeString, StreamOutput::writeGenericValue);
-        out.writeOptionalInstant(startedAt);
-        out.writeEnum(status);
-        out.writeOptionalLong(totalDocumentCount);
-        out.writeEnum(triggerMethod);
-        out.writeOptionalString(workerHostname);
-    }
-
     public boolean equals(Object other) {
         if (this == other) return true;
         if (other == null || getClass() != other.getClass()) return false;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java
index ce6f7f0dbf2b2..f46d915a7123f 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.engine.DocumentMissingException;
@@ -68,6 +69,7 @@
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.application.connector.ConnectorIndexService.CONNECTOR_INDEX_NAME;
+import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN;
 
 /**
  * A service that manages persistent {@link ConnectorSyncJob} configurations.
@@ -76,7 +78,8 @@ public class ConnectorSyncJobIndexService {
 
     private static final Long ZERO = 0L;
 
-    private final Client client;
+    // The client to interact with the system index (internal user).
+    private final Client clientWithOrigin;
 
     public static final String CONNECTOR_SYNC_JOB_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN;
 
@@ -84,7 +87,7 @@ public class ConnectorSyncJobIndexService {
      * @param client A client for executing actions on the connectors sync jobs index.
      */
     public ConnectorSyncJobIndexService(Client client) {
-        this.client = client;
+        this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN);
     }
 
     /**
@@ -149,7 +152,7 @@ public void createConnectorSyncJob(
 
                     indexRequest.source(syncJob.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS));
 
-                    client.index(
+                    clientWithOrigin.index(
                         indexRequest,
                         l.delegateFailureAndWrap(
                             (ll, indexResponse) -> ll.onResponse(new PostConnectorSyncJobAction.Response(indexResponse.getId()))
@@ -175,7 +178,7 @@ public void deleteConnectorSyncJob(String connectorSyncJobId, ActionListener<Del
             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
 
         try {
-            client.delete(
+            clientWithOrigin.delete(
                 deleteRequest,
                 new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(connectorSyncJobId, listener, (l, deleteResponse) -> {
                     if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) {
@@ -205,7 +208,7 @@ public void checkInConnectorSyncJob(String connectorSyncJobId, ActionListener<Up
         ).doc(Map.of(ConnectorSyncJob.LAST_SEEN_FIELD.getPreferredName(), newLastSeen));
 
         try {
-            client.update(
+            clientWithOrigin.update(
                 updateRequest,
                 new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(connectorSyncJobId, listener, (l, updateResponse) -> {
                     if (updateResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) {
@@ -230,7 +233,7 @@ public void getConnectorSyncJob(String connectorSyncJobId, ActionListener<Connec
         final GetRequest getRequest = new GetRequest(CONNECTOR_SYNC_JOB_INDEX_NAME).id(connectorSyncJobId).realtime(true);
 
         try {
-            client.get(
+            clientWithOrigin.get(
                 getRequest,
                 new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(connectorSyncJobId, listener, (l, getResponse) -> {
                     if (getResponse.isExists() == false) {
@@ -306,7 +309,7 @@ public void cancelConnectorSyncJob(String connectorSyncJobId, ActionListener<Upd
                     WriteRequest.RefreshPolicy.IMMEDIATE
                 ).doc(syncJobFieldsToUpdate);
 
-                client.update(
+                clientWithOrigin.update(
                     updateRequest,
                     new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(
                         connectorSyncJobId,
@@ -355,7 +358,7 @@ public void listConnectorSyncJobs(
 
             final SearchRequest searchRequest = new SearchRequest(CONNECTOR_SYNC_JOB_INDEX_NAME).source(searchSource);
 
-            client.search(searchRequest, new ActionListener<>() {
+            clientWithOrigin.search(searchRequest, new ActionListener<>() {
                 @Override
                 public void onResponse(SearchResponse searchResponse) {
                     try {
@@ -474,7 +477,7 @@ public void updateConnectorSyncJobIngestionStats(
         ).doc(fieldsToUpdate);
 
         try {
-            client.update(
+            clientWithOrigin.update(
                 updateRequest,
                 new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(syncJobId, listener, (l, updateResponse) -> {
                     if (updateResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) {
@@ -501,7 +504,7 @@ private void getSyncJobConnectorInfo(String connectorId, ConnectorSyncJobType jo
 
             final GetRequest request = new GetRequest(CONNECTOR_INDEX_NAME, connectorId);
 
-            client.get(request, new ActionListener<>() {
+            clientWithOrigin.get(request, new ActionListener<>() {
                 @Override
                 public void onResponse(GetResponse response) {
                     final boolean connectorDoesNotExist = response.isExists() == false;
@@ -594,7 +597,7 @@ public void updateConnectorSyncJobError(String connectorSyncJobId, String error,
                         )
                     );
 
-                client.update(
+                clientWithOrigin.update(
                     updateRequest,
                     new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(
                         connectorSyncJobId,
@@ -629,7 +632,7 @@ public void deleteAllSyncJobsByConnectorId(String connectorId, ActionListener<Bu
             )
         ).setRefresh(true).setIndicesOptions(IndicesOptions.fromOptions(true, true, false, false));
 
-        client.execute(DeleteByQueryAction.INSTANCE, deleteByQueryRequest, listener.delegateFailureAndWrap((l, r) -> {
+        clientWithOrigin.execute(DeleteByQueryAction.INSTANCE, deleteByQueryRequest, listener.delegateFailureAndWrap((l, r) -> {
             final List<BulkItemResponse.Failure> bulkDeleteFailures = r.getBulkFailures();
             if (bulkDeleteFailures.isEmpty() == false) {
                 l.onFailure(
@@ -681,7 +684,7 @@ public void claimConnectorSyncJob(
                     WriteRequest.RefreshPolicy.IMMEDIATE
                 ).doc(document);
 
-                client.update(
+                clientWithOrigin.update(
                     updateRequest,
                     new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(
                         connectorSyncJobId,
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java
index 160eda3aeef7c..658d48e9752d0 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java
@@ -28,7 +28,7 @@
 
 public class CancelConnectorSyncJobAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/sync_job/cancel";
+    public static final String NAME = "cluster:admin/xpack/connector/sync_job/cancel";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<ConnectorUpdateActionResponse>(NAME);
 
     private CancelConnectorSyncJobAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java
index aa6dea50de464..ae44813354eb9 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java
@@ -28,7 +28,7 @@
 
 public class CheckInConnectorSyncJobAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/sync_job/check_in";
+    public static final String NAME = "cluster:admin/xpack/connector/sync_job/check_in";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private CheckInConnectorSyncJobAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java
index b108116a5e68c..84e3183c2830c 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java
@@ -31,7 +31,7 @@
 
 public class ClaimConnectorSyncJobAction {
     public static final ParseField CONNECTOR_SYNC_JOB_ID_FIELD = new ParseField("connector_sync_job_id");
-    public static final String NAME = "indices:data/write/xpack/connector/sync_job/claim";
+    public static final String NAME = "cluster:admin/xpack/connector/sync_job/claim";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private ClaimConnectorSyncJobAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java
index bb83fd78151df..11a3334aed4e6 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java
@@ -8,19 +8,14 @@
 package org.elasticsearch.xpack.application.connector.syncjob.action;
 
 import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.IndicesRequest;
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry;
 
 import java.io.IOException;
 
 /**
  * Abstract base class for action requests targeting the connector sync job index.
- * Implements {@link org.elasticsearch.action.IndicesRequest} to ensure index-level privilege support.
- * This class defines the connectors sync job index as the target for all derived action requests.
  */
-public abstract class ConnectorSyncJobActionRequest extends ActionRequest implements IndicesRequest {
+public abstract class ConnectorSyncJobActionRequest extends ActionRequest {
 
     public ConnectorSyncJobActionRequest() {
         super();
@@ -29,14 +24,4 @@ public ConnectorSyncJobActionRequest() {
     public ConnectorSyncJobActionRequest(StreamInput in) throws IOException {
         super(in);
     }
-
-    @Override
-    public String[] indices() {
-        return new String[] { ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN };
-    }
-
-    @Override
-    public IndicesOptions indicesOptions() {
-        return IndicesOptions.lenientExpandHidden();
-    }
 }
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobAction.java
index c9c84ead2e4b0..935aa1fd6b7d1 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobAction.java
@@ -28,7 +28,7 @@
 
 public class DeleteConnectorSyncJobAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/sync_job/delete";
+    public static final String NAME = "cluster:admin/xpack/connector/sync_job/delete";
     public static final ActionType<AcknowledgedResponse> INSTANCE = new ActionType<>(NAME);
 
     private DeleteConnectorSyncJobAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobAction.java
index 7ff82b9881fb4..485e516c80c39 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobAction.java
@@ -29,7 +29,7 @@
 
 public class GetConnectorSyncJobAction {
 
-    public static final String NAME = "indices:data/read/xpack/connector/sync_job/get";
+    public static final String NAME = "cluster:admin/xpack/connector/sync_job/get";
     public static final ActionType<GetConnectorSyncJobAction.Response> INSTANCE = new ActionType<>(NAME);
 
     private GetConnectorSyncJobAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsAction.java
index 5b34e05ae37f0..04b765f3b5687 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsAction.java
@@ -33,7 +33,7 @@
 
 public class ListConnectorSyncJobsAction {
 
-    public static final String NAME = "indices:data/read/xpack/connector/sync_job/list";
+    public static final String NAME = "cluster:admin/xpack/connector/sync_job/list";
     public static final ActionType<ListConnectorSyncJobsAction.Response> INSTANCE = new ActionType<>(NAME);
 
     private ListConnectorSyncJobsAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobAction.java
index 8c1d24e466daa..0d17a6dba6c35 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobAction.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.application.connector.Connector;
-import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry;
 import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob;
 import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTriggerMethod;
 import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobType;
@@ -32,7 +31,7 @@
 
 public class PostConnectorSyncJobAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/sync_job/post";
+    public static final String NAME = "cluster:admin/xpack/connector/sync_job/post";
     public static final ActionType<PostConnectorSyncJobAction.Response> INSTANCE = new ActionType<>(NAME);
 
     private PostConnectorSyncJobAction() {/* no instances */}
@@ -140,14 +139,6 @@ public boolean equals(Object o) {
         public int hashCode() {
             return Objects.hash(id, jobType, triggerMethod);
         }
-
-        @Override
-        public String[] indices() {
-            // Creating a new sync job requires reading from connector index
-            return new String[] {
-                ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN,
-                ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN };
-        }
     }
 
     public static class Response extends ActionResponse implements ToXContentObject {
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java
index 2235ba7cfe720..0d5f57c202d9f 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java
@@ -28,7 +28,7 @@
 
 public class UpdateConnectorSyncJobErrorAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/sync_job/update_error";
+    public static final String NAME = "cluster:admin/xpack/connector/sync_job/update_error";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private UpdateConnectorSyncJobErrorAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java
index 0fd9b6dec8184..c890ca0d69bc6 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java
@@ -35,7 +35,7 @@
 
 public class UpdateConnectorSyncJobIngestionStatsAction {
 
-    public static final String NAME = "indices:data/write/xpack/connector/sync_job/update_stats";
+    public static final String NAME = "cluster:admin/xpack/connector/sync_job/update_stats";
     public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
 
     private UpdateConnectorSyncJobIngestionStatsAction() {/* no instances */}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsAction.java
index 62f9f3fd46cc4..11397583ce5b9 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsAction.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -34,8 +33,6 @@ public class ListQueryRulesetsAction {
     public static final String NAME = "cluster:admin/xpack/query_rules/list";
     public static final ActionType<ListQueryRulesetsAction.Response> INSTANCE = new ActionType<>(NAME);
 
-    public static final NodeFeature QUERY_RULE_LIST_TYPES = new NodeFeature("query_rule_list_types");
-
     private ListQueryRulesetsAction() {/* no instances */}
 
     public static class Request extends ActionRequest implements ToXContentObject {
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetAction.java
index 28f4a3b38dd59..c661bc9467e6e 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetAction.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -35,8 +34,6 @@
 
 public class TestQueryRulesetAction {
 
-    public static final NodeFeature QUERY_RULES_TEST_API = new NodeFeature("query_rules.test");
-
     // TODO - We'd like to transition this to require less stringent permissions
     public static final ActionType<TestQueryRulesetAction.Response> TYPE = new ActionType<>("cluster:admin/xpack/query_rules/test");
 
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilder.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilder.java
index 5b27cc7a3e05a..528204f4132ea 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilder.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilder.java
@@ -9,7 +9,6 @@
 
 import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.RankDocsQueryBuilder;
 import org.elasticsearch.license.LicenseUtils;
@@ -45,12 +44,10 @@
 public final class QueryRuleRetrieverBuilder extends CompoundRetrieverBuilder<QueryRuleRetrieverBuilder> {
 
     public static final String NAME = "rule";
-    public static final NodeFeature QUERY_RULE_RETRIEVERS_SUPPORTED = new NodeFeature("query_rule_retriever_supported");
 
     public static final ParseField RULESET_IDS_FIELD = new ParseField("ruleset_ids");
     public static final ParseField MATCH_CRITERIA_FIELD = new ParseField("match_criteria");
     public static final ParseField RETRIEVER_FIELD = new ParseField("retriever");
-    public static final ParseField RANK_WINDOW_SIZE_FIELD = new ParseField("rank_window_size");
 
     @SuppressWarnings("unchecked")
     public static final ConstructingObjectParser<QueryRuleRetrieverBuilder, RetrieverParserContext> PARSER = new ConstructingObjectParser<>(
@@ -77,9 +74,6 @@ public final class QueryRuleRetrieverBuilder extends CompoundRetrieverBuilder<Qu
     }
 
     public static QueryRuleRetrieverBuilder fromXContent(XContentParser parser, RetrieverParserContext context) throws IOException {
-        if (context.clusterSupportsFeature(QUERY_RULE_RETRIEVERS_SUPPORTED) == false) {
-            throw new ParsingException(parser.getTokenLocation(), "unknown retriever [" + NAME + "]");
-        }
         if (EnterpriseSearch.QUERY_RULES_RETRIEVER_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) {
             throw LicenseUtils.newComplianceException("Query Rules");
         }
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java
index 2219a2bebbd25..60aacd03de1f3 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java
@@ -19,8 +19,6 @@
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.get.GetRequest;
@@ -321,10 +319,6 @@ private void deleteSearchApplication(String resourceName, ActionListener<DeleteR
         }
     }
 
-    GetAliasesResponse getAlias(String searchAliasName) {
-        return client.admin().indices().getAliases(new GetAliasesRequest(searchAliasName)).actionGet();
-    }
-
     private void removeAlias(String searchAliasName, ActionListener<AcknowledgedResponse> listener) {
         IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest().addAliasAction(
             IndicesAliasesRequest.AliasActions.remove().aliases(searchAliasName).indices("*")
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/DeleteSearchApplicationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/DeleteSearchApplicationAction.java
index 93ad2f46b5583..98ae7cdef770f 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/DeleteSearchApplicationAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/DeleteSearchApplicationAction.java
@@ -35,7 +35,7 @@ private DeleteSearchApplicationAction() {/* no instances */}
     public static class Request extends ActionRequest implements ToXContentObject {
         private final String name;
 
-        public static ParseField NAME_FIELD = new ParseField("name");
+        public static final ParseField NAME_FIELD = new ParseField("name");
 
         public Request(StreamInput in) throws IOException {
             super(in);
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/PutSearchApplicationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/PutSearchApplicationAction.java
index 75bec0d61649f..ff829b6f53016 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/PutSearchApplicationAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/PutSearchApplicationAction.java
@@ -101,8 +101,8 @@ public int hashCode() {
             return Objects.hash(searchApp, create);
         }
 
-        public static ParseField SEARCH_APPLICATION = new ParseField("searchApp");
-        public static ParseField CREATE = new ParseField("create");
+        public static final ParseField SEARCH_APPLICATION = new ParseField("searchApp");
+        public static final ParseField CREATE = new ParseField("create");
 
         @SuppressWarnings("unchecked")
         private static final ConstructingObjectParser<Request, String> PARSER = new ConstructingObjectParser<>(
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java
index 12abca3a78591..6189db51df183 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteResponse;
-import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Tuple;
@@ -56,6 +55,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.MANAGED_CONNECTOR_INDEX_PREFIX;
 import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.getRandomConnectorFeatures;
 import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.getRandomCronExpression;
 import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.randomConnectorFeatureEnabled;
@@ -111,13 +111,66 @@ public void testDeleteConnector() throws Exception {
         }
 
         String connectorIdToDelete = connectorIds.get(0);
-        DeleteResponse resp = awaitDeleteConnector(connectorIdToDelete, false);
+        UpdateResponse resp = awaitDeleteConnector(connectorIdToDelete, false);
         assertThat(resp.status(), equalTo(RestStatus.OK));
         expectThrows(ResourceNotFoundException.class, () -> awaitGetConnector(connectorIdToDelete));
 
         expectThrows(ResourceNotFoundException.class, () -> awaitDeleteConnector(connectorIdToDelete, false));
     }
 
+    public void testDeleteConnector_expectSoftDeletion() throws Exception {
+        int numConnectors = 5;
+        List<String> connectorIds = new ArrayList<>();
+        List<Connector> connectors = new ArrayList<>();
+        for (int i = 0; i < numConnectors; i++) {
+            Connector connector = ConnectorTestUtils.getRandomConnector();
+            ConnectorCreateActionResponse resp = awaitCreateConnector(null, connector);
+            connectorIds.add(resp.getId());
+            connectors.add(connector);
+        }
+
+        String connectorIdToDelete = connectorIds.get(0);
+        UpdateResponse resp = awaitDeleteConnector(connectorIdToDelete, false);
+        assertThat(resp.status(), equalTo(RestStatus.OK));
+        expectThrows(ResourceNotFoundException.class, () -> awaitGetConnector(connectorIdToDelete));
+
+        expectThrows(ResourceNotFoundException.class, () -> awaitDeleteConnector(connectorIdToDelete, false));
+
+        Connector softDeletedConnector = awaitGetSoftDeletedConnector(connectorIdToDelete);
+        assertThat(softDeletedConnector.getConnectorId(), equalTo(connectorIdToDelete));
+        assertThat(softDeletedConnector.getServiceType(), equalTo(connectors.get(0).getServiceType()));
+    }
+
+    public void testDeleteConnector_expectSoftDeletionMultipleConnectors() throws Exception {
+        int numConnectors = 5;
+        List<String> connectorIds = new ArrayList<>();
+        for (int i = 0; i < numConnectors; i++) {
+            Connector connector = ConnectorTestUtils.getRandomConnector();
+            ConnectorCreateActionResponse resp = awaitCreateConnector(null, connector);
+            connectorIds.add(resp.getId());
+        }
+
+        // Delete all of them
+        for (int i = 0; i < numConnectors; i++) {
+            String connectorIdToDelete = connectorIds.get(i);
+            UpdateResponse resp = awaitDeleteConnector(connectorIdToDelete, false);
+            assertThat(resp.status(), equalTo(RestStatus.OK));
+        }
+
+        // Connectors were deleted from main index
+        for (int i = 0; i < numConnectors; i++) {
+            String connectorId = connectorIds.get(i);
+            expectThrows(ResourceNotFoundException.class, () -> awaitGetConnector(connectorId));
+        }
+
+        // Soft deleted connectors available in system index
+        for (int i = 0; i < numConnectors; i++) {
+            String connectorId = connectorIds.get(i);
+            Connector softDeletedConnector = awaitGetSoftDeletedConnector(connectorId);
+            assertThat(softDeletedConnector.getConnectorId(), equalTo(connectorId));
+        }
+    }
+
     public void testUpdateConnectorConfiguration_FullConfiguration() throws Exception {
         Connector connector = ConnectorTestUtils.getRandomConnector();
         String connectorId = randomUUID();
@@ -648,8 +701,8 @@ public void testUpdateConnectorScheduling_OnlyFullSchedule() throws Exception {
         assertThat(initialScheduling.getIncremental(), equalTo(indexedConnector.getScheduling().getIncremental()));
     }
 
-    public void testUpdateConnectorIndexName() throws Exception {
-        Connector connector = ConnectorTestUtils.getRandomConnector();
+    public void testUpdateConnectorIndexName_ForSelfManagedConnector() throws Exception {
+        Connector connector = ConnectorTestUtils.getRandomSelfManagedConnector();
         String connectorId = randomUUID();
 
         ConnectorCreateActionResponse resp = awaitCreateConnector(connectorId, connector);
@@ -669,8 +722,8 @@ public void testUpdateConnectorIndexName() throws Exception {
         assertThat(newIndexName, equalTo(indexedConnector.getIndexName()));
     }
 
-    public void testUpdateConnectorIndexName_WithTheSameIndexName() throws Exception {
-        Connector connector = ConnectorTestUtils.getRandomConnector();
+    public void testUpdateConnectorIndexName_ForSelfManagedConnector_WithTheSameIndexName() throws Exception {
+        Connector connector = ConnectorTestUtils.getRandomSelfManagedConnector();
         String connectorId = randomUUID();
 
         ConnectorCreateActionResponse resp = awaitCreateConnector(connectorId, connector);
@@ -685,6 +738,42 @@ public void testUpdateConnectorIndexName_WithTheSameIndexName() throws Exception
         assertThat(updateResponse.getResult(), equalTo(DocWriteResponse.Result.NOOP));
     }
 
+    public void testUpdateConnectorIndexName_ForManagedConnector_WithIllegalIndexName() throws Exception {
+        Connector connector = ConnectorTestUtils.getRandomElasticManagedConnector();
+        String connectorId = randomUUID();
+
+        ConnectorCreateActionResponse resp = awaitCreateConnector(connectorId, connector);
+        assertThat(resp.status(), anyOf(equalTo(RestStatus.CREATED), equalTo(RestStatus.OK)));
+
+        UpdateConnectorIndexNameAction.Request updateIndexNameRequest = new UpdateConnectorIndexNameAction.Request(
+            connectorId,
+            "wrong-prefix-" + randomAlphaOfLengthBetween(3, 10)
+        );
+
+        expectThrows(ElasticsearchStatusException.class, () -> awaitUpdateConnectorIndexName(updateIndexNameRequest));
+    }
+
+    public void testUpdateConnectorIndexName_ForManagedConnector_WithPrefixedIndexName() throws Exception {
+        Connector connector = ConnectorTestUtils.getRandomElasticManagedConnector();
+        String connectorId = randomUUID();
+
+        ConnectorCreateActionResponse resp = awaitCreateConnector(connectorId, connector);
+        assertThat(resp.status(), anyOf(equalTo(RestStatus.CREATED), equalTo(RestStatus.OK)));
+
+        String newIndexName = MANAGED_CONNECTOR_INDEX_PREFIX + randomAlphaOfLengthBetween(3, 10);
+
+        UpdateConnectorIndexNameAction.Request updateIndexNameRequest = new UpdateConnectorIndexNameAction.Request(
+            connectorId,
+            newIndexName
+        );
+
+        DocWriteResponse updateResponse = awaitUpdateConnectorIndexName(updateIndexNameRequest);
+        assertThat(updateResponse.status(), equalTo(RestStatus.OK));
+
+        Connector indexedConnector = awaitGetConnector(connectorId);
+        assertThat(newIndexName, equalTo(indexedConnector.getIndexName()));
+    }
+
     public void testUpdateConnectorServiceType() throws Exception {
         Connector connector = ConnectorTestUtils.getRandomConnector();
         String connectorId = randomUUID();
@@ -756,7 +845,7 @@ public void testUpdateConnectorNameOrDescription() throws Exception {
     }
 
     public void testUpdateConnectorNative() throws Exception {
-        Connector connector = ConnectorTestUtils.getRandomConnector();
+        Connector connector = ConnectorTestUtils.getRandomConnectorWithDetachedIndex();
         String connectorId = randomUUID();
 
         ConnectorCreateActionResponse resp = awaitCreateConnector(connectorId, connector);
@@ -773,6 +862,39 @@ public void testUpdateConnectorNative() throws Exception {
         assertThat(isNative, equalTo(indexedConnector.isNative()));
     }
 
+    public void testUpdateConnectorNativeTrue_WhenIllegalIndexPrefix() throws Exception {
+        Connector connector = ConnectorTestUtils.getRandomConnectorWithAttachedIndex("wrong-prefix-" + randomAlphaOfLength(10));
+        String connectorId = randomUUID();
+
+        ConnectorCreateActionResponse resp = awaitCreateConnector(connectorId, connector);
+        assertThat(resp.status(), anyOf(equalTo(RestStatus.CREATED), equalTo(RestStatus.OK)));
+
+        boolean isNative = true;
+
+        UpdateConnectorNativeAction.Request updateNativeRequest = new UpdateConnectorNativeAction.Request(connectorId, isNative);
+
+        expectThrows(ElasticsearchStatusException.class, () -> awaitUpdateConnectorNative(updateNativeRequest));
+    }
+
+    public void testUpdateConnectorNativeTrue_WithCorrectIndexPrefix() throws Exception {
+        Connector connector = ConnectorTestUtils.getRandomConnectorWithAttachedIndex(
+            MANAGED_CONNECTOR_INDEX_PREFIX + randomAlphaOfLength(10)
+        );
+        String connectorId = randomUUID();
+
+        ConnectorCreateActionResponse resp = awaitCreateConnector(connectorId, connector);
+        assertThat(resp.status(), anyOf(equalTo(RestStatus.CREATED), equalTo(RestStatus.OK)));
+
+        boolean isNative = true;
+
+        UpdateConnectorNativeAction.Request updateNativeRequest = new UpdateConnectorNativeAction.Request(connectorId, isNative);
+        DocWriteResponse updateResponse = awaitUpdateConnectorNative(updateNativeRequest);
+        assertThat(updateResponse.status(), equalTo(RestStatus.OK));
+
+        Connector indexedConnector = awaitGetConnector(connectorId);
+        assertThat(isNative, equalTo(indexedConnector.isNative()));
+    }
+
     public void testUpdateConnectorStatus() throws Exception {
         Connector connector = ConnectorTestUtils.getRandomConnector();
         String connectorId = randomUUID();
@@ -827,13 +949,13 @@ public void testUpdateConnectorApiKeyIdOrApiKeySecretId() throws Exception {
         assertThat(updateApiKeyIdRequest.getApiKeySecretId(), equalTo(indexedConnector.getApiKeySecretId()));
     }
 
-    private DeleteResponse awaitDeleteConnector(String connectorId, boolean deleteConnectorSyncJobs) throws Exception {
+    private UpdateResponse awaitDeleteConnector(String connectorId, boolean deleteConnectorSyncJobs) throws Exception {
         CountDownLatch latch = new CountDownLatch(1);
-        final AtomicReference<DeleteResponse> resp = new AtomicReference<>(null);
+        final AtomicReference<UpdateResponse> resp = new AtomicReference<>(null);
         final AtomicReference<Exception> exc = new AtomicReference<>(null);
         connectorIndexService.deleteConnector(connectorId, deleteConnectorSyncJobs, new ActionListener<>() {
             @Override
-            public void onResponse(DeleteResponse deleteResponse) {
+            public void onResponse(UpdateResponse deleteResponse) {
                 resp.set(deleteResponse);
                 latch.countDown();
             }
@@ -886,11 +1008,19 @@ public void onFailure(Exception e) {
         return resp.get();
     }
 
+    private Connector awaitGetSoftDeletedConnector(String connectorId) throws Exception {
+        return awaitGetConnector(connectorId, true);
+    }
+
     private Connector awaitGetConnector(String connectorId) throws Exception {
+        return awaitGetConnector(connectorId, false);
+    }
+
+    private Connector awaitGetConnector(String connectorId, Boolean isDeleted) throws Exception {
         CountDownLatch latch = new CountDownLatch(1);
         final AtomicReference<Connector> resp = new AtomicReference<>(null);
         final AtomicReference<Exception> exc = new AtomicReference<>(null);
-        connectorIndexService.getConnector(connectorId, new ActionListener<>() {
+        connectorIndexService.getConnector(connectorId, isDeleted, new ActionListener<>() {
             @Override
             public void onResponse(ConnectorSearchResult connectorResult) {
                 // Serialize the sourceRef to Connector class for unit tests
@@ -924,11 +1054,23 @@ private ConnectorIndexService.ConnectorResult awaitListConnector(
         List<String> names,
         List<String> serviceTypes,
         String searchQuery
+    ) throws Exception {
+        return awaitListConnector(from, size, indexNames, names, serviceTypes, searchQuery, false);
+    }
+
+    private ConnectorIndexService.ConnectorResult awaitListConnector(
+        int from,
+        int size,
+        List<String> indexNames,
+        List<String> names,
+        List<String> serviceTypes,
+        String searchQuery,
+        Boolean isDeleted
     ) throws Exception {
         CountDownLatch latch = new CountDownLatch(1);
         final AtomicReference<ConnectorIndexService.ConnectorResult> resp = new AtomicReference<>(null);
         final AtomicReference<Exception> exc = new AtomicReference<>(null);
-        connectorIndexService.listConnectors(from, size, indexNames, names, serviceTypes, searchQuery, new ActionListener<>() {
+        connectorIndexService.listConnectors(from, size, indexNames, names, serviceTypes, searchQuery, isDeleted, new ActionListener<>() {
             @Override
             public void onResponse(ConnectorIndexService.ConnectorResult result) {
                 resp.set(result);
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIngestPipelineTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIngestPipelineTests.java
index f4a92e51e8c6a..c3d4bf8b72ff5 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIngestPipelineTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIngestPipelineTests.java
@@ -50,7 +50,7 @@ public void testToXContent() throws IOException {
         String content = XContentHelper.stripWhitespace("""
                 {
                     "extract_binary_content": true,
-                    "name": "ent-search-generic-ingestion",
+                    "name": "search-default-ingestion",
                     "reduce_whitespace": true,
                     "run_ml_inference": false
                 }
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistryTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistryTests.java
index a4c7015afafcb..068b99626af9d 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistryTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistryTests.java
@@ -132,10 +132,7 @@ public void testThatNonExistingComponentTemplatesAreAddedImmediately() throws Ex
         ClusterChangedEvent event = createClusterChangedEvent(
             Collections.emptyMap(),
             Collections.emptyMap(),
-            Collections.singletonMap(
-                ConnectorTemplateRegistry.ENT_SEARCH_GENERIC_PIPELINE_NAME,
-                ConnectorTemplateRegistry.REGISTRY_VERSION
-            ),
+            Collections.singletonMap(ConnectorTemplateRegistry.SEARCH_DEFAULT_PIPELINE_NAME, ConnectorTemplateRegistry.REGISTRY_VERSION),
             Collections.emptyMap(),
             nodes
         );
@@ -169,10 +166,7 @@ public void testThatVersionedOldComponentTemplatesAreUpgraded() throws Exception
                 ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-settings",
                 ConnectorTemplateRegistry.REGISTRY_VERSION - 1
             ),
-            Collections.singletonMap(
-                ConnectorTemplateRegistry.ENT_SEARCH_GENERIC_PIPELINE_NAME,
-                ConnectorTemplateRegistry.REGISTRY_VERSION
-            ),
+            Collections.singletonMap(ConnectorTemplateRegistry.SEARCH_DEFAULT_PIPELINE_NAME, ConnectorTemplateRegistry.REGISTRY_VERSION),
             Collections.emptyMap(),
             nodes
         );
@@ -189,10 +183,7 @@ public void testThatUnversionedOldComponentTemplatesAreUpgraded() throws Excepti
         ClusterChangedEvent event = createClusterChangedEvent(
             Collections.emptyMap(),
             Collections.singletonMap(ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-mappings", null),
-            Collections.singletonMap(
-                ConnectorTemplateRegistry.ENT_SEARCH_GENERIC_PIPELINE_NAME,
-                ConnectorTemplateRegistry.REGISTRY_VERSION
-            ),
+            Collections.singletonMap(ConnectorTemplateRegistry.SEARCH_DEFAULT_PIPELINE_NAME, ConnectorTemplateRegistry.REGISTRY_VERSION),
             Collections.emptyMap(),
             nodes
         );
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java
index f052ef79d82fb..c563bc0a14ee3 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java
@@ -343,6 +343,18 @@ public static Connector getRandomConnectorWithDetachedIndex() {
         return getRandomConnectorBuilder().setIndexName(null).build();
     }
 
+    public static Connector getRandomConnectorWithAttachedIndex(String indexName) {
+        return getRandomConnectorBuilder().setIndexName(indexName).build();
+    }
+
+    public static Connector getRandomSelfManagedConnector() {
+        return getRandomConnectorBuilder().setIsNative(false).build();
+    }
+
+    public static Connector getRandomElasticManagedConnector() {
+        return getRandomConnectorBuilder().setIsNative(true).build();
+    }
+
     public static Connector getRandomConnectorWithServiceTypeNotDefined() {
         return getRandomConnectorBuilder().setServiceType(null).build();
     }
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java
index 734c6eaf86965..bb37a2b1e80b4 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java
@@ -9,43 +9,20 @@
 
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
-import org.junit.Before;
 
 import java.io.IOException;
-import java.util.List;
 
-import static java.util.Collections.emptyList;
 import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 import static org.hamcrest.CoreMatchers.equalTo;
 
 public class ConnectorTests extends ESTestCase {
 
-    private NamedWriteableRegistry namedWriteableRegistry;
-
-    @Before
-    public void registerNamedObjects() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
-
-        List<NamedWriteableRegistry.Entry> namedWriteables = searchModule.getNamedWriteables();
-        namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables);
-    }
-
-    public final void testRandomSerialization() throws IOException {
-        for (int runs = 0; runs < 10; runs++) {
-            Connector testInstance = ConnectorTestUtils.getRandomConnector();
-            assertTransportSerialization(testInstance);
-        }
-    }
-
     public void testToXContent() throws IOException {
         String connectorId = "test-connector";
         String content = XContentHelper.stripWhitespace("""
@@ -225,7 +202,7 @@ public void testToXContent() throws IOException {
                "name":"test-name",
                "pipeline":{
                   "extract_binary_content":true,
-                  "name":"ent-search-generic-ingestion",
+                  "name":"search-default-ingestion",
                   "reduce_whitespace":true,
                   "run_ml_inference":false
                },
@@ -286,7 +263,7 @@ public void testToContent_WithNullValues() throws IOException {
                "name": null,
                "pipeline":{
                   "extract_binary_content":true,
-                  "name":"ent-search-generic-ingestion",
+                  "name":"search-default-ingestion",
                   "reduce_whitespace":true,
                   "run_ml_inference":false
                },
@@ -350,7 +327,7 @@ public void testToXContent_withOptionalFieldsMissing() throws IOException {
                "name": null,
                "pipeline":{
                   "extract_binary_content":true,
-                  "name":"ent-search-generic-ingestion",
+                  "name":"search-default-ingestion",
                   "reduce_whitespace":true,
                   "run_ml_inference":false
                },
@@ -383,14 +360,4 @@ public void testToXContent_withOptionalFieldsMissing() throws IOException {
         assertToXContentEquivalent(originalBytes, toXContent(parsed, XContentType.JSON, humanReadable), XContentType.JSON);
         assertThat(parsed.getApiKeySecretId(), equalTo(null));
     }
-
-    private void assertTransportSerialization(Connector testInstance) throws IOException {
-        Connector deserializedInstance = copyInstance(testInstance);
-        assertNotSame(testInstance, deserializedInstance);
-        assertThat(testInstance, equalTo(deserializedInstance));
-    }
-
-    private Connector copyInstance(Connector instance) throws IOException {
-        return copyWriteable(instance, namedWriteableRegistry, Connector::new);
-    }
 }
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java
deleted file mode 100644
index 2b8e8735fa2cc..0000000000000
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.application.connector.action;
-
-import org.elasticsearch.TransportVersion;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.test.AbstractBWCSerializationTestCase;
-import org.elasticsearch.xcontent.XContentParser;
-
-import java.io.IOException;
-
-public class GetConnectorActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase<GetConnectorAction.Request> {
-
-    @Override
-    protected Writeable.Reader<GetConnectorAction.Request> instanceReader() {
-        return GetConnectorAction.Request::new;
-    }
-
-    @Override
-    protected GetConnectorAction.Request createTestInstance() {
-        return new GetConnectorAction.Request(randomAlphaOfLengthBetween(1, 10));
-    }
-
-    @Override
-    protected GetConnectorAction.Request mutateInstance(GetConnectorAction.Request instance) throws IOException {
-        return randomValueOtherThan(instance, this::createTestInstance);
-    }
-
-    @Override
-    protected GetConnectorAction.Request doParseInstance(XContentParser parser) throws IOException {
-        return GetConnectorAction.Request.parse(parser);
-    }
-
-    @Override
-    protected GetConnectorAction.Request mutateInstanceForVersion(GetConnectorAction.Request instance, TransportVersion version) {
-        return new GetConnectorAction.Request(instance.getConnectorId());
-    }
-}
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java
deleted file mode 100644
index 3390ece073e5f..0000000000000
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.application.connector.action;
-
-import org.elasticsearch.TransportVersion;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.test.AbstractBWCSerializationTestCase;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.application.EnterpriseSearchModuleTestUtils;
-import org.elasticsearch.xpack.core.action.util.PageParams;
-
-import java.io.IOException;
-import java.util.List;
-
-public class ListConnectorActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase<ListConnectorAction.Request> {
-    @Override
-    protected Writeable.Reader<ListConnectorAction.Request> instanceReader() {
-        return ListConnectorAction.Request::new;
-    }
-
-    @Override
-    protected ListConnectorAction.Request createTestInstance() {
-        PageParams pageParams = EnterpriseSearchModuleTestUtils.randomPageParams();
-        return new ListConnectorAction.Request(
-            pageParams,
-            List.of(generateRandomStringArray(10, 10, false)),
-            List.of(generateRandomStringArray(10, 10, false)),
-            List.of(generateRandomStringArray(10, 10, false)),
-            randomAlphaOfLengthBetween(3, 10)
-        );
-    }
-
-    @Override
-    protected ListConnectorAction.Request mutateInstance(ListConnectorAction.Request instance) throws IOException {
-        return randomValueOtherThan(instance, this::createTestInstance);
-    }
-
-    @Override
-    protected ListConnectorAction.Request doParseInstance(XContentParser parser) throws IOException {
-        return ListConnectorAction.Request.parse(parser);
-    }
-
-    @Override
-    protected ListConnectorAction.Request mutateInstanceForVersion(ListConnectorAction.Request instance, TransportVersion version) {
-        return new ListConnectorAction.Request(
-            instance.getPageParams(),
-            instance.getIndexNames(),
-            instance.getConnectorNames(),
-            instance.getConnectorServiceTypes(),
-            instance.getConnectorSearchQuery()
-        );
-    }
-}
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PostConnectorActionTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PostConnectorActionTests.java
index 0f0e83f2b9c51..e482bf3f6bb74 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PostConnectorActionTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PostConnectorActionTests.java
@@ -20,7 +20,7 @@ public void testValidate_WhenConnectorIdAndIndexNamePresent_ExpectNoValidationEr
         PostConnectorAction.Request request = new PostConnectorAction.Request(
             randomAlphaOfLength(10),
             randomAlphaOfLength(10),
-            randomBoolean(),
+            false,
             randomAlphaOfLength(10),
             randomAlphaOfLength(10),
             randomAlphaOfLength(10)
@@ -30,6 +30,24 @@ public void testValidate_WhenConnectorIdAndIndexNamePresent_ExpectNoValidationEr
         assertThat(exception, nullValue());
     }
 
+    public void testValidate_WrongIndexNamePresentForManagedConnector_ExpectValidationError() {
+        PostConnectorAction.Request requestWithIllegalIndexName = new PostConnectorAction.Request(
+            randomAlphaOfLength(10),
+            "wrong-prefix-" + randomAlphaOfLength(10),
+            true,
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(10)
+        );
+        ActionRequestValidationException exception = requestWithIllegalIndexName.validate();
+
+        assertThat(exception, notNullValue());
+        assertThat(
+            exception.getMessage(),
+            containsString("Index attached to an Elastic-managed connector must start with the prefix: [content-]")
+        );
+    }
+
     public void testValidate_WhenMalformedIndexName_ExpectValidationError() {
         PostConnectorAction.Request requestWithMissingConnectorId = new PostConnectorAction.Request(
             randomAlphaOfLength(10),
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PutConnectorActionTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PutConnectorActionTests.java
index 873e102e40931..10ab049413565 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PutConnectorActionTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PutConnectorActionTests.java
@@ -21,7 +21,7 @@ public void testValidate_WhenConnectorIdAndIndexNamePresent_ExpectNoValidationEr
             randomAlphaOfLength(10),
             randomAlphaOfLength(10),
             randomAlphaOfLength(10),
-            randomBoolean(),
+            false,
             randomAlphaOfLength(10),
             randomAlphaOfLength(10),
             randomAlphaOfLength(10)
@@ -31,6 +31,25 @@ public void testValidate_WhenConnectorIdAndIndexNamePresent_ExpectNoValidationEr
         assertThat(exception, nullValue());
     }
 
+    public void testValidate_WrongIndexNamePresentForManagedConnector_ExpectValidationError() {
+        PutConnectorAction.Request requestWithIllegalIndexName = new PutConnectorAction.Request(
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(10),
+            "wrong-prefix-" + randomAlphaOfLength(10),
+            true,
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(10)
+        );
+        ActionRequestValidationException exception = requestWithIllegalIndexName.validate();
+
+        assertThat(exception, notNullValue());
+        assertThat(
+            exception.getMessage(),
+            containsString("Index attached to an Elastic-managed connector must start with the prefix: [content-]")
+        );
+    }
+
     public void testValidate_WhenMalformedIndexName_ExpectValidationError() {
         PutConnectorAction.Request requestWithMissingConnectorId = new PutConnectorAction.Request(
             randomAlphaOfLength(10),
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretResponseBWCSerializingTests.java
index 964c5e15d845d..5523ad4f50fe2 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretResponseBWCSerializingTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretResponseBWCSerializingTests.java
@@ -8,22 +8,14 @@
 package org.elasticsearch.xpack.application.connector.secrets.action;
 
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.xpack.application.connector.Connector;
 import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsTestUtils;
 import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase;
 
 import java.io.IOException;
-import java.util.List;
 
 public class DeleteConnectorSecretResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase<DeleteConnectorSecretResponse> {
 
-    @Override
-    public NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(List.of(new NamedWriteableRegistry.Entry(Connector.class, Connector.NAME, Connector::new)));
-    }
-
     @Override
     protected Writeable.Reader<DeleteConnectorSecretResponse> instanceReader() {
         return DeleteConnectorSecretResponse::new;
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretResponseBWCSerializingTests.java
index 4448024814df3..c5ea4cbc8e550 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretResponseBWCSerializingTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretResponseBWCSerializingTests.java
@@ -8,22 +8,14 @@
 package org.elasticsearch.xpack.application.connector.secrets.action;
 
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.xpack.application.connector.Connector;
 import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsTestUtils;
 import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase;
 
 import java.io.IOException;
-import java.util.List;
 
 public class GetConnectorSecretResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase<GetConnectorSecretResponse> {
 
-    @Override
-    public NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(List.of(new NamedWriteableRegistry.Entry(Connector.class, Connector.NAME, Connector::new)));
-    }
-
     @Override
     protected Writeable.Reader<GetConnectorSecretResponse> instanceReader() {
         return GetConnectorSecretResponse::new;
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java
index 81b05ce25e177..449dd258359c4 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java
@@ -8,17 +8,14 @@
 package org.elasticsearch.xpack.application.connector.syncjob;
 
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.application.connector.Connector;
 import org.elasticsearch.xpack.application.connector.ConnectorSyncStatus;
-import org.junit.Before;
 
 import java.io.IOException;
 import java.time.Instant;
-import java.util.List;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
@@ -26,22 +23,6 @@
 
 public class ConnectorSyncJobTests extends ESTestCase {
 
-    private NamedWriteableRegistry namedWriteableRegistry;
-
-    @Before
-    public void registerNamedObjects() {
-        namedWriteableRegistry = new NamedWriteableRegistry(
-            List.of(new NamedWriteableRegistry.Entry(Connector.class, Connector.NAME, Connector::new))
-        );
-    }
-
-    public final void testRandomSerialization() throws IOException {
-        for (int run = 0; run < 10; run++) {
-            ConnectorSyncJob syncJob = ConnectorSyncJobTestUtils.getRandomConnectorSyncJob();
-            assertTransportSerialization(syncJob);
-        }
-    }
-
     public void testFromXContent_WithAllFields_AllSet() throws IOException {
         String content = XContentHelper.stripWhitespace("""
             {
@@ -77,7 +58,7 @@ public void testFromXContent_WithAllFields_AllSet() throws IOException {
                     "language": "english",
                     "pipeline": {
                         "extract_binary_content": true,
-                        "name": "ent-search-generic-ingestion",
+                        "name": "search-default-ingestion",
                         "reduce_whitespace": true,
                         "run_ml_inference": false
                     },
@@ -160,7 +141,7 @@ public void testFromXContent_WithOnlyNonNullableFieldsSet_DoesNotThrow() throws
                     "language": "english",
                     "pipeline": {
                         "extract_binary_content": true,
-                        "name": "ent-search-generic-ingestion",
+                        "name": "search-default-ingestion",
                         "reduce_whitespace": true,
                         "run_ml_inference": false
                     },
@@ -218,7 +199,7 @@ public void testFromXContent_WithAllNullableFieldsSetToNull_DoesNotThrow() throw
                     "language": "english",
                     "pipeline": {
                         "extract_binary_content": true,
-                        "name": "ent-search-generic-ingestion",
+                        "name": "search-default-ingestion",
                         "reduce_whitespace": true,
                         "run_ml_inference": false
                     },
@@ -275,7 +256,7 @@ public void testSyncJobConnectorFromXContent_WithAllFieldsSet() throws IOExcepti
                 "language": "english",
                 "pipeline": {
                     "extract_binary_content": true,
-                    "name": "ent-search-generic-ingestion",
+                    "name": "search-default-ingestion",
                     "reduce_whitespace": true,
                     "run_ml_inference": false
                 },
@@ -332,14 +313,4 @@ public void testSyncJobConnectorFromXContent_WithAllNonOptionalFieldsSet_DoesNot
 
         ConnectorSyncJob.syncJobConnectorFromXContentBytes(new BytesArray(content), null, XContentType.JSON);
     }
-
-    private void assertTransportSerialization(ConnectorSyncJob testInstance) throws IOException {
-        ConnectorSyncJob deserializedInstance = copyInstance(testInstance);
-        assertNotSame(testInstance, deserializedInstance);
-        assertThat(testInstance, equalTo(deserializedInstance));
-    }
-
-    private ConnectorSyncJob copyInstance(ConnectorSyncJob instance) throws IOException {
-        return copyWriteable(instance, namedWriteableRegistry, ConnectorSyncJob::new);
-    }
 }
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionResponseBWCSerializingTests.java
index 00f6e7cf57fc1..ff915090f3c20 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionResponseBWCSerializingTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionResponseBWCSerializingTests.java
@@ -8,23 +8,15 @@
 package org.elasticsearch.xpack.application.connector.syncjob.action;
 
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.xpack.application.connector.Connector;
 import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils;
 import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase;
 
 import java.io.IOException;
-import java.util.List;
 
 public class GetConnectorSyncJobActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase<
     GetConnectorSyncJobAction.Response> {
 
-    @Override
-    public NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(List.of(new NamedWriteableRegistry.Entry(Connector.class, Connector.NAME, Connector::new)));
-    }
-
     @Override
     protected Writeable.Reader<GetConnectorSyncJobAction.Response> instanceReader() {
         return GetConnectorSyncJobAction.Response::new;
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsActionResponseBWCSerializingTests.java
index bc7b6320dddbe..ead5975d590a6 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsActionResponseBWCSerializingTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsActionResponseBWCSerializingTests.java
@@ -8,23 +8,15 @@
 package org.elasticsearch.xpack.application.connector.syncjob.action;
 
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.xpack.application.connector.Connector;
 import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils;
 import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase;
 
 import java.io.IOException;
-import java.util.List;
 
 public class ListConnectorSyncJobsActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase<
     ListConnectorSyncJobsAction.Response> {
 
-    @Override
-    protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(List.of(new NamedWriteableRegistry.Entry(Connector.class, Connector.NAME, Connector::new)));
-    }
-
     @Override
     protected Writeable.Reader<ListConnectorSyncJobsAction.Response> instanceReader() {
         return ListConnectorSyncJobsAction.Response::new;
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilderTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilderTests.java
index 3081dcf11d95e..3c169d9a76772 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilderTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilderTests.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Predicates;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.retriever.RetrieverBuilder;
@@ -53,10 +54,7 @@ protected QueryRuleRetrieverBuilder createTestInstance() {
     protected QueryRuleRetrieverBuilder doParseInstance(XContentParser parser) throws IOException {
         return (QueryRuleRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder(
             parser,
-            new RetrieverParserContext(
-                new SearchUsage(),
-                nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED || nf == QueryRuleRetrieverBuilder.QUERY_RULE_RETRIEVERS_SUPPORTED
-            )
+            new RetrieverParserContext(new SearchUsage(), Predicates.never())
         );
     }
 
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexServiceTests.java
index 6e9d33b45041b..0c4202666d60a 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexServiceTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexServiceTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.cluster.metadata.Metadata;
@@ -246,7 +247,10 @@ public void testDeleteSearchApplication() throws Exception {
         DeleteResponse resp = awaitDeleteSearchApplication("my_search_app_4");
         assertThat(resp.status(), equalTo(RestStatus.OK));
         expectThrows(ResourceNotFoundException.class, () -> awaitGetSearchApplication("my_search_app_4"));
-        GetAliasesResponse response = searchAppService.getAlias("my_search_app_4");
+        GetAliasesResponse response = client().admin()
+            .indices()
+            .getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "my_search_app_4"))
+            .actionGet();
         assertTrue(response.getAliases().isEmpty());
 
         {
diff --git a/x-pack/plugin/eql/build.gradle b/x-pack/plugin/eql/build.gradle
index 9ae67f0e27c2b..72ab7dabf0fcb 100644
--- a/x-pack/plugin/eql/build.gradle
+++ b/x-pack/plugin/eql/build.gradle
@@ -8,9 +8,9 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 esplugin {
-  name 'x-pack-eql'
-  description 'The Elasticsearch plugin that powers EQL for Elasticsearch'
-  classname 'org.elasticsearch.xpack.eql.plugin.EqlPlugin'
+  name = 'x-pack-eql'
+  description = 'The Elasticsearch plugin that powers EQL for Elasticsearch'
+  classname ='org.elasticsearch.xpack.eql.plugin.EqlPlugin'
   extendedPlugins = ['x-pack-ql', 'lang-painless']
 }
 
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java
index 90244d9b2c019..3557114e2f4c7 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java
@@ -33,6 +33,9 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+
 public abstract class BaseEqlSpecTestCase extends RemoteClusterAwareEqlRestTestCase {
 
     protected static final String PARAM_FORMATTING = "%2$s";
@@ -52,6 +55,9 @@ public abstract class BaseEqlSpecTestCase extends RemoteClusterAwareEqlRestTestC
      */
     private final int size;
     private final int maxSamplesPerKey;
+    private final Boolean allowPartialSearchResults;
+    private final Boolean allowPartialSequenceResults;
+    private final Boolean expectShardFailures;
 
     @Before
     public void setup() throws Exception {
@@ -104,7 +110,16 @@ protected static List<Object[]> asArray(List<EqlSpec> specs) {
             }
 
             results.add(
-                new Object[] { spec.query(), name, spec.expectedEventIds(), spec.joinKeys(), spec.size(), spec.maxSamplesPerKey() }
+                new Object[] {
+                    spec.query(),
+                    name,
+                    spec.expectedEventIds(),
+                    spec.joinKeys(),
+                    spec.size(),
+                    spec.maxSamplesPerKey(),
+                    spec.allowPartialSearchResults(),
+                    spec.allowPartialSequenceResults(),
+                    spec.expectShardFailures() }
             );
         }
 
@@ -118,7 +133,10 @@ protected static List<Object[]> asArray(List<EqlSpec> specs) {
         List<long[]> eventIds,
         String[] joinKeys,
         Integer size,
-        Integer maxSamplesPerKey
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
     ) {
         this.index = index;
 
@@ -128,6 +146,9 @@ protected static List<Object[]> asArray(List<EqlSpec> specs) {
         this.joinKeys = joinKeys;
         this.size = size == null ? -1 : size;
         this.maxSamplesPerKey = maxSamplesPerKey == null ? -1 : maxSamplesPerKey;
+        this.allowPartialSearchResults = allowPartialSearchResults;
+        this.allowPartialSequenceResults = allowPartialSequenceResults;
+        this.expectShardFailures = expectShardFailures;
     }
 
     public void test() throws Exception {
@@ -137,6 +158,7 @@ public void test() throws Exception {
     private void assertResponse(ObjectPath response) throws Exception {
         List<Map<String, Object>> events = response.evaluate("hits.events");
         List<Map<String, Object>> sequences = response.evaluate("hits.sequences");
+        Object shardFailures = response.evaluate("shard_failures");
 
         if (events != null) {
             assertEvents(events);
@@ -145,6 +167,7 @@ private void assertResponse(ObjectPath response) throws Exception {
         } else {
             fail("No events or sequences found");
         }
+        assertShardFailures(shardFailures);
     }
 
     protected ObjectPath runQuery(String index, String query) throws Exception {
@@ -163,6 +186,32 @@ protected ObjectPath runQuery(String index, String query) throws Exception {
         if (maxSamplesPerKey > 0) {
             builder.field("max_samples_per_key", maxSamplesPerKey);
         }
+        boolean allowPartialResultsInBody = randomBoolean();
+        if (allowPartialSearchResults != null) {
+            if (allowPartialResultsInBody) {
+                builder.field("allow_partial_search_results", String.valueOf(allowPartialSearchResults));
+                if (allowPartialSequenceResults != null) {
+                    builder.field("allow_partial_sequence_results", String.valueOf(allowPartialSequenceResults));
+                }
+            } else {
+                // these will be overwritten by the path params, that have higher priority than the query (JSON body) params
+                if (allowPartialSearchResults != null) {
+                    builder.field("allow_partial_search_results", randomBoolean());
+                }
+                if (allowPartialSequenceResults != null) {
+                    builder.field("allow_partial_sequence_results", randomBoolean());
+                }
+            }
+        } else {
+            // Tests that don't specify a setting for these parameters should always pass.
+            // These params should be irrelevant.
+            if (randomBoolean()) {
+                builder.field("allow_partial_search_results", randomBoolean());
+            }
+            if (randomBoolean()) {
+                builder.field("allow_partial_sequence_results", randomBoolean());
+            }
+        }
         builder.endObject();
 
         Request request = new Request("POST", "/" + index + "/_eql/search");
@@ -170,6 +219,23 @@ protected ObjectPath runQuery(String index, String query) throws Exception {
         if (ccsMinimizeRoundtrips != null) {
             request.addParameter("ccs_minimize_roundtrips", ccsMinimizeRoundtrips.toString());
         }
+        if (allowPartialSearchResults != null) {
+            if (allowPartialResultsInBody == false) {
+                request.addParameter("allow_partial_search_results", String.valueOf(allowPartialSearchResults));
+                if (allowPartialSequenceResults != null) {
+                    request.addParameter("allow_partial_sequence_results", String.valueOf(allowPartialSequenceResults));
+                }
+            }
+        } else {
+            // Tests that don't specify a setting for these parameters should always pass.
+            // These params should be irrelevant.
+            if (randomBoolean()) {
+                request.addParameter("allow_partial_search_results", String.valueOf(randomBoolean()));
+            }
+            if (randomBoolean()) {
+                request.addParameter("allow_partial_sequence_results", String.valueOf(randomBoolean()));
+            }
+        }
         int timeout = Math.toIntExact(timeout().millis());
         RequestConfig config = RequestConfig.copy(RequestConfig.DEFAULT)
             .setConnectionRequestTimeout(timeout)
@@ -182,6 +248,20 @@ protected ObjectPath runQuery(String index, String query) throws Exception {
         return ObjectPath.createFromResponse(client().performRequest(request));
     }
 
+    private void assertShardFailures(Object shardFailures) {
+        if (expectShardFailures != null) {
+            if (expectShardFailures) {
+                assertNotNull(shardFailures);
+                List<?> list = (List<?>) shardFailures;
+                assertThat(list.size(), is(greaterThan(0)));
+            } else {
+                assertNull(shardFailures);
+            }
+        } else {
+            assertNull(shardFailures);
+        }
+    }
+
     private void assertEvents(List<Map<String, Object>> events) {
         assertNotNull(events);
         logger.debug("Events {}", new Object() {
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java
index 1d51af574c810..4618bd8f4ff3d 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java
@@ -52,6 +52,7 @@
  */
 public class DataLoader {
     public static final String TEST_INDEX = "endgame-140";
+    public static final String TEST_SHARD_FAILURES_INDEX = "endgame-shard-failures";
     public static final String TEST_EXTRA_INDEX = "extra";
     public static final String TEST_NANOS_INDEX = "endgame-140-nanos";
     public static final String TEST_SAMPLE = "sample1,sample2,sample3";
@@ -103,6 +104,11 @@ public static void loadDatasetIntoEs(RestClient client, CheckedBiFunction<XConte
         //
         load(client, TEST_MISSING_EVENTS_INDEX, null, null, p);
         load(client, TEST_SAMPLE_MULTI, null, null, p);
+        //
+        // index with a runtime field ("broken", type long) that causes shard failures.
+        // the rest of the mapping is the same as TEST_INDEX
+        //
+        load(client, TEST_SHARD_FAILURES_INDEX, null, DataLoader::timestampToUnixMillis, p);
     }
 
     private static void load(
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlDateNanosSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlDateNanosSpecTestCase.java
index a368ea0d22aa9..583a3eda69953 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlDateNanosSpecTestCase.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlDateNanosSpecTestCase.java
@@ -27,9 +27,23 @@ public EqlDateNanosSpecTestCase(
         List<long[]> eventIds,
         String[] joinKeys,
         Integer size,
-        Integer maxSamplesPerKey
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
     ) {
-        this(TEST_NANOS_INDEX, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+        this(
+            TEST_NANOS_INDEX,
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 
     // constructor for multi-cluster tests
@@ -40,9 +54,23 @@ public EqlDateNanosSpecTestCase(
         List<long[]> eventIds,
         String[] joinKeys,
         Integer size,
-        Integer maxSamplesPerKey
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
     ) {
-        super(index, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+        super(
+            index,
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java
index 292fe6c895cee..cc858ded25f37 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java
@@ -27,9 +27,23 @@ public EqlExtraSpecTestCase(
         List<long[]> eventIds,
         String[] joinKeys,
         Integer size,
-        Integer maxSamplesPerKey
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
     ) {
-        this(TEST_EXTRA_INDEX, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+        this(
+            TEST_EXTRA_INDEX,
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 
     // constructor for multi-cluster tests
@@ -40,9 +54,23 @@ public EqlExtraSpecTestCase(
         List<long[]> eventIds,
         String[] joinKeys,
         Integer size,
-        Integer maxSamplesPerKey
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
     ) {
-        super(index, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+        super(
+            index,
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlMissingEventsSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlMissingEventsSpecTestCase.java
index cdda9e9e068f5..f62c2b29101db 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlMissingEventsSpecTestCase.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlMissingEventsSpecTestCase.java
@@ -27,9 +27,23 @@ public EqlMissingEventsSpecTestCase(
         List<long[]> eventIds,
         String[] joinKeys,
         Integer size,
-        Integer maxSamplesPerKey
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
     ) {
-        this(TEST_MISSING_EVENTS_INDEX, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+        this(
+            TEST_MISSING_EVENTS_INDEX,
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 
     // constructor for multi-cluster tests
@@ -40,9 +54,23 @@ public EqlMissingEventsSpecTestCase(
         List<long[]> eventIds,
         String[] joinKeys,
         Integer size,
-        Integer maxSamplesPerKey
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
     ) {
-        super(index, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+        super(
+            index,
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleMultipleEntriesTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleMultipleEntriesTestCase.java
index 6471e264a92fa..a38ccacb42f5f 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleMultipleEntriesTestCase.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleMultipleEntriesTestCase.java
@@ -21,9 +21,23 @@ public EqlSampleMultipleEntriesTestCase(
         List<long[]> eventIds,
         String[] joinKeys,
         Integer size,
-        Integer maxSamplesPerKey
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
     ) {
-        this(TEST_SAMPLE_MULTI, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+        this(
+            TEST_SAMPLE_MULTI,
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 
     public EqlSampleMultipleEntriesTestCase(
@@ -33,9 +47,23 @@ public EqlSampleMultipleEntriesTestCase(
         List<long[]> eventIds,
         String[] joinKeys,
         Integer size,
-        Integer maxSamplesPerKey
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
     ) {
-        super(index, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+        super(
+            index,
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 
     @ParametersFactory(shuffle = false, argumentFormatting = PARAM_FORMATTING)
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleTestCase.java
index dfae73b3602a7..4748bd0e3307b 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleTestCase.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleTestCase.java
@@ -15,8 +15,29 @@
 
 public abstract class EqlSampleTestCase extends BaseEqlSpecTestCase {
 
-    public EqlSampleTestCase(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
-        this(TEST_SAMPLE, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+    public EqlSampleTestCase(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
+    ) {
+        this(
+            TEST_SAMPLE,
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 
     public EqlSampleTestCase(
@@ -26,9 +47,23 @@ public EqlSampleTestCase(
         List<long[]> eventIds,
         String[] joinKeys,
         Integer size,
-        Integer maxSamplesPerKey
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
     ) {
-        super(index, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+        super(
+            index,
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 
     @ParametersFactory(shuffle = false, argumentFormatting = PARAM_FORMATTING)
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java
index db7ee05ff2239..4dd617bac0abd 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java
@@ -30,6 +30,9 @@ public class EqlSpec {
 
     private Integer size;
     private Integer maxSamplesPerKey;
+    private Boolean allowPartialSearchResults;
+    private Boolean allowPartialSequenceResults;
+    private Boolean expectShardFailures;
 
     public String name() {
         return name;
@@ -103,6 +106,30 @@ public void maxSamplesPerKey(Integer maxSamplesPerKey) {
         this.maxSamplesPerKey = maxSamplesPerKey;
     }
 
+    public Boolean allowPartialSearchResults() {
+        return allowPartialSearchResults;
+    }
+
+    public void allowPartialSearchResults(Boolean allowPartialSearchResults) {
+        this.allowPartialSearchResults = allowPartialSearchResults;
+    }
+
+    public Boolean allowPartialSequenceResults() {
+        return allowPartialSequenceResults;
+    }
+
+    public void allowPartialSequenceResults(Boolean allowPartialSequenceResults) {
+        this.allowPartialSequenceResults = allowPartialSequenceResults;
+    }
+
+    public Boolean expectShardFailures() {
+        return expectShardFailures;
+    }
+
+    public void expectShardFailures(Boolean expectShardFailures) {
+        this.expectShardFailures = expectShardFailures;
+    }
+
     @Override
     public String toString() {
         String str = "";
@@ -132,7 +159,15 @@ public String toString() {
         if (maxSamplesPerKey != null) {
             str = appendWithComma(str, "max_samples_per_key", "" + maxSamplesPerKey);
         }
-
+        if (allowPartialSearchResults != null) {
+            str = appendWithComma(str, "allow_partial_search_results", String.valueOf(allowPartialSearchResults));
+        }
+        if (allowPartialSequenceResults != null) {
+            str = appendWithComma(str, "allow_partial_sequence_results", String.valueOf(allowPartialSequenceResults));
+        }
+        if (expectShardFailures != null) {
+            str = appendWithComma(str, "expect_shard_failures", String.valueOf(expectShardFailures));
+        }
         return str;
     }
 
@@ -150,12 +185,22 @@ public boolean equals(Object other) {
 
         return Objects.equals(this.query(), that.query())
             && Objects.equals(size, that.size)
-            && Objects.equals(maxSamplesPerKey, that.maxSamplesPerKey);
+            && Objects.equals(maxSamplesPerKey, that.maxSamplesPerKey)
+            && Objects.equals(allowPartialSearchResults, that.allowPartialSearchResults)
+            && Objects.equals(allowPartialSequenceResults, that.allowPartialSequenceResults)
+            && Objects.equals(expectShardFailures, that.expectShardFailures);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(this.query, size, maxSamplesPerKey);
+        return Objects.hash(
+            this.query,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 
     private static String appendWithComma(String str, String name, String append) {
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecFailingShardsTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecFailingShardsTestCase.java
new file mode 100644
index 0000000000000..c490a2f703dcc
--- /dev/null
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecFailingShardsTestCase.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.test.eql;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import java.util.List;
+
+import static org.elasticsearch.test.eql.DataLoader.TEST_INDEX;
+import static org.elasticsearch.test.eql.DataLoader.TEST_SHARD_FAILURES_INDEX;
+
+public abstract class EqlSpecFailingShardsTestCase extends BaseEqlSpecTestCase {
+
+    @ParametersFactory(shuffle = false, argumentFormatting = PARAM_FORMATTING)
+    public static List<Object[]> readTestSpecs() throws Exception {
+
+        // Load EQL validation specs
+        return asArray(EqlSpecLoader.load("/test_failing_shards.toml"));
+    }
+
+    @Override
+    protected String tiebreaker() {
+        return "serial_event_id";
+    }
+
+    // constructor for "local" rest tests
+    public EqlSpecFailingShardsTestCase(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
+    ) {
+        this(
+            TEST_INDEX + "," + TEST_SHARD_FAILURES_INDEX,
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
+    }
+
+    // constructor for multi-cluster tests
+    public EqlSpecFailingShardsTestCase(
+        String index,
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearch,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
+    ) {
+        super(
+            index,
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearch,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
+    }
+}
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java
index a1f555563e29c..f86107cf3bac5 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java
@@ -76,6 +76,10 @@ private static Integer getInteger(TomlTable table, String key) {
         return null;
     }
 
+    private static Boolean getBoolean(TomlTable table, String key) {
+        return table.getBoolean(key);
+    }
+
     private static List<EqlSpec> readFromStream(InputStream is, Set<String> uniqueTestNames) throws Exception {
         List<EqlSpec> testSpecs = new ArrayList<>();
 
@@ -90,6 +94,9 @@ private static List<EqlSpec> readFromStream(InputStream is, Set<String> uniqueTe
             spec.note(getTrimmedString(table, "note"));
             spec.description(getTrimmedString(table, "description"));
             spec.size(getInteger(table, "size"));
+            spec.allowPartialSearchResults(getBoolean(table, "allow_partial_search_results"));
+            spec.allowPartialSequenceResults(getBoolean(table, "allow_partial_sequence_results"));
+            spec.expectShardFailures(getBoolean(table, "expect_shard_failures"));
 
             List<?> arr = table.getList("tags");
             if (arr != null) {
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java
index 7113924f79029..62a3ea72fe51f 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java
@@ -28,8 +28,29 @@ protected String tiebreaker() {
     }
 
     // constructor for "local" rest tests
-    public EqlSpecTestCase(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
-        this(TEST_INDEX, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+    public EqlSpecTestCase(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearch,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
+    ) {
+        this(
+            TEST_INDEX,
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearch,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 
     // constructor for multi-cluster tests
@@ -40,8 +61,22 @@ public EqlSpecTestCase(
         List<long[]> eventIds,
         String[] joinKeys,
         Integer size,
-        Integer maxSamplesPerKey
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearch,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
     ) {
-        super(index, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+        super(
+            index,
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearch,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 }
diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.data b/x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.data
new file mode 100644
index 0000000000000..18a1d05656d09
--- /dev/null
+++ b/x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.data
@@ -0,0 +1,14 @@
+[
+  {
+    "event_subtype_full": "already_running",
+    "event_type": "process",
+    "event_type_full": "process_event",
+    "opcode": 3,
+    "pid": 0,
+    "process_name": "System Idle Process",
+    "serial_event_id": 10000,
+    "subtype": "create",
+    "timestamp": 117444736000000000,
+    "unique_pid": 1
+  }
+]
diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.mapping b/x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.mapping
new file mode 100644
index 0000000000000..3b5039f4098af
--- /dev/null
+++ b/x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.mapping
@@ -0,0 +1,105 @@
+# Text patterns like "[runtime_random_keyword_type]" will get replaced at runtime with a random string type.
+# See DataLoader class for pattern replacements.
+{
+    "runtime":{
+          "broken":{
+            "type": "long",
+            "script": {
+              "lang": "painless",
+              "source": "emit(doc['non_existing'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
+            }
+        }
+    },
+    "properties" : {
+        "command_line" : {
+            "type" : "[runtime_random_keyword_type]"
+        },
+        "event_type" : {
+            "type" : "[runtime_random_keyword_type]"
+        },
+        "event" : {
+            "properties" : {
+                "category" : {
+                    "type" : "alias",
+                    "path" : "event_type"
+                },
+                "sequence" : {
+                    "type" : "alias",
+                    "path" : "serial_event_id"
+                }
+            }
+        },
+        "md5" : {
+            "type" : "[runtime_random_keyword_type]"
+        },
+        "parent_process_name": {
+            "type" : "[runtime_random_keyword_type]"
+        },
+        "parent_process_path": {
+            "type" : "[runtime_random_keyword_type]"
+        },
+        "pid" : {
+            "type" : "long"
+        },
+        "ppid" : {
+            "type" : "long"
+        },
+        "process_name": {
+            "type" : "[runtime_random_keyword_type]"
+        },
+        "process_path": {
+            "type" : "[runtime_random_keyword_type]"
+        },
+        "subtype" : {
+            "type" : "[runtime_random_keyword_type]"
+        },
+        "timestamp" : {
+            "type" : "date"
+        },
+        "@timestamp" : {
+            "type" : "date"
+        },
+        "user" : {
+            "type" : "[runtime_random_keyword_type]"
+        },
+        "user_name" : {
+            "type" : "[runtime_random_keyword_type]"
+        },
+        "user_domain": {
+            "type" : "[runtime_random_keyword_type]"
+        },
+        "hostname" : {
+            "type" : "text",
+            "fields" : {
+                "[runtime_random_keyword_type]" : {
+                    "type" : "[runtime_random_keyword_type]",
+                    "ignore_above" : 256
+                }
+            }
+        },
+        "opcode" : {
+            "type" : "long"
+        },
+        "file_name" : {
+            "type" : "text",
+            "fields" : {
+                "[runtime_random_keyword_type]" : {
+                    "type" : "[runtime_random_keyword_type]",
+                    "ignore_above" : 256
+                }
+            }
+        },
+        "file_path" : {
+          "type" : "[runtime_random_keyword_type]"
+        },
+        "serial_event_id" : {
+            "type" : "long"
+        },
+        "source_address" : {
+            "type" : "ip"
+        },
+        "exit_code" : {
+            "type" : "long"
+        }
+    }
+}
diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_failing_shards.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_failing_shards.toml
new file mode 100644
index 0000000000000..a551c66fd48bd
--- /dev/null
+++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_failing_shards.toml
@@ -0,0 +1,173 @@
+# this query doesn't touch the "broken" field, so it should not fail
+[[queries]]
+name = "eventQueryNoShardFailures"
+query = 'process where serial_event_id == 1'
+allow_partial_search_results = true
+expected_event_ids  = [1]
+expect_shard_failures = false
+
+
+[[queries]]
+name = "eventQueryShardFailures"
+query = 'process where serial_event_id == 1 or broken == 1'
+allow_partial_search_results = true
+expected_event_ids  = [1]
+expect_shard_failures = true
+
+
+[[queries]]
+name = "eventQueryShardFailuresOptionalField"
+query = 'process where serial_event_id == 1 and ?optional_field_default_null == null or broken == 1'
+allow_partial_search_results = true
+expected_event_ids  = [1]
+expect_shard_failures = true
+
+
+[[queries]]
+name = "eventQueryShardFailuresOptionalFieldMatching"
+query = 'process where serial_event_id == 2 and ?subtype == "create" or broken == 1'
+allow_partial_search_results = true
+expected_event_ids  = [2]
+expect_shard_failures = true
+
+
+# this query doesn't touch the "broken" field, so it should not fail
+[[queries]]
+name = "sequenceQueryNoShardFailures"
+query = '''
+sequence
+  [process where serial_event_id == 1]
+  [process where serial_event_id == 2]
+'''
+expected_event_ids  = [1, 2]
+expect_shard_failures = false
+
+
+# this query doesn't touch the "broken" field, so it should not fail
+[[queries]]
+name = "sequenceQueryNoShardFailuresAllowFalse"
+query = '''
+sequence
+  [process where serial_event_id == 1]
+  [process where serial_event_id == 2]
+'''
+allow_partial_search_results = false
+expected_event_ids  = [1, 2]
+expect_shard_failures = false
+
+
+# this query doesn't touch the "broken" field, so it should not fail
+[[queries]]
+name = "sequenceQueryNoShardFailuresAllowTrue"
+query = '''
+sequence
+  [process where serial_event_id == 1]
+  [process where serial_event_id == 2]
+'''
+allow_partial_search_results = true
+expected_event_ids  = [1, 2]
+expect_shard_failures = false
+
+
+[[queries]]
+name = "sequenceQueryMissingShards"
+query = '''
+sequence
+  [process where serial_event_id == 1 or broken == 1]
+  [process where serial_event_id == 2]
+'''
+allow_partial_search_results = true
+expected_event_ids  = []
+expect_shard_failures = true
+
+
+[[queries]]
+name = "sequenceQueryMissingShardsPartialResults"
+query = '''
+sequence
+  [process where serial_event_id == 1 or broken == 1]
+  [process where serial_event_id == 2]
+'''
+allow_partial_search_results = true
+allow_partial_sequence_results = true
+expected_event_ids  = [1, 2]
+expect_shard_failures = true
+
+
+[[queries]]
+name = "sequenceQueryMissingShardsPartialResultsOptional"
+query = '''
+sequence
+  [process where ?serial_event_id == 1 or broken == 1]
+  [process where serial_event_id == 2]
+'''
+allow_partial_search_results = true
+allow_partial_sequence_results = true
+expected_event_ids  = [1, 2]
+expect_shard_failures = true
+
+
+[[queries]]
+name = "sequenceQueryMissingShardsPartialResultsOptional2"
+query = '''
+sequence with maxspan=100000d
+  [process where serial_event_id == 1 and ?subtype == "create" or broken == 1]
+  [process where serial_event_id == 2]
+'''
+allow_partial_search_results = true
+allow_partial_sequence_results = true
+expected_event_ids  = [1, 2]
+expect_shard_failures = true
+
+
+[[queries]]
+name = "sequenceQueryMissingShardsPartialResultsOptionalMissing"
+query = '''
+sequence with maxspan=100000d
+  [process where serial_event_id == 1 and ?subtype == "create"]
+ ![process where broken == 1]
+  [process where serial_event_id == 2]
+'''
+allow_partial_search_results = true
+allow_partial_sequence_results = true
+expected_event_ids  = [1, -1, 2]
+expect_shard_failures = true
+
+
+[[queries]]
+name = "sequenceQueryMissingShardsPartialResultsOptionalMissing2"
+query = '''
+sequence with maxspan=100000d
+  [process where serial_event_id == 1 and ?subtype == "create" or broken == 1]
+ ![process where broken == 1]
+  [process where serial_event_id == 2]
+'''
+allow_partial_search_results = true
+allow_partial_sequence_results = true
+expected_event_ids  = [1, -1, 2]
+expect_shard_failures = true
+
+
+[[queries]]
+name = "sampleQueryMissingShardsPartialResults"
+query = '''
+sample by event_subtype_full
+  [process where serial_event_id == 1 or broken == 1]
+  [process where serial_event_id == 2]
+'''
+allow_partial_search_results = true
+expected_event_ids  = [1, 2]
+expect_shard_failures = true
+
+
+[[queries]]
+name = "sampleQueryMissingShardsPartialResultsOptional"
+query = '''
+sample by event_subtype_full
+  [process where serial_event_id == 1 and ?subtype == "create" or broken == 1]
+  [process where serial_event_id == 2]
+'''
+allow_partial_search_results = true
+expected_event_ids  = [1, 2]
+expect_shard_failures = true
+
diff --git a/x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java b/x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java
index 2a29572374fa8..60c7fb1c7ad25 100644
--- a/x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java
+++ b/x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java
@@ -407,7 +407,16 @@ private void assertMultiValueFunctionQuery(
         for (int id : ids) {
             eventIds.add(String.valueOf(id));
         }
-        request.setJsonEntity("{\"query\":\"" + query + "\"}");
+
+        StringBuilder payload = new StringBuilder("{\"query\":\"" + query + "\"");
+        if (randomBoolean()) {
+            payload.append(", \"allow_partial_search_results\": true");
+        }
+        if (randomBoolean()) {
+            payload.append(", \"allow_partial_sequence_results\": true");
+        }
+        payload.append("}");
+        request.setJsonEntity(payload.toString());
         assertResponse(query, eventIds, runEql(client, request));
         testedFunctions.add(functionName);
     }
diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java
index c20968871472f..5d6824232d80f 100644
--- a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java
+++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java
@@ -37,7 +37,28 @@ protected String getRemoteCluster() {
         return REMOTE_CLUSTER.getHttpAddresses();
     }
 
-    public EqlDateNanosIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
-        super(remoteClusterIndex(TEST_NANOS_INDEX), query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+    public EqlDateNanosIT(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
+    ) {
+        super(
+            remoteClusterIndex(TEST_NANOS_INDEX),
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 }
diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java
index 774c19d02adf0..79b095434814b 100644
--- a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java
+++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java
@@ -37,7 +37,28 @@ protected String getRemoteCluster() {
         return REMOTE_CLUSTER.getHttpAddresses();
     }
 
-    public EqlExtraIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
-        super(remoteClusterIndex(TEST_EXTRA_INDEX), query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+    public EqlExtraIT(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
+    ) {
+        super(
+            remoteClusterIndex(TEST_EXTRA_INDEX),
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 }
diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java
index 1502c250bd058..7673eec32ec55 100644
--- a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java
+++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java
@@ -37,7 +37,28 @@ protected String getRemoteCluster() {
         return REMOTE_CLUSTER.getHttpAddresses();
     }
 
-    public EqlSampleIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
-        super(remoteClusterPattern(TEST_SAMPLE), query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+    public EqlSampleIT(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
+    ) {
+        super(
+            remoteClusterPattern(TEST_SAMPLE),
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 }
diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java
index 795fe4e103a31..ac6f7fe508c99 100644
--- a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java
+++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java
@@ -43,8 +43,22 @@ public EqlSampleMultipleEntriesIT(
         List<long[]> eventIds,
         String[] joinKeys,
         Integer size,
-        Integer maxSamplesPerKey
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
     ) {
-        super(remoteClusterPattern(TEST_SAMPLE_MULTI), query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+        super(
+            remoteClusterPattern(TEST_SAMPLE_MULTI),
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 }
diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java
index 2cddecb644a1a..db0c03e8fdb6f 100644
--- a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java
+++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java
@@ -37,7 +37,28 @@ protected String getRemoteCluster() {
         return REMOTE_CLUSTER.getHttpAddresses();
     }
 
-    public EqlSpecIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
-        super(remoteClusterIndex(TEST_INDEX), query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+    public EqlSpecIT(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
+    ) {
+        super(
+            remoteClusterIndex(TEST_INDEX),
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 }
diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java
index 1df10fde7fde5..5e1fa224de58d 100644
--- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java
+++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java
@@ -27,7 +27,27 @@ protected String getTestRestCluster() {
         return cluster.getHttpAddresses();
     }
 
-    public EqlDateNanosIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
-        super(query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+    public EqlDateNanosIT(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
+    ) {
+        super(
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 }
diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java
index 8af8fcac087b5..cb92eddeb0410 100644
--- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java
+++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java
@@ -27,7 +27,27 @@ protected String getTestRestCluster() {
         return cluster.getHttpAddresses();
     }
 
-    public EqlExtraIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
-        super(query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+    public EqlExtraIT(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
+    ) {
+        super(
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 }
diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlMissingEventsIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlMissingEventsIT.java
index 05557fb4883b3..4f1faf3322e7f 100644
--- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlMissingEventsIT.java
+++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlMissingEventsIT.java
@@ -27,8 +27,28 @@ protected String getTestRestCluster() {
         return cluster.getHttpAddresses();
     }
 
-    public EqlMissingEventsIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
-        super(query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+    public EqlMissingEventsIT(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
+    ) {
+        super(
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 
 }
diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java
index dc2c653fad89e..c0bce3ffc9e4f 100644
--- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java
+++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java
@@ -27,8 +27,28 @@ protected String getTestRestCluster() {
         return cluster.getHttpAddresses();
     }
 
-    public EqlSampleIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
-        super(query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+    public EqlSampleIT(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
+    ) {
+        super(
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 
 }
diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java
index af1ade9120bbd..f50ee36095ae0 100644
--- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java
+++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java
@@ -33,9 +33,22 @@ public EqlSampleMultipleEntriesIT(
         List<long[]> eventIds,
         String[] joinKeys,
         Integer size,
-        Integer maxSamplesPerKey
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
     ) {
-        super(query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+        super(
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 
 }
diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecFailingShardsIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecFailingShardsIT.java
new file mode 100644
index 0000000000000..cf05811a77857
--- /dev/null
+++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecFailingShardsIT.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.eql;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.elasticsearch.test.TestClustersThreadFilter;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.eql.EqlSpecFailingShardsTestCase;
+import org.junit.ClassRule;
+
+import java.util.List;
+
+@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
+public class EqlSpecFailingShardsIT extends EqlSpecFailingShardsTestCase {
+
+    @ClassRule
+    public static final ElasticsearchCluster cluster = EqlTestCluster.CLUSTER;
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    public EqlSpecFailingShardsIT(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
+    ) {
+        super(
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
+    }
+}
diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java
index 7aac0ae336c8a..0aad5cc1b73da 100644
--- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java
+++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java
@@ -27,7 +27,27 @@ protected String getTestRestCluster() {
         return cluster.getHttpAddresses();
     }
 
-    public EqlSpecIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
-        super(query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+    public EqlSpecIT(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey,
+        Boolean allowPartialSearchResults,
+        Boolean allowPartialSequenceResults,
+        Boolean expectShardFailures
+    ) {
+        super(
+            query,
+            name,
+            eventIds,
+            joinKeys,
+            size,
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults,
+            expectShardFailures
+        );
     }
 }
diff --git a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml
index e49264d76d5e9..c7974f3b584b4 100644
--- a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml
+++ b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml
@@ -83,6 +83,34 @@ setup:
             id: 123
             valid: true
 
+  - do:
+      indices.create:
+        index:  eql_test_rebel
+        body:
+          mappings:
+            properties:
+              some_keyword:
+                type: keyword
+            runtime:
+              day_of_week:
+                type: keyword
+                script:
+                  source: "throw new IllegalArgumentException(\"rebel shards\")"
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: eql_test_rebel
+              _id:    "1"
+          - event:
+              - category: process
+            "@timestamp": 2020-02-03T12:34:56Z
+            user: SYSTEM
+            id: 123
+            valid: false
+            some_keyword: longer than normal
+
 ---
 # Testing round-trip and the basic shape of the response
 "Execute some EQL.":
@@ -478,3 +506,118 @@ setup:
           query: 'sequence with maxspan=10d [network where user == "ADMIN"] ![network where used == "SYSTEM"]'
   - match: { error.root_cause.0.type: "verification_exception" }
   - match: { error.root_cause.0.reason: "Found 1 problem\nline 1:75: Unknown column [used], did you mean [user]?" }
+
+
+---
+"Execute query shard failures and with allow_partial_search_results":
+  - do:
+      eql.search:
+        index: eql_test*
+        body:
+          query: 'process where user == "SYSTEM" and day_of_week == "Monday"'
+          fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]
+          allow_partial_search_results: true
+
+  - match: {timed_out: false}
+  - match: {hits.total.value: 1}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.events.0._source.user: "SYSTEM"}
+  - match: {hits.events.0._id: "1"}
+  - match: {hits.events.0.fields.@timestamp: ["1580733296000"]}
+  - match: {hits.events.0.fields.id: [123]}
+  - match: {hits.events.0.fields.valid: [false]}
+  - match: {hits.events.0.fields.day_of_week: ["Monday"]}
+  - match: {shard_failures.0.index: "eql_test_rebel"}
+
+
+---
+"Execute query shard failures and with allow_partial_search_results as request param":
+  - do:
+      eql.search:
+        index: eql_test*
+        allow_partial_search_results: true
+        body:
+          query: 'process where user == "SYSTEM" and day_of_week == "Monday"'
+          fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]
+
+  - match: {timed_out: false}
+  - match: {hits.total.value: 1}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.events.0._source.user: "SYSTEM"}
+  - match: {hits.events.0._id: "1"}
+  - match: {hits.events.0.fields.@timestamp: ["1580733296000"]}
+  - match: {hits.events.0.fields.id: [123]}
+  - match: {hits.events.0.fields.valid: [false]}
+  - match: {hits.events.0.fields.day_of_week: ["Monday"]}
+  - match: {shard_failures.0.index: "eql_test_rebel"}
+
+
+---
+"Execute sequence with shard failures and allow_partial_search_results=true":
+  - do:
+      eql.search:
+        index: eql_test*
+        body:
+          query: 'sequence [process where user == "SYSTEM" and day_of_week == "Monday"] [process where user == "SYSTEM" and day_of_week == "Tuesday"]'
+          fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]
+          allow_partial_search_results: true
+
+  - match: {timed_out: false}
+  - match: {hits.total.value: 0}
+  - match: {shard_failures.0.index: "eql_test_rebel"}
+
+
+---
+"Execute sequence with shard failures, allow_partial_search_results=true and allow_partial_sequence_results=true":
+  - do:
+      eql.search:
+        index: eql_test*
+        body:
+          query: 'sequence [process where user == "SYSTEM" and day_of_week == "Monday"] [process where user == "SYSTEM" and day_of_week == "Tuesday"]'
+          fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]
+          allow_partial_search_results: true
+          allow_partial_sequence_results: true
+
+  - match: {timed_out: false}
+  - match: {hits.total.value: 1}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.events.0._source.user: "SYSTEM"}
+  - match: {hits.sequences.0.events.0._id: "1"}
+  - match: {hits.sequences.0.events.0.fields.@timestamp: ["1580733296000"]}
+  - match: {hits.sequences.0.events.0.fields.id: [123]}
+  - match: {hits.sequences.0.events.0.fields.valid: [false]}
+  - match: {hits.sequences.0.events.0.fields.day_of_week: ["Monday"]}
+  - match: {hits.sequences.0.events.1._id: "2"}
+  - match: {hits.sequences.0.events.1.fields.@timestamp: ["1580819696000"]}
+  - match: {hits.sequences.0.events.1.fields.id: [123]}
+  - match: {hits.sequences.0.events.1.fields.valid: [true]}
+  - match: {hits.sequences.0.events.1.fields.day_of_week: ["Tuesday"]}
+  - match: {shard_failures.0.index: "eql_test_rebel"}
+
+
+---
+"Execute sequence with shard failures, allow_partial_search_results=true and allow_partial_sequence_results=true as query params":
+  - do:
+      eql.search:
+        index: eql_test*
+        allow_partial_search_results: true
+        allow_partial_sequence_results: true
+        body:
+          query: 'sequence [process where user == "SYSTEM" and day_of_week == "Monday"] [process where user == "SYSTEM" and day_of_week == "Tuesday"]'
+          fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]
+
+  - match: {timed_out: false}
+  - match: {hits.total.value: 1}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.events.0._source.user: "SYSTEM"}
+  - match: {hits.sequences.0.events.0._id: "1"}
+  - match: {hits.sequences.0.events.0.fields.@timestamp: ["1580733296000"]}
+  - match: {hits.sequences.0.events.0.fields.id: [123]}
+  - match: {hits.sequences.0.events.0.fields.valid: [false]}
+  - match: {hits.sequences.0.events.0.fields.day_of_week: ["Monday"]}
+  - match: {hits.sequences.0.events.1._id: "2"}
+  - match: {hits.sequences.0.events.1.fields.@timestamp: ["1580819696000"]}
+  - match: {hits.sequences.0.events.1.fields.id: [123]}
+  - match: {hits.sequences.0.events.1.fields.valid: [true]}
+  - match: {hits.sequences.0.events.1.fields.day_of_week: ["Tuesday"]}
+  - match: {shard_failures.0.index: "eql_test_rebel"}
diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java
new file mode 100644
index 0000000000000..da6bb6180428b
--- /dev/null
+++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java
@@ -0,0 +1,613 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.eql.action;
+
+import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsAction;
+import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.AbstractMultiClustersTestCase;
+import org.elasticsearch.xpack.eql.plugin.EqlPlugin;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class CCSPartialResultsIT extends AbstractMultiClustersTestCase {
+
+    static String REMOTE_CLUSTER = "cluster_a";
+
+    protected Collection<Class<? extends Plugin>> nodePlugins(String cluster) {
+        return Collections.singletonList(LocalStateEQLXPackPlugin.class);
+    }
+
+    protected final Client localClient() {
+        return client(LOCAL_CLUSTER);
+    }
+
+    @Override
+    protected List<String> remoteClusterAlias() {
+        return List.of(REMOTE_CLUSTER);
+    }
+
+    @Override
+    protected boolean reuseClusters() {
+        return false;
+    }
+
+    /**
+     *
+     * @return remote node name
+     */
+    private String createSchema() {
+        final Client remoteClient = client(REMOTE_CLUSTER);
+        final String remoteNode = cluster(REMOTE_CLUSTER).startDataOnlyNode();
+        final String remoteNode2 = cluster(REMOTE_CLUSTER).startDataOnlyNode();
+
+        assertAcked(
+            remoteClient.admin()
+                .indices()
+                .prepareCreate("test-1-remote")
+                .setSettings(
+                    Settings.builder()
+                        .put("index.routing.allocation.require._name", remoteNode)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .build()
+                )
+                .setMapping("@timestamp", "type=date"),
+            TimeValue.timeValueSeconds(60)
+        );
+
+        assertAcked(
+            remoteClient.admin()
+                .indices()
+                .prepareCreate("test-2-remote")
+                .setSettings(
+                    Settings.builder()
+                        .put("index.routing.allocation.require._name", remoteNode2)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .build()
+                )
+                .setMapping("@timestamp", "type=date"),
+            TimeValue.timeValueSeconds(60)
+        );
+
+        for (int i = 0; i < 5; i++) {
+            int val = i * 2;
+            remoteClient.prepareIndex("test-1-remote")
+                .setId(Integer.toString(i))
+                .setSource("@timestamp", 100000 + val, "event.category", "process", "key", "same", "value", val)
+                .get();
+        }
+        for (int i = 0; i < 5; i++) {
+            int val = i * 2 + 1;
+            remoteClient.prepareIndex("test-2-remote")
+                .setId(Integer.toString(i))
+                .setSource("@timestamp", 100000 + val, "event.category", "process", "key", "same", "value", val)
+                .get();
+        }
+
+        remoteClient.admin().indices().prepareRefresh().get();
+        return remoteNode;
+    }
+
+    // ------------------------------------------------------------------------
+    // queries with full cluster (no missing shards)
+    // ------------------------------------------------------------------------
+
+    public void testNoFailures() throws ExecutionException, InterruptedException, IOException {
+        createSchema();
+
+        // event query
+        var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("process where true")
+            .allowPartialSearchResults(randomBoolean())
+            .allowPartialSequenceResults(randomBoolean());
+        EqlSearchResponse response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().events().size(), equalTo(10));
+        for (int i = 0; i < 10; i++) {
+            assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + i));
+        }
+        assertThat(response.shardFailures().length, is(0));
+
+        // sequence query on both shards
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sequence [process where value == 1] [process where value == 2]")
+            .allowPartialSearchResults(randomBoolean())
+            .allowPartialSequenceResults(randomBoolean());
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        EqlSearchResponse.Sequence sequence = response.hits().sequences().get(0);
+        assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1"));
+        assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 2"));
+        assertThat(response.shardFailures().length, is(0));
+
+        // sequence query on the available shard only
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sequence [process where value == 1] [process where value == 3]")
+            .allowPartialSearchResults(randomBoolean())
+            .allowPartialSequenceResults(randomBoolean());
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        sequence = response.hits().sequences().get(0);
+        assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1"));
+        assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 3"));
+        assertThat(response.shardFailures().length, is(0));
+
+        // sequence query on the unavailable shard only
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sequence [process where value == 0] [process where value == 2]")
+            .allowPartialSearchResults(randomBoolean())
+            .allowPartialSequenceResults(randomBoolean());
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        sequence = response.hits().sequences().get(0);
+        assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 0"));
+        assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 2"));
+        assertThat(response.shardFailures().length, is(0));
+
+        // sequence query with missing event on unavailable shard
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]")
+            .allowPartialSearchResults(randomBoolean())
+            .allowPartialSequenceResults(randomBoolean());
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(0));
+
+        // sample query on both shards
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sample by key [process where value == 2] [process where value == 1]")
+            .allowPartialSearchResults(randomBoolean())
+            .allowPartialSequenceResults(randomBoolean());
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        EqlSearchResponse.Sequence sample = response.hits().sequences().get(0);
+        assertThat(sample.events().get(0).toString(), containsString("\"value\" : 2"));
+        assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1"));
+        assertThat(response.shardFailures().length, is(0));
+
+        // sample query on the available shard only
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sample by key [process where value == 3] [process where value == 1]")
+            .allowPartialSearchResults(randomBoolean())
+            .allowPartialSequenceResults(randomBoolean());
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        sample = response.hits().sequences().get(0);
+        assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3"));
+        assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1"));
+        assertThat(response.shardFailures().length, is(0));
+
+        // sample query on the unavailable shard only
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sample by key [process where value == 2] [process where value == 0]")
+            .allowPartialSearchResults(randomBoolean())
+            .allowPartialSequenceResults(randomBoolean());
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        sample = response.hits().sequences().get(0);
+        assertThat(sample.events().get(0).toString(), containsString("\"value\" : 2"));
+        assertThat(sample.events().get(1).toString(), containsString("\"value\" : 0"));
+        assertThat(response.shardFailures().length, is(0));
+
+    }
+
+    // ------------------------------------------------------------------------
+    // same queries, with missing shards and allow_partial_search_results=true
+    // and allow_partial_sequence_result=true
+    // ------------------------------------------------------------------------
+
+    public void testAllowPartialSearchAndSequence_event() throws ExecutionException, InterruptedException, IOException {
+        var remoteNode = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        cluster(REMOTE_CLUSTER).stopNode(remoteNode);
+
+        // event query
+        var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("process where true")
+            .allowPartialSearchResults(true);
+        var response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().events().size(), equalTo(5));
+        for (int i = 0; i < 5; i++) {
+            assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1)));
+        }
+        assertThat(response.shardFailures().length, is(1));
+    }
+
+    public void testAllowPartialSearchAndSequence_sequence() throws ExecutionException, InterruptedException, IOException {
+        var remoteNode = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        cluster(REMOTE_CLUSTER).stopNode(remoteNode);
+
+        // sequence query on both shards
+        var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sequence [process where value == 1] [process where value == 2]")
+            .allowPartialSearchResults(true)
+            .allowPartialSequenceResults(true);
+        var response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query on the available shard only
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sequence [process where value == 1] [process where value == 3]")
+            .allowPartialSearchResults(true)
+            .allowPartialSequenceResults(true);
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        var sequence = response.hits().sequences().get(0);
+        assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1"));
+        assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 3"));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query on the unavailable shard only
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sequence [process where value == 0] [process where value == 2]")
+            .allowPartialSearchResults(true)
+            .allowPartialSequenceResults(true);
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sequence with maxspan=10s  [process where value == 1] ![process where value == 2] [process where value == 3]")
+            .allowPartialSearchResults(true)
+            .allowPartialSequenceResults(true);
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        sequence = response.hits().sequences().get(0);
+        assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1"));
+        assertThat(sequence.events().get(2).toString(), containsString("\"value\" : 3"));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+    }
+
+    public void testAllowPartialSearchAndSequence_sample() throws ExecutionException, InterruptedException, IOException {
+        var remoteNode = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        cluster(REMOTE_CLUSTER).stopNode(remoteNode);
+
+        // sample query on both shards
+        var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sample by key [process where value == 2] [process where value == 1]")
+            .allowPartialSearchResults(true)
+            .allowPartialSequenceResults(true);
+        var response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sample query on the available shard only
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sample by key [process where value == 3] [process where value == 1]")
+            .allowPartialSearchResults(true)
+            .allowPartialSequenceResults(true);
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        var sample = response.hits().sequences().get(0);
+        assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3"));
+        assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1"));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sample query on the unavailable shard only
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sample by key [process where value == 2] [process where value == 0]")
+            .allowPartialSearchResults(true)
+            .allowPartialSequenceResults(true);
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+    }
+
+    // ------------------------------------------------------------------------
+    // same queries, with missing shards and allow_partial_search_results=true
+    // and default allow_partial_sequence_results (ie. false)
+    // ------------------------------------------------------------------------
+
+    public void testAllowPartialSearch_event() throws ExecutionException, InterruptedException, IOException {
+        var remoteNode = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        cluster(REMOTE_CLUSTER).stopNode(remoteNode);
+
+        // event query
+        var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("process where true")
+            .allowPartialSearchResults(true);
+        var response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().events().size(), equalTo(5));
+        for (int i = 0; i < 5; i++) {
+            assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1)));
+        }
+        assertThat(response.shardFailures().length, is(1));
+
+    }
+
+    public void testAllowPartialSearch_sequence() throws ExecutionException, InterruptedException, IOException {
+        var remoteNode = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        cluster(REMOTE_CLUSTER).stopNode(remoteNode);
+
+        // sequence query on both shards
+        var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sequence [process where value == 1] [process where value == 2]")
+            .allowPartialSearchResults(true);
+        var response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query on the available shard only
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sequence [process where value == 1] [process where value == 3]")
+            .allowPartialSearchResults(true);
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query on the unavailable shard only
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sequence [process where value == 0] [process where value == 2]")
+            .allowPartialSearchResults(true);
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sequence with maxspan=10s  [process where value == 1] ![process where value == 2] [process where value == 3]")
+            .allowPartialSearchResults(true);
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+    }
+
+    public void testAllowPartialSearch_sample() throws ExecutionException, InterruptedException, IOException {
+        var remoteNode = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        cluster(REMOTE_CLUSTER).stopNode(remoteNode);
+
+        // sample query on both shards
+        var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sample by key [process where value == 2] [process where value == 1]")
+            .allowPartialSearchResults(true);
+        var response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sample query on the available shard only
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sample by key [process where value == 3] [process where value == 1]")
+            .allowPartialSearchResults(true);
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        var sample = response.hits().sequences().get(0);
+        assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3"));
+        assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1"));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sample query on the unavailable shard only
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sample by key [process where value == 2] [process where value == 0]")
+            .allowPartialSearchResults(true);
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+    }
+
+    // ------------------------------------------------------------------------
+    // same queries, with missing shards and with default xpack.eql.default_allow_partial_results=true
+    // ------------------------------------------------------------------------
+
+    public void testClusterSetting_event() throws ExecutionException, InterruptedException, IOException {
+        var remoteNode = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        cluster(REMOTE_CLUSTER).stopNode(remoteNode);
+
+        cluster(REMOTE_CLUSTER).client()
+            .execute(
+                ClusterUpdateSettingsAction.INSTANCE,
+                new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings(
+                    Settings.builder().put(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey(), true)
+                )
+            )
+            .get();
+
+        // event query
+        var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("process where true");
+        var response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().events().size(), equalTo(5));
+        for (int i = 0; i < 5; i++) {
+            assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1)));
+        }
+        assertThat(response.shardFailures().length, is(1));
+
+        localClient().execute(
+            ClusterUpdateSettingsAction.INSTANCE,
+            new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings(
+                Settings.builder().putNull(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey())
+            )
+        ).get();
+    }
+
+    public void testClusterSetting_sequence() throws ExecutionException, InterruptedException, IOException {
+        var remoteNode = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        cluster(REMOTE_CLUSTER).stopNode(remoteNode);
+
+        cluster(REMOTE_CLUSTER).client()
+            .execute(
+                ClusterUpdateSettingsAction.INSTANCE,
+                new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings(
+                    Settings.builder().put(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey(), true)
+                )
+            )
+            .get();
+        // sequence query on both shards
+        var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sequence [process where value == 1] [process where value == 2]");
+        var response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query on the available shard only
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sequence [process where value == 1] [process where value == 3]");
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query on the unavailable shard only
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sequence [process where value == 0] [process where value == 2]");
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sequence with maxspan=10s  [process where value == 1] ![process where value == 2] [process where value == 3]");
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        localClient().execute(
+            ClusterUpdateSettingsAction.INSTANCE,
+            new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings(
+                Settings.builder().putNull(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey())
+            )
+        ).get();
+    }
+
+    public void testClusterSetting_sample() throws ExecutionException, InterruptedException, IOException {
+        var remoteNode = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        cluster(REMOTE_CLUSTER).stopNode(remoteNode);
+
+        cluster(REMOTE_CLUSTER).client()
+            .execute(
+                ClusterUpdateSettingsAction.INSTANCE,
+                new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings(
+                    Settings.builder().put(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey(), true)
+                )
+            )
+            .get();
+
+        // sample query on both shards
+        var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sample by key [process where value == 2] [process where value == 1]");
+        var response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sample query on the available shard only
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sample by key [process where value == 3] [process where value == 1]");
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        var sample = response.hits().sequences().get(0);
+        assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3"));
+        assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1"));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sample query on the unavailable shard only
+        request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*")
+            .query("sample by key [process where value == 2] [process where value == 0]");
+        response = localClient().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1-remote"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        localClient().execute(
+            ClusterUpdateSettingsAction.INSTANCE,
+            new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings(
+                Settings.builder().putNull(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey())
+            )
+        ).get();
+    }
+}
diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java
new file mode 100644
index 0000000000000..9048d11f4eddf
--- /dev/null
+++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java
@@ -0,0 +1,780 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.eql.action;
+
+import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsAction;
+import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.SearchService;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.xpack.core.async.GetAsyncResultRequest;
+import org.elasticsearch.xpack.eql.plugin.EqlAsyncGetResultAction;
+import org.elasticsearch.xpack.eql.plugin.EqlPlugin;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+
+public class PartialSearchResultsIT extends AbstractEqlIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return CollectionUtils.appendToCopy(super.nodePlugins(), MockTransportService.TestPlugin.class);
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal, otherSettings))
+            .put(SearchService.KEEPALIVE_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(randomIntBetween(100, 500)))
+            .build();
+    }
+
+    /**
+     *
+     * @return node name where the first index is
+     */
+    private String createSchema() {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        final List<String> dataNodes = internalCluster().clusterService()
+            .state()
+            .nodes()
+            .getDataNodes()
+            .values()
+            .stream()
+            .map(DiscoveryNode::getName)
+            .toList();
+        final String assignedNodeForIndex1 = randomFrom(dataNodes);
+
+        assertAcked(
+            indicesAdmin().prepareCreate("test-1")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.routing.allocation.include._name", assignedNodeForIndex1)
+                        .build()
+                )
+                .setMapping("@timestamp", "type=date")
+        );
+        assertAcked(
+            indicesAdmin().prepareCreate("test-2")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.routing.allocation.exclude._name", assignedNodeForIndex1)
+                        .build()
+                )
+                .setMapping("@timestamp", "type=date")
+        );
+
+        for (int i = 0; i < 5; i++) {
+            int val = i * 2;
+            prepareIndex("test-1").setId(Integer.toString(i))
+                .setSource("@timestamp", 100000 + val, "event.category", "process", "key", "same", "value", val)
+                .get();
+        }
+        for (int i = 0; i < 5; i++) {
+            int val = i * 2 + 1;
+            prepareIndex("test-2").setId(Integer.toString(i))
+                .setSource("@timestamp", 100000 + val, "event.category", "process", "key", "same", "value", val)
+                .get();
+        }
+        refresh();
+        return assignedNodeForIndex1;
+    }
+
+    public void testNoFailures() throws Exception {
+        createSchema();
+
+        // ------------------------------------------------------------------------
+        // queries with full cluster (no missing shards)
+        // ------------------------------------------------------------------------
+
+        // event query
+        var request = new EqlSearchRequest().indices("test-*")
+            .query("process where true")
+            .allowPartialSearchResults(randomBoolean())
+            .allowPartialSequenceResults(randomBoolean());
+        EqlSearchResponse response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().events().size(), equalTo(10));
+        for (int i = 0; i < 10; i++) {
+            assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + i));
+        }
+        assertThat(response.shardFailures().length, is(0));
+
+        // sequence query on both shards
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sequence [process where value == 1] [process where value == 2]")
+            .allowPartialSearchResults(randomBoolean())
+            .allowPartialSequenceResults(randomBoolean());
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        EqlSearchResponse.Sequence sequence = response.hits().sequences().get(0);
+        assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1"));
+        assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 2"));
+        assertThat(response.shardFailures().length, is(0));
+
+        // sequence query on the available shard only
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sequence [process where value == 1] [process where value == 3]")
+            .allowPartialSearchResults(randomBoolean())
+            .allowPartialSequenceResults(randomBoolean());
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        sequence = response.hits().sequences().get(0);
+        assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1"));
+        assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 3"));
+        assertThat(response.shardFailures().length, is(0));
+
+        // sequence query on the unavailable shard only
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sequence [process where value == 0] [process where value == 2]")
+            .allowPartialSearchResults(randomBoolean())
+            .allowPartialSequenceResults(randomBoolean());
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        sequence = response.hits().sequences().get(0);
+        assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 0"));
+        assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 2"));
+        assertThat(response.shardFailures().length, is(0));
+
+        // sequence query with missing event on unavailable shard
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]")
+            .allowPartialSearchResults(randomBoolean())
+            .allowPartialSequenceResults(randomBoolean());
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(0));
+
+        // sample query on both shards
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sample by key [process where value == 2] [process where value == 1]")
+            .allowPartialSearchResults(randomBoolean())
+            .allowPartialSequenceResults(randomBoolean());
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        EqlSearchResponse.Sequence sample = response.hits().sequences().get(0);
+        assertThat(sample.events().get(0).toString(), containsString("\"value\" : 2"));
+        assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1"));
+        assertThat(response.shardFailures().length, is(0));
+
+        // sample query on the available shard only
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sample by key [process where value == 3] [process where value == 1]")
+            .allowPartialSearchResults(randomBoolean())
+            .allowPartialSequenceResults(randomBoolean());
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        sample = response.hits().sequences().get(0);
+        assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3"));
+        assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1"));
+        assertThat(response.shardFailures().length, is(0));
+
+        // sample query on the unavailable shard only
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sample by key [process where value == 2] [process where value == 0]")
+            .allowPartialSearchResults(randomBoolean())
+            .allowPartialSequenceResults(randomBoolean());
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        sample = response.hits().sequences().get(0);
+        assertThat(sample.events().get(0).toString(), containsString("\"value\" : 2"));
+        assertThat(sample.events().get(1).toString(), containsString("\"value\" : 0"));
+        assertThat(response.shardFailures().length, is(0));
+
+    }
+
+    // ------------------------------------------------------------------------
+    // same queries, with missing shards. Let them fail
+    // allow_partial_sequence_results has no effect if allow_partial_sequence_results is not set to true.
+    // ------------------------------------------------------------------------
+
+    public void testFailures_event() throws Exception {
+        final String assignedNodeForIndex1 = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        internalCluster().stopNode(assignedNodeForIndex1);
+
+        // event query
+        shouldFail("process where true");
+
+    }
+
+    public void testFailures_sequence() throws Exception {
+        final String assignedNodeForIndex1 = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        internalCluster().stopNode(assignedNodeForIndex1);
+
+        // sequence query on both shards
+        shouldFail("sequence [process where value == 1] [process where value == 2]");
+
+        // sequence query on the available shard only
+        shouldFail("sequence [process where value == 1] [process where value == 3]");
+
+        // sequence query on the unavailable shard only
+        shouldFail("sequence [process where value == 0] [process where value == 2]");
+
+        // sequence query with missing event on unavailable shard.
+        shouldFail("sequence with maxspan=10s  [process where value == 1] ![process where value == 2] [process where value == 3]");
+    }
+
+    public void testFailures_sample() throws Exception {
+        final String assignedNodeForIndex1 = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        internalCluster().stopNode(assignedNodeForIndex1);
+
+        // sample query on both shards
+        shouldFail("sample by key [process where value == 2] [process where value == 1]");
+
+        // sample query on the available shard only
+        shouldFail("sample by key [process where value == 3] [process where value == 1]");
+
+        // sample query on the unavailable shard only
+        shouldFail("sample by key [process where value == 2] [process where value == 0]");
+
+    }
+
+    // ------------------------------------------------------------------------
+    // same queries, with missing shards and allow_partial_search_results=true
+    // and allow_partial_sequence_result=true
+    // ------------------------------------------------------------------------
+
+    public void testAllowPartialSearchAndSequenceResults_event() throws Exception {
+        final String assignedNodeForIndex1 = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        internalCluster().stopNode(assignedNodeForIndex1);
+
+        // event query
+        var request = new EqlSearchRequest().indices("test-*").query("process where true").allowPartialSearchResults(true);
+        var response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().events().size(), equalTo(5));
+        for (int i = 0; i < 5; i++) {
+            assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1)));
+        }
+        assertThat(response.shardFailures().length, is(1));
+
+    }
+
+    public void testAllowPartialSearchAndSequenceResults_sequence() throws Exception {
+        final String assignedNodeForIndex1 = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        internalCluster().stopNode(assignedNodeForIndex1);
+
+        // sequence query on both shards
+        var request = new EqlSearchRequest().indices("test-*")
+            .query("sequence [process where value == 1] [process where value == 2]")
+            .allowPartialSearchResults(true)
+            .allowPartialSequenceResults(true);
+        var response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query on the available shard only
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sequence [process where value == 1] [process where value == 3]")
+            .allowPartialSearchResults(true)
+            .allowPartialSequenceResults(true);
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        var sequence = response.hits().sequences().get(0);
+        assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1"));
+        assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 3"));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query on the unavailable shard only
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sequence [process where value == 0] [process where value == 2]")
+            .allowPartialSearchResults(true)
+            .allowPartialSequenceResults(true);
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sequence with maxspan=10s  [process where value == 1] ![process where value == 2] [process where value == 3]")
+            .allowPartialSearchResults(true)
+            .allowPartialSequenceResults(true);
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        sequence = response.hits().sequences().get(0);
+        assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1"));
+        assertThat(sequence.events().get(2).toString(), containsString("\"value\" : 3"));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+    }
+
+    public void testAllowPartialSearchAndSequenceResults_sample() throws Exception {
+        final String assignedNodeForIndex1 = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        internalCluster().stopNode(assignedNodeForIndex1);
+
+        // sample query on both shards
+        var request = new EqlSearchRequest().indices("test-*")
+            .query("sample by key [process where value == 2] [process where value == 1]")
+            .allowPartialSearchResults(true)
+            .allowPartialSequenceResults(true);
+        var response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sample query on the available shard only
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sample by key [process where value == 3] [process where value == 1]")
+            .allowPartialSearchResults(true)
+            .allowPartialSequenceResults(true);
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        var sample = response.hits().sequences().get(0);
+        assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3"));
+        assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1"));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sample query on the unavailable shard only
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sample by key [process where value == 2] [process where value == 0]")
+            .allowPartialSearchResults(true)
+            .allowPartialSequenceResults(true);
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+    }
+
+    // ------------------------------------------------------------------------
+    // same queries, with missing shards and allow_partial_search_results=true
+    // and default allow_partial_sequence_results (ie. false)
+    // ------------------------------------------------------------------------
+
+    public void testAllowPartialSearchResults_event() throws Exception {
+        final String assignedNodeForIndex1 = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        internalCluster().stopNode(assignedNodeForIndex1);
+
+        // event query
+        var request = new EqlSearchRequest().indices("test-*").query("process where true").allowPartialSearchResults(true);
+        var response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().events().size(), equalTo(5));
+        for (int i = 0; i < 5; i++) {
+            assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1)));
+        }
+        assertThat(response.shardFailures().length, is(1));
+
+    }
+
+    public void testAllowPartialSearchResults_sequence() throws Exception {
+        final String assignedNodeForIndex1 = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        internalCluster().stopNode(assignedNodeForIndex1);
+
+        // sequence query on both shards
+        var request = new EqlSearchRequest().indices("test-*")
+            .query("sequence [process where value == 1] [process where value == 2]")
+            .allowPartialSearchResults(true);
+        var response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query on the available shard only
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sequence [process where value == 1] [process where value == 3]")
+            .allowPartialSearchResults(true);
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query on the unavailable shard only
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sequence [process where value == 0] [process where value == 2]")
+            .allowPartialSearchResults(true);
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sequence with maxspan=10s  [process where value == 1] ![process where value == 2] [process where value == 3]")
+            .allowPartialSearchResults(true);
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+    }
+
+    public void testAllowPartialSearchResults_sample() throws Exception {
+        final String assignedNodeForIndex1 = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        internalCluster().stopNode(assignedNodeForIndex1);
+
+        // sample query on both shards
+        var request = new EqlSearchRequest().indices("test-*")
+            .query("sample by key [process where value == 2] [process where value == 1]")
+            .allowPartialSearchResults(true);
+        var response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sample query on the available shard only
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sample by key [process where value == 3] [process where value == 1]")
+            .allowPartialSearchResults(true);
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        var sample = response.hits().sequences().get(0);
+        assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3"));
+        assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1"));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sample query on the unavailable shard only
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sample by key [process where value == 2] [process where value == 0]")
+            .allowPartialSearchResults(true);
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+    }
+
+    // ------------------------------------------------------------------------
+    // same queries, this time async, with missing shards and allow_partial_search_results=true
+    // and default allow_partial_sequence_results (ie. false)
+    // ------------------------------------------------------------------------
+
+    public void testAsyncAllowPartialSearchResults_event() throws Exception {
+        final String assignedNodeForIndex1 = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        internalCluster().stopNode(assignedNodeForIndex1);
+
+        // event query
+        var response = runAsync("process where true", true);
+        assertThat(response.hits().events().size(), equalTo(5));
+        for (int i = 0; i < 5; i++) {
+            assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1)));
+        }
+        assertThat(response.shardFailures().length, is(1));
+
+    }
+
+    public void testAsyncAllowPartialSearchResults_sequence() throws Exception {
+        final String assignedNodeForIndex1 = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        internalCluster().stopNode(assignedNodeForIndex1);
+
+        // sequence query on both shards
+        var response = runAsync("sequence [process where value == 1] [process where value == 2]", true);
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query on the available shard only
+        response = runAsync("sequence [process where value == 1] [process where value == 3]", true);
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query on the unavailable shard only
+        response = runAsync("sequence [process where value == 0] [process where value == 2]", true);
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE
+        response = runAsync(
+            "sequence with maxspan=10s  [process where value == 1] ![process where value == 2] [process where value == 3]",
+            true
+        );
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+    }
+
+    public void testAsyncAllowPartialSearchResults_sample() throws Exception {
+        final String assignedNodeForIndex1 = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        internalCluster().stopNode(assignedNodeForIndex1);
+        // sample query on both shards
+        var response = runAsync("sample by key [process where value == 2] [process where value == 1]", true);
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sample query on the available shard only
+        response = runAsync("sample by key [process where value == 3] [process where value == 1]", true);
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        var sample = response.hits().sequences().get(0);
+        assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3"));
+        assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1"));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sample query on the unavailable shard only
+        response = runAsync("sample by key [process where value == 2] [process where value == 0]", true);
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+    }
+
+    // ------------------------------------------------------------------------
+    // same queries, with missing shards and with default xpack.eql.default_allow_partial_results=true
+    // ------------------------------------------------------------------------
+
+    public void testClusterSetting_event() throws Exception {
+        final String assignedNodeForIndex1 = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        internalCluster().stopNode(assignedNodeForIndex1);
+
+        client().execute(
+            ClusterUpdateSettingsAction.INSTANCE,
+            new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings(
+                Settings.builder().put(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey(), true)
+            )
+        ).get();
+
+        // event query
+        var request = new EqlSearchRequest().indices("test-*").query("process where true");
+        var response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().events().size(), equalTo(5));
+        for (int i = 0; i < 5; i++) {
+            assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1)));
+        }
+        assertThat(response.shardFailures().length, is(1));
+
+        client().execute(
+            ClusterUpdateSettingsAction.INSTANCE,
+            new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings(
+                Settings.builder().putNull(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey())
+            )
+        ).get();
+    }
+
+    public void testClusterSetting_sequence() throws Exception {
+        final String assignedNodeForIndex1 = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        internalCluster().stopNode(assignedNodeForIndex1);
+
+        client().execute(
+            ClusterUpdateSettingsAction.INSTANCE,
+            new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings(
+                Settings.builder().put(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey(), true)
+            )
+        ).get();
+        // sequence query on both shards
+        var request = new EqlSearchRequest().indices("test-*").query("sequence [process where value == 1] [process where value == 2]");
+        var response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query on the available shard only
+        request = new EqlSearchRequest().indices("test-*").query("sequence [process where value == 1] [process where value == 3]");
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query on the unavailable shard only
+        request = new EqlSearchRequest().indices("test-*").query("sequence [process where value == 0] [process where value == 2]");
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE
+        request = new EqlSearchRequest().indices("test-*")
+            .query("sequence with maxspan=10s  [process where value == 1] ![process where value == 2] [process where value == 3]");
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        client().execute(
+            ClusterUpdateSettingsAction.INSTANCE,
+            new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings(
+                Settings.builder().putNull(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey())
+            )
+        ).get();
+    }
+
+    public void testClusterSetting_sample() throws Exception {
+        final String assignedNodeForIndex1 = createSchema();
+        // ------------------------------------------------------------------------
+        // stop one of the nodes, make one of the shards unavailable
+        // ------------------------------------------------------------------------
+
+        internalCluster().stopNode(assignedNodeForIndex1);
+
+        client().execute(
+            ClusterUpdateSettingsAction.INSTANCE,
+            new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings(
+                Settings.builder().put(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey(), true)
+            )
+        ).get();
+
+        // sample query on both shards
+        var request = new EqlSearchRequest().indices("test-*").query("sample by key [process where value == 2] [process where value == 1]");
+        var response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sample query on the available shard only
+        request = new EqlSearchRequest().indices("test-*").query("sample by key [process where value == 3] [process where value == 1]");
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(1));
+        var sample = response.hits().sequences().get(0);
+        assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3"));
+        assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1"));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        // sample query on the unavailable shard only
+        request = new EqlSearchRequest().indices("test-*").query("sample by key [process where value == 2] [process where value == 0]");
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        assertThat(response.hits().sequences().size(), equalTo(0));
+        assertThat(response.shardFailures().length, is(1));
+        assertThat(response.shardFailures()[0].index(), is("test-1"));
+        assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException"));
+
+        client().execute(
+            ClusterUpdateSettingsAction.INSTANCE,
+            new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings(
+                Settings.builder().putNull(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey())
+            )
+        ).get();
+    }
+
+    private static EqlSearchResponse runAsync(String query, Boolean allowPartialSearchResults) throws InterruptedException,
+        ExecutionException {
+        EqlSearchRequest request;
+        EqlSearchResponse response;
+        request = new EqlSearchRequest().indices("test-*").query(query).waitForCompletionTimeout(TimeValue.ZERO);
+        if (allowPartialSearchResults != null) {
+            request = request.allowPartialSearchResults(allowPartialSearchResults);
+        }
+        response = client().execute(EqlSearchAction.INSTANCE, request).get();
+        while (response.isRunning()) {
+            GetAsyncResultRequest getResultsRequest = new GetAsyncResultRequest(response.id()).setKeepAlive(TimeValue.timeValueMinutes(10))
+                .setWaitForCompletionTimeout(TimeValue.timeValueMillis(10));
+            response = client().execute(EqlAsyncGetResultAction.INSTANCE, getResultsRequest).get();
+        }
+        return response;
+    }
+
+    private static void shouldFail(String query) throws InterruptedException {
+        EqlSearchRequest request = new EqlSearchRequest().indices("test-*").query(query);
+        if (randomBoolean()) {
+            request = request.allowPartialSearchResults(false);
+        }
+        if (randomBoolean()) {
+            request = request.allowPartialSequenceResults(randomBoolean());
+        }
+        try {
+            client().execute(EqlSearchAction.INSTANCE, request).get();
+            fail();
+        } catch (ExecutionException e) {
+            assertThat(e.getCause(), instanceOf(SearchPhaseExecutionException.class));
+        }
+    }
+}
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java
index 0aeddd525e317..0b1d5005c5f89 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java
@@ -44,8 +44,8 @@
 
 public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Replaceable, ToXContent {
 
-    public static long MIN_KEEP_ALIVE = TimeValue.timeValueMinutes(1).millis();
-    public static TimeValue DEFAULT_KEEP_ALIVE = TimeValue.timeValueDays(5);
+    public static final long MIN_KEEP_ALIVE = TimeValue.timeValueMinutes(1).millis();
+    public static final TimeValue DEFAULT_KEEP_ALIVE = TimeValue.timeValueDays(5);
     public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.fromOptions(true, true, true, false);
 
     private String[] indices;
@@ -63,6 +63,8 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re
     private List<FieldAndFormat> fetchFields;
     private Map<String, Object> runtimeMappings = emptyMap();
     private int maxSamplesPerKey = RequestDefaults.MAX_SAMPLES_PER_KEY;
+    private Boolean allowPartialSearchResults;
+    private Boolean allowPartialSequenceResults;
 
     // Async settings
     private TimeValue waitForCompletionTimeout = null;
@@ -83,6 +85,8 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re
     static final String KEY_FETCH_FIELDS = "fields";
     static final String KEY_RUNTIME_MAPPINGS = "runtime_mappings";
     static final String KEY_MAX_SAMPLES_PER_KEY = "max_samples_per_key";
+    static final String KEY_ALLOW_PARTIAL_SEARCH_RESULTS = "allow_partial_search_results";
+    static final String KEY_ALLOW_PARTIAL_SEQUENCE_RESULTS = "allow_partial_sequence_results";
 
     static final ParseField FILTER = new ParseField(KEY_FILTER);
     static final ParseField TIMESTAMP_FIELD = new ParseField(KEY_TIMESTAMP_FIELD);
@@ -97,6 +101,8 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re
     static final ParseField RESULT_POSITION = new ParseField(KEY_RESULT_POSITION);
     static final ParseField FETCH_FIELDS_FIELD = SearchSourceBuilder.FETCH_FIELDS_FIELD;
     static final ParseField MAX_SAMPLES_PER_KEY = new ParseField(KEY_MAX_SAMPLES_PER_KEY);
+    static final ParseField ALLOW_PARTIAL_SEARCH_RESULTS = new ParseField(KEY_ALLOW_PARTIAL_SEARCH_RESULTS);
+    static final ParseField ALLOW_PARTIAL_SEQUENCE_RESULTS = new ParseField(KEY_ALLOW_PARTIAL_SEQUENCE_RESULTS);
 
     private static final ObjectParser<EqlSearchRequest, Void> PARSER = objectParser(EqlSearchRequest::new);
 
@@ -135,6 +141,13 @@ public EqlSearchRequest(StreamInput in) throws IOException {
         if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_7_0)) {
             maxSamplesPerKey = in.readInt();
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS)) {
+            allowPartialSearchResults = in.readOptionalBoolean();
+            allowPartialSequenceResults = in.readOptionalBoolean();
+        } else {
+            allowPartialSearchResults = false;
+            allowPartialSequenceResults = false;
+        }
     }
 
     @Override
@@ -245,6 +258,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(KEY_RUNTIME_MAPPINGS, runtimeMappings);
         }
         builder.field(KEY_MAX_SAMPLES_PER_KEY, maxSamplesPerKey);
+        builder.field(KEY_ALLOW_PARTIAL_SEARCH_RESULTS, allowPartialSearchResults);
+        builder.field(KEY_ALLOW_PARTIAL_SEQUENCE_RESULTS, allowPartialSequenceResults);
 
         return builder;
     }
@@ -279,6 +294,8 @@ protected static <R extends EqlSearchRequest> ObjectParser<R, Void> objectParser
         parser.declareField(EqlSearchRequest::fetchFields, EqlSearchRequest::parseFetchFields, FETCH_FIELDS_FIELD, ValueType.VALUE_ARRAY);
         parser.declareObject(EqlSearchRequest::runtimeMappings, (p, c) -> p.map(), SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD);
         parser.declareInt(EqlSearchRequest::maxSamplesPerKey, MAX_SAMPLES_PER_KEY);
+        parser.declareBoolean(EqlSearchRequest::allowPartialSearchResults, ALLOW_PARTIAL_SEARCH_RESULTS);
+        parser.declareBoolean(EqlSearchRequest::allowPartialSequenceResults, ALLOW_PARTIAL_SEQUENCE_RESULTS);
         return parser;
     }
 
@@ -427,6 +444,24 @@ public EqlSearchRequest maxSamplesPerKey(int maxSamplesPerKey) {
         return this;
     }
 
+    public Boolean allowPartialSearchResults() {
+        return allowPartialSearchResults;
+    }
+
+    public EqlSearchRequest allowPartialSearchResults(Boolean val) {
+        this.allowPartialSearchResults = val;
+        return this;
+    }
+
+    public Boolean allowPartialSequenceResults() {
+        return allowPartialSequenceResults;
+    }
+
+    public EqlSearchRequest allowPartialSequenceResults(Boolean val) {
+        this.allowPartialSequenceResults = val;
+        return this;
+    }
+
     private static List<FieldAndFormat> parseFetchFields(XContentParser parser) throws IOException {
         List<FieldAndFormat> result = new ArrayList<>();
         Token token = parser.currentToken();
@@ -470,6 +505,10 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_7_0)) {
             out.writeInt(maxSamplesPerKey);
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS)) {
+            out.writeOptionalBoolean(allowPartialSearchResults);
+            out.writeOptionalBoolean(allowPartialSequenceResults);
+        }
     }
 
     @Override
@@ -496,7 +535,9 @@ public boolean equals(Object o) {
             && Objects.equals(resultPosition, that.resultPosition)
             && Objects.equals(fetchFields, that.fetchFields)
             && Objects.equals(runtimeMappings, that.runtimeMappings)
-            && Objects.equals(maxSamplesPerKey, that.maxSamplesPerKey);
+            && Objects.equals(maxSamplesPerKey, that.maxSamplesPerKey)
+            && Objects.equals(allowPartialSearchResults, that.allowPartialSearchResults)
+            && Objects.equals(allowPartialSequenceResults, that.allowPartialSequenceResults);
     }
 
     @Override
@@ -517,7 +558,9 @@ public int hashCode() {
             resultPosition,
             fetchFields,
             runtimeMappings,
-            maxSamplesPerKey
+            maxSamplesPerKey,
+            allowPartialSearchResults,
+            allowPartialSequenceResults
         );
     }
 
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java
index 2b7b8b074fa71..e068f652cd289 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java
@@ -7,8 +7,11 @@
 package org.elasticsearch.xpack.eql.action;
 
 import org.apache.lucene.search.TotalHits;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ShardOperationFailedException;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -17,6 +20,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.core.Nullable;
@@ -36,6 +40,7 @@
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -54,6 +59,7 @@ public class EqlSearchResponse extends ActionResponse implements ToXContentObjec
     private final String asyncExecutionId;
     private final boolean isRunning;
     private final boolean isPartial;
+    private final ShardSearchFailure[] shardFailures;
 
     private static final class Fields {
         static final String TOOK = "took";
@@ -62,6 +68,7 @@ private static final class Fields {
         static final String ID = "id";
         static final String IS_RUNNING = "is_running";
         static final String IS_PARTIAL = "is_partial";
+        static final String SHARD_FAILURES = "shard_failures";
     }
 
     private static final ParseField TOOK = new ParseField(Fields.TOOK);
@@ -70,8 +77,10 @@ private static final class Fields {
     private static final ParseField ID = new ParseField(Fields.ID);
     private static final ParseField IS_RUNNING = new ParseField(Fields.IS_RUNNING);
     private static final ParseField IS_PARTIAL = new ParseField(Fields.IS_PARTIAL);
+    private static final ParseField SHARD_FAILURES = new ParseField(Fields.SHARD_FAILURES);
 
     private static final InstantiatingObjectParser<EqlSearchResponse, Void> PARSER;
+
     static {
         InstantiatingObjectParser.Builder<EqlSearchResponse, Void> parser = InstantiatingObjectParser.builder(
             "eql/search_response",
@@ -84,11 +93,12 @@ private static final class Fields {
         parser.declareString(optionalConstructorArg(), ID);
         parser.declareBoolean(constructorArg(), IS_RUNNING);
         parser.declareBoolean(constructorArg(), IS_PARTIAL);
+        parser.declareObjectArray(optionalConstructorArg(), (p, c) -> ShardSearchFailure.EMPTY_ARRAY, SHARD_FAILURES);
         PARSER = parser.build();
     }
 
-    public EqlSearchResponse(Hits hits, long tookInMillis, boolean isTimeout) {
-        this(hits, tookInMillis, isTimeout, null, false, false);
+    public EqlSearchResponse(Hits hits, long tookInMillis, boolean isTimeout, ShardSearchFailure[] shardFailures) {
+        this(hits, tookInMillis, isTimeout, null, false, false, shardFailures);
     }
 
     public EqlSearchResponse(
@@ -97,7 +107,8 @@ public EqlSearchResponse(
         boolean isTimeout,
         String asyncExecutionId,
         boolean isRunning,
-        boolean isPartial
+        boolean isPartial,
+        ShardSearchFailure[] shardFailures
     ) {
         super();
         this.hits = hits == null ? Hits.EMPTY : hits;
@@ -106,6 +117,7 @@ public EqlSearchResponse(
         this.asyncExecutionId = asyncExecutionId;
         this.isRunning = isRunning;
         this.isPartial = isPartial;
+        this.shardFailures = shardFailures;
     }
 
     public EqlSearchResponse(StreamInput in) throws IOException {
@@ -116,6 +128,11 @@ public EqlSearchResponse(StreamInput in) throws IOException {
         asyncExecutionId = in.readOptionalString();
         isPartial = in.readBoolean();
         isRunning = in.readBoolean();
+        if (in.getTransportVersion().onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS)) {
+            shardFailures = in.readArray(ShardSearchFailure::readShardSearchFailure, ShardSearchFailure[]::new);
+        } else {
+            shardFailures = ShardSearchFailure.EMPTY_ARRAY;
+        }
     }
 
     public static EqlSearchResponse fromXContent(XContentParser parser) {
@@ -130,6 +147,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(asyncExecutionId);
         out.writeBoolean(isPartial);
         out.writeBoolean(isRunning);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS)) {
+            out.writeArray(shardFailures);
+        }
     }
 
     @Override
@@ -147,6 +167,13 @@ private XContentBuilder innerToXContent(XContentBuilder builder, Params params)
         builder.field(IS_RUNNING.getPreferredName(), isRunning);
         builder.field(TOOK.getPreferredName(), tookInMillis);
         builder.field(TIMED_OUT.getPreferredName(), isTimeout);
+        if (CollectionUtils.isEmpty(shardFailures) == false) {
+            builder.startArray(SHARD_FAILURES.getPreferredName());
+            for (ShardOperationFailedException shardFailure : ExceptionsHelper.groupBy(shardFailures)) {
+                shardFailure.toXContent(builder, params);
+            }
+            builder.endArray();
+        }
         hits.toXContent(builder, params);
         return builder;
     }
@@ -178,6 +205,10 @@ public boolean isPartial() {
         return isPartial;
     }
 
+    public ShardSearchFailure[] shardFailures() {
+        return shardFailures;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {
@@ -190,12 +221,13 @@ public boolean equals(Object o) {
         return Objects.equals(hits, that.hits)
             && Objects.equals(tookInMillis, that.tookInMillis)
             && Objects.equals(isTimeout, that.isTimeout)
-            && Objects.equals(asyncExecutionId, that.asyncExecutionId);
+            && Objects.equals(asyncExecutionId, that.asyncExecutionId)
+            && Arrays.equals(shardFailures, that.shardFailures);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(hits, tookInMillis, isTimeout, asyncExecutionId);
+        return Objects.hash(hits, tookInMillis, isTimeout, asyncExecutionId, Arrays.hashCode(shardFailures));
     }
 
     @Override
@@ -206,7 +238,7 @@ public String toString() {
     // Event
     public static class Event implements Writeable, ToXContentObject {
 
-        public static Event MISSING_EVENT = new Event("", "", new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), null, true);
+        public static final Event MISSING_EVENT = new Event("", "", new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), null, true);
 
         private static final class Fields {
             static final String INDEX = GetResult._INDEX;
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchTask.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchTask.java
index 2a1bc3b7adb67..0fc8e8c88d7d9 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchTask.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchTask.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.eql.action;
 
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.xpack.core.async.AsyncExecutionId;
@@ -39,7 +40,8 @@ public EqlSearchResponse getCurrentResult() {
             false,
             getExecutionId().getEncoded(),
             true,
-            true
+            true,
+            ShardSearchFailure.EMPTY_ARRAY
         );
     }
 }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java
index 58bfb9513add5..e920067def8c9 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java
@@ -14,8 +14,8 @@ private RequestDefaults() {}
     public static final String FIELD_TIMESTAMP = "@timestamp";
     public static final String FIELD_EVENT_CATEGORY = "event.category";
 
-    public static int SIZE = 10;
-    public static int FETCH_SIZE = 1000;
-    public static boolean CCS_MINIMIZE_ROUNDTRIPS = true;
-    public static int MAX_SAMPLES_PER_KEY = 1;
+    public static final int SIZE = 10;
+    public static final int FETCH_SIZE = 1000;
+    public static final boolean CCS_MINIMIZE_ROUNDTRIPS = true;
+    public static final int MAX_SAMPLES_PER_KEY = 1;
 }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java
index b26c815c1a2b5..672d6b87a8dbb 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java
@@ -167,7 +167,9 @@ public Executable assemble(
             criteria.subList(0, completionStage),
             criteria.get(completionStage),
             matcher,
-            listOfKeys
+            listOfKeys,
+            cfg.allowPartialSearchResults(),
+            cfg.allowPartialSequenceResults()
         );
 
         return w;
@@ -235,7 +237,8 @@ public Executable assemble(List<List<Attribute>> listOfKeys, List<PhysicalPlan>
             cfg.fetchSize(),
             limit,
             session.circuitBreaker(),
-            cfg.maxSamplesPerKey()
+            cfg.maxSamplesPerKey(),
+            cfg.allowPartialSearchResults()
         );
     }
 
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/AbstractPayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/AbstractPayload.java
index 823cd04d25f45..9fecf958b9714 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/AbstractPayload.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/AbstractPayload.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.eql.execution.payload;
 
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xpack.eql.session.Payload;
 
@@ -14,10 +15,12 @@ public abstract class AbstractPayload implements Payload {
 
     private final boolean timedOut;
     private final TimeValue timeTook;
+    private ShardSearchFailure[] shardFailures;
 
-    protected AbstractPayload(boolean timedOut, TimeValue timeTook) {
+    protected AbstractPayload(boolean timedOut, TimeValue timeTook, ShardSearchFailure[] shardFailures) {
         this.timedOut = timedOut;
         this.timeTook = timeTook;
+        this.shardFailures = shardFailures;
     }
 
     @Override
@@ -29,4 +32,9 @@ public boolean timedOut() {
     public TimeValue timeTook() {
         return timeTook;
     }
+
+    @Override
+    public ShardSearchFailure[] shardFailures() {
+        return shardFailures;
+    }
 }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/EventPayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/EventPayload.java
index a7845ca62dccc..6471bc0814f70 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/EventPayload.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/EventPayload.java
@@ -20,7 +20,7 @@ public class EventPayload extends AbstractPayload {
     private final List<Event> values;
 
     public EventPayload(SearchResponse response) {
-        super(response.isTimedOut(), response.getTook());
+        super(response.isTimedOut(), response.getTook(), response.getShardFailures());
 
         SearchHits hits = response.getHits();
         values = new ArrayList<>(hits.getHits().length);
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java
index 89f1c4d1eb041..b9b7cfd6b615a 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.search.SearchHit;
@@ -35,6 +36,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -44,6 +46,7 @@
 import static org.elasticsearch.common.Strings.EMPTY_ARRAY;
 import static org.elasticsearch.xpack.eql.execution.assembler.SampleQueryRequest.COMPOSITE_AGG_NAME;
 import static org.elasticsearch.xpack.eql.execution.search.RuntimeUtils.prepareRequest;
+import static org.elasticsearch.xpack.eql.util.SearchHitUtils.addShardFailures;
 
 public class SampleIterator implements Executable {
 
@@ -58,6 +61,7 @@ public class SampleIterator implements Executable {
     private final Limit limit;
     private final int maxSamplesPerKey;
     private long startTime;
+    private Map<String, ShardSearchFailure> shardFailures = new HashMap<>();
 
     // ---------- CIRCUIT BREAKER -----------
 
@@ -84,13 +88,16 @@ public class SampleIterator implements Executable {
      */
     private long previousTotalPageSize = 0;
 
+    private boolean allowPartialSearchResults;
+
     public SampleIterator(
         QueryClient client,
         List<SampleCriterion> criteria,
         int fetchSize,
         Limit limit,
         CircuitBreaker circuitBreaker,
-        int maxSamplesPerKey
+        int maxSamplesPerKey,
+        boolean allowPartialSearchResults
     ) {
         this.client = client;
         this.criteria = criteria;
@@ -100,6 +107,7 @@ public SampleIterator(
         this.limit = limit;
         this.circuitBreaker = circuitBreaker;
         this.maxSamplesPerKey = maxSamplesPerKey;
+        this.allowPartialSearchResults = allowPartialSearchResults;
     }
 
     @Override
@@ -147,6 +155,7 @@ private void advance(ActionListener<Payload> listener) {
 
     private void queryForCompositeAggPage(ActionListener<Payload> listener, final SampleQueryRequest request) {
         client.query(request, listener.delegateFailureAndWrap((delegate, r) -> {
+            addShardFailures(shardFailures, r);
             // either the fields values or the fields themselves are missing
             // or the filter applied on the eql query matches no documents
             if (r.hasAggregations() == false) {
@@ -209,13 +218,16 @@ private void finalStep(ActionListener<Payload> listener) {
             for (SampleCriterion criterion : criteria) {
                 SampleQueryRequest r = criterion.finalQuery();
                 r.singleKeyPair(compositeKeyValues, maxCriteria, maxSamplesPerKey);
-                searches.add(prepareRequest(r.searchSource(), false, EMPTY_ARRAY));
+                searches.add(prepareRequest(r.searchSource(), false, allowPartialSearchResults, EMPTY_ARRAY));
             }
             sampleKeys.add(new SequenceKey(compositeKeyValues.toArray()));
         }
 
         int initialSize = samples.size();
         client.multiQuery(searches, listener.delegateFailureAndWrap((delegate, r) -> {
+            for (MultiSearchResponse.Item item : r) {
+                addShardFailures(shardFailures, item.getResponse());
+            }
             List<List<SearchHit>> sample = new ArrayList<>(maxCriteria);
             MultiSearchResponse.Item[] response = r.getResponses();
             int docGroupsCounter = 1;
@@ -280,14 +292,23 @@ private void payload(ActionListener<Payload> listener) {
         log.trace("Sending payload for [{}] samples", samples.size());
 
         if (samples.isEmpty()) {
-            listener.onResponse(new EmptyPayload(Type.SAMPLE, timeTook()));
+            listener.onResponse(new EmptyPayload(Type.SAMPLE, timeTook(), shardFailures.values().toArray(new ShardSearchFailure[0])));
             return;
         }
 
         // get results through search (to keep using PIT)
         client.fetchHits(
             hits(samples),
-            ActionListeners.map(listener, listOfHits -> new SamplePayload(samples, listOfHits, false, timeTook()))
+            ActionListeners.map(
+                listener,
+                listOfHits -> new SamplePayload(
+                    samples,
+                    listOfHits,
+                    false,
+                    timeTook(),
+                    shardFailures.values().toArray(new ShardSearchFailure[0])
+                )
+            )
         );
     }
 
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SamplePayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SamplePayload.java
index 121f4c208273b..aee084dd88734 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SamplePayload.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SamplePayload.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.eql.execution.sample;
 
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event;
@@ -19,8 +20,14 @@ class SamplePayload extends AbstractPayload {
 
     private final List<org.elasticsearch.xpack.eql.action.EqlSearchResponse.Sequence> values;
 
-    SamplePayload(List<Sample> samples, List<List<SearchHit>> docs, boolean timedOut, TimeValue timeTook) {
-        super(timedOut, timeTook);
+    SamplePayload(
+        List<Sample> samples,
+        List<List<SearchHit>> docs,
+        boolean timedOut,
+        TimeValue timeTook,
+        ShardSearchFailure[] shardFailures
+    ) {
+        super(timedOut, timeTook, shardFailures);
         values = new ArrayList<>(samples.size());
 
         for (int i = 0; i < samples.size(); i++) {
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java
index 6cbe5298b5950..18623c17dcffb 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java
@@ -46,12 +46,14 @@ public class BasicQueryClient implements QueryClient {
     final Client client;
     final String[] indices;
     final List<FieldAndFormat> fetchFields;
+    private final boolean allowPartialSearchResults;
 
     public BasicQueryClient(EqlSession eqlSession) {
         this.cfg = eqlSession.configuration();
         this.client = eqlSession.client();
         this.indices = cfg.indices();
         this.fetchFields = cfg.fetchFields();
+        this.allowPartialSearchResults = cfg.allowPartialSearchResults();
     }
 
     @Override
@@ -60,11 +62,11 @@ public void query(QueryRequest request, ActionListener<SearchResponse> listener)
         // set query timeout
         searchSource.timeout(cfg.requestTimeout());
 
-        SearchRequest search = prepareRequest(searchSource, false, indices);
-        search(search, searchLogListener(listener, log));
+        SearchRequest search = prepareRequest(searchSource, false, allowPartialSearchResults, indices);
+        search(search, allowPartialSearchResults, searchLogListener(listener, log, allowPartialSearchResults));
     }
 
-    protected void search(SearchRequest search, ActionListener<SearchResponse> listener) {
+    protected void search(SearchRequest search, boolean allowPartialSearchResults, ActionListener<SearchResponse> listener) {
         if (cfg.isCancelled()) {
             listener.onFailure(new TaskCancelledException("cancelled"));
             return;
@@ -77,7 +79,7 @@ protected void search(SearchRequest search, ActionListener<SearchResponse> liste
         client.search(search, listener);
     }
 
-    protected void search(MultiSearchRequest search, ActionListener<MultiSearchResponse> listener) {
+    protected void search(MultiSearchRequest search, boolean allowPartialSearchResults, ActionListener<MultiSearchResponse> listener) {
         if (cfg.isCancelled()) {
             listener.onFailure(new TaskCancelledException("cancelled"));
             return;
@@ -91,7 +93,7 @@ protected void search(MultiSearchRequest search, ActionListener<MultiSearchRespo
             log.trace("About to execute multi-queries {} on {}", sj, indices);
         }
 
-        client.multiSearch(search, multiSearchLogListener(listener, log));
+        client.multiSearch(search, multiSearchLogListener(listener, allowPartialSearchResults, log));
     }
 
     @Override
@@ -147,11 +149,11 @@ public void fetchHits(Iterable<List<HitReference>> refs, ActionListener<List<Lis
                 builder.runtimeMappings(cfg.runtimeMappings());
             }
 
-            SearchRequest search = prepareRequest(builder, false, entry.getKey());
+            SearchRequest search = prepareRequest(builder, false, allowPartialSearchResults, entry.getKey());
             multiSearchBuilder.add(search);
         }
 
-        search(multiSearchBuilder.request(), listener.delegateFailureAndWrap((delegate, r) -> {
+        search(multiSearchBuilder.request(), allowPartialSearchResults, listener.delegateFailureAndWrap((delegate, r) -> {
             for (MultiSearchResponse.Item item : r.getResponses()) {
                 // check for failures
                 if (item.isFailure()) {
@@ -187,6 +189,6 @@ public void multiQuery(List<SearchRequest> searches, ActionListener<MultiSearchR
             request.indices(indices);
             multiSearchBuilder.add(request);
         }
-        search(multiSearchBuilder.request(), listener);
+        search(multiSearchBuilder.request(), allowPartialSearchResults, listener);
     }
 }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClient.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClient.java
index 2cfa4db37ee07..e338550c22458 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClient.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClient.java
@@ -56,38 +56,38 @@ public PITAwareQueryClient(EqlSession eqlSession) {
     }
 
     @Override
-    protected void search(SearchRequest search, ActionListener<SearchResponse> listener) {
+    protected void search(SearchRequest search, boolean allowPartialSearchResults, ActionListener<SearchResponse> listener) {
         // no pitId, ask for one
         if (pitId == null) {
-            openPIT(listener, () -> searchWithPIT(search, listener));
+            openPIT(listener, () -> searchWithPIT(search, listener, allowPartialSearchResults), allowPartialSearchResults);
         } else {
-            searchWithPIT(search, listener);
+            searchWithPIT(search, listener, allowPartialSearchResults);
         }
     }
 
-    private void searchWithPIT(SearchRequest request, ActionListener<SearchResponse> listener) {
+    private void searchWithPIT(SearchRequest request, ActionListener<SearchResponse> listener, boolean allowPartialSearchResults) {
         makeRequestPITCompatible(request);
         // get the pid on each response
-        super.search(request, pitListener(SearchResponse::pointInTimeId, listener));
+        super.search(request, allowPartialSearchResults, pitListener(SearchResponse::pointInTimeId, listener));
     }
 
     @Override
-    protected void search(MultiSearchRequest search, ActionListener<MultiSearchResponse> listener) {
+    protected void search(MultiSearchRequest search, boolean allowPartialSearchResults, ActionListener<MultiSearchResponse> listener) {
         // no pitId, ask for one
         if (pitId == null) {
-            openPIT(listener, () -> searchWithPIT(search, listener));
+            openPIT(listener, () -> searchWithPIT(search, allowPartialSearchResults, listener), allowPartialSearchResults);
         } else {
-            searchWithPIT(search, listener);
+            searchWithPIT(search, allowPartialSearchResults, listener);
         }
     }
 
-    private void searchWithPIT(MultiSearchRequest search, ActionListener<MultiSearchResponse> listener) {
+    private void searchWithPIT(MultiSearchRequest search, boolean allowPartialSearchResults, ActionListener<MultiSearchResponse> listener) {
         for (SearchRequest request : search.requests()) {
             makeRequestPITCompatible(request);
         }
 
         // get the pid on each request
-        super.search(search, pitListener(r -> {
+        super.search(search, allowPartialSearchResults, pitListener(r -> {
             // get pid
             for (MultiSearchResponse.Item item : r.getResponses()) {
                 // pick the first non-failing response
@@ -135,9 +135,10 @@ private <Response> ActionListener<Response> pitListener(
         );
     }
 
-    private <Response> void openPIT(ActionListener<Response> listener, Runnable runnable) {
+    private <Response> void openPIT(ActionListener<Response> listener, Runnable runnable, boolean allowPartialSearchResults) {
         OpenPointInTimeRequest request = new OpenPointInTimeRequest(indices).indicesOptions(IndexResolver.FIELD_CAPS_INDICES_OPTIONS)
-            .keepAlive(keepAlive);
+            .keepAlive(keepAlive)
+            .allowPartialSearchResults(allowPartialSearchResults);
         request.indexFilter(filter);
         client.execute(TransportOpenPointInTimeAction.TYPE, request, listener.delegateFailureAndWrap((l, r) -> {
             pitId = r.getPointInTimeId();
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java
index 40f7f7139efa1..92af8c562f840 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java
@@ -56,10 +56,14 @@ public final class RuntimeUtils {
 
     private RuntimeUtils() {}
 
-    public static ActionListener<SearchResponse> searchLogListener(ActionListener<SearchResponse> listener, Logger log) {
+    public static ActionListener<SearchResponse> searchLogListener(
+        ActionListener<SearchResponse> listener,
+        Logger log,
+        boolean allowPartialResults
+    ) {
         return listener.delegateFailureAndWrap((delegate, response) -> {
             ShardSearchFailure[] failures = response.getShardFailures();
-            if (CollectionUtils.isEmpty(failures) == false) {
+            if (CollectionUtils.isEmpty(failures) == false && allowPartialResults == false) {
                 delegate.onFailure(new EqlIllegalArgumentException(failures[0].reason(), failures[0].getCause()));
                 return;
             }
@@ -70,16 +74,22 @@ public static ActionListener<SearchResponse> searchLogListener(ActionListener<Se
         });
     }
 
-    public static ActionListener<MultiSearchResponse> multiSearchLogListener(ActionListener<MultiSearchResponse> listener, Logger log) {
+    public static ActionListener<MultiSearchResponse> multiSearchLogListener(
+        ActionListener<MultiSearchResponse> listener,
+        boolean allowPartialSearchResults,
+        Logger log
+    ) {
         return listener.delegateFailureAndWrap((delegate, items) -> {
             for (MultiSearchResponse.Item item : items) {
                 Exception failure = item.getFailure();
                 SearchResponse response = item.getResponse();
 
                 if (failure == null) {
-                    ShardSearchFailure[] failures = response.getShardFailures();
-                    if (CollectionUtils.isEmpty(failures) == false) {
-                        failure = new EqlIllegalArgumentException(failures[0].reason(), failures[0].getCause());
+                    if (allowPartialSearchResults == false) {
+                        ShardSearchFailure[] failures = response.getShardFailures();
+                        if (CollectionUtils.isEmpty(failures) == false) {
+                            failure = new EqlIllegalArgumentException(failures[0].reason(), failures[0].getCause());
+                        }
                     }
                 }
                 if (failure != null) {
@@ -170,11 +180,16 @@ public static HitExtractor createExtractor(FieldExtraction ref, EqlConfiguration
         throw new EqlIllegalArgumentException("Unexpected value reference {}", ref.getClass());
     }
 
-    public static SearchRequest prepareRequest(SearchSourceBuilder source, boolean includeFrozen, String... indices) {
+    public static SearchRequest prepareRequest(
+        SearchSourceBuilder source,
+        boolean includeFrozen,
+        boolean allowPartialSearchResults,
+        String... indices
+    ) {
         SearchRequest searchRequest = new SearchRequest();
         searchRequest.indices(indices);
         searchRequest.source(source);
-        searchRequest.allowPartialSearchResults(false);
+        searchRequest.allowPartialSearchResults(allowPartialSearchResults);
         searchRequest.indicesOptions(
             includeFrozen ? IndexResolver.FIELD_CAPS_FROZEN_INDICES_OPTIONS : IndexResolver.FIELD_CAPS_INDICES_OPTIONS
         );
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequencePayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequencePayload.java
index 45083babddbb4..b4a8edc79b3ad 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequencePayload.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequencePayload.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.eql.execution.sequence;
 
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event;
@@ -19,8 +20,14 @@ class SequencePayload extends AbstractPayload {
 
     private final List<org.elasticsearch.xpack.eql.action.EqlSearchResponse.Sequence> values;
 
-    SequencePayload(List<Sequence> sequences, List<List<SearchHit>> docs, boolean timedOut, TimeValue timeTook) {
-        super(timedOut, timeTook);
+    SequencePayload(
+        List<Sequence> sequences,
+        List<List<SearchHit>> docs,
+        boolean timedOut,
+        TimeValue timeTook,
+        ShardSearchFailure[] shardFailures
+    ) {
+        super(timedOut, timeTook, shardFailures);
         values = new ArrayList<>(sequences.size());
 
         for (int i = 0; i < sequences.size(); i++) {
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java
index eabf6df518ad4..fac8788db0f95 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
@@ -41,6 +42,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -51,6 +53,7 @@
 import static org.elasticsearch.action.ActionListener.runAfter;
 import static org.elasticsearch.xpack.eql.execution.ExecutionUtils.copySource;
 import static org.elasticsearch.xpack.eql.execution.search.RuntimeUtils.combineFilters;
+import static org.elasticsearch.xpack.eql.util.SearchHitUtils.addShardFailures;
 import static org.elasticsearch.xpack.eql.util.SearchHitUtils.qualifiedIndex;
 
 /**
@@ -103,6 +106,9 @@ protected boolean removeEldestEntry(Map.Entry<String, String> eldest) {
 
     private final boolean hasKeys;
     private final List<List<Attribute>> listOfKeys;
+    private final boolean allowPartialSearchResults;
+    private final boolean allowPartialSequenceResults;
+    private Map<String, ShardSearchFailure> shardFailures = new HashMap<>();
 
     // flag used for DESC sequences to indicate whether
     // the window needs to restart (since the DESC query still has results)
@@ -127,7 +133,10 @@ public TumblingWindow(
         List<SequenceCriterion> criteria,
         SequenceCriterion until,
         SequenceMatcher matcher,
-        List<List<Attribute>> listOfKeys
+        List<List<Attribute>> listOfKeys,
+        boolean allowPartialSearchResults,
+        boolean allowPartialSequenceResults
+
     ) {
         this.client = client;
 
@@ -141,6 +150,8 @@ public TumblingWindow(
         this.hasKeys = baseRequest.keySize() > 0;
         this.restartWindowFromTailQuery = baseRequest.descending();
         this.listOfKeys = listOfKeys;
+        this.allowPartialSearchResults = allowPartialSearchResults;
+        this.allowPartialSequenceResults = allowPartialSequenceResults;
     }
 
     @Override
@@ -158,6 +169,9 @@ public void execute(ActionListener<Payload> listener) {
      * Move the window while preserving the same base.
      */
     private void tumbleWindow(int currentStage, ActionListener<Payload> listener) {
+        if (allowPartialSequenceResults == false && shardFailures.isEmpty() == false) {
+            doPayload(listener);
+        }
         if (currentStage > matcher.firstPositiveStage && matcher.hasCandidates() == false) {
             if (restartWindowFromTailQuery) {
                 currentStage = matcher.firstPositiveStage;
@@ -224,6 +238,9 @@ public void checkMissingEvents(Runnable next, ActionListener<Payload> listener)
 
     private void doCheckMissingEvents(List<Sequence> batchToCheck, MultiSearchResponse p, ActionListener<Payload> listener, Runnable next) {
         MultiSearchResponse.Item[] responses = p.getResponses();
+        for (MultiSearchResponse.Item response : responses) {
+            addShardFailures(shardFailures, response.getResponse());
+        }
         int nextResponse = 0;
         for (Sequence sequence : batchToCheck) {
             boolean leading = true;
@@ -316,7 +333,14 @@ private List<SearchRequest> prepareQueryForMissingEvents(List<Sequence> toCheck)
                     }
                     addKeyFilter(i, sequence, builder);
                     RuntimeUtils.combineFilters(builder, range);
-                    result.add(RuntimeUtils.prepareRequest(builder.size(1).trackTotalHits(false), false, Strings.EMPTY_ARRAY));
+                    result.add(
+                        RuntimeUtils.prepareRequest(
+                            builder.size(1).trackTotalHits(false),
+                            false,
+                            allowPartialSearchResults,
+                            Strings.EMPTY_ARRAY
+                        )
+                    );
                 } else {
                     leading = false;
                 }
@@ -361,6 +385,7 @@ private void advance(int stage, ActionListener<Payload> listener) {
      * Execute the base query.
      */
     private void baseCriterion(int baseStage, SearchResponse r, ActionListener<Payload> listener) {
+        addShardFailures(shardFailures, r);
         SequenceCriterion base = criteria.get(baseStage);
         SearchHits hits = r.getHits();
 
@@ -731,8 +756,10 @@ private void doPayload(ActionListener<Payload> listener) {
 
         log.trace("Sending payload for [{}] sequences", completed.size());
 
-        if (completed.isEmpty()) {
-            listener.onResponse(new EmptyPayload(Type.SEQUENCE, timeTook()));
+        if (completed.isEmpty() || (allowPartialSequenceResults == false && shardFailures.isEmpty() == false)) {
+            listener.onResponse(
+                new EmptyPayload(Type.SEQUENCE, timeTook(), shardFailures.values().toArray(new ShardSearchFailure[shardFailures.size()]))
+            );
             return;
         }
 
@@ -741,7 +768,13 @@ private void doPayload(ActionListener<Payload> listener) {
             if (criteria.get(matcher.firstPositiveStage).descending()) {
                 Collections.reverse(completed);
             }
-            return new SequencePayload(completed, addMissingEventPlaceholders(listOfHits), false, timeTook());
+            return new SequencePayload(
+                completed,
+                addMissingEventPlaceholders(listOfHits),
+                false,
+                timeTook(),
+                shardFailures.values().toArray(new ShardSearchFailure[0])
+            );
         }));
     }
 
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java
index 084a5e74a47e8..210f88c991539 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java
@@ -60,6 +60,20 @@ public class EqlPlugin extends Plugin implements ActionPlugin, CircuitBreakerPlu
         Setting.Property.DeprecatedWarning
     );
 
+    public static final Setting<Boolean> DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS = Setting.boolSetting(
+        "xpack.eql.default_allow_partial_results",
+        false,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    public static final Setting<Boolean> DEFAULT_ALLOW_PARTIAL_SEQUENCE_RESULTS = Setting.boolSetting(
+        "xpack.eql.default_allow_partial_sequence_results",
+        false,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
     public EqlPlugin() {}
 
     @Override
@@ -86,7 +100,7 @@ private Collection<Object> createComponents(Client client, Settings settings, Cl
      */
     @Override
     public List<Setting<?>> getSettings() {
-        return List.of(EQL_ENABLED_SETTING);
+        return List.of(EQL_ENABLED_SETTING, DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS, DEFAULT_ALLOW_PARTIAL_SEQUENCE_RESULTS);
     }
 
     @Override
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java
index e24a4749f45cd..65def24563e5e 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java
@@ -64,6 +64,12 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
             }
             eqlRequest.keepOnCompletion(request.paramAsBoolean("keep_on_completion", eqlRequest.keepOnCompletion()));
             eqlRequest.ccsMinimizeRoundtrips(request.paramAsBoolean("ccs_minimize_roundtrips", eqlRequest.ccsMinimizeRoundtrips()));
+            eqlRequest.allowPartialSearchResults(
+                request.paramAsBoolean("allow_partial_search_results", eqlRequest.allowPartialSearchResults())
+            );
+            eqlRequest.allowPartialSequenceResults(
+                request.paramAsBoolean("allow_partial_sequence_results", eqlRequest.allowPartialSequenceResults())
+            );
         }
 
         return channel -> {
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java
index c0141da2432ce..582352722fc58 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java
@@ -10,6 +10,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
@@ -144,7 +145,8 @@ public EqlSearchResponse initialResponse(EqlSearchTask task) {
             false,
             task.getExecutionId().getEncoded(),
             true,
-            true
+            true,
+            ShardSearchFailure.EMPTY_ARRAY
         );
     }
 
@@ -231,6 +233,12 @@ public static void operation(
                 request.indicesOptions(),
                 request.fetchSize(),
                 request.maxSamplesPerKey(),
+                request.allowPartialSearchResults() == null
+                    ? defaultAllowPartialSearchResults(clusterService)
+                    : request.allowPartialSearchResults(),
+                request.allowPartialSequenceResults() == null
+                    ? defaultAllowPartialSequenceResults(clusterService)
+                    : request.allowPartialSequenceResults(),
                 clientId,
                 new TaskId(nodeId, task.getId()),
                 task
@@ -244,12 +252,34 @@ public static void operation(
         }
     }
 
+    private static boolean defaultAllowPartialSearchResults(ClusterService clusterService) {
+        if (clusterService.getClusterSettings() == null) {
+            return EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getDefault(Settings.EMPTY);
+        }
+        return clusterService.getClusterSettings().get(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS);
+    }
+
+    private static boolean defaultAllowPartialSequenceResults(ClusterService clusterService) {
+        if (clusterService.getClusterSettings() == null) {
+            return EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEQUENCE_RESULTS.getDefault(Settings.EMPTY);
+        }
+        return clusterService.getClusterSettings().get(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEQUENCE_RESULTS);
+    }
+
     static EqlSearchResponse createResponse(Results results, AsyncExecutionId id) {
         EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(results.events(), results.sequences(), results.totalHits());
         if (id != null) {
-            return new EqlSearchResponse(hits, results.tookTime().getMillis(), results.timedOut(), id.getEncoded(), false, false);
+            return new EqlSearchResponse(
+                hits,
+                results.tookTime().getMillis(),
+                results.timedOut(),
+                id.getEncoded(),
+                false,
+                false,
+                results.shardFailures()
+            );
         } else {
-            return new EqlSearchResponse(hits, results.tookTime().getMillis(), results.timedOut());
+            return new EqlSearchResponse(hits, results.tookTime().getMillis(), results.timedOut(), results.shardFailures());
         }
     }
 
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EmptyPayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EmptyPayload.java
index 9822285465087..33ed5799cd073 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EmptyPayload.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EmptyPayload.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.eql.session;
 
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.core.TimeValue;
 
 import java.util.List;
@@ -17,14 +18,16 @@ public class EmptyPayload implements Payload {
 
     private final Type type;
     private final TimeValue timeTook;
+    private final ShardSearchFailure[] shardFailures;
 
     public EmptyPayload(Type type) {
-        this(type, TimeValue.ZERO);
+        this(type, TimeValue.ZERO, ShardSearchFailure.EMPTY_ARRAY);
     }
 
-    public EmptyPayload(Type type, TimeValue timeTook) {
+    public EmptyPayload(Type type, TimeValue timeTook, ShardSearchFailure[] shardFailures) {
         this.type = type;
         this.timeTook = timeTook;
+        this.shardFailures = shardFailures;
     }
 
     @Override
@@ -46,4 +49,10 @@ public TimeValue timeTook() {
     public List<?> values() {
         return emptyList();
     }
+
+    @Override
+    public ShardSearchFailure[] shardFailures() {
+        return shardFailures;
+    }
+
 }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java
index 8dd8220fb63bc..8242b0b533ad3 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java
@@ -30,6 +30,8 @@ public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu
     private final EqlSearchTask task;
     private final int fetchSize;
     private final int maxSamplesPerKey;
+    private final boolean allowPartialSearchResults;
+    private final boolean allowPartialSequenceResults;
 
     @Nullable
     private final QueryBuilder filter;
@@ -50,6 +52,8 @@ public EqlConfiguration(
         IndicesOptions indicesOptions,
         int fetchSize,
         int maxSamplesPerKey,
+        boolean allowPartialSearchResults,
+        boolean allowPartialSequenceResults,
         String clientId,
         TaskId taskId,
         EqlSearchTask task
@@ -67,6 +71,8 @@ public EqlConfiguration(
         this.task = task;
         this.fetchSize = fetchSize;
         this.maxSamplesPerKey = maxSamplesPerKey;
+        this.allowPartialSearchResults = allowPartialSearchResults;
+        this.allowPartialSequenceResults = allowPartialSequenceResults;
     }
 
     public String[] indices() {
@@ -89,6 +95,14 @@ public int maxSamplesPerKey() {
         return maxSamplesPerKey;
     }
 
+    public boolean allowPartialSearchResults() {
+        return allowPartialSearchResults;
+    }
+
+    public boolean allowPartialSequenceResults() {
+        return allowPartialSequenceResults;
+    }
+
     public QueryBuilder filter() {
         return filter;
     }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Payload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Payload.java
index 1d82478e6db26..05e614714a5aa 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Payload.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Payload.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.eql.session;
 
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.core.TimeValue;
 
 import java.util.List;
@@ -29,4 +30,6 @@ enum Type {
     TimeValue timeTook();
 
     List<?> values();
+
+    ShardSearchFailure[] shardFailures();
 }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Results.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Results.java
index bb76c08c801cb..13886470f21f5 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Results.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Results.java
@@ -9,6 +9,7 @@
 
 import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.search.TotalHits.Relation;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event;
 import org.elasticsearch.xpack.eql.action.EqlSearchResponse.Sequence;
@@ -23,18 +24,28 @@ public class Results {
     private final boolean timedOut;
     private final TimeValue tookTime;
     private final Type type;
+    private ShardSearchFailure[] shardFailures;
 
     public static Results fromPayload(Payload payload) {
         List<?> values = payload.values();
-        return new Results(new TotalHits(values.size(), Relation.EQUAL_TO), payload.timeTook(), false, values, payload.resultType());
+        payload.shardFailures();
+        return new Results(
+            new TotalHits(values.size(), Relation.EQUAL_TO),
+            payload.timeTook(),
+            false,
+            values,
+            payload.resultType(),
+            payload.shardFailures()
+        );
     }
 
-    Results(TotalHits totalHits, TimeValue tookTime, boolean timedOut, List<?> results, Type type) {
+    Results(TotalHits totalHits, TimeValue tookTime, boolean timedOut, List<?> results, Type type, ShardSearchFailure[] shardFailures) {
         this.totalHits = totalHits;
         this.tookTime = tookTime;
         this.timedOut = timedOut;
         this.results = results;
         this.type = type;
+        this.shardFailures = shardFailures;
     }
 
     public TotalHits totalHits() {
@@ -51,6 +62,10 @@ public List<Sequence> sequences() {
         return (type == Type.SEQUENCE || type == Type.SAMPLE) ? (List<Sequence>) results : null;
     }
 
+    public ShardSearchFailure[] shardFailures() {
+        return shardFailures;
+    }
+
     public TimeValue tookTime() {
         return tookTime;
     }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/SearchHitUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/SearchHitUtils.java
index 91795ac15b53e..2b5ec9718cfc4 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/SearchHitUtils.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/SearchHitUtils.java
@@ -7,8 +7,12 @@
 
 package org.elasticsearch.xpack.eql.util;
 
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.search.SearchHit;
 
+import java.util.Map;
+
 import static org.elasticsearch.transport.RemoteClusterAware.buildRemoteIndexName;
 
 public final class SearchHitUtils {
@@ -16,4 +20,12 @@ public final class SearchHitUtils {
     public static String qualifiedIndex(SearchHit hit) {
         return buildRemoteIndexName(hit.getClusterAlias(), hit.getIndex());
     }
+
+    public static void addShardFailures(Map<String, ShardSearchFailure> shardFailures, SearchResponse r) {
+        if (r.getShardFailures() != null) {
+            for (ShardSearchFailure shardFailure : r.getShardFailures()) {
+                shardFailures.put(shardFailure.toString(), shardFailure);
+            }
+        }
+    }
 }
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java
index a1aa8e4bd98d7..75884fab4dbb3 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java
@@ -51,6 +51,8 @@ private EqlTestUtils() {}
         null,
         123,
         1,
+        false,
+        true,
         "",
         new TaskId("test", 123),
         null
@@ -69,6 +71,8 @@ public static EqlConfiguration randomConfiguration() {
             randomIndicesOptions(),
             randomIntBetween(1, 1000),
             randomIntBetween(1, 1000),
+            randomBoolean(),
+            randomBoolean(),
             randomAlphaOfLength(16),
             new TaskId(randomAlphaOfLength(10), randomNonNegativeLong()),
             randomTask()
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java
index 0ff9fa9131b27..1a06aead910c8 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java
@@ -80,6 +80,8 @@ protected EqlSearchRequest createTestInstance() {
                 .waitForCompletionTimeout(randomTimeValue())
                 .keepAlive(randomTimeValue())
                 .keepOnCompletion(randomBoolean())
+                .allowPartialSearchResults(randomBoolean())
+                .allowPartialSequenceResults(randomBoolean())
                 .fetchFields(randomFetchFields)
                 .runtimeMappings(randomRuntimeMappings())
                 .resultPosition(randomFrom("tail", "head"))
@@ -136,6 +138,12 @@ protected EqlSearchRequest mutateInstanceForVersion(EqlSearchRequest instance, T
         mutatedInstance.runtimeMappings(version.onOrAfter(TransportVersions.V_7_13_0) ? instance.runtimeMappings() : emptyMap());
         mutatedInstance.resultPosition(version.onOrAfter(TransportVersions.V_7_17_8) ? instance.resultPosition() : "tail");
         mutatedInstance.maxSamplesPerKey(version.onOrAfter(TransportVersions.V_8_7_0) ? instance.maxSamplesPerKey() : 1);
+        mutatedInstance.allowPartialSearchResults(
+            version.onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS) ? instance.allowPartialSearchResults() : false
+        );
+        mutatedInstance.allowPartialSequenceResults(
+            version.onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS) ? instance.allowPartialSequenceResults() : false
+        );
 
         return mutatedInstance;
     }
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java
index 6cb283d11848e..fa118a5256df1 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java
@@ -9,6 +9,7 @@
 import org.apache.lucene.search.TotalHits;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.document.DocumentField;
@@ -190,7 +191,7 @@ public static EqlSearchResponse createRandomEventsResponse(TotalHits totalHits,
             hits = new EqlSearchResponse.Hits(randomEvents(xType), null, totalHits);
         }
         if (randomBoolean()) {
-            return new EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean());
+            return new EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean(), ShardSearchFailure.EMPTY_ARRAY);
         } else {
             return new EqlSearchResponse(
                 hits,
@@ -198,7 +199,8 @@ public static EqlSearchResponse createRandomEventsResponse(TotalHits totalHits,
                 randomBoolean(),
                 randomAlphaOfLength(10),
                 randomBoolean(),
-                randomBoolean()
+                randomBoolean(),
+                ShardSearchFailure.EMPTY_ARRAY
             );
         }
     }
@@ -222,7 +224,7 @@ public static EqlSearchResponse createRandomSequencesResponse(TotalHits totalHit
             hits = new EqlSearchResponse.Hits(null, seq, totalHits);
         }
         if (randomBoolean()) {
-            return new EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean());
+            return new EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean(), ShardSearchFailure.EMPTY_ARRAY);
         } else {
             return new EqlSearchResponse(
                 hits,
@@ -230,7 +232,8 @@ public static EqlSearchResponse createRandomSequencesResponse(TotalHits totalHit
                 randomBoolean(),
                 randomAlphaOfLength(10),
                 randomBoolean(),
-                randomBoolean()
+                randomBoolean(),
+                ShardSearchFailure.EMPTY_ARRAY
             );
         }
     }
@@ -273,7 +276,8 @@ protected EqlSearchResponse mutateInstanceForVersion(EqlSearchResponse instance,
             instance.isTimeout(),
             instance.id(),
             instance.isRunning(),
-            instance.isPartial()
+            instance.isPartial(),
+            ShardSearchFailure.EMPTY_ARRAY
         );
     }
 
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/LocalStateEQLXPackPlugin.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/LocalStateEQLXPackPlugin.java
index 4d5201f544d72..33573b99546fb 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/LocalStateEQLXPackPlugin.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/LocalStateEQLXPackPlugin.java
@@ -7,26 +7,41 @@
 
 package org.elasticsearch.xpack.eql.action;
 
+import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.indices.breaker.BreakerSettings;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.plugins.CircuitBreakerPlugin;
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 import org.elasticsearch.xpack.eql.plugin.EqlPlugin;
 import org.elasticsearch.xpack.ql.plugin.QlPlugin;
 
 import java.nio.file.Path;
 
-public class LocalStateEQLXPackPlugin extends LocalStateCompositeXPackPlugin {
+public class LocalStateEQLXPackPlugin extends LocalStateCompositeXPackPlugin implements CircuitBreakerPlugin {
+
+    private final EqlPlugin eqlPlugin;
 
     public LocalStateEQLXPackPlugin(final Settings settings, final Path configPath) {
         super(settings, configPath);
         LocalStateEQLXPackPlugin thisVar = this;
-        plugins.add(new EqlPlugin() {
+        this.eqlPlugin = new EqlPlugin() {
             @Override
             protected XPackLicenseState getLicenseState() {
                 return thisVar.getLicenseState();
             }
-        });
+        };
+        plugins.add(eqlPlugin);
         plugins.add(new QlPlugin());
     }
 
+    @Override
+    public BreakerSettings getCircuitBreaker(Settings settings) {
+        return eqlPlugin.getCircuitBreaker(settings);
+    }
+
+    @Override
+    public void setCircuitBreaker(CircuitBreaker circuitBreaker) {
+        eqlPlugin.setCircuitBreaker(circuitBreaker);
+    }
 }
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java
index 7bb6a228f6e48..abd928b04a9c7 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java
@@ -141,7 +141,15 @@ public void testImplicitTiebreakerBeingSet() {
             booleanArrayOf(stages, false),
             NOOP_CIRCUIT_BREAKER
         );
-        TumblingWindow window = new TumblingWindow(client, criteria, null, matcher, Collections.emptyList());
+        TumblingWindow window = new TumblingWindow(
+            client,
+            criteria,
+            null,
+            matcher,
+            Collections.emptyList(),
+            randomBoolean(),
+            randomBoolean()
+        );
         window.execute(wrap(p -> {}, ex -> { throw ExceptionsHelper.convertToRuntime(ex); }));
     }
 }
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java
index a8ed842e94c44..f6aa851b2fff0 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java
@@ -277,7 +277,15 @@ public void test() throws Exception {
         );
 
         QueryClient testClient = new TestQueryClient();
-        TumblingWindow window = new TumblingWindow(testClient, criteria, null, matcher, Collections.emptyList());
+        TumblingWindow window = new TumblingWindow(
+            testClient,
+            criteria,
+            null,
+            matcher,
+            Collections.emptyList(),
+            randomBoolean(),
+            randomBoolean()
+        );
 
         // finally make the assertion at the end of the listener
         window.execute(ActionTestUtils.assertNoFailureListener(this::checkResults));
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java
index dc132659417ff..80b1ff97b725d 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java
@@ -89,7 +89,7 @@ public void query(QueryRequest r, ActionListener<SearchResponse> l) {}
 
             @Override
             public void fetchHits(Iterable<List<HitReference>> refs, ActionListener<List<List<SearchHit>>> listener) {}
-        }, mockCriteria(), randomIntBetween(10, 500), new Limit(1000, 0), CIRCUIT_BREAKER, 1);
+        }, mockCriteria(), randomIntBetween(10, 500), new Limit(1000, 0), CIRCUIT_BREAKER, 1, randomBoolean());
 
         CIRCUIT_BREAKER.startBreaking();
         iterator.pushToStack(new SampleIterator.Page(CB_STACK_SIZE_PRECISION - 1));
@@ -142,7 +142,8 @@ public void fetchHits(Iterable<List<HitReference>> refs, ActionListener<List<Lis
                 randomIntBetween(10, 500),
                 new Limit(1000, 0),
                 eqlCircuitBreaker,
-                1
+                1,
+                randomBoolean()
             );
 
             // unfortunately, mocking an actual result set it extremely complicated
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java
index f1c5d483d4002..9a78dd4573133 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java
@@ -102,6 +102,8 @@ public void testQueryFilterUsedInPitAndSearches() {
                 null,
                 123,
                 1,
+                randomBoolean(),
+                randomBoolean(),
                 "",
                 new TaskId("test", 123),
                 new EqlSearchTask(
@@ -168,7 +170,15 @@ public void fetchHits(Iterable<List<HitReference>> refs, ActionListener<List<Lis
             }
 
             SequenceMatcher matcher = new SequenceMatcher(stages, false, TimeValue.MINUS_ONE, null, booleanArrayOf(stages, false), cb);
-            TumblingWindow window = new TumblingWindow(eqlClient, criteria, null, matcher, Collections.emptyList());
+            TumblingWindow window = new TumblingWindow(
+                eqlClient,
+                criteria,
+                null,
+                matcher,
+                Collections.emptyList(),
+                randomBoolean(),
+                randomBoolean()
+            );
             window.execute(wrap(response -> {
                 // do nothing, we don't care about the query results
             }, ex -> { fail("Shouldn't have failed"); }));
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java
index fe1fca45364e3..58448d981fcca 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java
@@ -146,7 +146,15 @@ public void testCircuitBreakerTumblingWindow() {
             booleanArrayOf(stages, false),
             CIRCUIT_BREAKER
         );
-        TumblingWindow window = new TumblingWindow(client, criteria, null, matcher, Collections.emptyList());
+        TumblingWindow window = new TumblingWindow(
+            client,
+            criteria,
+            null,
+            matcher,
+            Collections.emptyList(),
+            randomBoolean(),
+            randomBoolean()
+        );
         window.execute(ActionTestUtils.assertNoFailureListener(p -> {}));
 
         CIRCUIT_BREAKER.startBreaking();
@@ -228,7 +236,15 @@ private void assertMemoryCleared(
                 booleanArrayOf(sequenceFiltersCount, false),
                 eqlCircuitBreaker
             );
-            TumblingWindow window = new TumblingWindow(eqlClient, criteria, null, matcher, Collections.emptyList());
+            TumblingWindow window = new TumblingWindow(
+                eqlClient,
+                criteria,
+                null,
+                matcher,
+                Collections.emptyList(),
+                randomBoolean(),
+                randomBoolean()
+            );
             window.execute(ActionListener.noop());
 
             assertTrue(esClient.searchRequestsRemainingCount() == 0); // ensure all the search requests have been asked for
@@ -271,7 +287,15 @@ public void testEqlCBCleanedUp_on_ParentCBBreak() {
                 booleanArrayOf(sequenceFiltersCount, false),
                 eqlCircuitBreaker
             );
-            TumblingWindow window = new TumblingWindow(eqlClient, criteria, null, matcher, Collections.emptyList());
+            TumblingWindow window = new TumblingWindow(
+                eqlClient,
+                criteria,
+                null,
+                matcher,
+                Collections.emptyList(),
+                randomBoolean(),
+                randomBoolean()
+            );
             window.execute(wrap(p -> fail(), ex -> assertTrue(ex instanceof CircuitBreakingException)));
         }
         assertCriticalWarnings("[indices.breaker.total.limit] setting of [0%] is below the recommended minimum of 50.0% of the heap");
@@ -329,6 +353,8 @@ private QueryClient buildQueryClient(ESMockClient esClient, CircuitBreaker eqlCi
             null,
             123,
             1,
+            randomBoolean(),
+            randomBoolean(),
             "",
             new TaskId("test", 123),
             new EqlSearchTask(
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/PITFailureTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/PITFailureTests.java
index 1a2f00463b49b..2eee6a262e73c 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/PITFailureTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/PITFailureTests.java
@@ -83,6 +83,8 @@ public void testHandlingPitFailure() {
                 null,
                 123,
                 1,
+                randomBoolean(),
+                randomBoolean(),
                 "",
                 new TaskId("test", 123),
                 new EqlSearchTask(
@@ -132,7 +134,15 @@ public void testHandlingPitFailure() {
             );
 
             SequenceMatcher matcher = new SequenceMatcher(1, false, TimeValue.MINUS_ONE, null, booleanArrayOf(1, false), cb);
-            TumblingWindow window = new TumblingWindow(eqlClient, criteria, null, matcher, Collections.emptyList());
+            TumblingWindow window = new TumblingWindow(
+                eqlClient,
+                criteria,
+                null,
+                matcher,
+                Collections.emptyList(),
+                randomBoolean(),
+                randomBoolean()
+            );
             window.execute(
                 wrap(
                     p -> { fail("Search succeeded despite PIT failure"); },
diff --git a/x-pack/plugin/esql-core/build.gradle b/x-pack/plugin/esql-core/build.gradle
index 796ec1d137155..c18c10840d221 100644
--- a/x-pack/plugin/esql-core/build.gradle
+++ b/x-pack/plugin/esql-core/build.gradle
@@ -3,9 +3,9 @@ apply plugin: 'elasticsearch.internal-test-artifact'
 apply plugin: 'elasticsearch.publish'
 
 esplugin {
-  name 'x-pack-esql-core'
-  description 'Elasticsearch infrastructure plugin for ESQL'
-  classname 'org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin'
+  name = 'x-pack-esql-core'
+  description = 'Elasticsearch infrastructure plugin for ESQL'
+  classname = 'org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin'
   extendedPlugins = ['x-pack-core']
 }
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/ParsingException.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/ParsingException.java
deleted file mode 100644
index bce3f848c9387..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/ParsingException.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.core;
-
-import org.elasticsearch.xpack.esql.core.tree.Source;
-
-import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
-
-public class ParsingException extends QlClientException {
-    private final int line;
-    private final int charPositionInLine;
-
-    public ParsingException(String message, Exception cause, int line, int charPositionInLine) {
-        super(message, cause);
-        this.line = line;
-        this.charPositionInLine = charPositionInLine;
-    }
-
-    public ParsingException(String message, Object... args) {
-        this(Source.EMPTY, message, args);
-    }
-
-    public ParsingException(Source source, String message, Object... args) {
-        super(message, args);
-        this.line = source.source().getLineNumber();
-        this.charPositionInLine = source.source().getColumnNumber();
-    }
-
-    public ParsingException(Exception cause, Source source, String message, Object... args) {
-        super(cause, message, args);
-        this.line = source.source().getLineNumber();
-        this.charPositionInLine = source.source().getColumnNumber();
-    }
-
-    public int getLineNumber() {
-        return line;
-    }
-
-    public int getColumnNumber() {
-        return charPositionInLine + 1;
-    }
-
-    public String getErrorMessage() {
-        return super.getMessage();
-    }
-
-    @Override
-    public String getMessage() {
-        return format("line {}:{}: {}", getLineNumber(), getColumnNumber(), getErrorMessage());
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java
index a092e17931237..8a075e8887512 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java
@@ -174,8 +174,12 @@ public Stream<Attribute> parallelStream() {
     }
 
     @Override
-    public boolean equals(Object o) {
-        return delegate.equals(o);
+    public boolean equals(Object obj) {
+        if (obj instanceof AttributeSet as) {
+            obj = as.delegate;
+        }
+
+        return delegate.equals(obj);
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java
index 00765a8c0528c..b254612a700df 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java
@@ -78,12 +78,20 @@ public Expression(Source source, List<Expression> children) {
         super(source, children);
     }
 
-    // whether the expression can be evaluated statically (folded) or not
+    /**
+     * Whether the expression can be evaluated statically, aka "folded", or not.
+     */
     public boolean foldable() {
         return false;
     }
 
-    public Object fold() {
+    /**
+     * Evaluate this expression statically to a constant. It is an error to call
+     * this if {@link #foldable} returns false.
+     */
+    public Object fold(FoldContext ctx) {
+        // TODO After removing FoldContext.unbounded from non-test code examine all calls
+        // for places we should use instanceof Literal instead
         throw new QlIllegalArgumentException("Should not fold expression");
     }
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java
index 4e4338aad3704..739333ded0fde 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java
@@ -107,10 +107,10 @@ public static boolean foldable(List<? extends Expression> exps) {
         return true;
     }
 
-    public static List<Object> fold(List<? extends Expression> exps) {
+    public static List<Object> fold(FoldContext ctx, List<? extends Expression> exps) {
         List<Object> folded = new ArrayList<>(exps.size());
         for (Expression exp : exps) {
-            folded.add(exp.fold());
+            folded.add(exp.fold(ctx));
         }
 
         return folded;
@@ -135,7 +135,7 @@ public static String name(Expression e) {
     /**
      * Is this {@linkplain Expression} <strong>guaranteed</strong> to have
      * only the {@code null} value. {@linkplain Expression}s that
-     * {@link Expression#fold()} to {@code null} <strong>may</strong>
+     * {@link Expression#fold} to {@code null} <strong>may</strong>
      * return {@code false} here, but should <strong>eventually</strong> be folded
      * into a {@link Literal} containing {@code null} which will return
      * {@code true} from here.
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FoldContext.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FoldContext.java
new file mode 100644
index 0000000000000..25da44c5fd226
--- /dev/null
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FoldContext.java
@@ -0,0 +1,178 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.breaker.CircuitBreakingException;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.MemorySizeValue;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.xpack.esql.core.QlClientException;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.Objects;
+
+/**
+ * Context passed to {@link Expression#fold}. This is not thread safe.
+ */
+public class FoldContext {
+    private static final long SMALL = MemorySizeValue.parseBytesSizeValueOrHeapRatio("5%", "small").getBytes();
+
+    /**
+     * {@link Expression#fold} using less than 5% of heap. Fine in tests but otherwise
+     * calling this is a signal that you either, shouldn't be calling {@link Expression#fold}
+     * at all, or should pass in a shared {@link FoldContext} made by {@code Configuration}.
+     */
+    public static FoldContext small() {
+        return new FoldContext(SMALL);
+    }
+
+    private final long initialAllowedBytes;
+    private long allowedBytes;
+
+    public FoldContext(long allowedBytes) {
+        this.initialAllowedBytes = allowedBytes;
+        this.allowedBytes = allowedBytes;
+    }
+
+    /**
+     * The maximum allowed bytes. {@link #allowedBytes()} will be the same as this
+     * for an unused context.
+     */
+    public long initialAllowedBytes() {
+        return initialAllowedBytes;
+    }
+
+    /**
+     * The remaining allowed bytes.
+     */
+    long allowedBytes() {
+        return allowedBytes;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == null || getClass() != o.getClass()) return false;
+        FoldContext that = (FoldContext) o;
+        return initialAllowedBytes == that.initialAllowedBytes && allowedBytes == that.allowedBytes;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(initialAllowedBytes, allowedBytes);
+    }
+
+    @Override
+    public String toString() {
+        return "FoldContext[" + allowedBytes + '/' + initialAllowedBytes + ']';
+    }
+
+    /**
+     * Track an allocation. Best to call this <strong>before</strong> allocating
+     * if possible, but after is ok if the allocation is small.
+     * <p>
+     *     Note that, unlike {@link CircuitBreaker}, you don't <strong>have</strong>
+     *     to free this allocation later. This is important because the query plan
+     *     doesn't implement {@link Releasable} so it <strong>can't</strong> free
+     *     consistently. But when you have to allocate big chunks of memory during
+     *     folding and know that you are returning the memory it is kindest to
+     *     call this with a negative number, effectively giving those bytes back.
+     * </p>
+     */
+    public void trackAllocation(Source source, long bytes) {
+        allowedBytes -= bytes;
+        assert allowedBytes <= initialAllowedBytes : "returned more bytes than it used";
+        if (allowedBytes < 0) {
+            throw new FoldTooMuchMemoryException(source, bytes, initialAllowedBytes);
+        }
+    }
+
+    /**
+     * Adapt this into a {@link CircuitBreaker} suitable for building bounded local
+     * DriverContext. This is absolutely an abuse of the {@link CircuitBreaker} contract
+     * and only methods used by BlockFactory are implemented. And this'll throw a
+     * {@link FoldTooMuchMemoryException} instead of the standard {@link CircuitBreakingException}.
+     * This works for the common folding implementation though.
+     */
+    public CircuitBreaker circuitBreakerView(Source source) {
+        return new CircuitBreaker() {
+            @Override
+            public void circuitBreak(String fieldName, long bytesNeeded) {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public void addEstimateBytesAndMaybeBreak(long bytes, String label) throws CircuitBreakingException {
+                trackAllocation(source, bytes);
+            }
+
+            @Override
+            public void addWithoutBreaking(long bytes) {
+                assert bytes <= 0 : "we only expect this to be used for deallocation";
+                allowedBytes -= bytes;
+                assert allowedBytes <= initialAllowedBytes : "returned more bytes than it used";
+            }
+
+            @Override
+            public long getUsed() {
+                /*
+                 * This isn't expected to be used by we can implement it so we may as
+                 * well. Maybe it'll be useful for debugging one day.
+                 */
+                return initialAllowedBytes - allowedBytes;
+            }
+
+            @Override
+            public long getLimit() {
+                /*
+                 * This isn't expected to be used by we can implement it so we may as
+                 * well. Maybe it'll be useful for debugging one day.
+                 */
+                return initialAllowedBytes;
+            }
+
+            @Override
+            public double getOverhead() {
+                return 1.0;
+            }
+
+            @Override
+            public long getTrippedCount() {
+                return 0;
+            }
+
+            @Override
+            public String getName() {
+                return REQUEST;
+            }
+
+            @Override
+            public Durability getDurability() {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public void setLimitAndOverhead(long limit, double overhead) {
+                throw new UnsupportedOperationException();
+            }
+        };
+    }
+
+    public static class FoldTooMuchMemoryException extends QlClientException {
+        protected FoldTooMuchMemoryException(Source source, long bytesForExpression, long initialAllowedBytes) {
+            super(
+                "line {}:{}: Folding query used more than {}. The expression that pushed past the limit is [{}] which needed {}.",
+                source.source().getLineNumber(),
+                source.source().getColumnNumber(),
+                ByteSizeValue.ofBytes(initialAllowedBytes),
+                source.text(),
+                ByteSizeValue.ofBytes(bytesForExpression)
+            );
+        }
+    }
+}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Foldables.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Foldables.java
index 601758bca5918..233113c3fe1b8 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Foldables.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Foldables.java
@@ -10,9 +10,9 @@
 
 public abstract class Foldables {
 
-    public static Object valueOf(Expression e) {
+    public static Object valueOf(FoldContext ctx, Expression e) {
         if (e.foldable()) {
-            return e.fold();
+            return e.fold(ctx);
         }
         throw new QlIllegalArgumentException("Cannot determine value for {}", e);
     }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java
index 53f559c5c82fe..afe616489d81d 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java
@@ -98,7 +98,7 @@ public boolean resolved() {
     }
 
     @Override
-    public Object fold() {
+    public Object fold(FoldContext ctx) {
         return value;
     }
 
@@ -138,7 +138,7 @@ public String nodeString() {
      * Utility method for creating a literal out of a foldable expression.
      * Throws an exception if the expression is not foldable.
      */
-    public static Literal of(Expression foldable) {
+    public static Literal of(FoldContext ctx, Expression foldable) {
         if (foldable.foldable() == false) {
             throw new QlIllegalArgumentException("Foldable expression required for Literal creation; received unfoldable " + foldable);
         }
@@ -147,7 +147,7 @@ public static Literal of(Expression foldable) {
             return (Literal) foldable;
         }
 
-        return new Literal(foldable.source(), foldable.fold(), foldable.dataType());
+        return new Literal(foldable.source(), foldable.fold(ctx), foldable.dataType());
     }
 
     public static Literal of(Expression source, Object value) {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java
index b817ec17c7bda..842f3c0ddadd7 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java
@@ -133,6 +133,14 @@ public static TypeResolution isFoldable(Expression e, String operationName, Para
         return TypeResolution.TYPE_RESOLVED;
     }
 
+    /**
+     * Is this {@link Expression#foldable()} and not {@code null}.
+     *
+     * @deprecated instead of calling this, check for a {@link Literal} containing
+     *             {@code null}. Foldable expressions will be folded by other rules,
+     *             eventually, to a {@link Literal}.
+     */
+    @Deprecated
     public static TypeResolution isNotNullAndFoldable(Expression e, String operationName, ParamOrdinal paramOrd) {
         TypeResolution resolution = isFoldable(e, operationName, paramOrd);
 
@@ -140,7 +148,7 @@ public static TypeResolution isNotNullAndFoldable(Expression e, String operation
             return resolution;
         }
 
-        if (e.dataType() == DataType.NULL || e.fold() == null) {
+        if (e.dataType() == DataType.NULL || e.fold(FoldContext.small()) == null) {
             resolution = new TypeResolution(
                 format(
                     null,
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/Function.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/Function.java
index a1afcdbf1f77c..cad5c631088f2 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/Function.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/Function.java
@@ -6,7 +6,6 @@
  */
 package org.elasticsearch.xpack.esql.core.expression.function;
 
-import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
@@ -43,11 +42,6 @@ public Nullability nullable() {
         return Expressions.nullable(children());
     }
 
-    /** Return true if this function can be executed under the provided {@link XPackLicenseState}, otherwise false.*/
-    public boolean checkLicense(XPackLicenseState state) {
-        return true;
-    }
-
     @Override
     public int hashCode() {
         return Objects.hash(getClass(), children());
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java
index 8704a42ed33e2..36517b1be9ce7 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
 
@@ -53,5 +54,5 @@ public boolean foldable() {
     }
 
     @Override
-    public abstract Object fold();
+    public abstract Object fold(FoldContext ctx);
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/BinaryPredicate.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/BinaryPredicate.java
index be5caedacd50a..bf5549b31e5fa 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/BinaryPredicate.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/BinaryPredicate.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.esql.core.expression.predicate;
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 
@@ -29,8 +30,8 @@ protected BinaryPredicate(Source source, Expression left, Expression right, F fu
 
     @SuppressWarnings("unchecked")
     @Override
-    public R fold() {
-        return function().apply((T) left().fold(), (U) right().fold());
+    public R fold(FoldContext ctx) {
+        return function().apply((T) left().fold(ctx), (U) right().fold(ctx));
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java
index 2ee4866d35443..a4e4685f764e8 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java
@@ -8,6 +8,8 @@
 
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
@@ -85,24 +87,37 @@ public ZoneId zoneId() {
         return zoneId;
     }
 
+    /**
+     * In case that the range is empty due to foldable, invalid bounds, but the bounds themselves are not yet folded, the optimizer will
+     * need two passes to fold this.
+     * That's because we shouldn't perform folding when trying to determine foldability.
+     */
     @Override
     public boolean foldable() {
         if (lower.foldable() && upper.foldable()) {
-            return areBoundariesInvalid() || value.foldable();
-        }
+            if (value().foldable()) {
+                return true;
+            }
 
+            // We cannot fold the bounds here; but if they're already literals, we can check if the range is always empty.
+            if (lower() instanceof Literal l && upper() instanceof Literal u) {
+                return areBoundariesInvalid(l.value(), u.value());
+            }
+        }
         return false;
     }
 
     @Override
-    public Object fold() {
-        if (areBoundariesInvalid()) {
+    public Object fold(FoldContext ctx) {
+        Object lowerValue = lower.fold(ctx);
+        Object upperValue = upper.fold(ctx);
+        if (areBoundariesInvalid(lowerValue, upperValue)) {
             return Boolean.FALSE;
         }
 
-        Object val = value.fold();
-        Integer lowerCompare = BinaryComparison.compare(lower.fold(), val);
-        Integer upperCompare = BinaryComparison.compare(val, upper().fold());
+        Object val = value.fold(ctx);
+        Integer lowerCompare = BinaryComparison.compare(lower.fold(ctx), val);
+        Integer upperCompare = BinaryComparison.compare(val, upper().fold(ctx));
         boolean lowerComparsion = lowerCompare == null ? false : (includeLower ? lowerCompare <= 0 : lowerCompare < 0);
         boolean upperComparsion = upperCompare == null ? false : (includeUpper ? upperCompare <= 0 : upperCompare < 0);
         return lowerComparsion && upperComparsion;
@@ -112,9 +127,7 @@ public Object fold() {
      * Check whether the boundaries are invalid ( upper &lt; lower) or not.
      * If they are, the value does not have to be evaluated.
      */
-    protected boolean areBoundariesInvalid() {
-        Object lowerValue = lower.fold();
-        Object upperValue = upper.fold();
+    protected boolean areBoundariesInvalid(Object lowerValue, Object upperValue) {
         if (DataType.isDateTime(value.dataType()) || DataType.isDateTime(lower.dataType()) || DataType.isDateTime(upper.dataType())) {
             try {
                 if (upperValue instanceof String upperString) {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java
index c4983b49a6bc8..218f61856accc 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
@@ -56,8 +57,8 @@ protected TypeResolution resolveType() {
     }
 
     @Override
-    public Object fold() {
-        return apply(field().fold());
+    public Object fold(FoldContext ctx) {
+        return apply(field().fold(ctx));
     }
 
     private static Boolean apply(Object input) {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java
index 9879a1f5ffc29..f5542ff7c3de5 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable;
@@ -49,8 +50,8 @@ protected IsNotNull replaceChild(Expression newChild) {
     }
 
     @Override
-    public Object fold() {
-        return field().fold() != null && DataType.isNull(field().dataType()) == false;
+    public Object fold(FoldContext ctx) {
+        return DataType.isNull(field().dataType()) == false && field().fold(ctx) != null;
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java
index d88945045b03e..bb85791a9f85e 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable;
@@ -45,8 +46,8 @@ protected IsNull replaceChild(Expression newChild) {
     }
 
     @Override
-    public Object fold() {
-        return field().fold() == null || DataType.isNull(field().dataType());
+    public Object fold(FoldContext ctx) {
+        return DataType.isNull(field().dataType()) || field().fold(ctx) == null;
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Neg.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Neg.java
index 9a8a14f320cd6..b0e79704f5fda 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Neg.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Neg.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -53,8 +54,8 @@ protected TypeResolution resolveType() {
     }
 
     @Override
-    public Object fold() {
-        return Arithmetics.negate((Number) field().fold());
+    public Object fold(FoldContext ctx) {
+        return Arithmetics.negate((Number) field().fold(ctx));
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RegexMatch.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RegexMatch.java
index 0f9116ade5a31..a4a0a6217161e 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RegexMatch.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RegexMatch.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.esql.core.expression.predicate.regex;
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -62,7 +63,7 @@ public boolean foldable() {
     }
 
     @Override
-    public Boolean fold() {
+    public Boolean fold(FoldContext ctx) {
         throw new UnsupportedOperationException();
     }
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java
index 468d076c1b7ef..e0f4f6b032662 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
-import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MultiMatchQueryPredicate;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
@@ -22,7 +21,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardLike;
 import org.elasticsearch.xpack.esql.core.querydsl.query.BoolQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.ExistsQuery;
-import org.elasticsearch.xpack.esql.core.querydsl.query.MultiMatchQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.NotQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.querydsl.query.RegexQuery;
@@ -71,18 +69,6 @@ private static Query translateField(RegexMatch e, String targetFieldName) {
         }
     }
 
-    public static class MultiMatches extends ExpressionTranslator<MultiMatchQueryPredicate> {
-
-        @Override
-        protected Query asQuery(MultiMatchQueryPredicate q, TranslatorHandler handler) {
-            return doTranslate(q, handler);
-        }
-
-        public static Query doTranslate(MultiMatchQueryPredicate q, TranslatorHandler handler) {
-            return new MultiMatchQuery(q.source(), q.query(), q.fields(), q);
-        }
-    }
-
     public static class BinaryLogic extends ExpressionTranslator<
         org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic> {
 
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java
index 511c7f4b1d2f8..ade79c8168076 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java
@@ -30,7 +30,7 @@
 
 public class AttributeMapTests extends ESTestCase {
 
-    private static Attribute a(String name) {
+    static Attribute a(String name) {
         return new UnresolvedAttribute(Source.EMPTY, name);
     }
 
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeSetTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeSetTests.java
new file mode 100644
index 0000000000000..0e97773fb90d2
--- /dev/null
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeSetTests.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.List;
+
+import static org.elasticsearch.xpack.esql.core.expression.AttributeMapTests.a;
+
+public class AttributeSetTests extends ESTestCase {
+
+    public void testEquals() {
+        Attribute a1 = a("1");
+        Attribute a2 = a("2");
+
+        AttributeSet first = new AttributeSet(List.of(a1, a2));
+        assertEquals(first, first);
+
+        AttributeSet second = new AttributeSet();
+        second.add(a1);
+        second.add(a2);
+
+        assertEquals(first, second);
+        assertEquals(second, first);
+
+        AttributeSet third = new AttributeSet();
+        third.add(a("1"));
+        third.add(a("2"));
+
+        assertNotEquals(first, third);
+        assertNotEquals(third, first);
+
+        assertEquals(AttributeSet.EMPTY, AttributeSet.EMPTY);
+        assertEquals(AttributeSet.EMPTY, first.intersect(third));
+        assertEquals(third.intersect(first), AttributeSet.EMPTY);
+    }
+}
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/FoldContextTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/FoldContextTests.java
new file mode 100644
index 0000000000000..2080f4007777c
--- /dev/null
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/FoldContextTests.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class FoldContextTests extends ESTestCase {
+    public void testEq() {
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(randomFoldContext(), this::copy, this::mutate);
+    }
+
+    private FoldContext randomFoldContext() {
+        FoldContext ctx = new FoldContext(randomNonNegativeLong());
+        if (randomBoolean()) {
+            ctx.trackAllocation(Source.EMPTY, randomLongBetween(0, ctx.initialAllowedBytes()));
+        }
+        return ctx;
+    }
+
+    private FoldContext copy(FoldContext ctx) {
+        FoldContext copy = new FoldContext(ctx.initialAllowedBytes());
+        copy.trackAllocation(Source.EMPTY, ctx.initialAllowedBytes() - ctx.allowedBytes());
+        return copy;
+    }
+
+    private FoldContext mutate(FoldContext ctx) {
+        if (randomBoolean()) {
+            FoldContext differentInitial = new FoldContext(ctx.initialAllowedBytes() + 1);
+            differentInitial.trackAllocation(Source.EMPTY, differentInitial.initialAllowedBytes() - ctx.allowedBytes());
+            assertThat(differentInitial.allowedBytes(), equalTo(ctx.allowedBytes()));
+            return differentInitial;
+        } else {
+            FoldContext differentAllowed = new FoldContext(ctx.initialAllowedBytes());
+            long allowed = randomValueOtherThan(ctx.allowedBytes(), () -> randomLongBetween(0, ctx.initialAllowedBytes()));
+            differentAllowed.trackAllocation(Source.EMPTY, ctx.initialAllowedBytes() - allowed);
+            assertThat(differentAllowed.allowedBytes(), equalTo(allowed));
+            return differentAllowed;
+        }
+    }
+
+    public void testTrackAllocation() {
+        FoldContext ctx = new FoldContext(10);
+        ctx.trackAllocation(Source.synthetic("shouldn't break"), 10);
+        Exception e = expectThrows(
+            FoldContext.FoldTooMuchMemoryException.class,
+            () -> ctx.trackAllocation(Source.synthetic("should break"), 1)
+        );
+        assertThat(
+            e.getMessage(),
+            equalTo(
+                "line -1:-1: Folding query used more than 10b. "
+                    + "The expression that pushed past the limit is [should break] which needed 1b."
+            )
+        );
+    }
+
+    public void testCircuitBreakerViewBreaking() {
+        FoldContext ctx = new FoldContext(10);
+        ctx.circuitBreakerView(Source.synthetic("shouldn't break")).addEstimateBytesAndMaybeBreak(10, "test");
+        Exception e = expectThrows(
+            FoldContext.FoldTooMuchMemoryException.class,
+            () -> ctx.circuitBreakerView(Source.synthetic("should break")).addEstimateBytesAndMaybeBreak(1, "test")
+        );
+        assertThat(
+            e.getMessage(),
+            equalTo(
+                "line -1:-1: Folding query used more than 10b. "
+                    + "The expression that pushed past the limit is [should break] which needed 1b."
+            )
+        );
+    }
+
+    public void testCircuitBreakerViewWithoutBreaking() {
+        FoldContext ctx = new FoldContext(10);
+        CircuitBreaker view = ctx.circuitBreakerView(Source.synthetic("shouldn't break"));
+        view.addEstimateBytesAndMaybeBreak(10, "test");
+        view.addWithoutBreaking(-1);
+        assertThat(view.getUsed(), equalTo(9L));
+    }
+
+    public void testToString() {
+        // Random looking numbers are indeed random. Just so we have consistent numbers to assert on in toString.
+        FoldContext ctx = new FoldContext(123);
+        ctx.trackAllocation(Source.EMPTY, 22);
+        assertThat(ctx.toString(), equalTo("FoldContext[101/123]"));
+    }
+}
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java
index ed4c6282368ca..cd15ed5a94cfc 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.esql.core.expression.predicate;
 
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.DateUtils;
@@ -211,7 +212,11 @@ public void testAreBoundariesInvalid() {
                 (Boolean) test[7],
                 ZoneId.systemDefault()
             );
-            assertEquals("failed on test " + i + ": " + Arrays.toString(test), test[8], range.areBoundariesInvalid());
+            assertEquals(
+                "failed on test " + i + ": " + Arrays.toString(test),
+                test[8],
+                range.areBoundariesInvalid(range.lower().fold(FoldContext.small()), range.upper().fold(FoldContext.small()))
+            );
         }
     }
 
@@ -226,5 +231,4 @@ private static DataType randomNumericType() {
     private static DataType randomTextType() {
         return randomFrom(KEYWORD, TEXT);
     }
-
 }
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java
index d323174d2d3d9..91b0564a5b404 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Range;
@@ -104,7 +105,7 @@ public void testFoldExcludingRangeToFalse() {
 
         Range r = rangeOf(fa, SIX, false, FIVE, true);
         assertTrue(r.foldable());
-        assertEquals(Boolean.FALSE, r.fold());
+        assertEquals(Boolean.FALSE, r.fold(FoldContext.small()));
     }
 
     // 6 < a <= 5.5 -> FALSE
@@ -113,7 +114,7 @@ public void testFoldExcludingRangeWithDifferentTypesToFalse() {
 
         Range r = rangeOf(fa, SIX, false, L(5.5d), true);
         assertTrue(r.foldable());
-        assertEquals(Boolean.FALSE, r.fold());
+        assertEquals(Boolean.FALSE, r.fold(FoldContext.small()));
     }
 
     // Conjunction
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/TestUtils.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/TestUtils.java
index b37ca0431ec2d..40321fddebdfe 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/TestUtils.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/TestUtils.java
@@ -15,6 +15,8 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 
+import java.util.regex.Pattern;
+
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength;
 import static org.elasticsearch.test.ESTestCase.randomBoolean;
@@ -26,6 +28,8 @@
 public final class TestUtils {
     private TestUtils() {}
 
+    private static final Pattern WS_PATTERN = Pattern.compile("\\s");
+
     public static Literal of(Object value) {
         return of(Source.EMPTY, value);
     }
@@ -59,4 +63,9 @@ public static FieldAttribute getFieldAttribute(String name) {
     public static FieldAttribute getFieldAttribute(String name, DataType dataType) {
         return new FieldAttribute(EMPTY, name, new EsField(name + "f", dataType, emptyMap(), true));
     }
+
+    /** Similar to {@link String#strip()}, but removes the WS throughout the entire string. */
+    public static String stripThrough(String input) {
+        return WS_PATTERN.matcher(input).replaceAll(StringUtils.EMPTY);
+    }
 }
diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle
index 02f9752d21e09..d7fa786a24e7c 100644
--- a/x-pack/plugin/esql/build.gradle
+++ b/x-pack/plugin/esql/build.gradle
@@ -19,9 +19,9 @@ apply plugin: 'elasticsearch.string-templates'
 apply plugin: 'elasticsearch.publish'
 
 esplugin {
-  name 'x-pack-esql'
-  description 'The plugin that powers ESQL for Elasticsearch'
-  classname 'org.elasticsearch.xpack.esql.plugin.EsqlPlugin'
+  name = 'x-pack-esql'
+  description = 'The plugin that powers ESQL for Elasticsearch'
+  classname ='org.elasticsearch.xpack.esql.plugin.EsqlPlugin'
   extendedPlugins = ['x-pack-esql-core', 'lang-painless', 'x-pack-ml']
 }
 
@@ -276,15 +276,16 @@ tasks.register("regen") {
   }
 }
 
-tasks.named("spotlessJava") { dependsOn stringTemplates }
+tasks.named("spotlessJava") { dependsOn "stringTemplates" }
 tasks.named('checkstyleMain').configure {
   excludes = [ "**/*.java.st" ]
   exclude { normalize(it.file.toString()).contains("src/main/generated-src/generated") }
   exclude { normalize(it.file.toString()).contains("src/main/generated") }
 }
 
-def prop(Type, type, TYPE, BYTES, Array) {
+def prop(Name, Type, type, TYPE, BYTES, Array) {
   return [
+    "Name" : Name,
     "Type" : Type,
     "type" : type,
     "TYPE" : TYPE,
@@ -296,15 +297,19 @@ def prop(Type, type, TYPE, BYTES, Array) {
     "double" : type == "double" ? "true" : "",
     "BytesRef" : type == "BytesRef" ? "true" : "",
     "boolean" : type == "boolean" ? "true" : "",
+    "nanosMillis" : Name == "NanosMillis" ? "true" : "",
+    "millisNanos" : Name == "MillisNanos" ? "true" : "",
   ]
 }
 
 tasks.named('stringTemplates').configure {
-  var intProperties = prop("Int", "int", "INT", "Integer.BYTES", "IntArray")
-  var longProperties = prop("Long", "long", "LONG", "Long.BYTES", "LongArray")
-  var doubleProperties = prop("Double", "double", "DOUBLE", "Double.BYTES", "DoubleArray")
-  var bytesRefProperties = prop("BytesRef", "BytesRef", "BYTES_REF", "org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF", "")
-  var booleanProperties = prop("Boolean", "boolean", "BOOLEAN", "Byte.BYTES", "BitArray")
+  var intProperties = prop("Int", "Int", "int", "INT", "Integer.BYTES", "IntArray")
+  var longProperties = prop("Long", "Long", "long", "LONG", "Long.BYTES", "LongArray")
+  var nanosMillisProperties = prop("NanosMillis", "Long", "long", "LONG", "Long.BYTES", "LongArray")
+  var millisNanosProperties = prop("MillisNanos", "Long", "long", "LONG", "Long.BYTES", "LongArray")
+  var doubleProperties = prop("Double", "Double", "double", "DOUBLE", "Double.BYTES", "DoubleArray")
+  var bytesRefProperties = prop("BytesRef", "BytesRef", "BytesRef", "BYTES_REF", "org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF", "")
+  var booleanProperties = prop("Boolean", "Boolean", "boolean", "BOOLEAN", "Byte.BYTES", "BitArray")
 
   File inInputFile = file("src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/X-InEvaluator.java.st")
   template {
@@ -322,6 +327,16 @@ tasks.named('stringTemplates').configure {
     it.inputFile =  inInputFile
     it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InLongEvaluator.java"
   }
+  template {
+    it.properties = nanosMillisProperties
+    it.inputFile =  inInputFile
+    it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InNanosMillisEvaluator.java"
+  }
+  template {
+    it.properties = millisNanosProperties
+    it.inputFile =  inInputFile
+    it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InMillisNanosEvaluator.java"
+  }
   template {
     it.properties = doubleProperties
     it.inputFile =  inInputFile
diff --git a/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Fixed.java b/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Fixed.java
index 62703fa400ff7..1f10abf3b9fb0 100644
--- a/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Fixed.java
+++ b/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Fixed.java
@@ -11,7 +11,6 @@
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
-import java.util.function.Function;
 
 /**
  * Used on parameters on methods annotated with {@link Evaluator} to indicate
@@ -27,12 +26,23 @@
     boolean includeInToString() default true;
 
     /**
-     * Should the Evaluator's factory build this per evaluator with a
-     * {@code Function<DriverContext, T>} or just take fixed implementation?
-     * This is typically set to {@code true} to use the {@link Function}
-     * to make "scratch" objects which have to be isolated in a single thread.
-     * This is typically set to {@code false} when the parameter is simply
-     * immutable and can be shared.
+     * Defines the scope of the parameter.
+     * - SINGLETON (default) will build a single instance and share it across all evaluators
+     * - THREAD_LOCAL will build a new instance for each evaluator thread
      */
-    boolean build() default false;
+    Scope scope() default Scope.SINGLETON;
+
+    /**
+     * Defines the parameter scope
+     */
+    enum Scope {
+        /**
+         * Should be used for immutable parameters that can be shared across different threads
+         */
+        SINGLETON,
+        /**
+         * Should be used for mutable or not thread safe parameters
+         */
+        THREAD_LOCAL,
+    }
 }
diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java
index 5869eff23a9ab..b4a0cf9127f23 100644
--- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java
+++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java
@@ -16,6 +16,7 @@
 import com.squareup.javapoet.TypeSpec;
 
 import org.elasticsearch.compute.ann.Fixed;
+import org.elasticsearch.compute.ann.Fixed.Scope;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -725,7 +726,7 @@ public String closeInvocation() {
         }
     }
 
-    private record FixedProcessFunctionArg(TypeName type, String name, boolean includeInToString, boolean build, boolean releasable)
+    private record FixedProcessFunctionArg(TypeName type, String name, boolean includeInToString, Scope scope, boolean releasable)
         implements
             ProcessFunctionArg {
         @Override
@@ -762,12 +763,18 @@ public void implementFactoryCtor(MethodSpec.Builder builder) {
         }
 
         private TypeName factoryFieldType() {
-            return build ? ParameterizedTypeName.get(ClassName.get(Function.class), DRIVER_CONTEXT, type.box()) : type;
+            return switch (scope) {
+                case SINGLETON -> type;
+                case THREAD_LOCAL -> ParameterizedTypeName.get(ClassName.get(Function.class), DRIVER_CONTEXT, type.box());
+            };
         }
 
         @Override
         public String factoryInvocation(MethodSpec.Builder factoryMethodBuilder) {
-            return build ? name + ".apply(context)" : name;
+            return switch (scope) {
+                case SINGLETON -> name;
+                case THREAD_LOCAL -> name + ".apply(context)";
+            };
         }
 
         @Override
@@ -1020,7 +1027,7 @@ private ProcessFunction(
                             type,
                             name,
                             fixed.includeInToString(),
-                            fixed.build(),
+                            fixed.scope(),
                             Types.extendsSuper(types, v.asType(), "org.elasticsearch.core.Releasable")
                         )
                     );
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregator.java
index 602fd29433193..bd77bd7ff1e46 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregator.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregator.java
@@ -130,8 +130,20 @@ public static class GroupingState implements Releasable {
         private final BytesRefHash bytes;
 
         private GroupingState(BigArrays bigArrays) {
-            values = new LongLongHash(1, bigArrays);
-            bytes = new BytesRefHash(1, bigArrays);
+            LongLongHash _values = null;
+            BytesRefHash _bytes = null;
+            try {
+                _values = new LongLongHash(1, bigArrays);
+                _bytes = new BytesRefHash(1, bigArrays);
+
+                values = _values;
+                bytes = _bytes;
+
+                _values = null;
+                _bytes = null;
+            } finally {
+                Releasables.closeExpectNoException(_values, _bytes);
+            }
         }
 
         void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java
index 3d600bec1bd65..896662dddf1eb 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java
@@ -122,7 +122,7 @@ public BooleanBlock filter(int... positions) {
                 int valueCount = getValueCount(pos);
                 int first = getFirstValueIndex(pos);
                 if (valueCount == 1) {
-                    builder.appendBoolean(getBoolean(getFirstValueIndex(pos)));
+                    builder.appendBoolean(getBoolean(first));
                 } else {
                     builder.beginPositionEntry();
                     for (int c = 0; c < valueCount; c++) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java
index f50135aa51dd4..5bcb1b0ec5095 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java
@@ -110,7 +110,7 @@ public BytesRefBlock filter(int... positions) {
                 int valueCount = getValueCount(pos);
                 int first = getFirstValueIndex(pos);
                 if (valueCount == 1) {
-                    builder.appendBytesRef(getBytesRef(getFirstValueIndex(pos), scratch));
+                    builder.appendBytesRef(getBytesRef(first, scratch));
                 } else {
                     builder.beginPositionEntry();
                     for (int c = 0; c < valueCount; c++) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java
index eceec30348749..20bd42da98c71 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java
@@ -101,7 +101,7 @@ public DoubleBlock filter(int... positions) {
                 int valueCount = getValueCount(pos);
                 int first = getFirstValueIndex(pos);
                 if (valueCount == 1) {
-                    builder.appendDouble(getDouble(getFirstValueIndex(pos)));
+                    builder.appendDouble(getDouble(first));
                 } else {
                     builder.beginPositionEntry();
                     for (int c = 0; c < valueCount; c++) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayBlock.java
index 56f0cedb5f15e..c0941557dc4fe 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayBlock.java
@@ -101,7 +101,7 @@ public FloatBlock filter(int... positions) {
                 int valueCount = getValueCount(pos);
                 int first = getFirstValueIndex(pos);
                 if (valueCount == 1) {
-                    builder.appendFloat(getFloat(getFirstValueIndex(pos)));
+                    builder.appendFloat(getFloat(first));
                 } else {
                     builder.beginPositionEntry();
                     for (int c = 0; c < valueCount; c++) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java
index 2e10d09e1a410..8ced678bc90b0 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java
@@ -101,7 +101,7 @@ public IntBlock filter(int... positions) {
                 int valueCount = getValueCount(pos);
                 int first = getFirstValueIndex(pos);
                 if (valueCount == 1) {
-                    builder.appendInt(getInt(getFirstValueIndex(pos)));
+                    builder.appendInt(getInt(first));
                 } else {
                     builder.beginPositionEntry();
                     for (int c = 0; c < valueCount; c++) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java
index 776fa363f6080..fb631ab326ce7 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java
@@ -101,7 +101,7 @@ public LongBlock filter(int... positions) {
                 int valueCount = getValueCount(pos);
                 int first = getFirstValueIndex(pos);
                 if (valueCount == 1) {
-                    builder.appendLong(getLong(getFirstValueIndex(pos)));
+                    builder.appendLong(getLong(first));
                 } else {
                     builder.beginPositionEntry();
                     for (int c = 0; c < valueCount; c++) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st
index a8884c58116f3..1cef234b2238f 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st
@@ -244,8 +244,20 @@ $endif$
 $if(long||double)$
             values = new LongLongHash(1, bigArrays);
 $elseif(BytesRef)$
-            values = new LongLongHash(1, bigArrays);
-            bytes = new BytesRefHash(1, bigArrays);
+            LongLongHash _values = null;
+            BytesRefHash _bytes = null;
+            try {
+                _values = new LongLongHash(1, bigArrays);
+                _bytes = new BytesRefHash(1, bigArrays);
+
+                values = _values;
+                bytes = _bytes;
+
+                _values = null;
+                _bytes = null;
+            } finally {
+                Releasables.closeExpectNoException(_values, _bytes);
+            }
 $elseif(int||float)$
             values = new LongHash(1, bigArrays);
 $endif$
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/PointType.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/PointType.java
index 5395ca0b85163..fb45f869c4133 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/PointType.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/PointType.java
@@ -7,12 +7,11 @@
 
 package org.elasticsearch.compute.aggregation.spatial;
 
-import org.apache.lucene.geo.GeoEncodingUtils;
-import org.apache.lucene.geo.XYEncodingUtils;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.Rectangle;
 import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor;
 import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor.WrapLongitude;
+import org.elasticsearch.lucene.spatial.CoordinateEncoder;
 
 import java.util.Optional;
 
@@ -23,26 +22,6 @@ public Optional<Rectangle> computeEnvelope(Geometry geo) {
             return SpatialEnvelopeVisitor.visitGeo(geo, WrapLongitude.WRAP);
         }
 
-        @Override
-        public double decodeX(int encoded) {
-            return GeoEncodingUtils.decodeLongitude(encoded);
-        }
-
-        @Override
-        public double decodeY(int encoded) {
-            return GeoEncodingUtils.decodeLatitude(encoded);
-        }
-
-        @Override
-        public int encodeX(double decoded) {
-            return GeoEncodingUtils.encodeLongitude(decoded);
-        }
-
-        @Override
-        public int encodeY(double decoded) {
-            return GeoEncodingUtils.encodeLatitude(decoded);
-        }
-
         // Geo encodes the longitude in the lower 32 bits and the latitude in the upper 32 bits.
         @Override
         public int extractX(long encoded) {
@@ -53,6 +32,11 @@ public int extractX(long encoded) {
         public int extractY(long encoded) {
             return SpatialAggregationUtils.extractFirst(encoded);
         }
+
+        @Override
+        public CoordinateEncoder encoder() {
+            return CoordinateEncoder.GEO;
+        }
     },
     CARTESIAN {
         @Override
@@ -60,26 +44,6 @@ public Optional<Rectangle> computeEnvelope(Geometry geo) {
             return SpatialEnvelopeVisitor.visitCartesian(geo);
         }
 
-        @Override
-        public double decodeX(int encoded) {
-            return XYEncodingUtils.decode(encoded);
-        }
-
-        @Override
-        public double decodeY(int encoded) {
-            return XYEncodingUtils.decode(encoded);
-        }
-
-        @Override
-        public int encodeX(double decoded) {
-            return XYEncodingUtils.encode((float) decoded);
-        }
-
-        @Override
-        public int encodeY(double decoded) {
-            return XYEncodingUtils.encode((float) decoded);
-        }
-
         @Override
         public int extractX(long encoded) {
             return SpatialAggregationUtils.extractFirst(encoded);
@@ -89,19 +53,18 @@ public int extractX(long encoded) {
         public int extractY(long encoded) {
             return SpatialAggregationUtils.extractSecond(encoded);
         }
+
+        @Override
+        public CoordinateEncoder encoder() {
+            return CoordinateEncoder.CARTESIAN;
+        }
     };
 
     public abstract Optional<Rectangle> computeEnvelope(Geometry geo);
 
-    public abstract double decodeX(int encoded);
-
-    public abstract double decodeY(int encoded);
-
-    public abstract int encodeX(double decoded);
-
-    public abstract int encodeY(double decoded);
-
     public abstract int extractX(long encoded);
 
     public abstract int extractY(long encoded);
+
+    public abstract CoordinateEncoder encoder();
 }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingState.java
index 9ce0ccdda0ff5..cb765e4d6757e 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingState.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingState.java
@@ -72,10 +72,10 @@ public void add(int groupId, Geometry geometry) {
             .ifPresent(
                 r -> add(
                     groupId,
-                    pointType.encodeX(r.getMinX()),
-                    pointType.encodeX(r.getMaxX()),
-                    pointType.encodeY(r.getMaxY()),
-                    pointType.encodeY(r.getMinY())
+                    pointType.encoder().encodeX(r.getMinX()),
+                    pointType.encoder().encodeX(r.getMaxX()),
+                    pointType.encoder().encodeY(r.getMaxY()),
+                    pointType.encoder().encodeY(r.getMinY())
                 )
             );
     }
@@ -122,10 +122,10 @@ public Block toBlock(IntVector selected, DriverContext driverContext) {
                         new BytesRef(
                             WellKnownBinary.toWKB(
                                 new Rectangle(
-                                    pointType.decodeX(minXs.get(si)),
-                                    pointType.decodeX(maxXs.get(si)),
-                                    pointType.decodeY(maxYs.get(si)),
-                                    pointType.decodeY(minYs.get(si))
+                                    pointType.encoder().decodeX(minXs.get(si)),
+                                    pointType.encoder().decodeX(maxXs.get(si)),
+                                    pointType.encoder().decodeY(maxYs.get(si)),
+                                    pointType.encoder().decodeY(minYs.get(si))
                                 ),
                                 ByteOrder.LITTLE_ENDIAN
                             )
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java
index 3dd7a6d4acde2..41bc50abcf6bc 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java
@@ -91,8 +91,8 @@ public void add(int groupId, Geometry geo) {
                 SpatialAggregationUtils.encodePositiveLongitude(geoPointVisitor.getMinPosX()),
                 SpatialAggregationUtils.encodeNegativeLongitude(geoPointVisitor.getMaxNegX()),
                 SpatialAggregationUtils.encodePositiveLongitude(geoPointVisitor.getMaxPosX()),
-                POINT_TYPE.encodeY(geoPointVisitor.getMaxY()),
-                POINT_TYPE.encodeY(geoPointVisitor.getMinY())
+                POINT_TYPE.encoder().encodeY(geoPointVisitor.getMaxY()),
+                POINT_TYPE.encoder().encodeY(geoPointVisitor.getMinY())
             );
         }
     }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentState.java
index 0eea9b79f73ea..3dc150d1702a2 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentState.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentState.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.Rectangle;
 import org.elasticsearch.geometry.utils.WellKnownBinary;
+import org.elasticsearch.lucene.spatial.CoordinateEncoder;
 
 import java.nio.ByteOrder;
 
@@ -46,10 +47,10 @@ public void add(Geometry geo) {
         pointType.computeEnvelope(geo)
             .ifPresent(
                 r -> add(
-                    pointType.encodeX(r.getMinX()),
-                    pointType.encodeX(r.getMaxX()),
-                    pointType.encodeY(r.getMaxY()),
-                    pointType.encodeY(r.getMinY())
+                    pointType.encoder().encodeX(r.getMinX()),
+                    pointType.encoder().encodeX(r.getMaxX()),
+                    pointType.encoder().encodeY(r.getMaxY()),
+                    pointType.encoder().encodeY(r.getMinY())
                 )
             );
     }
@@ -74,8 +75,9 @@ public Block toBlock(DriverContext driverContext) {
     }
 
     private byte[] toWKB() {
+        CoordinateEncoder encoder = pointType.encoder();
         return WellKnownBinary.toWKB(
-            new Rectangle(pointType.decodeX(minX), pointType.decodeX(maxX), pointType.decodeY(maxY), pointType.decodeY(minY)),
+            new Rectangle(encoder.decodeX(minX), encoder.decodeX(maxX), encoder.decodeY(maxY), encoder.decodeY(minY)),
             ByteOrder.LITTLE_ENDIAN
         );
     }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java
index 99200d2ed99f5..0d6163636fcde 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java
@@ -53,8 +53,8 @@ public void add(Geometry geo) {
                 SpatialAggregationUtils.encodePositiveLongitude(geoPointVisitor.getMinPosX()),
                 SpatialAggregationUtils.encodeNegativeLongitude(geoPointVisitor.getMaxNegX()),
                 SpatialAggregationUtils.encodePositiveLongitude(geoPointVisitor.getMaxPosX()),
-                POINT_TYPE.encodeY(geoPointVisitor.getMaxY()),
-                POINT_TYPE.encodeY(geoPointVisitor.getMinY())
+                POINT_TYPE.encoder().encodeY(geoPointVisitor.getMaxY()),
+                POINT_TYPE.encoder().encodeY(geoPointVisitor.getMinY())
             );
         }
     }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java
index 1e06cf1ea4450..edf54a829deba 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java
@@ -212,10 +212,46 @@ default boolean mvSortedAscending() {
     /**
      * Expand multivalued fields into one row per value. Returns the same block if there aren't any multivalued
      * fields to expand. The returned block needs to be closed by the caller to release the block's resources.
-     * TODO: pass BlockFactory
      */
     Block expand();
 
+    /**
+     * Build a {@link Block} with a {@code null} inserted {@code before} each
+     * listed position.
+     * <p>
+     *     Note: {@code before} must be non-decreasing.
+     * </p>
+     */
+    default Block insertNulls(IntVector before) {
+        // TODO remove default and scatter to implementation where it can be a lot more efficient
+        int myCount = getPositionCount();
+        int beforeCount = before.getPositionCount();
+        try (Builder builder = elementType().newBlockBuilder(myCount + beforeCount, blockFactory())) {
+            int beforeP = 0;
+            int nextNull = before.getInt(beforeP);
+            for (int mainP = 0; mainP < myCount; mainP++) {
+                while (mainP == nextNull) {
+                    builder.appendNull();
+                    beforeP++;
+                    if (beforeP >= beforeCount) {
+                        builder.copyFrom(this, mainP, myCount);
+                        return builder.build();
+                    }
+                    nextNull = before.getInt(beforeP);
+                }
+                // This line right below this is the super inefficient one.
+                builder.copyFrom(this, mainP, mainP + 1);
+            }
+            assert nextNull == myCount;
+            while (beforeP < beforeCount) {
+                nextNull = before.getInt(beforeP++);
+                assert nextNull == myCount;
+                builder.appendNull();
+            }
+            return builder.build();
+        }
+    }
+
     /**
      * Builds {@link Block}s. Typically, you use one of it's direct supinterfaces like {@link IntBlock.Builder}.
      * This is {@link Releasable} and should be released after building the block or if building the block fails.
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st
index e855e6d6296d8..16e2a62b9d030 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st
@@ -149,7 +149,7 @@ $endif$
                 int valueCount = getValueCount(pos);
                 int first = getFirstValueIndex(pos);
                 if (valueCount == 1) {
-                    builder.append$Type$(get$Type$(getFirstValueIndex(pos)$if(BytesRef)$, scratch$endif$));
+                    builder.append$Type$(get$Type$(first$if(BytesRef)$, scratch$endif$));
                 } else {
                     builder.beginPositionEntry();
                     for (int c = 0; c < valueCount; c++) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java
index c071b5055df76..ad7a4f169d92a 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java
@@ -9,10 +9,11 @@
 
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ToXContent;
@@ -167,24 +168,24 @@ public DriverSleeps sleeps() {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).object(ob -> {
-            ob.append((b, p) -> {
-                b.timestampFieldsFromUnixEpochMillis("start_millis", "start", startMillis);
-                b.timestampFieldsFromUnixEpochMillis("stop_millis", "stop", stopMillis);
-                b.field("took_nanos", tookNanos);
-                if (b.humanReadable()) {
-                    b.field("took_time", TimeValue.timeValueNanos(tookNanos));
-                }
-                b.field("cpu_nanos", cpuNanos);
-                if (b.humanReadable()) {
-                    b.field("cpu_time", TimeValue.timeValueNanos(cpuNanos));
-                }
-                b.field("iterations", iterations);
-                return b;
-            });
-            ob.array("operators", operators.iterator());
-            ob.field("sleeps", sleeps);
-        });
+        return Iterators.concat(ChunkedToXContentHelper.startObject(), Iterators.single((b, p) -> {
+            b.timestampFieldsFromUnixEpochMillis("start_millis", "start", startMillis);
+            b.timestampFieldsFromUnixEpochMillis("stop_millis", "stop", stopMillis);
+            b.field("took_nanos", tookNanos);
+            if (b.humanReadable()) {
+                b.field("took_time", TimeValue.timeValueNanos(tookNanos));
+            }
+            b.field("cpu_nanos", cpuNanos);
+            if (b.humanReadable()) {
+                b.field("cpu_time", TimeValue.timeValueNanos(cpuNanos));
+            }
+            b.field("iterations", iterations);
+            return b;
+        }),
+            ChunkedToXContentHelper.array("operators", operators.iterator()),
+            Iterators.single((b, p) -> b.field("sleeps", sleeps)),
+            ChunkedToXContentHelper.endObject()
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/MergePositionsOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/MergePositionsOperator.java
index d42655446ca10..578e1c046954b 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/MergePositionsOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/MergePositionsOperator.java
@@ -20,21 +20,24 @@
 import java.util.Objects;
 
 /**
- * Combines values at the given blocks with the same positions into a single position for the blocks at the given channels
+ * Combines values at the given blocks with the same positions into a single position
+ * for the blocks at the given channels.
+ * <p>
  * Example, input pages consisting of three blocks:
- * positions    | field-1 | field-2 |
- * -----------------------------------
+ * </p>
+ * <pre>{@code
+ * | positions    | field-1 | field-2 |
+ * ------------------------------------
  * Page 1:
- * 1           |  a,b    |   2020  |
- * 1           |  c      |   2021  |
- * ---------------------------------
+ * | 1            |  a,b    |   2020  |
+ * | 1            |  c      |   2021  |
  * Page 2:
- * 2           |  a,e    |   2021  |
- * ---------------------------------
+ * | 2            |  a,e    |   2021  |
  * Page 3:
- * 4           |  d      |   null  |
- * ---------------------------------
+ * | 4            |  d      |   null  |
+ * }</pre>
  * Output:
+ * <pre>{@code
  * |  field-1   | field-2    |
  * ---------------------------
  * |  null      | null       |
@@ -42,6 +45,7 @@
  * |  a,e       | 2021       |
  * |  null      | null       |
  * |  d         | 2023       |
+ * }</pre>
  */
 public final class MergePositionsOperator implements Operator {
     private boolean finished = false;
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/RightChunkedLeftJoin.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/RightChunkedLeftJoin.java
new file mode 100644
index 0000000000000..f9895ff346b5c
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/RightChunkedLeftJoin.java
@@ -0,0 +1,253 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.operator.lookup;
+
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
+
+import java.util.Optional;
+import java.util.stream.IntStream;
+
+/**
+ * Performs a {@code LEFT JOIN} where many "right hand" pages are joined
+ * against a "left hand" {@link Page}. Each row on the "left hand" page
+ * is output at least once whether it appears in the "right hand" or not.
+ * And more than once if it appears in the "right hand" pages more than once.
+ * <p>
+ *     The "right hand" page contains a non-decreasing {@code positions}
+ *     column that controls which position in the "left hand" page the row
+ *     in the "right hand" page. This'll make more sense with a picture:
+ * </p>
+ * <pre>{@code
+ * "left hand"                 "right hand"
+ * | lhdata |             | positions | r1 | r2 |
+ * ----------             -----------------------
+ * |    l00 |             |         0 |  1 |  2 |
+ * |    l01 |             |         1 |  2 |  3 |
+ * |    l02 |             |         1 |  3 |  3 |
+ * |    ... |             |         3 |  9 |  9 |
+ * |    l99 |
+ * }</pre>
+ * <p>
+ *     Joins to:
+ * </p>
+ * <pre>{@code
+ * | lhdata  |  r1  |  r2  |
+ * -------------------------
+ * |     l00 |    1 |    2 |
+ * |     l01 |    2 |    3 |
+ * |     l01 |    3 |    3 |   <1>
+ * |     l02 | null | null |   <2>
+ * |     l03 |    9 |    9 |
+ * }</pre>
+ * <ol>
+ *     <li>{@code l01} is duplicated because it's positions appears twice in
+ *     the right hand page.</li>
+ *     <li>{@code l02}'s row is filled with {@code null}s because it's position
+ *     does not appear in the right hand page.</li>
+ * </ol>
+ * <p>
+ *     This supports joining many "right hand" pages against the same
+ *     "left hand" so long as the first value of the next {@code positions}
+ *     column is the same or greater than the last value of the previous
+ *     {@code positions} column. Large gaps are fine. Starting with the
+ *     same number as you ended on is fine. This looks like:
+ * </p>
+ * <pre>{@code
+ * "left hand"                 "right hand"
+ * | lhdata |             | positions | r1 | r2 |
+ * ----------             -----------------------
+ * |    l00 |                    page 1
+ * |    l01 |             |         0 |  1 |  2 |
+ * |    l02 |             |         1 |  3 |  3 |
+ * |    l03 |                    page 2
+ * |    l04 |             |         1 |  9 |  9 |
+ * |    l05 |             |         2 |  9 |  9 |
+ * |    l06 |                    page 3
+ * |    ... |             |         5 | 10 | 10 |
+ * |    l99 |             |         7 | 11 | 11 |
+ * }</pre>
+ * <p>
+ *     Which makes:
+ * </p>
+ * <pre>{@code
+ * | lhdata  |  r1  |  r2  |
+ * -------------------------
+ *         page 1
+ * |     l00 |    1 |    2 |
+ * |     l01 |    3 |    3 |
+ *         page 2
+ * |     l01 |    9 |    9 |
+ * |     l02 |    9 |    9 |
+ *         page 3
+ * |     l03 | null | null |
+ * |     l04 | null | null |
+ * |     l05 |   10 |   10 |
+ * |     l06 | null | null |
+ * |     l07 |   11 |   11 |
+ * }</pre>
+ * <p>
+ *     Note that the output pages are sized by the "right hand" pages with
+ *     {@code null}s inserted.
+ * </p>
+ * <p>
+ *     Finally, after all "right hand" pages have been joined this will produce
+ *     all remaining "left hand" rows joined against {@code null}.
+ *     Another picture:
+ * </p>
+ * <pre>{@code
+ * "left hand"                 "right hand"
+ * | lhdata |             | positions | r1 | r2 |
+ * ----------             -----------------------
+ * |    l00 |                    last page
+ * |    l01 |             |        96 |  1 |  2 |
+ * |    ... |             |        97 |  1 |  2 |
+ * |    l99 |
+ * }</pre>
+ * <p>
+ *     Which makes:
+ * </p>
+ * <pre>{@code
+ * | lhdata  |  r1  |  r2  |
+ * -------------------------
+ *     last matching page
+ * |     l96 |    1 |    2 |
+ * |     l97 |    2 |    3 |
+ *    trailing nulls page
+ * |     l98 | null | null |
+ * |     l99 | null | null |
+ * }</pre>
+ */
+class RightChunkedLeftJoin implements Releasable {
+    private final Page leftHand;
+    private final int mergedElementCount;
+    /**
+     * The next position that we'll emit <strong>or</strong> one more than the
+     * next position we'll emit. This is used to cover gaps between "right hand"
+     * pages and to detect if "right hand" pages "go backwards".
+     */
+    private int next = 0;
+
+    RightChunkedLeftJoin(Page leftHand, int mergedElementCounts) {
+        this.leftHand = leftHand;
+        this.mergedElementCount = mergedElementCounts;
+    }
+
+    Page join(Page rightHand) {
+        IntVector positions = rightHand.<IntBlock>getBlock(0).asVector();
+        if (positions.getInt(0) < next - 1) {
+            throw new IllegalArgumentException("maximum overlap is one position");
+        }
+        Block[] blocks = new Block[leftHand.getBlockCount() + mergedElementCount];
+        if (rightHand.getBlockCount() != mergedElementCount + 1) {
+            throw new IllegalArgumentException(
+                "expected right hand side with [" + (mergedElementCount + 1) + "] but got [" + rightHand.getBlockCount() + "]"
+            );
+        }
+        IntVector.Builder leftFilterBuilder = null;
+        IntVector leftFilter = null;
+        IntVector.Builder insertNullsBuilder = null;
+        IntVector insertNulls = null;
+        try {
+            leftFilterBuilder = positions.blockFactory().newIntVectorBuilder(positions.getPositionCount());
+            for (int p = 0; p < positions.getPositionCount(); p++) {
+                int pos = positions.getInt(p);
+                if (pos > next) {
+                    if (insertNullsBuilder == null) {
+                        insertNullsBuilder = positions.blockFactory().newIntVectorBuilder(pos - next);
+                    }
+                    for (int missing = next; missing < pos; missing++) {
+                        leftFilterBuilder.appendInt(missing);
+                        insertNullsBuilder.appendInt(p);
+                    }
+                }
+                leftFilterBuilder.appendInt(pos);
+                next = pos + 1;
+            }
+            leftFilter = leftFilterBuilder.build();
+            int[] leftFilterArray = toArray(leftFilter);
+            insertNulls = insertNullsBuilder == null ? null : insertNullsBuilder.build();
+
+            int b = 0;
+            while (b < leftHand.getBlockCount()) {
+                blocks[b] = leftHand.getBlock(b).filter(leftFilterArray);
+                b++;
+            }
+            int rb = 1; // Skip the positions column
+            while (b < blocks.length) {
+                Block block = rightHand.getBlock(rb);
+                if (insertNulls == null) {
+                    block.mustIncRef();
+                } else {
+                    block = block.insertNulls(insertNulls);
+                }
+                blocks[b] = block;
+                b++;
+                rb++;
+            }
+            Page result = new Page(blocks);
+            blocks = null;
+            return result;
+        } finally {
+            Releasables.close(
+                blocks == null ? null : Releasables.wrap(blocks),
+                leftFilter,
+                leftFilterBuilder,
+                insertNullsBuilder,
+                insertNulls
+            );
+        }
+    }
+
+    Optional<Page> noMoreRightHandPages() {
+        if (next == leftHand.getPositionCount()) {
+            return Optional.empty();
+        }
+        BlockFactory factory = leftHand.getBlock(0).blockFactory();
+        Block[] blocks = new Block[leftHand.getBlockCount() + mergedElementCount];
+        // TODO make a filter that takes a min and max?
+        int[] filter = IntStream.range(next, leftHand.getPositionCount()).toArray();
+        try {
+            int b = 0;
+            while (b < leftHand.getBlockCount()) {
+                blocks[b] = leftHand.getBlock(b).filter(filter);
+                b++;
+            }
+            while (b < blocks.length) {
+                blocks[b] = factory.newConstantNullBlock(leftHand.getPositionCount() - next);
+                b++;
+            }
+            Page result = new Page(blocks);
+            blocks = null;
+            return Optional.of(result);
+        } finally {
+            if (blocks != null) {
+                Releasables.close(blocks);
+            }
+        }
+    }
+
+    @Override
+    public void close() {
+        Releasables.close(leftHand::releaseBlocks);
+    }
+
+    private int[] toArray(IntVector vector) {
+        // TODO replace parameter to filter with vector and remove this
+        int[] array = new int[vector.getPositionCount()];
+        for (int p = 0; p < vector.getPositionCount(); p++) {
+            array[p] = vector.getInt(p);
+        }
+        return array;
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregatorFunctionTests.java
new file mode 100644
index 0000000000000..c0a91fe22b87b
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregatorFunctionTests.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BlockUtils;
+import org.elasticsearch.compute.operator.SequenceBytesRefBlockSourceOperator;
+import org.elasticsearch.compute.operator.SourceOperator;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+
+public class ValuesBytesRefAggregatorFunctionTests extends AggregatorFunctionTestCase {
+    @Override
+    protected SourceOperator simpleInput(BlockFactory blockFactory, int size) {
+        return new SequenceBytesRefBlockSourceOperator(
+            blockFactory,
+            IntStream.range(0, size).mapToObj(l -> new BytesRef(randomAlphaOfLengthBetween(0, 100)))
+        );
+    }
+
+    @Override
+    protected AggregatorFunctionSupplier aggregatorFunction(List<Integer> inputChannels) {
+        return new ValuesBytesRefAggregatorFunctionSupplier(inputChannels);
+    }
+
+    @Override
+    protected String expectedDescriptionOfAggregator() {
+        return "values of bytes";
+    }
+
+    @Override
+    public void assertSimpleOutput(List<Block> input, Block result) {
+        TreeSet<?> set = new TreeSet<>((List<?>) BlockUtils.toJavaObject(result, 0));
+        Object[] values = input.stream()
+            .flatMap(AggregatorFunctionTestCase::allBytesRefs)
+            .collect(Collectors.toSet())
+            .toArray(Object[]::new);
+        if (false == set.containsAll(Arrays.asList(values))) {
+            assertThat(set, containsInAnyOrder(values));
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesBytesRefGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesBytesRefGroupingAggregatorFunctionTests.java
new file mode 100644
index 0000000000000..fc9bc90828df3
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesBytesRefGroupingAggregatorFunctionTests.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BlockUtils;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.LongBytesRefTupleBlockSourceOperator;
+import org.elasticsearch.compute.operator.SourceOperator;
+import org.elasticsearch.core.Tuple;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public class ValuesBytesRefGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase {
+    @Override
+    protected AggregatorFunctionSupplier aggregatorFunction(List<Integer> inputChannels) {
+        return new ValuesBytesRefAggregatorFunctionSupplier(inputChannels);
+    }
+
+    @Override
+    protected String expectedDescriptionOfAggregator() {
+        return "values of bytes";
+    }
+
+    @Override
+    protected SourceOperator simpleInput(BlockFactory blockFactory, int size) {
+        return new LongBytesRefTupleBlockSourceOperator(
+            blockFactory,
+            IntStream.range(0, size).mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), new BytesRef(randomAlphaOfLengthBetween(0, 100))))
+        );
+    }
+
+    @Override
+    public void assertSimpleGroup(List<Page> input, Block result, int position, Long group) {
+        Object[] values = input.stream().flatMap(p -> allBytesRefs(p, group)).collect(Collectors.toSet()).toArray(Object[]::new);
+        Object resultValue = BlockUtils.toJavaObject(result, position);
+        switch (values.length) {
+            case 0 -> assertThat(resultValue, nullValue());
+            case 1 -> assertThat(resultValue, equalTo(values[0]));
+            default -> {
+                TreeSet<?> set = new TreeSet<>((List<?>) resultValue);
+                if (false == set.containsAll(Arrays.asList(values))) {
+                    assertThat(set, containsInAnyOrder(values));
+                }
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java
index 439ebe34c7d4a..33a294131c996 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java
@@ -224,6 +224,7 @@ public void testIntBlock() {
             try (IntBlock.Builder blockBuilder = blockFactory.newIntBlockBuilder(1)) {
                 IntBlock copy = blockBuilder.copyFrom(block, 0, block.getPositionCount()).build();
                 assertThat(copy, equalTo(block));
+                assertInsertNulls(block);
                 releaseAndAssertBreaker(block, copy);
             }
 
@@ -250,6 +251,7 @@ public void testIntBlock() {
                 assertSingleValueDenseBlock(vector.asBlock());
                 assertThat(vector.min(), equalTo(0));
                 assertThat(vector.max(), equalTo(positionCount - 1));
+                assertInsertNulls(vector.asBlock());
                 releaseAndAssertBreaker(vector.asBlock());
             }
         }
@@ -272,12 +274,14 @@ public void testIntBlockEmpty() {
             assertEmptyLookup(blockFactory, block);
             assertThat(block.asVector().min(), equalTo(Integer.MAX_VALUE));
             assertThat(block.asVector().max(), equalTo(Integer.MIN_VALUE));
+            assertInsertNulls(block);
             releaseAndAssertBreaker(block);
 
             try (IntVector.Builder vectorBuilder = blockFactory.newIntVectorBuilder(0)) {
                 IntVector vector = vectorBuilder.build();
                 assertThat(vector.min(), equalTo(Integer.MAX_VALUE));
                 assertThat(vector.max(), equalTo(Integer.MIN_VALUE));
+                assertInsertNulls(vector.asBlock());
                 releaseAndAssertBreaker(vector.asBlock());
             }
         }
@@ -317,6 +321,7 @@ public void testConstantIntBlock() {
             assertEmptyLookup(blockFactory, block);
             assertThat(block.asVector().min(), equalTo(value));
             assertThat(block.asVector().max(), equalTo(value));
+            assertInsertNulls(block);
             releaseAndAssertBreaker(block);
         }
     }
@@ -350,6 +355,7 @@ public void testLongBlock() {
             try (LongBlock.Builder blockBuilder = blockFactory.newLongBlockBuilder(1)) {
                 LongBlock copy = blockBuilder.copyFrom(block, 0, block.getPositionCount()).build();
                 assertThat(copy, equalTo(block));
+                assertInsertNulls(block);
                 releaseAndAssertBreaker(block, copy);
             }
 
@@ -372,6 +378,7 @@ public void testLongBlock() {
             LongStream.range(0, positionCount).forEach(vectorBuilder::appendLong);
             LongVector vector = vectorBuilder.build();
             assertSingleValueDenseBlock(vector.asBlock());
+            assertInsertNulls(vector.asBlock());
             releaseAndAssertBreaker(vector.asBlock());
         }
     }
@@ -408,6 +415,7 @@ public void testConstantLongBlock() {
                 b -> assertThat(b, instanceOf(ConstantNullBlock.class))
             );
             assertEmptyLookup(blockFactory, block);
+            assertInsertNulls(block);
             releaseAndAssertBreaker(block);
         }
     }
@@ -442,6 +450,7 @@ public void testDoubleBlock() {
             try (DoubleBlock.Builder blockBuilder = blockFactory.newDoubleBlockBuilder(1)) {
                 DoubleBlock copy = blockBuilder.copyFrom(block, 0, block.getPositionCount()).build();
                 assertThat(copy, equalTo(block));
+                assertInsertNulls(block);
                 releaseAndAssertBreaker(block, copy);
             }
 
@@ -466,6 +475,7 @@ public void testDoubleBlock() {
                 IntStream.range(0, positionCount).mapToDouble(ii -> 1.0 / ii).forEach(vectorBuilder::appendDouble);
                 DoubleVector vector = vectorBuilder.build();
                 assertSingleValueDenseBlock(vector.asBlock());
+                assertInsertNulls(vector.asBlock());
                 releaseAndAssertBreaker(vector.asBlock());
             }
         }
@@ -501,6 +511,7 @@ public void testConstantDoubleBlock() {
                 b -> assertThat(b, instanceOf(ConstantNullBlock.class))
             );
             assertEmptyLookup(blockFactory, block);
+            assertInsertNulls(block);
             releaseAndAssertBreaker(block);
         }
     }
@@ -536,6 +547,7 @@ public void testFloatBlock() {
             try (FloatBlock.Builder blockBuilder = blockFactory.newFloatBlockBuilder(1)) {
                 FloatBlock copy = blockBuilder.copyFrom(block, 0, block.getPositionCount()).build();
                 assertThat(copy, equalTo(block));
+                assertInsertNulls(block);
                 releaseAndAssertBreaker(block, copy);
             }
 
@@ -560,6 +572,7 @@ public void testFloatBlock() {
                 IntStream.range(0, positionCount).mapToDouble(ii -> 1.0 / ii).forEach(vectorBuilder::appendDouble);
                 DoubleVector vector = vectorBuilder.build();
                 assertSingleValueDenseBlock(vector.asBlock());
+                assertInsertNulls(vector.asBlock());
                 releaseAndAssertBreaker(vector.asBlock());
             }
         }
@@ -595,6 +608,7 @@ public void testConstantFloatBlock() {
                 b -> assertThat(b, instanceOf(ConstantNullBlock.class))
             );
             assertEmptyLookup(blockFactory, block);
+            assertInsertNulls(block);
             releaseAndAssertBreaker(block);
         }
     }
@@ -646,6 +660,7 @@ private void testBytesRefBlock(Supplier<BytesRef> byteArraySupplier, boolean cho
         try (BytesRefBlock.Builder blockBuilder = blockFactory.newBytesRefBlockBuilder(1)) {
             BytesRefBlock copy = blockBuilder.copyFrom(block, 0, block.getPositionCount()).build();
             assertThat(copy, equalTo(block));
+            assertInsertNulls(block);
             releaseAndAssertBreaker(block, copy);
         }
 
@@ -671,6 +686,7 @@ private void testBytesRefBlock(Supplier<BytesRef> byteArraySupplier, boolean cho
             IntStream.range(0, positionCount).mapToObj(ii -> new BytesRef(randomAlphaOfLength(5))).forEach(vectorBuilder::appendBytesRef);
             BytesRefVector vector = vectorBuilder.build();
             assertSingleValueDenseBlock(vector.asBlock());
+            assertInsertNulls(vector.asBlock());
             releaseAndAssertBreaker(vector.asBlock());
         }
     }
@@ -726,6 +742,7 @@ public void testBytesRefBlockBuilderWithNulls() {
                 }
             }
             assertKeepMask(block);
+            assertInsertNulls(block);
             releaseAndAssertBreaker(block);
         }
     }
@@ -765,6 +782,7 @@ public void testConstantBytesRefBlock() {
                 b -> assertThat(b, instanceOf(ConstantNullBlock.class))
             );
             assertEmptyLookup(blockFactory, block);
+            assertInsertNulls(block);
             releaseAndAssertBreaker(block);
         }
     }
@@ -810,6 +828,7 @@ public void testBooleanBlock() {
             try (BooleanBlock.Builder blockBuilder = blockFactory.newBooleanBlockBuilder(1)) {
                 BooleanBlock copy = blockBuilder.copyFrom(block, 0, block.getPositionCount()).build();
                 assertThat(copy, equalTo(block));
+                assertInsertNulls(block);
                 releaseAndAssertBreaker(block, copy);
             }
 
@@ -852,6 +871,7 @@ public void testBooleanBlock() {
                     assertTrue(vector.allFalse());
                 }
             }
+            assertInsertNulls(vector.asBlock());
             releaseAndAssertBreaker(vector.asBlock());
         }
     }
@@ -893,6 +913,7 @@ public void testConstantBooleanBlock() {
                 assertFalse(block.asVector().allTrue());
                 assertTrue(block.asVector().allFalse());
             }
+            assertInsertNulls(block);
             releaseAndAssertBreaker(block);
         }
     }
@@ -935,6 +956,7 @@ public void testConstantNullBlock() {
                 singletonList(null),
                 b -> assertThat(b, instanceOf(ConstantNullBlock.class))
             );
+            assertInsertNulls(block);
             releaseAndAssertBreaker(block);
         }
     }
@@ -1390,6 +1412,7 @@ <B extends Block, BB extends Block.Builder, T> void assertNullValues(
         asserter.accept(randomNonNullPosition, block);
         assertTrue(block.isNull(randomNullPosition));
         assertFalse(block.isNull(randomNonNullPosition));
+        assertInsertNulls(block);
         releaseAndAssertBreaker(block);
         if (block instanceof BooleanBlock bb) {
             try (ToMask mask = bb.toMask()) {
@@ -1409,6 +1432,7 @@ void assertZeroPositionsAndRelease(BooleanBlock block) {
     void assertZeroPositionsAndRelease(Block block) {
         assertThat(block.getPositionCount(), is(0));
         assertKeepMaskEmpty(block);
+        assertInsertNulls(block);
         releaseAndAssertBreaker(block);
     }
 
@@ -1451,6 +1475,36 @@ static void assertToMask(BooleanVector vector) {
         }
     }
 
+    static void assertInsertNulls(Block block) {
+        int maxNulls = Math.min(1000, block.getPositionCount() * 5);
+        List<Object> orig = new ArrayList<>(block.getPositionCount());
+        BlockTestUtils.readInto(orig, block);
+
+        int nullCount = 0;
+        try (IntVector.Builder beforeBuilder = block.blockFactory().newIntVectorBuilder(block.getPositionCount())) {
+            List<Object> expected = new ArrayList<>(block.getPositionCount());
+            for (int p = 0; p < block.getPositionCount(); p++) {
+                while (nullCount < maxNulls && randomBoolean()) {
+                    expected.add(null);
+                    beforeBuilder.appendInt(p);
+                    nullCount++;
+                }
+                expected.add(orig.get(p));
+            }
+            while (nullCount == 0 || (nullCount < maxNulls && randomBoolean())) {
+                expected.add(null);
+                beforeBuilder.appendInt(block.getPositionCount());
+                nullCount++;
+            }
+
+            try (IntVector before = beforeBuilder.build(); Block withNulls = block.insertNulls(before)) {
+                List<Object> actual = new ArrayList<>(block.getPositionCount());
+                BlockTestUtils.readInto(actual, withNulls);
+                assertThat(actual, equalTo(expected));
+            }
+        }
+    }
+
     void releaseAndAssertBreaker(Block... blocks) {
         assertThat(breaker.getUsed(), greaterThan(0L));
         Page[] pages = Arrays.stream(blocks).map(Page::new).toArray(Page[]::new);
@@ -1909,4 +1963,14 @@ static BooleanVector randomMask(int positions) {
             return builder.build();
         }
     }
+
+    /**
+     * A random {@link ElementType} for which we can build a {@link RandomBlock}.
+     */
+    public static ElementType randomElementType() {
+        return randomValueOtherThanMany(
+            e -> e == ElementType.UNKNOWN || e == ElementType.NULL || e == ElementType.DOC || e == ElementType.COMPOSITE,
+            () -> randomFrom(ElementType.values())
+        );
+    }
 }
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockMultiValuedTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockMultiValuedTests.java
index e37b2638b56f7..da7b8cd87db7d 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockMultiValuedTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockMultiValuedTests.java
@@ -29,6 +29,7 @@
 import java.util.function.IntUnaryOperator;
 import java.util.stream.IntStream;
 
+import static org.elasticsearch.compute.data.BasicBlockTests.assertInsertNulls;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.nullValue;
@@ -68,6 +69,7 @@ public void testMultiValued() {
 
             assertThat(b.block().mayHaveMultivaluedFields(), equalTo(b.values().stream().anyMatch(l -> l != null && l.size() > 1)));
             assertThat(b.block().doesHaveMultivaluedFields(), equalTo(b.values().stream().anyMatch(l -> l != null && l.size() > 1)));
+            assertInsertNulls(b.block());
         } finally {
             b.block().close();
         }
@@ -171,6 +173,16 @@ public void testMask() {
         }
     }
 
+    public void testInsertNull() {
+        int positionCount = randomIntBetween(1, 16 * 1024);
+        var b = BasicBlockTests.randomBlock(blockFactory(), elementType, positionCount, nullAllowed, 2, 10, 0, 0);
+        try {
+            assertInsertNulls(b.block());
+        } finally {
+            b.block().close();
+        }
+    }
+
     private void assertFiltered(boolean all, boolean shuffled) {
         int positionCount = randomIntBetween(1, 16 * 1024);
         var b = BasicBlockTests.randomBlock(blockFactory(), elementType, positionCount, nullAllowed, 0, 10, 0, 0);
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ComputeTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ComputeTestCase.java
index ce62fb9896eba..cf99c59bb4c71 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ComputeTestCase.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ComputeTestCase.java
@@ -22,6 +22,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.Consumer;
 
 import static org.hamcrest.Matchers.equalTo;
 
@@ -76,6 +77,16 @@ protected final BlockFactory crankyBlockFactory() {
         return blockFactory;
     }
 
+    protected final void testWithCrankyBlockFactory(Consumer<BlockFactory> run) {
+        try {
+            run.accept(crankyBlockFactory());
+            logger.info("cranky let us finish!");
+        } catch (CircuitBreakingException e) {
+            logger.info("cranky", e);
+            assertThat(e.getMessage(), equalTo(CrankyCircuitBreakerService.ERROR_MESSAGE));
+        }
+    }
+
     @After
     public final void allBreakersEmpty() throws Exception {
         // first check that all big arrays are released, which can affect breakers
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LimitOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LimitOperatorTests.java
index 8200529e18290..bbe4a07cc44bd 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LimitOperatorTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LimitOperatorTests.java
@@ -10,14 +10,13 @@
 import org.elasticsearch.compute.data.BasicBlockTests;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
-import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.compute.data.Page;
 import org.hamcrest.Matcher;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.LongStream;
 
+import static org.elasticsearch.compute.data.BasicBlockTests.randomElementType;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.sameInstance;
 
@@ -129,17 +128,6 @@ Block randomBlock(BlockFactory blockFactory, int size) {
         if (randomBoolean()) {
             return blockFactory.newConstantNullBlock(size);
         }
-        return BasicBlockTests.randomBlock(blockFactory, randomElement(), size, false, 1, 1, 0, 0).block();
-    }
-
-    static ElementType randomElement() {
-        List<ElementType> l = new ArrayList<>();
-        for (ElementType e : ElementType.values()) {
-            if (e == ElementType.UNKNOWN || e == ElementType.NULL || e == ElementType.DOC || e == ElementType.COMPOSITE) {
-                continue;
-            }
-            l.add(e);
-        }
-        return randomFrom(l);
+        return BasicBlockTests.randomBlock(blockFactory, randomElementType(), size, false, 1, 1, 0, 0).block();
     }
 }
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java
index be792a0ef2612..54db0453530bc 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java
@@ -143,6 +143,7 @@ public final void testSimpleWithCranky() {
 
         DriverContext driverContext = crankyDriverContext();
 
+        Exception exception = null;
         boolean driverStarted = false;
         try {
             Operator operator = simple().get(driverContext);
@@ -150,8 +151,8 @@ public final void testSimpleWithCranky() {
             drive(operator, input.iterator(), driverContext);
             // Either we get lucky and cranky doesn't throw and the test completes or we don't and it throws
         } catch (CircuitBreakingException e) {
-            logger.info("broken", e);
             assertThat(e.getMessage(), equalTo(CrankyCircuitBreakerService.ERROR_MESSAGE));
+            exception = e;
         }
         if (driverStarted == false) {
             // if drive hasn't even started then we need to release the input pages
@@ -159,7 +160,14 @@ public final void testSimpleWithCranky() {
         }
 
         // Note the lack of try/finally here - we're asserting that when the driver throws an exception we clear the breakers.
-        assertThat(inputFactoryContext.breaker().getUsed(), equalTo(0L));
+        long inputUsedBytes = inputFactoryContext.breaker().getUsed();
+        if (inputUsedBytes != 0L) {
+            fail(exception, "Expected no used bytes for input, found: " + inputUsedBytes);
+        }
+        long driverUsedBytes = driverContext.breaker().getUsed();
+        if (driverUsedBytes != 0L) {
+            fail(exception, "Expected no used bytes for driver, found: " + driverUsedBytes);
+        }
     }
 
     /**
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/lookup/RightChunkedLeftJoinTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/lookup/RightChunkedLeftJoinTests.java
new file mode 100644
index 0000000000000..1312b772dbfa1
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/lookup/RightChunkedLeftJoinTests.java
@@ -0,0 +1,434 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.operator.lookup;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.BasicBlockTests;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BlockTestUtils;
+import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.data.ElementType;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.ComputeTestCase;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.test.ListMatcher;
+
+import java.text.NumberFormat;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.IntStream;
+
+import static org.elasticsearch.test.ListMatcher.matchesList;
+import static org.elasticsearch.test.MapMatcher.assertMap;
+import static org.hamcrest.Matchers.equalTo;
+
+public class RightChunkedLeftJoinTests extends ComputeTestCase {
+    public void testNoGaps() {
+        testNoGaps(blockFactory());
+    }
+
+    public void testNoGapsCranky() {
+        testWithCrankyBlockFactory(this::testNoGaps);
+    }
+
+    private void testNoGaps(BlockFactory factory) {
+        int size = 100;
+        try (RightChunkedLeftJoin join = new RightChunkedLeftJoin(buildExampleLeftHand(factory, size), 2)) {
+            assertJoined(
+                factory,
+                join,
+                new int[][] {
+                    { 0, 1, 2 }, // formatter
+                    { 1, 2, 3 }, // formatter
+                    { 2, 3, 3 }, // formatter
+                    { 3, 9, 9 }, // formatter
+                },
+                new Object[][] {
+                    { "l00", 1, 2 }, // formatter
+                    { "l01", 2, 3 }, // formatter
+                    { "l02", 3, 3 }, // formatter
+                    { "l03", 9, 9 }, // formatter
+                }
+            );
+            assertTrailing(join, size, 4);
+        }
+    }
+
+    /**
+     * Test the first example in the main javadoc of {@link RightChunkedLeftJoin}.
+     */
+    public void testFirstExample() {
+        testFirstExample(blockFactory());
+    }
+
+    public void testFirstExampleCranky() {
+        testWithCrankyBlockFactory(this::testFirstExample);
+    }
+
+    private void testFirstExample(BlockFactory factory) {
+        try (RightChunkedLeftJoin join = new RightChunkedLeftJoin(buildExampleLeftHand(factory, 100), 2)) {
+            assertJoined(
+                factory,
+                join,
+                new int[][] {
+                    { 0, 1, 2 },  // formatter
+                    { 1, 2, 3 },  // formatter
+                    { 1, 3, 3 },  // formatter
+                    { 3, 9, 9 },  // formatter
+                },
+                new Object[][] {
+                    { "l00", 1, 2 },  // formatter
+                    { "l01", 2, 3 },  // formatter
+                    { "l01", 3, 3 },  // formatter
+                    { "l02", null, null },  // formatter
+                    { "l03", 9, 9 },  // formatter
+                }
+            );
+        }
+    }
+
+    public void testLeadingNulls() {
+        testLeadingNulls(blockFactory());
+    }
+
+    public void testLeadingNullsCranky() {
+        testWithCrankyBlockFactory(this::testLeadingNulls);
+    }
+
+    private void testLeadingNulls(BlockFactory factory) {
+        int size = 3;
+        try (RightChunkedLeftJoin join = new RightChunkedLeftJoin(buildExampleLeftHand(factory, size), 2)) {
+            assertJoined(
+                factory,
+                join,
+                new int[][] { { 2, 1, 2 } },
+                new Object[][] {
+                    { "l0", null, null }, // formatter
+                    { "l1", null, null }, // formatter
+                    { "l2", 1, 2 }, // formatter
+                }
+            );
+            assertTrailing(join, size, 3);
+        }
+    }
+
+    public void testSecondExample() {
+        testSecondExample(blockFactory());
+    }
+
+    public void testSecondExampleCranky() {
+        testWithCrankyBlockFactory(this::testSecondExample);
+    }
+
+    /**
+     * Test the second example in the main javadoc of {@link RightChunkedLeftJoin}.
+     */
+    private void testSecondExample(BlockFactory factory) {
+        int size = 100;
+        try (RightChunkedLeftJoin join = new RightChunkedLeftJoin(buildExampleLeftHand(factory, size), 2)) {
+            assertJoined(
+                factory,
+                join,
+                new int[][] {
+                    { 0, 1, 2 },  // formatter
+                    { 1, 3, 3 },  // formatter
+                },
+                new Object[][] {
+                    { "l00", 1, 2 },  // formatter
+                    { "l01", 3, 3 },  // formatter
+                }
+            );
+            assertJoined(
+                factory,
+                join,
+                new int[][] {
+                    { 1, 9, 9 },  // formatter
+                    { 2, 9, 9 },  // formatter
+                },
+                new Object[][] {
+                    { "l01", 9, 9 },  // formatter
+                    { "l02", 9, 9 },  // formatter
+                }
+            );
+            assertJoined(
+                factory,
+                join,
+                new int[][] {
+                    { 5, 10, 10 },  // formatter
+                    { 7, 11, 11 },  // formatter
+                },
+                new Object[][] {
+                    { "l03", null, null },  // formatter
+                    { "l04", null, null },  // formatter
+                    { "l05", 10, 10 },  // formatter
+                    { "l06", null, null },  // formatter
+                    { "l07", 11, 11 },  // formatter
+                }
+            );
+            assertTrailing(join, size, 8);
+        }
+    }
+
+    public void testThirdExample() {
+        testThirdExample(blockFactory());
+    }
+
+    public void testThirdExampleCranky() {
+        testWithCrankyBlockFactory(this::testThirdExample);
+    }
+
+    /**
+     * Test the third example in the main javadoc of {@link RightChunkedLeftJoin}.
+     */
+    private void testThirdExample(BlockFactory factory) {
+        int size = 100;
+        try (RightChunkedLeftJoin join = new RightChunkedLeftJoin(buildExampleLeftHand(factory, size), 2)) {
+            Page pre = buildPage(factory, IntStream.range(0, 96).mapToObj(p -> new int[] { p, p, p }).toArray(int[][]::new));
+            try {
+                join.join(pre).releaseBlocks();
+            } finally {
+                pre.releaseBlocks();
+            }
+            assertJoined(
+                factory,
+                join,
+                new int[][] {
+                    { 96, 1, 2 },  // formatter
+                    { 97, 3, 3 },  // formatter
+                },
+                new Object[][] {
+                    { "l96", 1, 2 },  // formatter
+                    { "l97", 3, 3 },  // formatter
+                }
+            );
+            assertTrailing(join, size, 98);
+        }
+    }
+
+    public void testRandom() {
+        testRandom(blockFactory());
+    }
+
+    public void testRandomCranky() {
+        testWithCrankyBlockFactory(this::testRandom);
+    }
+
+    private void testRandom(BlockFactory factory) {
+        int leftSize = between(100, 10000);
+        ElementType[] leftColumns = randomArray(1, 10, ElementType[]::new, BasicBlockTests::randomElementType);
+        ElementType[] rightColumns = randomArray(1, 10, ElementType[]::new, BasicBlockTests::randomElementType);
+
+        RandomPage left = randomPage(factory, leftColumns, leftSize);
+        try (RightChunkedLeftJoin join = new RightChunkedLeftJoin(left.page, rightColumns.length)) {
+            int rightSize = 5;
+            IntVector selected = randomPositions(factory, leftSize, rightSize);
+            RandomPage right = randomPage(factory, rightColumns, rightSize, selected.asBlock());
+
+            try {
+                Page joined = join.join(right.page);
+                try {
+                    assertThat(joined.getPositionCount(), equalTo(selected.max() + 1));
+
+                    List<List<Object>> actualColumns = new ArrayList<>();
+                    BlockTestUtils.readInto(actualColumns, joined);
+                    int rightRow = 0;
+                    for (int leftRow = 0; leftRow < joined.getPositionCount(); leftRow++) {
+                        List<Object> actualRow = new ArrayList<>();
+                        for (List<Object> actualColumn : actualColumns) {
+                            actualRow.add(actualColumn.get(leftRow));
+                        }
+                        ListMatcher matcher = ListMatcher.matchesList();
+                        for (int c = 0; c < leftColumns.length; c++) {
+                            matcher = matcher.item(unwrapSingletonLists(left.blocks[c].values().get(leftRow)));
+                        }
+                        if (selected.getInt(rightRow) == leftRow) {
+                            for (int c = 0; c < rightColumns.length; c++) {
+                                matcher = matcher.item(unwrapSingletonLists(right.blocks[c].values().get(rightRow)));
+                            }
+                            rightRow++;
+                        } else {
+                            for (int c = 0; c < rightColumns.length; c++) {
+                                matcher = matcher.item(null);
+                            }
+                        }
+                        assertMap(actualRow, matcher);
+                    }
+                } finally {
+                    joined.releaseBlocks();
+                }
+
+                int start = selected.max() + 1;
+                if (start >= left.page.getPositionCount()) {
+                    assertThat(join.noMoreRightHandPages().isPresent(), equalTo(false));
+                    return;
+                }
+                Page remaining = join.noMoreRightHandPages().get();
+                try {
+                    assertThat(remaining.getPositionCount(), equalTo(left.page.getPositionCount() - start));
+                    List<List<Object>> actualColumns = new ArrayList<>();
+                    BlockTestUtils.readInto(actualColumns, remaining);
+                    for (int leftRow = start; leftRow < left.page.getPositionCount(); leftRow++) {
+                        List<Object> actualRow = new ArrayList<>();
+                        for (List<Object> actualColumn : actualColumns) {
+                            actualRow.add(actualColumn.get(leftRow - start));
+                        }
+                        ListMatcher matcher = ListMatcher.matchesList();
+                        for (int c = 0; c < leftColumns.length; c++) {
+                            matcher = matcher.item(unwrapSingletonLists(left.blocks[c].values().get(leftRow)));
+                        }
+                        for (int c = 0; c < rightColumns.length; c++) {
+                            matcher = matcher.item(null);
+                        }
+                        assertMap(actualRow, matcher);
+                    }
+
+                } finally {
+                    remaining.releaseBlocks();
+                }
+            } finally {
+                right.page.releaseBlocks();
+            }
+        } finally {
+            left.page.releaseBlocks();
+        }
+    }
+
+    NumberFormat exampleNumberFormat(int size) {
+        NumberFormat nf = NumberFormat.getIntegerInstance(Locale.ROOT);
+        nf.setMinimumIntegerDigits((int) Math.ceil(Math.log10(size)));
+        return nf;
+    }
+
+    Page buildExampleLeftHand(BlockFactory factory, int size) {
+        NumberFormat nf = exampleNumberFormat(size);
+        try (BytesRefVector.Builder builder = factory.newBytesRefVectorBuilder(size)) {
+            for (int i = 0; i < size; i++) {
+                builder.appendBytesRef(new BytesRef("l" + nf.format(i)));
+            }
+            return new Page(builder.build().asBlock());
+        }
+    }
+
+    Page buildPage(BlockFactory factory, int[][] rows) {
+        try (
+            IntVector.Builder positions = factory.newIntVectorFixedBuilder(rows.length);
+            IntVector.Builder r1 = factory.newIntVectorFixedBuilder(rows.length);
+            IntVector.Builder r2 = factory.newIntVectorFixedBuilder(rows.length);
+        ) {
+            for (int[] row : rows) {
+                positions.appendInt(row[0]);
+                r1.appendInt(row[1]);
+                r2.appendInt(row[2]);
+            }
+            return new Page(positions.build().asBlock(), r1.build().asBlock(), r2.build().asBlock());
+        }
+    }
+
+    private void assertJoined(Page joined, Object[][] expected) {
+        try {
+            List<List<Object>> actualColumns = new ArrayList<>();
+            BlockTestUtils.readInto(actualColumns, joined);
+
+            for (int r = 0; r < expected.length; r++) {
+                List<Object> actualRow = new ArrayList<>();
+                for (int c = 0; c < actualColumns.size(); c++) {
+                    Object v = actualColumns.get(c).get(r);
+                    if (v instanceof BytesRef b) {
+                        v = b.utf8ToString();
+                    }
+                    actualRow.add(v);
+                }
+
+                ListMatcher rowMatcher = matchesList();
+                for (Object v : expected[r]) {
+                    rowMatcher = rowMatcher.item(v);
+                }
+                assertMap("row " + r, actualRow, rowMatcher);
+            }
+        } finally {
+            joined.releaseBlocks();
+        }
+    }
+
+    private void assertJoined(BlockFactory factory, RightChunkedLeftJoin join, int[][] rightRows, Object[][] expectRows) {
+        Page rightHand = buildPage(factory, rightRows);
+        try {
+            assertJoined(join.join(rightHand), expectRows);
+        } finally {
+            rightHand.releaseBlocks();
+        }
+    }
+
+    private void assertTrailing(RightChunkedLeftJoin join, int size, int next) {
+        NumberFormat nf = exampleNumberFormat(size);
+        if (size == next) {
+            assertThat(join.noMoreRightHandPages(), equalTo(Optional.empty()));
+        } else {
+            assertJoined(
+                join.noMoreRightHandPages().get(),
+                IntStream.range(next, size).mapToObj(p -> new Object[] { "l" + nf.format(p), null, null }).toArray(Object[][]::new)
+            );
+        }
+    }
+
+    Object unwrapSingletonLists(Object o) {
+        if (o instanceof List<?> l && l.size() == 1) {
+            return l.getFirst();
+        }
+        return o;
+    }
+
+    record RandomPage(Page page, BasicBlockTests.RandomBlock[] blocks) {};
+
+    RandomPage randomPage(BlockFactory factory, ElementType[] types, int positions, Block... prepend) {
+        BasicBlockTests.RandomBlock[] randomBlocks = new BasicBlockTests.RandomBlock[types.length];
+        Block[] blocks = new Block[prepend.length + types.length];
+        try {
+            for (int c = 0; c < prepend.length; c++) {
+                blocks[c] = prepend[c];
+            }
+            for (int c = 0; c < types.length; c++) {
+
+                int min = between(0, 3);
+                randomBlocks[c] = BasicBlockTests.randomBlock(
+                    factory,
+                    types[c],
+                    positions,
+                    randomBoolean(),
+                    min,
+                    between(min, min + 3),
+                    0,
+                    0
+                );
+                blocks[prepend.length + c] = randomBlocks[c].block();
+            }
+            Page p = new Page(blocks);
+            blocks = null;
+            return new RandomPage(p, randomBlocks);
+        } finally {
+            if (blocks != null) {
+                Releasables.close(blocks);
+            }
+        }
+    }
+
+    IntVector randomPositions(BlockFactory factory, int leftSize, int positionCount) {
+        Set<Integer> positions = new HashSet<>();
+        while (positions.size() < positionCount) {
+            positions.add(between(0, leftSize - 1));
+        }
+        int[] positionsArray = positions.stream().mapToInt(i -> i).sorted().toArray();
+        return factory.newIntArrayVector(positionsArray, positionsArray.length);
+    }
+}
diff --git a/x-pack/plugin/esql/qa/security/build.gradle b/x-pack/plugin/esql/qa/security/build.gradle
index 068a4fd8f4989..c04595ef22839 100644
--- a/x-pack/plugin/esql/qa/security/build.gradle
+++ b/x-pack/plugin/esql/qa/security/build.gradle
@@ -5,3 +5,9 @@ apply plugin: 'elasticsearch.internal-test-artifact'
 tasks.named('javaRestTest') {
   usesDefaultDistribution()
 }
+
+dependencies {
+  javaRestTestImplementation project(xpackModule('esql-core'))
+  javaRestTestImplementation project(xpackModule('esql'))
+  javaRestTestImplementation project(xpackModule('esql:qa:server'))
+}
diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java
index 9566aeb8f28dc..ce4aa8582929b 100644
--- a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java
+++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java
@@ -26,10 +26,13 @@
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
+import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
+import org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase;
 import org.junit.Before;
 import org.junit.ClassRule;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -87,9 +90,11 @@ private void indexDocument(String index, int id, double value, String org) throw
 
     @Before
     public void indexDocuments() throws IOException {
+        Settings lookupSettings = Settings.builder().put("index.mode", "lookup").build();
         String mapping = """
             "properties":{"value": {"type": "double"}, "org": {"type": "keyword"}}
             """;
+
         createIndex("index", Settings.EMPTY, mapping);
         indexDocument("index", 1, 10.0, "sales");
         indexDocument("index", 2, 20.0, "engineering");
@@ -110,6 +115,16 @@ public void indexDocuments() throws IOException {
         indexDocument("indexpartial", 2, 40.0, "sales");
         refresh("indexpartial");
 
+        createIndex("lookup-user1", lookupSettings, mapping);
+        indexDocument("lookup-user1", 1, 12.0, "engineering");
+        indexDocument("lookup-user1", 2, 31.0, "sales");
+        refresh("lookup-user1");
+
+        createIndex("lookup-user2", lookupSettings, mapping);
+        indexDocument("lookup-user2", 1, 32.0, "marketing");
+        indexDocument("lookup-user2", 2, 40.0, "sales");
+        refresh("lookup-user2");
+
         if (aliasExists("second-alias") == false) {
             Request aliasRequest = new Request("POST", "_aliases");
             aliasRequest.setJsonEntity("""
@@ -126,6 +141,17 @@ public void indexDocuments() throws IOException {
                             }
                           }
                         },
+                        {
+                          "add": {
+                            "alias": "lookup-first-alias",
+                            "index": "lookup-user1",
+                            "filter": {
+                                "term": {
+                                    "org": "sales"
+                                }
+                            }
+                          }
+                        },
                         {
                           "add": {
                             "alias": "second-alias",
@@ -229,22 +255,30 @@ public void testAliasFilter() throws Exception {
     public void testUnauthorizedIndices() throws IOException {
         ResponseException error;
         error = expectThrows(ResponseException.class, () -> runESQLCommand("user1", "from index-user2 | stats sum(value)"));
+        assertThat(error.getMessage(), containsString("Unknown index [index-user2]"));
         assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(400));
 
         error = expectThrows(ResponseException.class, () -> runESQLCommand("user2", "from index-user1 | stats sum(value)"));
+        assertThat(error.getMessage(), containsString("Unknown index [index-user1]"));
         assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(400));
 
         error = expectThrows(ResponseException.class, () -> runESQLCommand("alias_user2", "from index-user2 | stats sum(value)"));
+        assertThat(error.getMessage(), containsString("Unknown index [index-user2]"));
         assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(400));
 
         error = expectThrows(ResponseException.class, () -> runESQLCommand("metadata1_read2", "from index-user1 | stats sum(value)"));
+        assertThat(error.getMessage(), containsString("Unknown index [index-user1]"));
         assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(400));
     }
 
     public void testInsufficientPrivilege() {
-        Exception error = expectThrows(Exception.class, () -> runESQLCommand("metadata1_read2", "FROM index-user1 | STATS sum=sum(value)"));
+        ResponseException error = expectThrows(
+            ResponseException.class,
+            () -> runESQLCommand("metadata1_read2", "FROM index-user1 | STATS sum=sum(value)")
+        );
         logger.info("error", error);
         assertThat(error.getMessage(), containsString("Unknown index [index-user1]"));
+        assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));
     }
 
     public void testIndexPatternErrorMessageComparison_ESQL_SearchDSL() throws Exception {
@@ -511,6 +545,73 @@ record Listen(long timestamp, String songId, double duration) {
         }
     }
 
+    public void testLookupJoinIndexAllowed() throws Exception {
+        assumeTrue(
+            "Requires LOOKUP JOIN capability",
+            EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V10.capabilityName()))
+        );
+
+        Response resp = runESQLCommand(
+            "metadata1_read2",
+            "ROW x = 40.0 | EVAL value = x | LOOKUP JOIN `lookup-user2` ON value | KEEP x, org"
+        );
+        assertOK(resp);
+        Map<String, Object> respMap = entityAsMap(resp);
+        assertThat(
+            respMap.get("columns"),
+            equalTo(List.of(Map.of("name", "x", "type", "double"), Map.of("name", "org", "type", "keyword")))
+        );
+        assertThat(respMap.get("values"), equalTo(List.of(List.of(40.0, "sales"))));
+
+        // Alias, should find the index and the row
+        resp = runESQLCommand("alias_user1", "ROW x = 31.0 | EVAL value = x | LOOKUP JOIN `lookup-first-alias` ON value | KEEP x, org");
+        assertOK(resp);
+        respMap = entityAsMap(resp);
+        assertThat(
+            respMap.get("columns"),
+            equalTo(List.of(Map.of("name", "x", "type", "double"), Map.of("name", "org", "type", "keyword")))
+        );
+        assertThat(respMap.get("values"), equalTo(List.of(List.of(31.0, "sales"))));
+
+        // Alias, for a row that's filtered out
+        resp = runESQLCommand("alias_user1", "ROW x = 123.0 | EVAL value = x | LOOKUP JOIN `lookup-first-alias` ON value | KEEP x, org");
+        assertOK(resp);
+        respMap = entityAsMap(resp);
+        assertThat(
+            respMap.get("columns"),
+            equalTo(List.of(Map.of("name", "x", "type", "double"), Map.of("name", "org", "type", "keyword")))
+        );
+        assertThat(respMap.get("values"), equalTo(List.of(Arrays.asList(123.0, null))));
+    }
+
+    public void testLookupJoinIndexForbidden() throws Exception {
+        assumeTrue(
+            "Requires LOOKUP JOIN capability",
+            EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V10.capabilityName()))
+        );
+
+        var resp = expectThrows(
+            ResponseException.class,
+            () -> runESQLCommand("metadata1_read2", "FROM lookup-user2 | EVAL value = 10.0 | LOOKUP JOIN `lookup-user1` ON value | KEEP x")
+        );
+        assertThat(resp.getMessage(), containsString("Unknown index [lookup-user1]"));
+        assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));
+
+        resp = expectThrows(
+            ResponseException.class,
+            () -> runESQLCommand("metadata1_read2", "ROW x = 10.0 | EVAL value = x | LOOKUP JOIN `lookup-user1` ON value | KEEP x")
+        );
+        assertThat(resp.getMessage(), containsString("Unknown index [lookup-user1]"));
+        assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));
+
+        resp = expectThrows(
+            ResponseException.class,
+            () -> runESQLCommand("alias_user1", "ROW x = 10.0 | EVAL value = x | LOOKUP JOIN `lookup-user1` ON value | KEEP x")
+        );
+        assertThat(resp.getMessage(), containsString("Unknown index [lookup-user1]"));
+        assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));
+    }
+
     private void createEnrichPolicy() throws Exception {
         createIndex("songs", Settings.EMPTY, """
             "properties":{"song_id": {"type": "keyword"}, "title": {"type": "keyword"}, "artist": {"type": "keyword"} }
diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml b/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml
index 365a072edb74e..f46e7ef56f3a1 100644
--- a/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml
+++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml
@@ -35,15 +35,15 @@ user2:
 metadata1_read2:
   cluster: []
   indices:
-    - names: [ 'index-user1' ]
+    - names: [ 'index-user1', 'lookup-user1' ]
       privileges: [ 'view_index_metadata' ]
-    - names: [ 'index-user2' ]
+    - names: [ 'index-user2', 'lookup-user2' ]
       privileges: [ 'read' ]
 
 alias_user1:
   cluster: []
   indices:
-    - names: [ 'first-alias' ]
+    - names: [ 'first-alias', 'lookup-first-alias' ]
       privileges:
         - read
 
diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java
index 1120a69cc5166..f5d8f472fd59f 100644
--- a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java
+++ b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.esql.qa.mixed;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.TestFeatureService;
 import org.elasticsearch.xpack.esql.CsvSpecReader.CsvTestCase;
@@ -21,8 +20,7 @@
 import java.util.List;
 
 import static org.elasticsearch.xpack.esql.CsvTestUtils.isEnabled;
-import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V5;
-import static org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase.Mode.ASYNC;
+import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V10;
 
 public class MixedClusterEsqlSpecIT extends EsqlSpecTestCase {
     @ClassRule
@@ -49,10 +47,6 @@ protected static boolean oldClusterHasFeature(String featureId) {
         return oldClusterTestFeatureService.clusterHasFeature(featureId);
     }
 
-    protected static boolean oldClusterHasFeature(NodeFeature feature) {
-        return oldClusterHasFeature(feature.id());
-    }
-
     @AfterClass
     public static void cleanUp() {
         oldClusterTestFeatureService = null;
@@ -74,14 +68,6 @@ public MixedClusterEsqlSpecIT(
     protected void shouldSkipTest(String testName) throws IOException {
         super.shouldSkipTest(testName);
         assumeTrue("Test " + testName + " is skipped on " + bwcVersion, isEnabled(testName, instructions, bwcVersion));
-        if (mode == ASYNC) {
-            assumeTrue("Async is not supported on " + bwcVersion, supportsAsync());
-        }
-    }
-
-    @Override
-    protected boolean supportsAsync() {
-        return oldClusterHasFeature(ASYNC_QUERY_FEATURE_ID);
     }
 
     @Override
@@ -96,7 +82,7 @@ protected boolean supportsInferenceTestService() {
 
     @Override
     protected boolean supportsIndexModeLookup() throws IOException {
-        return hasCapabilities(List.of(JOIN_LOOKUP_V5.capabilityName()));
+        return hasCapabilities(List.of(JOIN_LOOKUP_V10.capabilityName()));
     }
 
     @Override
diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java
index 5c7f981c93a97..987a5334f903c 100644
--- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java
+++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java
@@ -48,7 +48,7 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V2;
-import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V5;
+import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V10;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST;
 import static org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase.Mode.SYNC;
@@ -124,7 +124,7 @@ protected void shouldSkipTest(String testName) throws IOException {
         assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName()));
         assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V2.capabilityName()));
         assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName()));
-        assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V5.capabilityName()));
+        assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V10.capabilityName()));
     }
 
     private TestFeatureService remoteFeaturesService() throws IOException {
@@ -283,8 +283,8 @@ protected boolean supportsInferenceTestService() {
 
     @Override
     protected boolean supportsIndexModeLookup() throws IOException {
-        // CCS does not yet support JOIN_LOOKUP_V5 and clusters falsely report they have this capability
-        // return hasCapabilities(List.of(JOIN_LOOKUP_V5.capabilityName()));
+        // CCS does not yet support JOIN_LOOKUP_V10 and clusters falsely report they have this capability
+        // return hasCapabilities(List.of(JOIN_LOOKUP_V10.capabilityName()));
         return false;
     }
 }
diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java
index 452f40baa34a8..6e43d40a3005a 100644
--- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java
+++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java
@@ -12,6 +12,7 @@
 import org.apache.http.HttpHost;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
@@ -37,9 +38,11 @@
 
 import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.test.MapMatcher.matchesMap;
+import static org.elasticsearch.xpack.esql.ccq.Clusters.REMOTE_CLUSTER_NAME;
 import static org.hamcrest.Matchers.any;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasKey;
 
 @ThreadLeakFilters(filters = TestClustersThreadFilter.class)
 public class MultiClustersIT extends ESRestTestCase {
@@ -395,6 +398,40 @@ public void testIndexPattern() throws Exception {
         }
     }
 
+    @SuppressWarnings("unchecked")
+    public void testStats() throws IOException {
+        assumeTrue("capabilities endpoint is not available", capabilitiesEndpointAvailable());
+
+        Request caps = new Request("GET", "_capabilities?method=GET&path=_cluster/stats&capabilities=esql-stats");
+        Response capsResponse = client().performRequest(caps);
+        Map<String, Object> capsResult = entityAsMap(capsResponse.getEntity());
+        assumeTrue("esql stats capability missing", capsResult.get("supported").equals(true));
+
+        run("FROM test-local-index,*:test-remote-index | STATS total = SUM(data) BY color | SORT color", includeCCSMetadata());
+        Request stats = new Request("GET", "_cluster/stats");
+        Response statsResponse = client().performRequest(stats);
+        Map<String, Object> result = entityAsMap(statsResponse.getEntity());
+        assertThat(result, hasKey("ccs"));
+        Map<String, Object> ccs = (Map<String, Object>) result.get("ccs");
+        assertThat(ccs, hasKey("_esql"));
+        Map<String, Object> esql = (Map<String, Object>) ccs.get("_esql");
+        assertThat(esql, hasKey("total"));
+        assertThat(esql, hasKey("success"));
+        assertThat(esql, hasKey("took"));
+        assertThat(esql, hasKey("remotes_per_search_max"));
+        assertThat(esql, hasKey("remotes_per_search_avg"));
+        assertThat(esql, hasKey("failure_reasons"));
+        assertThat(esql, hasKey("features"));
+        assertThat(esql, hasKey("clusters"));
+        Map<String, Object> clusters = (Map<String, Object>) esql.get("clusters");
+        assertThat(clusters, hasKey(REMOTE_CLUSTER_NAME));
+        assertThat(clusters, hasKey("(local)"));
+        Map<String, Object> clusterData = (Map<String, Object>) clusters.get(REMOTE_CLUSTER_NAME);
+        assertThat(clusterData, hasKey("total"));
+        assertThat(clusterData, hasKey("skipped"));
+        assertThat(clusterData, hasKey("took"));
+    }
+
     private RestClient remoteClusterClient() throws IOException {
         var clusterHosts = parseClusterHosts(remoteCluster.getHttpAddresses());
         return buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[0]));
@@ -404,6 +441,10 @@ private static boolean ccsMetadataAvailable() {
         return Clusters.localClusterVersion().onOrAfter(Version.V_8_16_0);
     }
 
+    private static boolean capabilitiesEndpointAvailable() {
+        return Clusters.localClusterVersion().onOrAfter(Version.V_8_15_0);
+    }
+
     private static boolean includeCCSMetadata() {
         return ccsMetadataAvailable() && randomBoolean();
     }
diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java
new file mode 100644
index 0000000000000..7c81f97714a6f
--- /dev/null
+++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.ccq;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.apache.http.HttpHost;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.test.TestClustersThreadFilter;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.xpack.esql.qa.rest.RequestIndexFilteringTestCase;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+import java.io.IOException;
+
+@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
+public class RequestIndexFilteringIT extends RequestIndexFilteringTestCase {
+
+    static ElasticsearchCluster remoteCluster = Clusters.remoteCluster();
+    static ElasticsearchCluster localCluster = Clusters.localCluster(remoteCluster);
+
+    @ClassRule
+    public static TestRule clusterRule = RuleChain.outerRule(remoteCluster).around(localCluster);
+    private static RestClient remoteClient;
+
+    @Override
+    protected String getTestRestCluster() {
+        return localCluster.getHttpAddresses();
+    }
+
+    @Before
+    public void setRemoteClient() throws IOException {
+        if (remoteClient == null) {
+            var clusterHosts = parseClusterHosts(remoteCluster.getHttpAddresses());
+            remoteClient = buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[0]));
+        }
+    }
+
+    @AfterClass
+    public static void closeRemoteClients() throws IOException {
+        try {
+            IOUtils.close(remoteClient);
+        } finally {
+            remoteClient = null;
+        }
+    }
+
+    @Override
+    protected void indexTimestampData(int docs, String indexName, String date, String differentiatorFieldName) throws IOException {
+        indexTimestampDataForClient(client(), docs, indexName, date, differentiatorFieldName);
+        indexTimestampDataForClient(remoteClient, docs, indexName, date, differentiatorFieldName);
+    }
+
+    @Override
+    protected String from(String... indexName) {
+        if (randomBoolean()) {
+            return "FROM *:" + String.join(",*:", indexName);
+        } else {
+            return "FROM " + String.join(",", indexName);
+        }
+    }
+
+    @After
+    public void wipeRemoteTestData() throws IOException {
+        try {
+            var response = remoteClient.performRequest(new Request("DELETE", "/test*"));
+            assertEquals(200, response.getStatusLine().getStatusCode());
+        } catch (ResponseException re) {
+            assertEquals(404, re.getResponse().getStatusLine().getStatusCode());
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/SemanticMatchIT.java b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/SemanticMatchIT.java
new file mode 100644
index 0000000000000..0ce84330b0b01
--- /dev/null
+++ b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/SemanticMatchIT.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.qa.multi_node;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.elasticsearch.test.TestClustersThreadFilter;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.xpack.esql.qa.rest.SemanticMatchTestCase;
+import org.junit.ClassRule;
+
+@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
+public class SemanticMatchIT extends SemanticMatchTestCase {
+    @ClassRule
+    public static ElasticsearchCluster cluster = Clusters.testCluster(spec -> spec.plugin("inference-service-test"));
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+}
diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/SemanticMatchIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/SemanticMatchIT.java
new file mode 100644
index 0000000000000..8edc2dbcf35a2
--- /dev/null
+++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/SemanticMatchIT.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.qa.single_node;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.elasticsearch.test.TestClustersThreadFilter;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.xpack.esql.qa.rest.SemanticMatchTestCase;
+import org.junit.ClassRule;
+
+@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
+public class SemanticMatchIT extends SemanticMatchTestCase {
+    @ClassRule
+    public static ElasticsearchCluster cluster = Clusters.testCluster(spec -> spec.plugin("inference-service-test"));
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+}
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java
index e7bce73e21cdd..18bfb6b8676ce 100644
--- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java
@@ -75,9 +75,6 @@
 @TimeoutSuite(millis = 30 * TimeUnits.MINUTE)
 public abstract class EsqlSpecTestCase extends ESRestTestCase {
 
-    // To avoid referencing the main module, we replicate EsqlFeatures.ASYNC_QUERY.id() here
-    protected static final String ASYNC_QUERY_FEATURE_ID = "esql.async_query";
-
     private static final Logger LOGGER = LogManager.getLogger(EsqlSpecTestCase.class);
     private final String fileName;
     private final String groupName;
@@ -140,10 +137,6 @@ public void setup() throws IOException {
         }
     }
 
-    protected boolean supportsAsync() {
-        return clusterHasFeature(ASYNC_QUERY_FEATURE_ID); // the Async API was introduced in 8.13.0
-    }
-
     @AfterClass
     public static void wipeTestData() throws IOException {
         try {
@@ -173,7 +166,8 @@ public final void test() throws Throwable {
 
     protected void shouldSkipTest(String testName) throws IOException {
         if (testCase.requiredCapabilities.contains("semantic_text_type")
-            || testCase.requiredCapabilities.contains("semantic_text_aggregations")) {
+            || testCase.requiredCapabilities.contains("semantic_text_aggregations")
+            || testCase.requiredCapabilities.contains("semantic_text_field_caps")) {
             assumeTrue("Inference test service needs to be supported for semantic_text", supportsInferenceTestService());
         }
         checkCapabilities(adminClient(), testFeatureService, testName, testCase);
@@ -199,7 +193,7 @@ protected static boolean hasCapabilities(List<String> requiredCapabilities) thro
         return hasCapabilities(adminClient(), requiredCapabilities);
     }
 
-    protected static boolean hasCapabilities(RestClient client, List<String> requiredCapabilities) throws IOException {
+    public static boolean hasCapabilities(RestClient client, List<String> requiredCapabilities) throws IOException {
         if (requiredCapabilities.isEmpty()) {
             return true;
         }
@@ -280,7 +274,6 @@ protected boolean deduplicateExactWarnings() {
 
     private Map<String, Object> runEsql(RequestObjectBuilder requestObject, AssertWarnings assertWarnings) throws IOException {
         if (mode == Mode.ASYNC) {
-            assert supportsAsync();
             return RestEsqlTestCase.runEsqlAsync(requestObject, assertWarnings);
         } else {
             return RestEsqlTestCase.runEsqlSync(requestObject, assertWarnings);
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java
index 406997b66dbf0..ac5a3d4be27f3 100644
--- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java
@@ -11,9 +11,11 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.esql.AssertWarnings;
+import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.junit.After;
 import org.junit.Assert;
 
@@ -29,12 +31,13 @@
 import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.entityToMap;
 import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.requestObjectBuilder;
 import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.oneOf;
 
 public abstract class RequestIndexFilteringTestCase extends ESRestTestCase {
 
@@ -48,6 +51,10 @@ public void wipeTestData() throws IOException {
         }
     }
 
+    protected String from(String... indexName) {
+        return "FROM " + String.join(",", indexName);
+    }
+
     public void testTimestampFilterFromQuery() throws IOException {
         int docsTest1 = 50;
         int docsTest2 = 30;
@@ -55,7 +62,7 @@ public void testTimestampFilterFromQuery() throws IOException {
         indexTimestampData(docsTest2, "test2", "2023-11-26", "id2");
 
         // filter includes both indices in the result (all columns, all rows)
-        RestEsqlTestCase.RequestObjectBuilder builder = timestampFilter("gte", "2023-01-01").query("FROM test*");
+        RestEsqlTestCase.RequestObjectBuilder builder = timestampFilter("gte", "2023-01-01").query(from("test*"));
         Map<String, Object> result = runEsql(builder);
         assertMap(
             result,
@@ -69,7 +76,7 @@ public void testTimestampFilterFromQuery() throws IOException {
         );
 
         // filter includes only test1. Columns from test2 are filtered out, as well (not only rows)!
-        builder = timestampFilter("gte", "2024-01-01").query("FROM test*");
+        builder = timestampFilter("gte", "2024-01-01").query(from("test*"));
         assertMap(
             runEsql(builder),
             matchesMap().entry(
@@ -82,7 +89,7 @@ public void testTimestampFilterFromQuery() throws IOException {
 
         // filter excludes both indices (no rows); the first analysis step fails because there are no columns, a second attempt succeeds
         // after eliminating the index filter. All columns are returned.
-        builder = timestampFilter("gte", "2025-01-01").query("FROM test*");
+        builder = timestampFilter("gte", "2025-01-01").query(from("test*"));
         assertMap(
             runEsql(builder),
             matchesMap().entry(
@@ -102,7 +109,7 @@ public void testFieldExistsFilter_KeepWildcard() throws IOException {
         indexTimestampData(docsTest2, "test2", "2023-11-26", "id2");
 
         // filter includes only test1. Columns and rows of test2 are filtered out
-        RestEsqlTestCase.RequestObjectBuilder builder = existsFilter("id1").query("FROM test*");
+        RestEsqlTestCase.RequestObjectBuilder builder = existsFilter("id1").query(from("test*"));
         Map<String, Object> result = runEsql(builder);
         assertMap(
             result,
@@ -115,7 +122,7 @@ public void testFieldExistsFilter_KeepWildcard() throws IOException {
         );
 
         // filter includes only test1. Columns from test2 are filtered out, as well (not only rows)!
-        builder = existsFilter("id1").query("FROM test* METADATA _index | KEEP _index, id*");
+        builder = existsFilter("id1").query(from("test*") + " METADATA _index | KEEP _index, id*");
         result = runEsql(builder);
         assertMap(
             result,
@@ -128,7 +135,7 @@ public void testFieldExistsFilter_KeepWildcard() throws IOException {
         @SuppressWarnings("unchecked")
         var values = (List<List<Object>>) result.get("values");
         for (List<Object> row : values) {
-            assertThat(row.get(0), equalTo("test1"));
+            assertThat(row.get(0), oneOf("test1", "remote_cluster:test1"));
             assertThat(row.get(1), instanceOf(Integer.class));
         }
     }
@@ -141,7 +148,7 @@ public void testFieldExistsFilter_With_ExplicitUseOfDiscardedIndexFields() throw
 
         // test2 is explicitly used in a query with "SORT id2" even if the index filter should discard test2
         RestEsqlTestCase.RequestObjectBuilder builder = existsFilter("id1").query(
-            "FROM test* METADATA _index | SORT id2 | KEEP _index, id*"
+            from("test*") + " METADATA _index | SORT id2 | KEEP _index, id*"
         );
         Map<String, Object> result = runEsql(builder);
         assertMap(
@@ -156,7 +163,7 @@ public void testFieldExistsFilter_With_ExplicitUseOfDiscardedIndexFields() throw
         @SuppressWarnings("unchecked")
         var values = (List<List<Object>>) result.get("values");
         for (List<Object> row : values) {
-            assertThat(row.get(0), equalTo("test1"));
+            assertThat(row.get(0), oneOf("test1", "remote_cluster:test1"));
             assertThat(row.get(1), instanceOf(Integer.class));
             assertThat(row.get(2), nullValue());
         }
@@ -171,54 +178,64 @@ public void testFieldNameTypo() throws IOException {
         // idx field name is explicitly used, though it doesn't exist in any of the indices. First test - without filter
         ResponseException e = expectThrows(
             ResponseException.class,
-            () -> runEsql(requestObjectBuilder().query("FROM test* | WHERE idx == 123"))
+            () -> runEsql(requestObjectBuilder().query(from("test*") + " | WHERE idx == 123"))
         );
         assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
         assertThat(e.getMessage(), containsString("verification_exception"));
         assertThat(e.getMessage(), containsString("Found 1 problem"));
-        assertThat(e.getMessage(), containsString("line 1:20: Unknown column [idx]"));
+        assertThat(e.getMessage(), containsString("Unknown column [idx]"));
 
-        e = expectThrows(ResponseException.class, () -> runEsql(requestObjectBuilder().query("FROM test1 | WHERE idx == 123")));
+        e = expectThrows(ResponseException.class, () -> runEsql(requestObjectBuilder().query(from("test1") + " | WHERE idx == 123")));
         assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
         assertThat(e.getMessage(), containsString("verification_exception"));
         assertThat(e.getMessage(), containsString("Found 1 problem"));
-        assertThat(e.getMessage(), containsString("line 1:20: Unknown column [idx]"));
+        assertThat(e.getMessage(), containsString("Unknown column [idx]"));
 
         e = expectThrows(
             ResponseException.class,
-            () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM test* | WHERE idx == 123"))
+            () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("test*") + " | WHERE idx == 123"))
         );
         assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
         assertThat(e.getMessage(), containsString("Found 1 problem"));
-        assertThat(e.getMessage(), containsString("line 1:20: Unknown column [idx]"));
+        assertThat(e.getMessage(), containsString("Unknown column [idx]"));
 
         e = expectThrows(
             ResponseException.class,
-            () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM test2 | WHERE idx == 123"))
+            () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("test2") + " | WHERE idx == 123"))
         );
         assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
         assertThat(e.getMessage(), containsString("Found 1 problem"));
-        assertThat(e.getMessage(), containsString("line 1:20: Unknown column [idx]"));
+        assertThat(e.getMessage(), containsString("Unknown column [idx]"));
     }
 
     public void testIndicesDontExist() throws IOException {
         int docsTest1 = 0; // we are interested only in the created index, not necessarily that it has data
         indexTimestampData(docsTest1, "test1", "2024-11-26", "id1");
 
-        ResponseException e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM foo")));
+        ResponseException e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo"))));
         assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
         assertThat(e.getMessage(), containsString("verification_exception"));
-        assertThat(e.getMessage(), containsString("Unknown index [foo]"));
+        assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo]"), containsString("Unknown index [remote_cluster:foo]")));
 
-        e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM foo*")));
+        e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo*"))));
         assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
         assertThat(e.getMessage(), containsString("verification_exception"));
-        assertThat(e.getMessage(), containsString("Unknown index [foo*]"));
+        assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo*]"), containsString("Unknown index [remote_cluster:foo*]")));
 
-        e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM foo,test1")));
+        e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo", "test1"))));
         assertEquals(404, e.getResponse().getStatusLine().getStatusCode());
         assertThat(e.getMessage(), containsString("index_not_found_exception"));
-        assertThat(e.getMessage(), containsString("no such index [foo]"));
+        assertThat(e.getMessage(), anyOf(containsString("no such index [foo]"), containsString("no such index [remote_cluster:foo]")));
+
+        if (EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled()) {
+            e = expectThrows(
+                ResponseException.class,
+                () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("test1") + " | LOOKUP JOIN foo ON id1"))
+            );
+            assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
+            assertThat(e.getMessage(), containsString("verification_exception"));
+            assertThat(e.getMessage(), containsString("Unknown index [foo]"));
+        }
     }
 
     private static RestEsqlTestCase.RequestObjectBuilder timestampFilter(String op, String date) throws IOException {
@@ -240,6 +257,11 @@ public Map<String, Object> runEsql(RestEsqlTestCase.RequestObjectBuilder request
     }
 
     protected void indexTimestampData(int docs, String indexName, String date, String differentiatorFieldName) throws IOException {
+        indexTimestampDataForClient(client(), docs, indexName, date, differentiatorFieldName);
+    }
+
+    protected void indexTimestampDataForClient(RestClient client, int docs, String indexName, String date, String differentiatorFieldName)
+        throws IOException {
         Request createIndex = new Request("PUT", indexName);
         createIndex.setJsonEntity("""
             {
@@ -262,7 +284,7 @@ protected void indexTimestampData(int docs, String indexName, String date, Strin
                 }
               }
             }""".replace("%differentiator_field_name%", differentiatorFieldName));
-        Response response = client().performRequest(createIndex);
+        Response response = client.performRequest(createIndex);
         assertThat(
             entityToMap(response.getEntity(), XContentType.JSON),
             matchesMap().entry("shards_acknowledged", true).entry("index", indexName).entry("acknowledged", true)
@@ -280,7 +302,7 @@ protected void indexTimestampData(int docs, String indexName, String date, Strin
             bulk.addParameter("refresh", "true");
             bulk.addParameter("filter_path", "errors");
             bulk.setJsonEntity(b.toString());
-            response = client().performRequest(bulk);
+            response = client.performRequest(bulk);
             Assert.assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8));
         }
     }
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/SemanticMatchTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/SemanticMatchTestCase.java
new file mode 100644
index 0000000000000..b9314645cf2c9
--- /dev/null
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/SemanticMatchTestCase.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.qa.rest;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.hamcrest.core.StringContains.containsString;
+
+public abstract class SemanticMatchTestCase extends ESRestTestCase {
+    public void testWithMultipleInferenceIds() throws IOException {
+        assumeTrue("semantic text capability not available", EsqlCapabilities.Cap.SEMANTIC_TEXT_TYPE.isEnabled());
+
+        String query = """
+            from test-semantic1,test-semantic2
+            | where match(semantic_text_field, "something")
+            """;
+        ResponseException re = expectThrows(ResponseException.class, () -> runEsqlQuery(query));
+
+        assertThat(re.getMessage(), containsString("Field [semantic_text_field] has multiple inference IDs associated with it"));
+
+        assertEquals(400, re.getResponse().getStatusLine().getStatusCode());
+    }
+
+    public void testWithInferenceNotConfigured() {
+        assumeTrue("semantic text capability not available", EsqlCapabilities.Cap.SEMANTIC_TEXT_TYPE.isEnabled());
+
+        String query = """
+            from test-semantic3
+            | where match(semantic_text_field, "something")
+            """;
+        ResponseException re = expectThrows(ResponseException.class, () -> runEsqlQuery(query));
+
+        assertThat(re.getMessage(), containsString("Inference endpoint not found"));
+        assertEquals(404, re.getResponse().getStatusLine().getStatusCode());
+    }
+
+    @Before
+    public void setUpIndices() throws IOException {
+        var settings = Settings.builder().build();
+
+        String mapping1 = """
+                "properties": {
+                  "semantic_text_field": {
+                   "type": "semantic_text",
+                   "inference_id": "test_sparse_inference"
+                  }
+                }
+            """;
+        createIndex(adminClient(), "test-semantic1", settings, mapping1);
+
+        String mapping2 = """
+                 "properties": {
+                   "semantic_text_field": {
+                    "type": "semantic_text",
+                    "inference_id": "test_dense_inference"
+                   }
+                 }
+            """;
+        createIndex(adminClient(), "test-semantic2", settings, mapping2);
+
+        String mapping3 = """
+                 "properties": {
+                   "semantic_text_field": {
+                    "type": "semantic_text",
+                    "inference_id": "inexistent"
+                   }
+                 }
+            """;
+        createIndex(adminClient(), "test-semantic3", settings, mapping3);
+    }
+
+    @Before
+    public void setUpTextEmbeddingInferenceEndpoint() throws IOException {
+        Request request = new Request("PUT", "_inference/text_embedding/test_dense_inference");
+        request.setJsonEntity("""
+                  {
+                   "service": "test_service",
+                   "service_settings": {
+                     "model": "my_model",
+                     "api_key": "abc64"
+                   },
+                   "task_settings": {
+                   }
+                 }
+            """);
+        adminClient().performRequest(request);
+    }
+
+    @After
+    public void wipeData() throws IOException {
+        adminClient().performRequest(new Request("DELETE", "*"));
+
+        try {
+            adminClient().performRequest(new Request("DELETE", "_inference/test_dense_inference"));
+        } catch (ResponseException e) {
+            // 404 here means the endpoint was not created
+            if (e.getResponse().getStatusLine().getStatusCode() != 404) {
+                throw e;
+            }
+        }
+    }
+
+    private Map<String, Object> runEsqlQuery(String query) throws IOException {
+        RestEsqlTestCase.RequestObjectBuilder builder = RestEsqlTestCase.requestObjectBuilder().query(query);
+        return RestEsqlTestCase.runEsqlSync(builder);
+    }
+}
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java
index 7adafa908ce4f..7ebc08a5a3ffe 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.test.VersionUtils;
@@ -63,7 +64,7 @@
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO;
 
 public final class CsvTestUtils {
-    private static final int MAX_WIDTH = 20;
+    private static final int MAX_WIDTH = 80;
     private static final CsvPreference CSV_SPEC_PREFERENCES = new CsvPreference.Builder('"', '|', "\r\n").build();
     private static final String NULL_VALUE = "null";
     private static final char ESCAPE_CHAR = '\\';
@@ -478,7 +479,15 @@ public enum Type {
         GEO_POINT(x -> x == null ? null : GEO.wktToWkb(x), BytesRef.class),
         CARTESIAN_POINT(x -> x == null ? null : CARTESIAN.wktToWkb(x), BytesRef.class),
         GEO_SHAPE(x -> x == null ? null : GEO.wktToWkb(x), BytesRef.class),
-        CARTESIAN_SHAPE(x -> x == null ? null : CARTESIAN.wktToWkb(x), BytesRef.class);
+        CARTESIAN_SHAPE(x -> x == null ? null : CARTESIAN.wktToWkb(x), BytesRef.class),
+        UNSUPPORTED(Type::convertUnsupported, Void.class);
+
+        private static Void convertUnsupported(String s) {
+            if (s != null) {
+                throw new IllegalArgumentException(Strings.format("Unsupported type should always be null, was '%s'", s));
+            }
+            return null;
+        }
 
         private static final Map<String, Type> LOOKUP = new HashMap<>();
 
@@ -486,6 +495,9 @@ public enum Type {
             for (Type value : Type.values()) {
                 LOOKUP.put(value.name(), value);
             }
+            // Types with a different field caps family type
+            LOOKUP.put("SEMANTIC_TEXT", TEXT);
+
             // widen smaller types
             LOOKUP.put("SHORT", INTEGER);
             LOOKUP.put("BYTE", INTEGER);
@@ -536,6 +548,9 @@ public static Type asType(String name) {
         }
 
         public static Type asType(ElementType elementType, Type actualType) {
+            if (actualType == Type.UNSUPPORTED) {
+                return UNSUPPORTED;
+            }
             return switch (elementType) {
                 case INT -> INTEGER;
                 case LONG -> LONG;
@@ -551,11 +566,11 @@ public static Type asType(ElementType elementType, Type actualType) {
         }
 
         private static Type bytesRefBlockType(Type actualType) {
-            if (actualType == GEO_POINT || actualType == CARTESIAN_POINT || actualType == GEO_SHAPE || actualType == CARTESIAN_SHAPE) {
-                return actualType;
-            } else {
-                return KEYWORD;
-            }
+            return switch (actualType) {
+                case NULL -> NULL;
+                case GEO_POINT, CARTESIAN_POINT, GEO_SHAPE, CARTESIAN_SHAPE -> actualType;
+                default -> KEYWORD;
+            };
         }
 
         Object convert(String value) {
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java
index abfe90f80e372..fbd4f9feca78d 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -50,64 +51,72 @@
 
 public class CsvTestsDataLoader {
     private static final int BULK_DATA_SIZE = 100_000;
-    private static final TestsDataset EMPLOYEES = new TestsDataset("employees", "mapping-default.json", "employees.csv").noSubfields();
-    private static final TestsDataset EMPLOYEES_INCOMPATIBLE = new TestsDataset(
+    private static final TestDataset EMPLOYEES = new TestDataset("employees", "mapping-default.json", "employees.csv").noSubfields();
+    private static final TestDataset EMPLOYEES_INCOMPATIBLE = new TestDataset(
         "employees_incompatible",
         "mapping-default-incompatible.json",
         "employees_incompatible.csv"
     ).noSubfields();
-    private static final TestsDataset HOSTS = new TestsDataset("hosts");
-    private static final TestsDataset APPS = new TestsDataset("apps");
-    private static final TestsDataset APPS_SHORT = APPS.withIndex("apps_short").withTypeMapping(Map.of("id", "short"));
-    private static final TestsDataset LANGUAGES = new TestsDataset("languages");
-    private static final TestsDataset LANGUAGES_LOOKUP = LANGUAGES.withIndex("languages_lookup")
+    private static final TestDataset HOSTS = new TestDataset("hosts");
+    private static final TestDataset APPS = new TestDataset("apps");
+    private static final TestDataset APPS_SHORT = APPS.withIndex("apps_short").withTypeMapping(Map.of("id", "short"));
+    private static final TestDataset LANGUAGES = new TestDataset("languages");
+    private static final TestDataset LANGUAGES_LOOKUP = LANGUAGES.withIndex("languages_lookup")
         .withSetting("languages_lookup-settings.json");
-    private static final TestsDataset LANGUAGES_LOOKUP_NON_UNIQUE_KEY = LANGUAGES_LOOKUP.withIndex("languages_lookup_non_unique_key")
+    private static final TestDataset LANGUAGES_LOOKUP_NON_UNIQUE_KEY = LANGUAGES_LOOKUP.withIndex("languages_lookup_non_unique_key")
         .withData("languages_non_unique_key.csv");
-    private static final TestsDataset ALERTS = new TestsDataset("alerts");
-    private static final TestsDataset UL_LOGS = new TestsDataset("ul_logs");
-    private static final TestsDataset SAMPLE_DATA = new TestsDataset("sample_data");
-    private static final TestsDataset MV_SAMPLE_DATA = new TestsDataset("mv_sample_data");
-    private static final TestsDataset SAMPLE_DATA_STR = SAMPLE_DATA.withIndex("sample_data_str")
+    private static final TestDataset LANGUAGES_NESTED_FIELDS = new TestDataset(
+        "languages_nested_fields",
+        "mapping-languages_nested_fields.json",
+        "languages_nested_fields.csv"
+    ).withSetting("languages_lookup-settings.json");
+    private static final TestDataset ALERTS = new TestDataset("alerts");
+    private static final TestDataset UL_LOGS = new TestDataset("ul_logs");
+    private static final TestDataset SAMPLE_DATA = new TestDataset("sample_data");
+    private static final TestDataset MV_SAMPLE_DATA = new TestDataset("mv_sample_data");
+    private static final TestDataset SAMPLE_DATA_STR = SAMPLE_DATA.withIndex("sample_data_str")
         .withTypeMapping(Map.of("client_ip", "keyword"));
-    private static final TestsDataset SAMPLE_DATA_TS_LONG = SAMPLE_DATA.withIndex("sample_data_ts_long")
+    private static final TestDataset SAMPLE_DATA_TS_LONG = SAMPLE_DATA.withIndex("sample_data_ts_long")
         .withData("sample_data_ts_long.csv")
         .withTypeMapping(Map.of("@timestamp", "long"));
-    private static final TestsDataset SAMPLE_DATA_TS_NANOS = SAMPLE_DATA.withIndex("sample_data_ts_nanos")
+    private static final TestDataset SAMPLE_DATA_TS_NANOS = SAMPLE_DATA.withIndex("sample_data_ts_nanos")
         .withData("sample_data_ts_nanos.csv")
         .withTypeMapping(Map.of("@timestamp", "date_nanos"));
-    private static final TestsDataset MISSING_IP_SAMPLE_DATA = new TestsDataset("missing_ip_sample_data");
-    private static final TestsDataset CLIENT_IPS = new TestsDataset("clientips");
-    private static final TestsDataset CLIENT_IPS_LOOKUP = CLIENT_IPS.withIndex("clientips_lookup")
+    private static final TestDataset MISSING_IP_SAMPLE_DATA = new TestDataset("missing_ip_sample_data");
+    private static final TestDataset CLIENT_IPS = new TestDataset("clientips");
+    private static final TestDataset CLIENT_IPS_LOOKUP = CLIENT_IPS.withIndex("clientips_lookup")
         .withSetting("clientips_lookup-settings.json");
-    private static final TestsDataset MESSAGE_TYPES = new TestsDataset("message_types");
-    private static final TestsDataset MESSAGE_TYPES_LOOKUP = MESSAGE_TYPES.withIndex("message_types_lookup")
+    private static final TestDataset MESSAGE_TYPES = new TestDataset("message_types");
+    private static final TestDataset MESSAGE_TYPES_LOOKUP = MESSAGE_TYPES.withIndex("message_types_lookup")
         .withSetting("message_types_lookup-settings.json");
-    private static final TestsDataset CLIENT_CIDR = new TestsDataset("client_cidr");
-    private static final TestsDataset AGES = new TestsDataset("ages");
-    private static final TestsDataset HEIGHTS = new TestsDataset("heights");
-    private static final TestsDataset DECADES = new TestsDataset("decades");
-    private static final TestsDataset AIRPORTS = new TestsDataset("airports");
-    private static final TestsDataset AIRPORTS_MP = AIRPORTS.withIndex("airports_mp").withData("airports_mp.csv");
-    private static final TestsDataset AIRPORTS_NO_DOC_VALUES = new TestsDataset("airports_no_doc_values").withData("airports.csv");
-    private static final TestsDataset AIRPORTS_NOT_INDEXED = new TestsDataset("airports_not_indexed").withData("airports.csv");
-    private static final TestsDataset AIRPORTS_NOT_INDEXED_NOR_DOC_VALUES = new TestsDataset("airports_not_indexed_nor_doc_values")
-        .withData("airports.csv");
-    private static final TestsDataset AIRPORTS_WEB = new TestsDataset("airports_web");
-    private static final TestsDataset DATE_NANOS = new TestsDataset("date_nanos");
-    private static final TestsDataset COUNTRIES_BBOX = new TestsDataset("countries_bbox");
-    private static final TestsDataset COUNTRIES_BBOX_WEB = new TestsDataset("countries_bbox_web");
-    private static final TestsDataset AIRPORT_CITY_BOUNDARIES = new TestsDataset("airport_city_boundaries");
-    private static final TestsDataset CARTESIAN_MULTIPOLYGONS = new TestsDataset("cartesian_multipolygons");
-    private static final TestsDataset MULTIVALUE_GEOMETRIES = new TestsDataset("multivalue_geometries");
-    private static final TestsDataset MULTIVALUE_POINTS = new TestsDataset("multivalue_points");
-    private static final TestsDataset DISTANCES = new TestsDataset("distances");
-    private static final TestsDataset K8S = new TestsDataset("k8s", "k8s-mappings.json", "k8s.csv").withSetting("k8s-settings.json");
-    private static final TestsDataset ADDRESSES = new TestsDataset("addresses");
-    private static final TestsDataset BOOKS = new TestsDataset("books");
-    private static final TestsDataset SEMANTIC_TEXT = new TestsDataset("semantic_text").withInferenceEndpoint(true);
-
-    public static final Map<String, TestsDataset> CSV_DATASET_MAP = Map.ofEntries(
+    private static final TestDataset CLIENT_CIDR = new TestDataset("client_cidr");
+    private static final TestDataset AGES = new TestDataset("ages");
+    private static final TestDataset HEIGHTS = new TestDataset("heights");
+    private static final TestDataset DECADES = new TestDataset("decades");
+    private static final TestDataset AIRPORTS = new TestDataset("airports");
+    private static final TestDataset AIRPORTS_MP = AIRPORTS.withIndex("airports_mp").withData("airports_mp.csv");
+    private static final TestDataset AIRPORTS_NO_DOC_VALUES = new TestDataset("airports_no_doc_values").withData("airports.csv");
+    private static final TestDataset AIRPORTS_NOT_INDEXED = new TestDataset("airports_not_indexed").withData("airports.csv");
+    private static final TestDataset AIRPORTS_NOT_INDEXED_NOR_DOC_VALUES = new TestDataset("airports_not_indexed_nor_doc_values").withData(
+        "airports.csv"
+    );
+    private static final TestDataset AIRPORTS_WEB = new TestDataset("airports_web");
+    private static final TestDataset DATE_NANOS = new TestDataset("date_nanos");
+    private static final TestDataset COUNTRIES_BBOX = new TestDataset("countries_bbox");
+    private static final TestDataset COUNTRIES_BBOX_WEB = new TestDataset("countries_bbox_web");
+    private static final TestDataset AIRPORT_CITY_BOUNDARIES = new TestDataset("airport_city_boundaries");
+    private static final TestDataset CARTESIAN_MULTIPOLYGONS = new TestDataset("cartesian_multipolygons");
+    private static final TestDataset CARTESIAN_MULTIPOLYGONS_NO_DOC_VALUES = new TestDataset("cartesian_multipolygons_no_doc_values")
+        .withData("cartesian_multipolygons.csv");
+    private static final TestDataset MULTIVALUE_GEOMETRIES = new TestDataset("multivalue_geometries");
+    private static final TestDataset MULTIVALUE_POINTS = new TestDataset("multivalue_points");
+    private static final TestDataset DISTANCES = new TestDataset("distances");
+    private static final TestDataset K8S = new TestDataset("k8s", "k8s-mappings.json", "k8s.csv").withSetting("k8s-settings.json");
+    private static final TestDataset ADDRESSES = new TestDataset("addresses");
+    private static final TestDataset BOOKS = new TestDataset("books").withSetting("books-settings.json");
+    private static final TestDataset SEMANTIC_TEXT = new TestDataset("semantic_text").withInferenceEndpoint(true);
+
+    public static final Map<String, TestDataset> CSV_DATASET_MAP = Map.ofEntries(
         Map.entry(EMPLOYEES.indexName, EMPLOYEES),
         Map.entry(EMPLOYEES_INCOMPATIBLE.indexName, EMPLOYEES_INCOMPATIBLE),
         Map.entry(HOSTS.indexName, HOSTS),
@@ -116,6 +125,7 @@ public class CsvTestsDataLoader {
         Map.entry(LANGUAGES.indexName, LANGUAGES),
         Map.entry(LANGUAGES_LOOKUP.indexName, LANGUAGES_LOOKUP),
         Map.entry(LANGUAGES_LOOKUP_NON_UNIQUE_KEY.indexName, LANGUAGES_LOOKUP_NON_UNIQUE_KEY),
+        Map.entry(LANGUAGES_NESTED_FIELDS.indexName, LANGUAGES_NESTED_FIELDS),
         Map.entry(UL_LOGS.indexName, UL_LOGS),
         Map.entry(SAMPLE_DATA.indexName, SAMPLE_DATA),
         Map.entry(MV_SAMPLE_DATA.indexName, MV_SAMPLE_DATA),
@@ -142,6 +152,7 @@ public class CsvTestsDataLoader {
         Map.entry(COUNTRIES_BBOX_WEB.indexName, COUNTRIES_BBOX_WEB),
         Map.entry(AIRPORT_CITY_BOUNDARIES.indexName, AIRPORT_CITY_BOUNDARIES),
         Map.entry(CARTESIAN_MULTIPOLYGONS.indexName, CARTESIAN_MULTIPOLYGONS),
+        Map.entry(CARTESIAN_MULTIPOLYGONS_NO_DOC_VALUES.indexName, CARTESIAN_MULTIPOLYGONS_NO_DOC_VALUES),
         Map.entry(MULTIVALUE_GEOMETRIES.indexName, MULTIVALUE_GEOMETRIES),
         Map.entry(MULTIVALUE_POINTS.indexName, MULTIVALUE_POINTS),
         Map.entry(DATE_NANOS.indexName, DATE_NANOS),
@@ -256,12 +267,12 @@ public static void main(String[] args) throws IOException {
         }
     }
 
-    public static Set<TestsDataset> availableDatasetsForEs(RestClient client, boolean supportsIndexModeLookup) throws IOException {
+    public static Set<TestDataset> availableDatasetsForEs(RestClient client, boolean supportsIndexModeLookup) throws IOException {
         boolean inferenceEnabled = clusterHasInferenceEndpoint(client);
 
-        Set<TestsDataset> testDataSets = new HashSet<>();
+        Set<TestDataset> testDataSets = new HashSet<>();
 
-        for (TestsDataset dataset : CSV_DATASET_MAP.values()) {
+        for (TestDataset dataset : CSV_DATASET_MAP.values()) {
             if ((inferenceEnabled || dataset.requiresInferenceEndpoint == false)
                 && (supportsIndexModeLookup || isLookupDataset(dataset) == false)) {
                 testDataSets.add(dataset);
@@ -271,7 +282,7 @@ public static Set<TestsDataset> availableDatasetsForEs(RestClient client, boolea
         return testDataSets;
     }
 
-    public static boolean isLookupDataset(TestsDataset dataset) throws IOException {
+    public static boolean isLookupDataset(TestDataset dataset) throws IOException {
         Settings settings = dataset.readSettingsFile();
         String mode = settings.get("index.mode");
         return (mode != null && mode.equalsIgnoreCase("lookup"));
@@ -353,13 +364,13 @@ private static void loadEnrichPolicy(RestClient client, String policyName, Strin
         client.performRequest(request);
     }
 
-    private static void load(RestClient client, TestsDataset dataset, Logger logger, IndexCreator indexCreator) throws IOException {
+    private static void load(RestClient client, TestDataset dataset, Logger logger, IndexCreator indexCreator) throws IOException {
         final String mappingName = "/" + dataset.mappingFileName;
         URL mapping = CsvTestsDataLoader.class.getResource(mappingName);
         if (mapping == null) {
             throw new IllegalArgumentException("Cannot find resource " + mappingName);
         }
-        final String dataName = "/" + dataset.dataFileName;
+        final String dataName = "/data/" + dataset.dataFileName;
         URL data = CsvTestsDataLoader.class.getResource(dataName);
         if (data == null) {
             throw new IllegalArgumentException("Cannot find resource " + dataName);
@@ -594,25 +605,32 @@ private static void forceMerge(RestClient client, Set<String> indices, Logger lo
         }
     }
 
-    public record TestsDataset(
+    public record MultiIndexTestDataset(String indexPattern, List<TestDataset> datasets) {
+        public static MultiIndexTestDataset of(TestDataset testsDataset) {
+            return new MultiIndexTestDataset(testsDataset.indexName, List.of(testsDataset));
+        }
+
+    }
+
+    public record TestDataset(
         String indexName,
         String mappingFileName,
         String dataFileName,
         String settingFileName,
         boolean allowSubFields,
-        Map<String, String> typeMapping,
+        @Nullable Map<String, String> typeMapping,
         boolean requiresInferenceEndpoint
     ) {
-        public TestsDataset(String indexName, String mappingFileName, String dataFileName) {
+        public TestDataset(String indexName, String mappingFileName, String dataFileName) {
             this(indexName, mappingFileName, dataFileName, null, true, null, false);
         }
 
-        public TestsDataset(String indexName) {
+        public TestDataset(String indexName) {
             this(indexName, "mapping-" + indexName + ".json", indexName + ".csv", null, true, null, false);
         }
 
-        public TestsDataset withIndex(String indexName) {
-            return new TestsDataset(
+        public TestDataset withIndex(String indexName) {
+            return new TestDataset(
                 indexName,
                 mappingFileName,
                 dataFileName,
@@ -623,8 +641,8 @@ public TestsDataset withIndex(String indexName) {
             );
         }
 
-        public TestsDataset withData(String dataFileName) {
-            return new TestsDataset(
+        public TestDataset withData(String dataFileName) {
+            return new TestDataset(
                 indexName,
                 mappingFileName,
                 dataFileName,
@@ -635,8 +653,8 @@ public TestsDataset withData(String dataFileName) {
             );
         }
 
-        public TestsDataset withSetting(String settingFileName) {
-            return new TestsDataset(
+        public TestDataset withSetting(String settingFileName) {
+            return new TestDataset(
                 indexName,
                 mappingFileName,
                 dataFileName,
@@ -647,8 +665,8 @@ public TestsDataset withSetting(String settingFileName) {
             );
         }
 
-        public TestsDataset noSubfields() {
-            return new TestsDataset(
+        public TestDataset noSubfields() {
+            return new TestDataset(
                 indexName,
                 mappingFileName,
                 dataFileName,
@@ -659,8 +677,8 @@ public TestsDataset noSubfields() {
             );
         }
 
-        public TestsDataset withTypeMapping(Map<String, String> typeMapping) {
-            return new TestsDataset(
+        public TestDataset withTypeMapping(Map<String, String> typeMapping) {
+            return new TestDataset(
                 indexName,
                 mappingFileName,
                 dataFileName,
@@ -671,8 +689,8 @@ public TestsDataset withTypeMapping(Map<String, String> typeMapping) {
             );
         }
 
-        public TestsDataset withInferenceEndpoint(boolean needsInference) {
-            return new TestsDataset(indexName, mappingFileName, dataFileName, settingFileName, allowSubFields, typeMapping, needsInference);
+        public TestDataset withInferenceEndpoint(boolean needsInference) {
+            return new TestDataset(indexName, mappingFileName, dataFileName, settingFileName, allowSubFields, typeMapping, needsInference);
         }
 
         private Settings readSettingsFile() throws IOException {
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
index 18ce9d7e3e057..7e25fb29fdb78 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
@@ -41,6 +41,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Range;
@@ -61,6 +62,7 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.parser.QueryParam;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
@@ -70,6 +72,7 @@
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
 import org.elasticsearch.xpack.esql.session.Configuration;
+import org.elasticsearch.xpack.esql.session.QueryBuilderResolver;
 import org.elasticsearch.xpack.esql.stats.Metrics;
 import org.elasticsearch.xpack.esql.stats.SearchStats;
 import org.elasticsearch.xpack.versionfield.Version;
@@ -349,8 +352,14 @@ public String toString() {
 
     public static final Configuration TEST_CFG = configuration(new QueryPragmas(Settings.EMPTY));
 
+    public static LogicalOptimizerContext unboundLogicalOptimizerContext() {
+        return new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG, FoldContext.small());
+    }
+
     public static final Verifier TEST_VERIFIER = new Verifier(new Metrics(new EsqlFunctionRegistry()), new XPackLicenseState(() -> 0L));
 
+    public static final QueryBuilderResolver MOCK_QUERY_BUILDER_RESOLVER = new MockQueryBuilderResolver();
+
     private EsqlTestUtils() {}
 
     public static Configuration configuration(QueryPragmas pragmas, String query) {
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/MockQueryBuilderResolver.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/MockQueryBuilderResolver.java
new file mode 100644
index 0000000000000..7af3a89108fc0
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/MockQueryBuilderResolver.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.session.QueryBuilderResolver;
+import org.elasticsearch.xpack.esql.session.Result;
+
+import java.util.function.BiConsumer;
+
+public class MockQueryBuilderResolver extends QueryBuilderResolver {
+    public MockQueryBuilderResolver() {
+        super(null, null, null, null);
+    }
+
+    @Override
+    public void resolveQueryBuilders(
+        LogicalPlan plan,
+        ActionListener<Result> listener,
+        BiConsumer<LogicalPlan, ActionListener<Result>> callback
+    ) {
+        callback.accept(plan, listener);
+    }
+}
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/books-settings.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/books-settings.json
new file mode 100644
index 0000000000000..b324c27b40653
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/books-settings.json
@@ -0,0 +1,5 @@
+{
+  "index": {
+    "number_of_shards": 3
+  }
+}
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/books.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/books.csv
deleted file mode 100644
index 1deefaa3c6475..0000000000000
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/books.csv
+++ /dev/null
@@ -1,80 +0,0 @@
-book_no:keyword,title:text,author:text,year:integer,publisher:text,ratings:float,description:text
-2924,A Gentle Creature and Other Stories: White Nights\, A Gentle Creature\, and The Dream of a Ridiculous Man (The World's Classics),[Fyodor Dostoevsky, Alan Myers, W. J. Leatherbarrow],2009,Oxford Paperbacks,4.00,In these stories Dostoevsky explores both the figure of the dreamer divorced from reality and also his own ambiguous attitude to utopianism\, themes central to many of his great novels. This new translation captures the power and lyricism of Dostoevsky's writing\, while the introduction examines the stories in relation to one another and to his novels.
-7670,A Middle English Reader and Vocabulary,[Kenneth Sisam, J. R. R. Tolkien],2011,Courier Corporation,4.33,This highly respected anthology of medieval English literature features poetry\, prose and popular tales from Arthurian legend and classical mythology. Includes notes on each extract\, appendices\, and an extensive glossary by J. R. R. Tolkien.
-7381,A Psychic in the Heartland: The Extraordinary Experiences of a Small Town Doctor,Bettilu Stein Faulkner,2003,Red Wheel/Weiser,4.50,The true story of a small-town doctor destined to live his life along two paths: one as a successful physician\, the other as a psychic with ever more interesting adventures. Experiencing a wide range of spiritual phenomena\, Dr. Riblet Hout learned about the connection between the healer and the healed\, our individual missions on earth\, free will\, and our relationship with God. He also paints a vivid picture of life on the other side as well as the moment of transition from physical life to afterlife.
-2883,A Summer of Faulkner: As I Lay Dying/The Sound and the Fury/Light in August (Oprah's Book Club),William Faulkner,2005,Vintage Books,3.89,Presents three novels\, including As I Lay Dying\, in which the Bundren family journeys across Mississippi to bury their mother\, The Sound and the Fury\, in which Caddy Compson's story is narrated by her three brothers\, and Light in August\, in which th
-4023,A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings,[Walter Scheps, Agnes Perkins, Charles Adolph Huttar, John Ronald Reuel Tolkien],1975,Open Court Publishing,4.67,The structure\, content\, and character of Tolkien's The Hobbit and The Lord of the Rings are dealt with in ten critical essays.
-2382,A Wizard of Earthsea (Earthsea Trilogy Ser.),Ursula K. Le Guin,1991,Atheneum Books for Young Readers,4.01,A boy grows to manhood while attempting to subdue the evil he unleashed on the world as an apprentice to the Master Wizard.
-7541,A Writer's Diary (Volume 1: 1873-1876),Fyodor Dostoevsky,1997,Northwestern University Press,4.50,Winner of the AATSEEL Outstanding Translation Award This is the first paperback edition of the complete collection of writings that has been called Dostoevsky's boldest experiment with literary form\, it is a uniquely encyclopedic forum of fictional and nonfictional genres. The Diary's radical format was matched by the extreme range of its contents. In a single frame it incorporated an astonishing variety of material: short stories\, humorous sketches\, reports on sensational crimes\, historical predictions\, portraits of famous people\, autobiographical pieces\, and plans for stories\, some of which were never written while others appeared in the Diary itself.
-7400,Anna Karenina: Television Tie-In Edition (Signet classics),[Leo Tolstoy, SBP Editors],2019,Samaira Book Publishers,4.45,The Russian novelist and moral philosopher Leo Tolstoy (1828-1910) ranks as one of the world s great writers\, and his 'War and Peace' has been called the greatest novel ever written. But during his long lifetime\, Tolstoy also wrote enough shorter works to fill many volumes. The message in all his stories is presented with such humour that the reader hardly realises that it is strongly didactic. These stories give a snapshot of Russia and its people in the late nineteenth century.
-4917,Autumn of the Patriarch,Gabriel Garcia Marquez,2014,Penguin UK,4.33,Gabriel Garcia Marquez\, winner of the 1982 Nobel Prize for Literature and author of One Hundred Years of Solitude\, explores the loneliness of power in Autumn of the Patriarch. 'Over the weekend the vultures got into the presidential palace by pecking through the screens on the balcony windows and the flapping of their wings stirred up the stagnant time inside' As the citizens of an unnamed Caribbean nation creep through dusty corridors in search of their tyrannical leader\, they cannot comprehend that the frail and withered man lying dead on the floor can be the self-styled General of the Universe. Their arrogant\, manically violent leader\, known for serving up traitors to dinner guests and drowning young children at sea\, can surely not die the humiliating death of a mere mortal? Tracing the demands of a man whose egocentric excesses mask the loneliness of isolation and whose lies have become so ingrained that they are indistinguishable from truth\, Marquez has created a fantastical portrait of despotism that rings with an air of reality. 'Delights with its quirky humanity and black humour and impresses by its total originality' Vogue 'Captures perfectly the moral squalor and political paralysis that enshrouds a society awaiting the death of a long-term dictator' Guardian 'Marquez writes in this lyrical\, magical language that no-one else can do' Salman Rushdie
-9896,Barn burning (A tale blazer book),William Faulkner,1979,Perfection Learning,3.50,Reprinted from Collected Stories of William Faulkner\, by permission of Random House\, Inc.
-9607,Beowolf: The monsters and the critics,John Ronald Reuel Tolkien,1997,HarperCollins UK,4.12,A collection of seven essays by J.R.R. Tolkien arising out of Tolkien's work in medieval literature
-1985,Brothers Karamazov,Fyodor Dostoevsky,2015,First Avenue Editions,5.00,Four brothers reunite in their hometown in Russia. The murder of their father forces the brothers to question their beliefs about each other\, religion\, and morality.
-2713,Collected Stories of William Faulkner,William Faulkner,1995,Vintage,4.53,A collection of short stories focuses on the people of rural Mississippi
-2464,Conversations with Kurt Vonnegut (Literary Conversations),Kurt Vonnegut,1988,Univ. Press of Mississippi,4.40,Gathers interviews with Vonnegut from each period of his career and offers a brief profile of his life and accomplishments
-8534,Crime and Punishment (Oxford World's Classics),Fyodor Dostoevsky,2017,Oxford University Press,4.38,'One death\, in exchange for thousands of lives - it's simple arithmetic!' A new translation of Dostoevsky's epic masterpiece\, Crime and Punishment (1866). The impoverished student Raskolnikov decides to free himself from debt by killing an old moneylender\, an act he sees as elevating himself above conventional morality. Like Napoleon he will assert his will and his crime will be justified by its elimination of 'vermin' for the sake of the greater good. But Raskolnikov is torn apart by fear\, guilt\, and a growing conscience under the influence of his love for Sonya. Meanwhile the police detective Porfiry is on his trial. It is a powerfully psychological novel\, in which the St Petersburg setting\, Dostoevsky's own circumstances\, and contemporary social problems all play their part.
-8605,Dead Souls,Nikolai Gogol,1997,Vintage,4.28,Chichikov\, an amusing and often confused schemer\, buys deceased serfs' names from landholders' poll tax lists hoping to mortgage them for profit
-6970,Domestic Goddesses,Edith Vonnegut,1998,Pomegranate,4.67,In this immensely charming and insightful book\, artist Edith Vonnegut takes issue with traditional art imagery in which women are shown as weak and helpless. Through twenty-seven of her own paintings interspersed with her text\, she poignantly -- and humorously -- illustrates her maxim that the lives of mothers and homemakers are filled with endless challenges and vital decisions that should be portrayed with the dignity they deserve. In Vonnegut's paintings\, one woman bravely blocks the sun from harming a child (Sun Block) while another vacuums the stairs with angelic figures singing her praises (Electrolux). In contrasting her own Domestic Goddesses with the diaphanous women of classical art (seven paintings by masters such as Titian and Botticelli are included)\, she 'expresses the importance of traditional roles of women so cleverly and with such joy that her message and images will be forever emblazoned on our collective psyche.
-4814,El Coronel No Tiene Quien Le Escriba / No One Writes to the Colonel (Spanish Edition),Gabriel Garcia Marquez,2005,Harper Collins,4.45,Written with compassionate realism and wit\, the stories in this mesmerizing collection depict the disparities of town and village life in South America\, of the frightfully poor and outrageously rich\, of memories and illusions\, and of lost opportunities and present joys.
-4636,FINAL WITNESS,Simon Tolkien,2004,Random House Digital\, Inc.,3.94,The murder of Lady Anne Robinson by two intruders causes a schism in the victim's family when her son convinces police that his father's beautiful personal assistant hired the killers\, while his father\, the British minister of defense\, refuses to believe his son and marries the accused. A first novel. Reprint.
-2936,Fellowship of the Ring 2ND Edition,John Ronald Reuel Tolkien,2008,HarperCollins UK,4.43,Sauron\, the Dark Lord\, has gathered to him all the Rings of Power - the means by which he intends to rule Middle-earth. All he lacks in his plans for dominion is the One Ring - the ring that rules them all - which has fallen into the hands of the hobbit\, Bilbo Baggins. In a sleepy village in the Shire\, young Frodo Baggins finds himself faced with an immense task\, as his elderly cousin Bilbo entrusts the Ring to his care. Frodo must leave his home and make a perilous journey across Middle-earth to the Cracks of Doom\, there to destroy the Ring and foil the Dark Lord in his evil purpose. JRR Tolkien's great work of imaginative fiction has been labelled both a heroic romance and a classic fantasy fiction. By turns comic and homely\, epic and diabolic\, the narrative moves through countless changes of scene and character in an imaginary world which is totally convincing in its detail.
-8956,GOD BLESS YOU MR. ROSEWATER : Or Pearls Before Swine,Kurt Vonnegut,1970,New York : Dell,4.00,A lawyer schemes to gain control of a large fortune by having the present claimant declared insane.
-6818,Hadji Murad,Leo Tolstoy,2022,Hachette UK,3.88,'How truth thickens and deepens when it migrates from didactic fable to the raw experience of a visceral awakening is one of the thrills of Tolstoy's stories' Sharon Cameron in her preface to Hadji Murad and Other Stories This\, the third volume of Tolstoy's shorter fiction concentrates on his later stories\, including one of his greatest\, 'Hadji Murad'. In the stark form of homily that shapes these later works\, life considered as one's own has no rational meaning. From the chain of events that follows in the wake of two schoolboys' deception in 'The Forged Coupon' to the disillusionment of the narrator in 'After the Ball' we see\, in Virginia Woolf's observation\, that Tolstoy puts at the centre of his writing one 'who gathers into himself all experience\, turns the world round between his fingers\, and never ceases to ask\, even as he enjoys it\, what is the meaning of it'. The riverrun edition reissues the translation of Louise and Aylmer Maude\, whose influential versions of Tolstoy first brought his work to a wide readership in English.
-3950,Hocus,Kurt Vonnegut,1997,Penguin,4.67,Tarkington College\, a small\, exclusive college in upstate New York\, is turned upside down when ten thousand prisoners from the maximum security prison across Lake Mohiga break out and head for the college
-5404,Intruder in the dust,William Faulkner,2011,Vintage,3.18,A classic Faulkner novel which explores the lives of a family of characters in the South. An aging black who has long refused to adopt the black's traditionally servile attitude is wrongfully accused of murdering a white man.
-5578,Intruder in the dust: A novel,William Faulkner,1991,Vintage,3.18,Dramatizes the events that surround the murder of a white man in a volatile Southern community
-6380,La hojarasca (Spanish Edition),Gabriel Garcia Marquez,1979,Harper Collins,3.75,Translated from the Spanish by Gregory Rabassa
-5335,Letters of J R R Tolkien,J.R.R. Tolkien,2014,HarperCollins,4.70,This collection will entertain all who appreciate the art of masterful letter writing. The Letters of J.R.R Tolkien sheds much light on Tolkien's creative genius and grand design for the creation of a whole new world: Middle-earth. Featuring a radically expanded index\, this volume provides a valuable research tool for all fans wishing to trace the evolution of THE HOBBIT and THE LORD OF THE RINGS.
-3870,My First 100 Words in Spanish/English (My First 100 Words Pull-Tab Book),Keith Faulkner,1998,Libros Para Ninos,4.50,Learning a foreign language has never been this much fun! Just pull the sturdy tabs and change the words under the pictures from English to Spanish and back again to English!
-4502,O'Brian's Bride,Colleen Faulkner,1995,Zebra Books,5.00,Abandoning her pampered English life to marry a man in the American colonies\, Elizabeth finds her new world shattered when her husband is killed in an accident\, leaving her in charge of a business on the untamed frontier. Original.
-7635,Oliphaunt (Beastly Verse),J. R. R. Tolkien,1989,Contemporary Books,2.50,A poem in which an elephant describes himself and his way of life. On board pages.
-3254,Pearl and Sir Orfeo,[John Ronald Reuel Tolkien, Christopher Tolkien],1995,Harpercollins Pub Limited,5.00,Three epic poems from 14th century England speak of life during the age of chivalry. Translated from medieval English.
-3677,Planet of Exile,Ursula K. Le Guin,1979,Orion,4.20,PLAYAWAY: An alliance between the powerful Tevars and the brown-skinned\, clairvoyant Farbons must take place if the two colonies are to withstand the fierce attack of the nomadic tribes from the north of the planet Eltanin.
-4289,Poems from the Hobbit,J R R Tolkien,1999,HarperCollins Publishers,4.00,A collection of J.R.R. Tolkien's Hobbit poems in a miniature hardback volume complete with illustrations by Tolkien himself. Far over misty mountains cold To dungeons deep and caverns old We must away ere break of day To seek the pale enchanted gold. J.R.R. Tolkien's acclaimed The Hobbit contains 12 poems which are themselves masterpieces of writing. This miniature book\, illustrated with 30 of Tolkien's own paintings and drawings from the book -- some quite rare and all in full colour -- includes all the poems\, plus Gollum's eight riddles in verse\, and will be a perfect keepsake for lovers of The Hobbit and of accomplished poetry.
-6151,Pop! Went Another Balloon: A Magical Counting Storybook (Magical Counting Storybooks),[Keith Faulkner, Rory Tyger],2003,Dutton Childrens Books,5.00,Toby the turtle goes from in-line skates to a motorcycle to a rocketship with a handful of balloons that pop\, one by one\, along the way.
-3535,Rainbow's End: A Magical Story and Moneybox,[Keith Faulkner, Beverlie Manson],2003,Barrons Juveniles,4.00,In this combination picture storybook and coin bank\, the unusual front cover shows an illustration from the story that's embellished with five transparent plastic windows. Opening the book\, children will find a story about a poor little ballerina who is crying because her dancing shoes are worn and she has no money to replace them. Full color. Consumable.
-8423,Raising Faithful Kids in a Fast-Paced World,Paul Faulkner,1995,Howard Publishing Company,5.00,To find help for struggling parents\, Dr. Paul Faulkner--renowned family counselor and popular speaker--interviewed 30 successful families who have managed to raise faithful kids while also maintaining demanding careers. The invaluable strategies and methods he gleaned are now available in this powerful book delivered in Dr. Faulkner's warm\, humorous style.
-1463,Realms of Tolkien: Images of Middle-earth,J. R. R. Tolkien,1997,HarperCollins Publishers,4.00,Twenty new and familiar Tolkien artists are represented in this fabulous volume\, breathing an extraordinary variety of life into 58 different scenes\, each of which is accompanied by appropriate passage from The Hobbit and The Lord of the Rings and The Silmarillion
-6323,Resurrection (The Penguin classics),Leo Tolstoy,2009,Penguin,3.25,Leo Tolstoy's last completed novel\, Resurrection is an intimate\, psychological tale of guilt\, anger and forgiveness Serving on the jury at a murder trial\, Prince Dmitri Nekhlyudov is devastated when he sees the prisoner - Katyusha\, a young maid he seduced and abandoned years before. As Dmitri faces the consequences of his actions\, he decides to give up his life of wealth and luxury to devote himself to rescuing Katyusha\, even if it means following her into exile in Siberia. But can a man truly find redemption by saving another person? Tolstoy's most controversial novel\, Resurrection (1899) is a scathing indictment of injustice\, corruption and hypocrisy at all levels of society. Creating a vast panorama of Russian life\, from peasants to aristocrats\, bureaucrats to convicts\, it reveals Tolstoy's magnificent storytelling powers. Anthony Briggs' superb new translation preserves Tolstoy's gripping realism and satirical humour. In his introduction\, Briggs discusses the true story behind Resurrection\, Tolstoy's political and religious reasons for writing the novel\, his gift for characterization and the compelling psychological portrait of Dmitri. This edition also includes a chronology\, notes and a summary of chapters. For more than seventy years\, Penguin has been the leading publisher of classic literature in the English-speaking world. With more than 1\,700 titles\, Penguin Classics represents a global bookshelf of the best works throughout history and across genres and disciplines. Readers trust the series to provide authoritative texts enhanced by introductions and notes by distinguished scholars and contemporary authors\, as well as up-to-date translations by award-winning translators.
-2714,Return of the King Being the Third Part of The Lord of the Rings,J. R. R. Tolkien,2012,HarperCollins,4.60,Concluding the story begun in The Hobbit\, this is the final part of Tolkien s epic masterpiece\, The Lord of the Rings\, featuring an exclusive cover image from the film\, the definitive text\, and a detailed map of Middle-earth. The armies of the Dark Lord Sauron are massing as his evil shadow spreads ever wider. Men\, Dwarves\, Elves and Ents unite forces to do battle agains the Dark. Meanwhile\, Frodo and Sam struggle further into Mordor in their heroic quest to destroy the One Ring. The devastating conclusion of J.R.R. Tolkien s classic tale of magic and adventure\, begun in The Fellowship of the Ring and The Two Towers\, features the definitive edition of the text and includes the Appendices and a revised Index in full. To celebrate the release of the first of Peter Jackson s two-part film adaptation of The Hobbit\, THE HOBBIT: AN UNEXPECTED JOURNEY\, this third part of The Lord of the Rings is available for a limited time with an exclusive cover image from Peter Jackson s award-winning trilogy.
-7350,Return of the Shadow,[John Ronald Reuel Tolkien, Christopher Tolkien],2000,Mariner Books,5.00,In this sixth volume of The History of Middle-earth the story reaches The Lord of the Rings. In The Return of the Shadow (an abandoned title for the first volume) Christopher Tolkien describes\, with full citation of the earliest notes\, outline plans\, and narrative drafts\, the intricate evolution of The Fellowship of the Ring and the gradual emergence of the conceptions that transformed what J.R.R. Tolkien for long believed would be a far shorter book\, 'a sequel to The Hobbit'. The enlargement of Bilbo's 'magic ring' into the supremely potent and dangerous Ruling Ring of the Dark Lord is traced and the precise moment is seen when\, in an astonishing and unforeseen leap in the earliest narrative\, a Black Rider first rode into the Shire\, his significance still unknown. The character of the hobbit called Trotter (afterwards Strider or Aragorn) is developed while his indentity remains an absolute puzzle\, and the suspicion only very slowly becomes certainty that he must after all be a Man. The hobbits\, Frodo's companions\, undergo intricate permutations of name and personality\, and other major figures appear in strange modes: a sinister Treebeard\, in league with the Enemy\, a ferocious and malevolent Farmer Maggot. The story in this book ends at the point where J.R.R. Tolkien halted in the story for a long time\, as the Company of the Ring\, still lacking Legolas and Gimli\, stood before the tomb of Balin in the Mines of Moria. The Return of the Shadow is illustrated with reproductions of the first maps and notable pages from the earliest manuscripts.
-6760,Roverandom,J. R. R. Tolkien,1999,Mariner Books,4.38,Rover\, a dog who has been turned into a toy dog encounters rival wizards and experiences various adventures on the moon with giant spiders\, dragon moths\, and the Great White Dragon. By the author of The Hobbit. Reprint.
-8873,Searoad: Chronicles of Klatsand,Ursula K. Le Guin,2004,Shambhala Publications,5.00,A series of interlinking tales and a novella by the author of the Earthsea trilogy portrays the triumphs and struggles of several generations of women who independently control Klatsand\, a small resort town on the Oregon coast. Reprint.
-2378,Selected Letters of Lucretia Coffin Mott (Women in American History),[Lucretia Mott, Holly Byers Ochoa, Carol Faulkner],2002,University of Illinois Press,5.00,Dedicated to reform of almost every kind - temperance\, peace\, equal rights\, woman suffrage\, nonresistance\, and the abolition of slavery - Mott viewed women's rights as only one element of a broad-based reform agenda for American society.
-1502,Selected Passages from Correspondence with Friends,Nikolai Vasilevich Gogol,2009,Vanderbilt University Press,4.00,Nikolai Gogol wrote some letters to his friends\, none of which were a nose of high rank. Many are reproduced here (the letters\, not noses).
-5996,Smith of Wooten Manor & Farmer Giles of Ham,John Ronald Reuel Tolkien,1969,Del Rey,4.91,Two bewitching fantasies by J.R.R. Tolkien\, beloved author of THE HOBBIT. In SMITH OF WOOTTON MAJOR\, Tolkien explores the gift of fantasy\, and what it means to the life and character of the man who receives it. And FARMER GILES OF HAM tells a delightfully ribald mock-heroic tale\, where a dragon who invades a town refuses to fight\, and a farmer is chosen to slay him.
-2301,Smith of Wootton Major & Farmer Giles of Ham,John Ronald Reuel Tolkien,1969,Del Rey,5.00,Two bewitching fantasies by J.R.R. Tolkien\, beloved author of THE HOBBIT. In SMITH OF WOOTTON MAJOR\, Tolkien explores the gift of fantasy\, and what it means to the life and character of the man who receives it. And FARMER GILES OF HAM tells a delightfully ribald mock-heroic tale\, where a dragon who invades a town refuses to fight\, and a farmer is chosen to slay him.
-2236,Steering the Craft,Ursula K. Le Guin,2015,Houghton Mifflin Harcourt,4.73,A revised and updated guide to the essentials of a writer's craft\, presented by a brilliant practitioner of the art Completely revised and rewritten to address the challenges and opportunities of the modern era\, this handbook is a short\, deceptively simple guide to the craft of writing. Le Guin lays out ten chapters that address the most fundamental components of narrative\, from the sound of language to sentence construction to point of view. Each chapter combines illustrative examples from the global canon with Le Guin's own witty commentary and an exercise that the writer can do solo or in a group. She also offers a comprehensive guide to working in writing groups\, both actual and online. Masterly and concise\, Steering the Craft deserves a place on every writer's shelf.
-4724,THE UNVANQUISHED,William Faulkner,2011,Vintage,3.50,Set in Mississippi during the Civil War and Reconstruction\, THE UNVANQUISHED focuses on the Sartoris family\, who\, with their code of personal responsibility and courage\, stand for the best of the Old South's traditions.
-5948,That We Are Gentle Creatures,Fyodor Dostoevsky,2009,OUP Oxford,4.33,In the stories in this volume Dostoevsky explores both the figure of the dreamer divorced from reality and also his own ambiguous attitude to utopianism\, themes central to many of his great novels. In White Nights the apparent idyll of the dreamer's romantic fantasies disguises profound loneliness and estrangement from 'living life'. Despite his sentimental friendship with Nastenka\, his final withdrawal into the world of the imagination anticipates the retreat into the 'underground' of many of Dostoevsky's later intellectual heroes. A Gentle Creature and The Dream of a Ridiculous Man show how such withdrawal from reality can end in spiritual desolation and moral indifference and how\, in Dostoevsky's view\, the tragedy of the alienated individual can be resolved only by the rediscovery of a sense of compassion and responsibility towards fellow human beings. This new translation captures the power and lyricism of Dostoevsky's writing\, while the introduction examines the stories in relation to one another and to his novels. ABOUT THE SERIES: For over 100 years Oxford World's Classics has made available the widest range of literature from around the globe. Each affordable volume reflects Oxford's commitment to scholarship\, providing the most accurate text plus a wealth of other valuable features\, including expert introductions by leading authorities\, helpful notes to clarify the text\, up-to-date bibliographies for further study\, and much more.
-1937,The Best Short Stories of Dostoevsky (Modern Library),Fyodor Dostoevsky,2012,Modern Library,4.33,This collection\, unique to the Modern Library\, gathers seven of Dostoevsky's key works and shows him to be equally adept at the short story as with the novel. Exploring many of the same themes as in his longer works\, these small masterpieces move from the tender and romantic White Nights\, an archetypal nineteenth-century morality tale of pathos and loss\, to the famous Notes from the Underground\, a story of guilt\, ineffectiveness\, and uncompromising cynicism\, and the first major work of existential literature. Among Dostoevsky's prototypical characters is Yemelyan in The Honest Thief\, whose tragedy turns on an inability to resist crime. Presented in chronological order\, in David Magarshack's celebrated translation\, this is the definitive edition of Dostoevsky's best stories.
-2776,The Devil and Other Stories (Oxford World's Classics),Leo Tolstoy,2003,OUP Oxford,5.00,'It is impossible to explain why Yevgeny chose Liza Annenskaya\, as it is always impossible to explain why a man chooses this and not that woman.' This collection of eleven stories spans virtually the whole of Tolstoy's creative life. While each is unique in form\, as a group they are representative of his style\, and touch on the central themes that surface in War and Peace and Anna Karenina. Stories as different as 'The Snowstorm'\, 'Lucerne'\, 'The Diary of a Madman'\, and 'The Devil' are grounded in autobiographical experience. They deal with journeys of self-discovery and the moral and religious questioning that characterizes Tolstoy's works of criticism and philosophy. 'Strider' and 'Father Sergy'\, as well as reflecting Tolstoy's own experiences\, also reveal profound psychological insights. These stories range over much of the Russian world of the nineteenth century\, from the nobility to the peasantry\, the military to the clergy\, from merchants and cobblers to a horse and a tree. Together they present a fascinating picture of Tolstoy's skill and artistry. ABOUT THE SERIES: For over 100 years Oxford World's Classics has made available the widest range of literature from around the globe. Each affordable volume reflects Oxford's commitment to scholarship\, providing the most accurate text plus a wealth of other valuable features\, including expert introductions by leading authorities\, helpful notes to clarify the text\, up-to-date bibliographies for further study\, and much more.
-4231,The Dispossessed,Ursula K. Le Guin,1974,Harpercollins,4.26,Frequently reissued with the same ISBN\, but with slightly differing bibliographical details.
-7480,The Hobbit,J. R. R. Tolkien,2012,Mariner Books,4.64,Celebrating 75 years of one of the world's most treasured classics with an all new trade paperback edition. Repackaged with new cover art. 500\,000 first printing.
-6405,The Hobbit or There and Back Again,J. R. R. Tolkien,2012,Mariner Books,4.63,Celebrating 75 years of one of the world's most treasured classics with an all new trade paperback edition. Repackaged with new cover art. 500\,000 first printing.
-2540,The Inspector General (Language - Russian) (Russian Edition),[Nicolai Gogol, Thomas Seltzer],2014,CreateSpace,3.50,The Inspector-General is a national institution. To place a purely literary valuation upon it and call it the greatest of Russian comedies would not convey the significance of its position either in Russian literature or in Russian life itself. There is no other single work in the modern literature of any language that carries with it the wealth of associations which the Inspector-General does to the educated Russian.
-2951,The Insulted and Injured,Fyodor Dostoevsky,2011,Wm. B. Eerdmans Publishing,4.00,The Insulted and Injured\, which came out in 1861\, was Fyodor Dostoevsky's first major work of fiction after his Siberian exile and the first of the long novels that made him famous. Set in nineteenth-century Petersburg\, this gripping novel features a vividly drawn set of characters - including Vanya (Dostoevsky's semi-autobiographical hero)\, Natasha (the woman he loves)\, and Alyosha (Natasha's aristocratic lover) - all suffering from the cruelly selfish machinations of Alyosha's father\, the dark and powerful Prince Valkovsky. Boris Jakim's fresh English-language rendering of this gem in the Doestoevsky canon is both more colorful and more accurate than any earlier translation. --from back cover.
-2130,The J. R. R. Tolkien Audio Collection,[John Ronald Reuel Tolkien, Christopher Tolkien],2002,HarperCollins Publishers,4.89,For generations\, J R R Tolkien's words have brought to thrilling life a world of hobbits\, magic\, and historic myth\, woken from its foggy slumber within our minds. Here\, he tells the tales in his own voice.
-9801,The Karamazov Brothers (Oxford World's Classics),Fyodor Dostoevsky,2008,Oxford University Press,4.40,A remarkable work showing the author's power to depict Russian character and his understanding of human nature. Driven by intense\, uncontrollable emotions of rage and revenge\, the four Karamazov brothers all become involved in the brutal murder of their despicable father.
-5469,The Lays of Beleriand,[John Ronald Reuel Tolkien, Christopher Tolkien],2002,Harpercollins Pub Limited,4.42,The third volume that contains the early myths and legends which led to the writing of Tolkien's epic tale of war\, The Silmarillion. This\, the third volume of The History of Middle-earth\, gives us a priviledged insight into the creation of the mythology of Middle-earth\, through the alliterative verse tales of two of the most crucial stories in Tolkien's world -- those of Turien and Luthien. The first of the poems is the unpublished Lay of The Children of Hurin\, narrating on a grand scale the tragedy of Turin Turambar. The second is the moving Lay of Leithian\, the chief source of the tale of Beren and Luthien in The Silmarillion\, telling of the Quest of the Silmaril and the encounter with Morgoth in his subterranean fortress. Accompanying the poems are commentaries on the evolution of the history of the Elder Days. Also included is the notable criticism of The Lay of The Leithian by CS Lewis\, who read the poem in 1929.
-2675,The Lord of the Rings - Boxed Set,J.R.R. Tolkien,2012,HarperCollins,4.56,This beautiful gift edition of The Hobbit\, J.R.R. Tolkien's classic prelude to his Lord of the Rings trilogy\, features cover art\, illustrations\, and watercolor paintings by the artist Alan Lee. Bilbo Baggins is a hobbit who enjoys a comfortable\, unambitious life\, rarely traveling any farther than his pantry or cellar. But his contentment is disturbed when the wizard Gandalf and a company of dwarves arrive on his doorstep one day to whisk him away on an adventure. They have launched a plot to raid the treasure hoard guarded by Smaug the Magnificent\, a large and very dangerous dragon. Bilbo reluctantly joins their quest\, unaware that on his journey to the Lonely Mountain he will encounter both a magic ring and a frightening creature known as Gollum. Written for J.R.R. Tolkien's own children\, The Hobbit has sold many millions of copies worldwide and established itself as a modern classic.
-7140,The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1),[J. R. R. Tolkien, Alan Lee],2002,HarperSport,4.75,A selection of stunning poster paintings from the celebrated Tolkien artist Alan Lee - the man behind many of the striking images from The Lord of The Rings movie. The 50 paintings contained within the centenary edition of The Lord of the Rings in 1992 have themselves become classics and Alan Lee's interpretations are hailed as the most faithful to Tolkien's own vision. This new poster collection\, a perfect complement to volume one\, reproduces six more of the most popular paintings from the book in a format suitable either for hanging as posters or mounting and framing.
-5127,The Overcoat, Nikolai Gogol,1992,Courier Corporation,3.75,Four short stories include a satirical tale of Russian bureaucrats and a portrayal of an elderly couple living in the secluded countryside.
-8875,The Two Towers,John Ronald Reuel Tolkien,2007,HarperCollins UK,4.64,The second volume in The Lord of the Rings\, This title is also available as a film.
-4977,The Unvanquished,William Faulkner,2011,Vintage,3.50,Set in Mississippi during the Civil War and Reconstruction\, THE UNVANQUISHED focuses on the Sartoris family\, who\, with their code of personal responsibility and courage\, stand for the best of the Old South's traditions.
-4382,The Wolves of Witchmaker,Carole Guinane,2001,iUniverse,5.00,Polly Lavender is mysteriously lured onto Witchmaker's grounds along with her best friends Tony Rico\, Gracie Reene\, and Zeus\, the wolf they rescued as a pup. The three must quickly learn to master the art of magic because they have been chosen to lead Witchmaker Prep against a threat that has grim consequences.
-7912,The Word For World is Forest,Ursula K. Le Guin,2015,Gollancz,4.22,When the inhabitants of a peaceful world are conquered by the bloodthirsty yumens\, their existence is irrevocably altered. Forced into servitude\, the Athsheans find themselves at the mercy of their brutal masters. Desperation causes the Athsheans\, led by Selver\, to retaliate against their captors\, abandoning their strictures against violence. But in defending their lives\, they have endangered the very foundations of their society. For every blow against the invaders is a blow to the humanity of the Athsheans. And once the killing starts\, there is no turning back.
-1211,The brothers Karamazov,Fyodor Dostoevsky,2003,Bantam Classics,1.00,In 1880 Dostoevsky completed The Brothers Karamazov\, the literary effort for which he had been preparing all his life. Compelling\, profound\, complex\, it is the story of a patricide and of the four sons who each had a motive for murder: Dmitry\, the sensualist\, Ivan\, the intellectual\, Alyosha\, the mystic\, and twisted\, cunning Smerdyakov\, the bastard child. Frequently lurid\, nightmarish\, always brilliant\, the novel plunges the reader into a sordid love triangle\, a pathological obsession\, and a gripping courtroom drama. But throughout the whole\, Dostoevsky searhes for the truth--about man\, about life\, about the existence of God. A terrifying answer to man's eternal questions\, this monumental work remains the crowning achievement of perhaps the finest novelist of all time. From the Paperback edition.
-8086,The grand inquisitor (Milestones of thought),Fyodor Dostoevsky,1981,A&C Black,4.09,Dostoevsky's portrayal of the Catholic Church during the Inquisition is a plea for the power of pure faith\, and a critique of the tyrannies of institutionalized religion. This is an except from the Brothers Karamazov which stands alone as a statement of philiosophy and a warning about the surrender of freedom for the sake of comfort.
-8077,The unvanquished,William Faulkner,2011,Vintage,4.00,Set in Mississippi during the Civil War and Reconstruction\, THE UNVANQUISHED focuses on the Sartoris family\, who\, with their code of personal responsibility and courage\, stand for the best of the Old South's traditions.
-8480,The wind's twelve quarters: Short stories,Ursula K. Le Guin,2017,HarperCollins,5.00,The recipient of numerous literary prizes\, including the National Book Award\, the Kafka Award\, and the Pushcart Prize\, Ursula K. Le Guin is renowned for her lyrical writing\, rich characters\, and diverse worlds. The Wind's Twelve Quarters collects seventeen powerful stories\, each with an introduction by the author\, ranging from fantasy to intriguing scientific concepts\, from medieval settings to the future. Including an insightful foreword by Le Guin\, describing her experience\, her inspirations\, and her approach to writing\, this stunning collection explores human values\, relationships\, and survival\, and showcases the myriad talents of one of the most provocative writers of our time.
-2847,To Love A Dark Stranger (Lovegram Historical Romance),Colleen Faulkner,1997,Zebra Books,5.00,Bestselling author Colleen Faulkner's tumultuous saga of royal intrigue and forbidden desire sweeps from the magnificent estates of the aristocracy to the shadowy streets of London to King Charles II's glittering Restoration court.
-3293,Universe by Design,Danny Faulkner,2004,New Leaf Publishing Group,4.25,Views the stars and planets from a creationist standpoint\, addresses common misconceptions and difficulties about relativity and cosmology\, and discusses problems with the big bang theory with many analogies\, examples\, diagrams\, and illustrations. Original.
-5327,War and Peace,Leo Tolstoy,2016,Lulu.com,3.84,Covering the period from the French invasion under Napoleon into Russia. Although not covering solely the war itself\, the serialized novel does cover the effects the war had on Russian society from the common person right up to the Tsar himself. The book starts to move more to a philosophical consideration on war and peace near the end making the book as a whole an important piece of literature.
-4536,War and Peace (Signet Classics),[Leo Tolstoy, Pat Conroy, John Hockenberry],2012,Signet Classics,4.75,Presents the classical epic of the Napoleonic Wars and their effects on four Russian families.
-9032,War and Peace: A Novel (6 Volumes),Tolstoy Leo,2013,Hardpress Publishing,3.81,Unlike some other reproductions of classic texts (1) We have not used OCR(Optical Character Recognition)\, as this leads to bad quality books with introduced typos. (2) In books where there are images such as portraits\, maps\, sketches etc We have endeavoured to keep the quality of these images\, so they represent accurately the original artefact. Although occasionally there may be certain imperfections with these old texts\, we feel they deserve to be made available for future generations to enjoy.
-5119,William Faulkner,William Faulkner,2011,Vintage,4.00,This invaluable volume\, which has been republished to commemorate the one-hundredth anniversary of Faulkner's birth\, contains some of the greatest short fiction by a writer who defined the course of American literature. Its forty-five stories fall into three categories: those not included in Faulkner's earlier collections\, previously unpublished short fiction\, and stories that were later expanded into such novels as The Unvanquished\, The Hamlet\, and Go Down\, Moses. With its Introduction and extensive notes by the biographer Joseph Blotner\, Uncollected Stories of William Faulkner is an essential addition to its author's canon--as well as a book of some of the most haunting\, harrowing\, and atmospheric short fiction written in the twentieth century.
-8615,Winter notes on summer impressions,Fyodor Dostoevsky,2018,Alma Books,4.75,In June 1862\, Dostoevsky left Petersburg on his first excursion to Western Europe. Ostensibly making the trip to consult Western specialists about his epilepsy\, he also wished to see first-hand the source of the Western ideas he believed were corrupting Russia. Over the course of his journey he visited a number of major cities\, including Berlin\, Paris\, London\, Florence\, Milan and Vienna.His record of the trip\, Winter Notes on Summer Impressions - first published in the February 1863 issue of Vremya\, the periodical he edited - is the chrysalis out of which many elements of his later masterpieces developed.
-6478,Woman-The Full Story: A Dynamic Celebration of Freedoms,Michele Guinness,2003,Zondervan,5.00,What does it mean to be a woman today? What have women inherited from their radical\, risk-taking sisters of the past? And how does God view this half of humanity? Michele Guinness invites us on an adventure of discovery\, exploring the biblical texts\, the annals of history and the experiences of women today in search of the challenges and achievements\, failures and joys\, of women throughout the ages.
-8678,Worlds of Exile and Illusion: Three Complete Novels of the Hainish Series in One Volume--Rocannon's World\, Planet of Exile\, City of Illusions,Ursula K. Le Guin,2016,Orb Books,4.41,Worlds of Exile and Illusion contains three novels in the Hainish Series from Ursula K. Le Guin\, one of the greatest science fiction writers and many times the winner of the Hugo and Nebula Awards. Her career as a novelist was launched by the three novels contained here. These books\, Rocannon's World\, Planet of Exile\, and City of Illusions\, are set in the same universe as Le Guin's groundbreaking classic\, The Left Hand of Darkness. At the Publisher's request\, this title is being sold without Digital Rights Management Software (DRM) applied.
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/bucket.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/bucket.csv-spec
index b29c489910f65..8cfde2bb9bde7 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/bucket.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/bucket.csv-spec
@@ -145,6 +145,24 @@ AVG(salary):double | bucket:date
 // end::bucket_in_agg-result[]
 ;
 
+bucketWithOffset#[skip:-8.13.99, reason:BUCKET renamed in 8.14]
+// tag::bucketWithOffset[]
+FROM employees
+| STATS dates = MV_SORT(VALUES(birth_date)) BY b = BUCKET(birth_date + 1 HOUR, 1 YEAR) - 1 HOUR
+| EVAL d_count = MV_COUNT(dates)
+| SORT d_count, b
+| LIMIT 3
+// end::bucketWithOffset[]
+;
+
+// tag::bucketWithOffset-result[]
+dates:date                                                                                              |b:date                  |d_count:integer
+1965-01-03T00:00:00.000Z                                                                                |1964-12-31T23:00:00.000Z|1
+[1955-01-21T00:00:00.000Z, 1955-08-20T00:00:00.000Z, 1955-08-28T00:00:00.000Z, 1955-10-04T00:00:00.000Z]|1954-12-31T23:00:00.000Z|4
+[1957-04-04T00:00:00.000Z, 1957-05-23T00:00:00.000Z, 1957-05-25T00:00:00.000Z, 1957-12-03T00:00:00.000Z]|1956-12-31T23:00:00.000Z|4
+// end::bucketWithOffset-result[]
+;
+
 docsBucketMonth#[skip:-8.13.99, reason:BUCKET renamed in 8.14]
 //tag::docsBucketMonth[]
 FROM employees
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/addresses.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/addresses.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/addresses.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/addresses.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ages.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/ages.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/ages.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/ages.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/airport_city_boundaries.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/airport_city_boundaries.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/airport_city_boundaries.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/airport_city_boundaries.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/airports.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/airports.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/airports.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/airports.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/airports_mp.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/airports_mp.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/airports_mp.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/airports_mp.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/airports_web.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/airports_web.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/airports_web.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/airports_web.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/alerts.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/alerts.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/alerts.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/alerts.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/apps.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/apps.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/apps.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/apps.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/books.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/books.csv
new file mode 100644
index 0000000000000..1cb01687e6511
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/books.csv
@@ -0,0 +1,80 @@
+_id:keyword,book_no:keyword,title:text,author:text,year:integer,publisher:text,ratings:float,description:text
+0,2924,A Gentle Creature and Other Stories: White Nights\, A Gentle Creature\, and The Dream of a Ridiculous Man (The World's Classics),[Fyodor Dostoevsky, Alan Myers, W. J. Leatherbarrow],2009,Oxford Paperbacks,4.00,In these stories Dostoevsky explores both the figure of the dreamer divorced from reality and also his own ambiguous attitude to utopianism\, themes central to many of his great novels. This new translation captures the power and lyricism of Dostoevsky's writing\, while the introduction examines the stories in relation to one another and to his novels.
+1,7670,A Middle English Reader and Vocabulary,[Kenneth Sisam, J. R. R. Tolkien],2011,Courier Corporation,4.33,This highly respected anthology of medieval English literature features poetry\, prose and popular tales from Arthurian legend and classical mythology. Includes notes on each extract\, appendices\, and an extensive glossary by J. R. R. Tolkien.
+2,7381,A Psychic in the Heartland: The Extraordinary Experiences of a Small Town Doctor,Bettilu Stein Faulkner,2003,Red Wheel/Weiser,4.50,The true story of a small-town doctor destined to live his life along two paths: one as a successful physician\, the other as a psychic with ever more interesting adventures. Experiencing a wide range of spiritual phenomena\, Dr. Riblet Hout learned about the connection between the healer and the healed\, our individual missions on earth\, free will\, and our relationship with God. He also paints a vivid picture of life on the other side as well as the moment of transition from physical life to afterlife.
+3,2883,A Summer of Faulkner: As I Lay Dying/The Sound and the Fury/Light in August (Oprah's Book Club),William Faulkner,2005,Vintage Books,3.89,Presents three novels\, including As I Lay Dying\, in which the Bundren family journeys across Mississippi to bury their mother\, The Sound and the Fury\, in which Caddy Compson's story is narrated by her three brothers\, and Light in August\, in which th
+4,4023,A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings,[Walter Scheps, Agnes Perkins, Charles Adolph Huttar, John Ronald Reuel Tolkien],1975,Open Court Publishing,4.67,The structure\, content\, and character of Tolkien's The Hobbit and The Lord of the Rings are dealt with in ten critical essays.
+5,2382,A Wizard of Earthsea (Earthsea Trilogy Ser.),Ursula K. Le Guin,1991,Atheneum Books for Young Readers,4.01,A boy grows to manhood while attempting to subdue the evil he unleashed on the world as an apprentice to the Master Wizard.
+6,7541,A Writer's Diary (Volume 1: 1873-1876),Fyodor Dostoevsky,1997,Northwestern University Press,4.50,Winner of the AATSEEL Outstanding Translation Award This is the first paperback edition of the complete collection of writings that has been called Dostoevsky's boldest experiment with literary form\, it is a uniquely encyclopedic forum of fictional and nonfictional genres. The Diary's radical format was matched by the extreme range of its contents. In a single frame it incorporated an astonishing variety of material: short stories\, humorous sketches\, reports on sensational crimes\, historical predictions\, portraits of famous people\, autobiographical pieces\, and plans for stories\, some of which were never written while others appeared in the Diary itself.
+7,7400,Anna Karenina: Television Tie-In Edition (Signet classics),[Leo Tolstoy, SBP Editors],2019,Samaira Book Publishers,4.45,The Russian novelist and moral philosopher Leo Tolstoy (1828-1910) ranks as one of the world s great writers\, and his 'War and Peace' has been called the greatest novel ever written. But during his long lifetime\, Tolstoy also wrote enough shorter works to fill many volumes. The message in all his stories is presented with such humour that the reader hardly realises that it is strongly didactic. These stories give a snapshot of Russia and its people in the late nineteenth century.
+8,4917,Autumn of the Patriarch,Gabriel Garcia Marquez,2014,Penguin UK,4.33,Gabriel Garcia Marquez\, winner of the 1982 Nobel Prize for Literature and author of One Hundred Years of Solitude\, explores the loneliness of power in Autumn of the Patriarch. 'Over the weekend the vultures got into the presidential palace by pecking through the screens on the balcony windows and the flapping of their wings stirred up the stagnant time inside' As the citizens of an unnamed Caribbean nation creep through dusty corridors in search of their tyrannical leader\, they cannot comprehend that the frail and withered man lying dead on the floor can be the self-styled General of the Universe. Their arrogant\, manically violent leader\, known for serving up traitors to dinner guests and drowning young children at sea\, can surely not die the humiliating death of a mere mortal? Tracing the demands of a man whose egocentric excesses mask the loneliness of isolation and whose lies have become so ingrained that they are indistinguishable from truth\, Marquez has created a fantastical portrait of despotism that rings with an air of reality. 'Delights with its quirky humanity and black humour and impresses by its total originality' Vogue 'Captures perfectly the moral squalor and political paralysis that enshrouds a society awaiting the death of a long-term dictator' Guardian 'Marquez writes in this lyrical\, magical language that no-one else can do' Salman Rushdie
+9,9896,Barn burning (A tale blazer book),William Faulkner,1979,Perfection Learning,3.50,Reprinted from Collected Stories of William Faulkner\, by permission of Random House\, Inc.
+10,9607,Beowolf: The monsters and the critics,John Ronald Reuel Tolkien,1997,HarperCollins UK,4.12,A collection of seven essays by J.R.R. Tolkien arising out of Tolkien's work in medieval literature
+11,1985,Brothers Karamazov,Fyodor Dostoevsky,2015,First Avenue Editions,5.00,Four brothers reunite in their hometown in Russia. The murder of their father forces the brothers to question their beliefs about each other\, religion\, and morality.
+12,2713,Collected Stories of William Faulkner,William Faulkner,1995,Vintage,4.53,A collection of short stories focuses on the people of rural Mississippi
+13,2464,Conversations with Kurt Vonnegut (Literary Conversations),Kurt Vonnegut,1988,Univ. Press of Mississippi,4.40,Gathers interviews with Vonnegut from each period of his career and offers a brief profile of his life and accomplishments
+14,8534,Crime and Punishment (Oxford World's Classics),Fyodor Dostoevsky,2017,Oxford University Press,4.38,'One death\, in exchange for thousands of lives - it's simple arithmetic!' A new translation of Dostoevsky's epic masterpiece\, Crime and Punishment (1866). The impoverished student Raskolnikov decides to free himself from debt by killing an old moneylender\, an act he sees as elevating himself above conventional morality. Like Napoleon he will assert his will and his crime will be justified by its elimination of 'vermin' for the sake of the greater good. But Raskolnikov is torn apart by fear\, guilt\, and a growing conscience under the influence of his love for Sonya. Meanwhile the police detective Porfiry is on his trial. It is a powerfully psychological novel\, in which the St Petersburg setting\, Dostoevsky's own circumstances\, and contemporary social problems all play their part.
+15,8605,Dead Souls,Nikolai Gogol,1997,Vintage,4.28,Chichikov\, an amusing and often confused schemer\, buys deceased serfs' names from landholders' poll tax lists hoping to mortgage them for profit
+16,6970,Domestic Goddesses,Edith Vonnegut,1998,Pomegranate,4.67,In this immensely charming and insightful book\, artist Edith Vonnegut takes issue with traditional art imagery in which women are shown as weak and helpless. Through twenty-seven of her own paintings interspersed with her text\, she poignantly -- and humorously -- illustrates her maxim that the lives of mothers and homemakers are filled with endless challenges and vital decisions that should be portrayed with the dignity they deserve. In Vonnegut's paintings\, one woman bravely blocks the sun from harming a child (Sun Block) while another vacuums the stairs with angelic figures singing her praises (Electrolux). In contrasting her own Domestic Goddesses with the diaphanous women of classical art (seven paintings by masters such as Titian and Botticelli are included)\, she 'expresses the importance of traditional roles of women so cleverly and with such joy that her message and images will be forever emblazoned on our collective psyche.
+17,4814,El Coronel No Tiene Quien Le Escriba / No One Writes to the Colonel (Spanish Edition),Gabriel Garcia Marquez,2005,Harper Collins,4.45,Written with compassionate realism and wit\, the stories in this mesmerizing collection depict the disparities of town and village life in South America\, of the frightfully poor and outrageously rich\, of memories and illusions\, and of lost opportunities and present joys.
+18,4636,FINAL WITNESS,Simon Tolkien,2004,Random House Digital\, Inc.,3.94,The murder of Lady Anne Robinson by two intruders causes a schism in the victim's family when her son convinces police that his father's beautiful personal assistant hired the killers\, while his father\, the British minister of defense\, refuses to believe his son and marries the accused. A first novel. Reprint.
+19,2936,Fellowship of the Ring 2ND Edition,John Ronald Reuel Tolkien,2008,HarperCollins UK,4.43,Sauron\, the Dark Lord\, has gathered to him all the Rings of Power - the means by which he intends to rule Middle-earth. All he lacks in his plans for dominion is the One Ring - the ring that rules them all - which has fallen into the hands of the hobbit\, Bilbo Baggins. In a sleepy village in the Shire\, young Frodo Baggins finds himself faced with an immense task\, as his elderly cousin Bilbo entrusts the Ring to his care. Frodo must leave his home and make a perilous journey across Middle-earth to the Cracks of Doom\, there to destroy the Ring and foil the Dark Lord in his evil purpose. JRR Tolkien's great work of imaginative fiction has been labelled both a heroic romance and a classic fantasy fiction. By turns comic and homely\, epic and diabolic\, the narrative moves through countless changes of scene and character in an imaginary world which is totally convincing in its detail.
+20,8956,GOD BLESS YOU MR. ROSEWATER : Or Pearls Before Swine,Kurt Vonnegut,1970,New York : Dell,4.00,A lawyer schemes to gain control of a large fortune by having the present claimant declared insane.
+21,6818,Hadji Murad,Leo Tolstoy,2022,Hachette UK,3.88,'How truth thickens and deepens when it migrates from didactic fable to the raw experience of a visceral awakening is one of the thrills of Tolstoy's stories' Sharon Cameron in her preface to Hadji Murad and Other Stories This\, the third volume of Tolstoy's shorter fiction concentrates on his later stories\, including one of his greatest\, 'Hadji Murad'. In the stark form of homily that shapes these later works\, life considered as one's own has no rational meaning. From the chain of events that follows in the wake of two schoolboys' deception in 'The Forged Coupon' to the disillusionment of the narrator in 'After the Ball' we see\, in Virginia Woolf's observation\, that Tolstoy puts at the centre of his writing one 'who gathers into himself all experience\, turns the world round between his fingers\, and never ceases to ask\, even as he enjoys it\, what is the meaning of it'. The riverrun edition reissues the translation of Louise and Aylmer Maude\, whose influential versions of Tolstoy first brought his work to a wide readership in English.
+22,3950,Hocus,Kurt Vonnegut,1997,Penguin,4.67,Tarkington College\, a small\, exclusive college in upstate New York\, is turned upside down when ten thousand prisoners from the maximum security prison across Lake Mohiga break out and head for the college
+23,5404,Intruder in the dust,William Faulkner,2011,Vintage,3.18,A classic Faulkner novel which explores the lives of a family of characters in the South. An aging black who has long refused to adopt the black's traditionally servile attitude is wrongfully accused of murdering a white man.
+24,5578,Intruder in the dust: A novel,William Faulkner,1991,Vintage,3.18,Dramatizes the events that surround the murder of a white man in a volatile Southern community
+25,6380,La hojarasca (Spanish Edition),Gabriel Garcia Marquez,1979,Harper Collins,3.75,Translated from the Spanish by Gregory Rabassa
+26,5335,Letters of J R R Tolkien,J.R.R. Tolkien,2014,HarperCollins,4.70,This collection will entertain all who appreciate the art of masterful letter writing. The Letters of J.R.R Tolkien sheds much light on Tolkien's creative genius and grand design for the creation of a whole new world: Middle-earth. Featuring a radically expanded index\, this volume provides a valuable research tool for all fans wishing to trace the evolution of THE HOBBIT and THE LORD OF THE RINGS.
+27,3870,My First 100 Words in Spanish/English (My First 100 Words Pull-Tab Book),Keith Faulkner,1998,Libros Para Ninos,4.50,Learning a foreign language has never been this much fun! Just pull the sturdy tabs and change the words under the pictures from English to Spanish and back again to English!
+28,4502,O'Brian's Bride,Colleen Faulkner,1995,Zebra Books,5.00,Abandoning her pampered English life to marry a man in the American colonies\, Elizabeth finds her new world shattered when her husband is killed in an accident\, leaving her in charge of a business on the untamed frontier. Original.
+29,7635,Oliphaunt (Beastly Verse),J. R. R. Tolkien,1989,Contemporary Books,2.50,A poem in which an elephant describes himself and his way of life. On board pages.
+30,3254,Pearl and Sir Orfeo,[John Ronald Reuel Tolkien, Christopher Tolkien],1995,Harpercollins Pub Limited,5.00,Three epic poems from 14th century England speak of life during the age of chivalry. Translated from medieval English.
+31,3677,Planet of Exile,Ursula K. Le Guin,1979,Orion,4.20,PLAYAWAY: An alliance between the powerful Tevars and the brown-skinned\, clairvoyant Farbons must take place if the two colonies are to withstand the fierce attack of the nomadic tribes from the north of the planet Eltanin.
+32,4289,Poems from the Hobbit,J R R Tolkien,1999,HarperCollins Publishers,4.00,A collection of J.R.R. Tolkien's Hobbit poems in a miniature hardback volume complete with illustrations by Tolkien himself. Far over misty mountains cold To dungeons deep and caverns old We must away ere break of day To seek the pale enchanted gold. J.R.R. Tolkien's acclaimed The Hobbit contains 12 poems which are themselves masterpieces of writing. This miniature book\, illustrated with 30 of Tolkien's own paintings and drawings from the book -- some quite rare and all in full colour -- includes all the poems\, plus Gollum's eight riddles in verse\, and will be a perfect keepsake for lovers of The Hobbit and of accomplished poetry.
+33,6151,Pop! Went Another Balloon: A Magical Counting Storybook (Magical Counting Storybooks),[Keith Faulkner, Rory Tyger],2003,Dutton Childrens Books,5.00,Toby the turtle goes from in-line skates to a motorcycle to a rocketship with a handful of balloons that pop\, one by one\, along the way.
+34,3535,Rainbow's End: A Magical Story and Moneybox,[Keith Faulkner, Beverlie Manson],2003,Barrons Juveniles,4.00,In this combination picture storybook and coin bank\, the unusual front cover shows an illustration from the story that's embellished with five transparent plastic windows. Opening the book\, children will find a story about a poor little ballerina who is crying because her dancing shoes are worn and she has no money to replace them. Full color. Consumable.
+35,8423,Raising Faithful Kids in a Fast-Paced World,Paul Faulkner,1995,Howard Publishing Company,5.00,To find help for struggling parents\, Dr. Paul Faulkner--renowned family counselor and popular speaker--interviewed 30 successful families who have managed to raise faithful kids while also maintaining demanding careers. The invaluable strategies and methods he gleaned are now available in this powerful book delivered in Dr. Faulkner's warm\, humorous style.
+36,1463,Realms of Tolkien: Images of Middle-earth,J. R. R. Tolkien,1997,HarperCollins Publishers,4.00,Twenty new and familiar Tolkien artists are represented in this fabulous volume\, breathing an extraordinary variety of life into 58 different scenes\, each of which is accompanied by appropriate passage from The Hobbit and The Lord of the Rings and The Silmarillion
+37,6323,Resurrection (The Penguin classics),Leo Tolstoy,2009,Penguin,3.25,Leo Tolstoy's last completed novel\, Resurrection is an intimate\, psychological tale of guilt\, anger and forgiveness Serving on the jury at a murder trial\, Prince Dmitri Nekhlyudov is devastated when he sees the prisoner - Katyusha\, a young maid he seduced and abandoned years before. As Dmitri faces the consequences of his actions\, he decides to give up his life of wealth and luxury to devote himself to rescuing Katyusha\, even if it means following her into exile in Siberia. But can a man truly find redemption by saving another person? Tolstoy's most controversial novel\, Resurrection (1899) is a scathing indictment of injustice\, corruption and hypocrisy at all levels of society. Creating a vast panorama of Russian life\, from peasants to aristocrats\, bureaucrats to convicts\, it reveals Tolstoy's magnificent storytelling powers. Anthony Briggs' superb new translation preserves Tolstoy's gripping realism and satirical humour. In his introduction\, Briggs discusses the true story behind Resurrection\, Tolstoy's political and religious reasons for writing the novel\, his gift for characterization and the compelling psychological portrait of Dmitri. This edition also includes a chronology\, notes and a summary of chapters. For more than seventy years\, Penguin has been the leading publisher of classic literature in the English-speaking world. With more than 1\,700 titles\, Penguin Classics represents a global bookshelf of the best works throughout history and across genres and disciplines. Readers trust the series to provide authoritative texts enhanced by introductions and notes by distinguished scholars and contemporary authors\, as well as up-to-date translations by award-winning translators.
+38,2714,Return of the King Being the Third Part of The Lord of the Rings,J. R. R. Tolkien,2012,HarperCollins,4.60,Concluding the story begun in The Hobbit\, this is the final part of Tolkien s epic masterpiece\, The Lord of the Rings\, featuring an exclusive cover image from the film\, the definitive text\, and a detailed map of Middle-earth. The armies of the Dark Lord Sauron are massing as his evil shadow spreads ever wider. Men\, Dwarves\, Elves and Ents unite forces to do battle agains the Dark. Meanwhile\, Frodo and Sam struggle further into Mordor in their heroic quest to destroy the One Ring. The devastating conclusion of J.R.R. Tolkien s classic tale of magic and adventure\, begun in The Fellowship of the Ring and The Two Towers\, features the definitive edition of the text and includes the Appendices and a revised Index in full. To celebrate the release of the first of Peter Jackson s two-part film adaptation of The Hobbit\, THE HOBBIT: AN UNEXPECTED JOURNEY\, this third part of The Lord of the Rings is available for a limited time with an exclusive cover image from Peter Jackson s award-winning trilogy.
+39,7350,Return of the Shadow,[John Ronald Reuel Tolkien, Christopher Tolkien],2000,Mariner Books,5.00,In this sixth volume of The History of Middle-earth the story reaches The Lord of the Rings. In The Return of the Shadow (an abandoned title for the first volume) Christopher Tolkien describes\, with full citation of the earliest notes\, outline plans\, and narrative drafts\, the intricate evolution of The Fellowship of the Ring and the gradual emergence of the conceptions that transformed what J.R.R. Tolkien for long believed would be a far shorter book\, 'a sequel to The Hobbit'. The enlargement of Bilbo's 'magic ring' into the supremely potent and dangerous Ruling Ring of the Dark Lord is traced and the precise moment is seen when\, in an astonishing and unforeseen leap in the earliest narrative\, a Black Rider first rode into the Shire\, his significance still unknown. The character of the hobbit called Trotter (afterwards Strider or Aragorn) is developed while his indentity remains an absolute puzzle\, and the suspicion only very slowly becomes certainty that he must after all be a Man. The hobbits\, Frodo's companions\, undergo intricate permutations of name and personality\, and other major figures appear in strange modes: a sinister Treebeard\, in league with the Enemy\, a ferocious and malevolent Farmer Maggot. The story in this book ends at the point where J.R.R. Tolkien halted in the story for a long time\, as the Company of the Ring\, still lacking Legolas and Gimli\, stood before the tomb of Balin in the Mines of Moria. The Return of the Shadow is illustrated with reproductions of the first maps and notable pages from the earliest manuscripts.
+40,6760,Roverandom,J. R. R. Tolkien,1999,Mariner Books,4.38,Rover\, a dog who has been turned into a toy dog encounters rival wizards and experiences various adventures on the moon with giant spiders\, dragon moths\, and the Great White Dragon. By the author of The Hobbit. Reprint.
+41,8873,Searoad: Chronicles of Klatsand,Ursula K. Le Guin,2004,Shambhala Publications,5.00,A series of interlinking tales and a novella by the author of the Earthsea trilogy portrays the triumphs and struggles of several generations of women who independently control Klatsand\, a small resort town on the Oregon coast. Reprint.
+42,2378,Selected Letters of Lucretia Coffin Mott (Women in American History),[Lucretia Mott, Holly Byers Ochoa, Carol Faulkner],2002,University of Illinois Press,5.00,Dedicated to reform of almost every kind - temperance\, peace\, equal rights\, woman suffrage\, nonresistance\, and the abolition of slavery - Mott viewed women's rights as only one element of a broad-based reform agenda for American society.
+43,1502,Selected Passages from Correspondence with Friends,Nikolai Vasilevich Gogol,2009,Vanderbilt University Press,4.00,Nikolai Gogol wrote some letters to his friends\, none of which were a nose of high rank. Many are reproduced here (the letters\, not noses).
+44,5996,Smith of Wooten Manor & Farmer Giles of Ham,John Ronald Reuel Tolkien,1969,Del Rey,4.91,Two bewitching fantasies by J.R.R. Tolkien\, beloved author of THE HOBBIT. In SMITH OF WOOTTON MAJOR\, Tolkien explores the gift of fantasy\, and what it means to the life and character of the man who receives it. And FARMER GILES OF HAM tells a delightfully ribald mock-heroic tale\, where a dragon who invades a town refuses to fight\, and a farmer is chosen to slay him.
+45,2301,Smith of Wootton Major & Farmer Giles of Ham,John Ronald Reuel Tolkien,1969,Del Rey,5.00,Two bewitching fantasies by J.R.R. Tolkien\, beloved author of THE HOBBIT. In SMITH OF WOOTTON MAJOR\, Tolkien explores the gift of fantasy\, and what it means to the life and character of the man who receives it. And FARMER GILES OF HAM tells a delightfully ribald mock-heroic tale\, where a dragon who invades a town refuses to fight\, and a farmer is chosen to slay him.
+46,2236,Steering the Craft,Ursula K. Le Guin,2015,Houghton Mifflin Harcourt,4.73,A revised and updated guide to the essentials of a writer's craft\, presented by a brilliant practitioner of the art Completely revised and rewritten to address the challenges and opportunities of the modern era\, this handbook is a short\, deceptively simple guide to the craft of writing. Le Guin lays out ten chapters that address the most fundamental components of narrative\, from the sound of language to sentence construction to point of view. Each chapter combines illustrative examples from the global canon with Le Guin's own witty commentary and an exercise that the writer can do solo or in a group. She also offers a comprehensive guide to working in writing groups\, both actual and online. Masterly and concise\, Steering the Craft deserves a place on every writer's shelf.
+47,4724,THE UNVANQUISHED,William Faulkner,2011,Vintage,3.50,Set in Mississippi during the Civil War and Reconstruction\, THE UNVANQUISHED focuses on the Sartoris family\, who\, with their code of personal responsibility and courage\, stand for the best of the Old South's traditions.
+48,5948,That We Are Gentle Creatures,Fyodor Dostoevsky,2009,OUP Oxford,4.33,In the stories in this volume Dostoevsky explores both the figure of the dreamer divorced from reality and also his own ambiguous attitude to utopianism\, themes central to many of his great novels. In White Nights the apparent idyll of the dreamer's romantic fantasies disguises profound loneliness and estrangement from 'living life'. Despite his sentimental friendship with Nastenka\, his final withdrawal into the world of the imagination anticipates the retreat into the 'underground' of many of Dostoevsky's later intellectual heroes. A Gentle Creature and The Dream of a Ridiculous Man show how such withdrawal from reality can end in spiritual desolation and moral indifference and how\, in Dostoevsky's view\, the tragedy of the alienated individual can be resolved only by the rediscovery of a sense of compassion and responsibility towards fellow human beings. This new translation captures the power and lyricism of Dostoevsky's writing\, while the introduction examines the stories in relation to one another and to his novels. ABOUT THE SERIES: For over 100 years Oxford World's Classics has made available the widest range of literature from around the globe. Each affordable volume reflects Oxford's commitment to scholarship\, providing the most accurate text plus a wealth of other valuable features\, including expert introductions by leading authorities\, helpful notes to clarify the text\, up-to-date bibliographies for further study\, and much more.
+49,1937,The Best Short Stories of Dostoevsky (Modern Library),Fyodor Dostoevsky,2012,Modern Library,4.33,This collection\, unique to the Modern Library\, gathers seven of Dostoevsky's key works and shows him to be equally adept at the short story as with the novel. Exploring many of the same themes as in his longer works\, these small masterpieces move from the tender and romantic White Nights\, an archetypal nineteenth-century morality tale of pathos and loss\, to the famous Notes from the Underground\, a story of guilt\, ineffectiveness\, and uncompromising cynicism\, and the first major work of existential literature. Among Dostoevsky's prototypical characters is Yemelyan in The Honest Thief\, whose tragedy turns on an inability to resist crime. Presented in chronological order\, in David Magarshack's celebrated translation\, this is the definitive edition of Dostoevsky's best stories.
+50,2776,The Devil and Other Stories (Oxford World's Classics),Leo Tolstoy,2003,OUP Oxford,5.00,'It is impossible to explain why Yevgeny chose Liza Annenskaya\, as it is always impossible to explain why a man chooses this and not that woman.' This collection of eleven stories spans virtually the whole of Tolstoy's creative life. While each is unique in form\, as a group they are representative of his style\, and touch on the central themes that surface in War and Peace and Anna Karenina. Stories as different as 'The Snowstorm'\, 'Lucerne'\, 'The Diary of a Madman'\, and 'The Devil' are grounded in autobiographical experience. They deal with journeys of self-discovery and the moral and religious questioning that characterizes Tolstoy's works of criticism and philosophy. 'Strider' and 'Father Sergy'\, as well as reflecting Tolstoy's own experiences\, also reveal profound psychological insights. These stories range over much of the Russian world of the nineteenth century\, from the nobility to the peasantry\, the military to the clergy\, from merchants and cobblers to a horse and a tree. Together they present a fascinating picture of Tolstoy's skill and artistry. ABOUT THE SERIES: For over 100 years Oxford World's Classics has made available the widest range of literature from around the globe. Each affordable volume reflects Oxford's commitment to scholarship\, providing the most accurate text plus a wealth of other valuable features\, including expert introductions by leading authorities\, helpful notes to clarify the text\, up-to-date bibliographies for further study\, and much more.
+51,4231,The Dispossessed,Ursula K. Le Guin,1974,Harpercollins,4.26,Frequently reissued with the same ISBN\, but with slightly differing bibliographical details.
+52,7480,The Hobbit,J. R. R. Tolkien,2012,Mariner Books,4.64,Celebrating 75 years of one of the world's most treasured classics with an all new trade paperback edition. Repackaged with new cover art. 500\,000 first printing.
+53,6405,The Hobbit or There and Back Again,J. R. R. Tolkien,2012,Mariner Books,4.63,Celebrating 75 years of one of the world's most treasured classics with an all new trade paperback edition. Repackaged with new cover art. 500\,000 first printing.
+54,2540,The Inspector General (Language - Russian) (Russian Edition),[Nicolai Gogol, Thomas Seltzer],2014,CreateSpace,3.50,The Inspector-General is a national institution. To place a purely literary valuation upon it and call it the greatest of Russian comedies would not convey the significance of its position either in Russian literature or in Russian life itself. There is no other single work in the modern literature of any language that carries with it the wealth of associations which the Inspector-General does to the educated Russian.
+55,2951,The Insulted and Injured,Fyodor Dostoevsky,2011,Wm. B. Eerdmans Publishing,4.00,The Insulted and Injured\, which came out in 1861\, was Fyodor Dostoevsky's first major work of fiction after his Siberian exile and the first of the long novels that made him famous. Set in nineteenth-century Petersburg\, this gripping novel features a vividly drawn set of characters - including Vanya (Dostoevsky's semi-autobiographical hero)\, Natasha (the woman he loves)\, and Alyosha (Natasha's aristocratic lover) - all suffering from the cruelly selfish machinations of Alyosha's father\, the dark and powerful Prince Valkovsky. Boris Jakim's fresh English-language rendering of this gem in the Doestoevsky canon is both more colorful and more accurate than any earlier translation. --from back cover.
+56,2130,The J. R. R. Tolkien Audio Collection,[John Ronald Reuel Tolkien, Christopher Tolkien],2002,HarperCollins Publishers,4.89,For generations\, J R R Tolkien's words have brought to thrilling life a world of hobbits\, magic\, and historic myth\, woken from its foggy slumber within our minds. Here\, he tells the tales in his own voice.
+57,9801,The Karamazov Brothers (Oxford World's Classics),Fyodor Dostoevsky,2008,Oxford University Press,4.40,A remarkable work showing the author's power to depict Russian character and his understanding of human nature. Driven by intense\, uncontrollable emotions of rage and revenge\, the four Karamazov brothers all become involved in the brutal murder of their despicable father.
+58,5469,The Lays of Beleriand,[John Ronald Reuel Tolkien, Christopher Tolkien],2002,Harpercollins Pub Limited,4.42,The third volume that contains the early myths and legends which led to the writing of Tolkien's epic tale of war\, The Silmarillion. This\, the third volume of The History of Middle-earth\, gives us a priviledged insight into the creation of the mythology of Middle-earth\, through the alliterative verse tales of two of the most crucial stories in Tolkien's world -- those of Turien and Luthien. The first of the poems is the unpublished Lay of The Children of Hurin\, narrating on a grand scale the tragedy of Turin Turambar. The second is the moving Lay of Leithian\, the chief source of the tale of Beren and Luthien in The Silmarillion\, telling of the Quest of the Silmaril and the encounter with Morgoth in his subterranean fortress. Accompanying the poems are commentaries on the evolution of the history of the Elder Days. Also included is the notable criticism of The Lay of The Leithian by CS Lewis\, who read the poem in 1929.
+59,2675,The Lord of the Rings - Boxed Set,J.R.R. Tolkien,2012,HarperCollins,4.56,This beautiful gift edition of The Hobbit\, J.R.R. Tolkien's classic prelude to his Lord of the Rings trilogy\, features cover art\, illustrations\, and watercolor paintings by the artist Alan Lee. Bilbo Baggins is a hobbit who enjoys a comfortable\, unambitious life\, rarely traveling any farther than his pantry or cellar. But his contentment is disturbed when the wizard Gandalf and a company of dwarves arrive on his doorstep one day to whisk him away on an adventure. They have launched a plot to raid the treasure hoard guarded by Smaug the Magnificent\, a large and very dangerous dragon. Bilbo reluctantly joins their quest\, unaware that on his journey to the Lonely Mountain he will encounter both a magic ring and a frightening creature known as Gollum. Written for J.R.R. Tolkien's own children\, The Hobbit has sold many millions of copies worldwide and established itself as a modern classic.
+60,7140,The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1),[J. R. R. Tolkien, Alan Lee],2002,HarperSport,4.75,A selection of stunning poster paintings from the celebrated Tolkien artist Alan Lee - the man behind many of the striking images from The Lord of The Rings movie. The 50 paintings contained within the centenary edition of The Lord of the Rings in 1992 have themselves become classics and Alan Lee's interpretations are hailed as the most faithful to Tolkien's own vision. This new poster collection\, a perfect complement to volume one\, reproduces six more of the most popular paintings from the book in a format suitable either for hanging as posters or mounting and framing.
+61,5127,The Overcoat, Nikolai Gogol,1992,Courier Corporation,3.75,Four short stories include a satirical tale of Russian bureaucrats and a portrayal of an elderly couple living in the secluded countryside.
+62,8875,The Two Towers,John Ronald Reuel Tolkien,2007,HarperCollins UK,4.64,The second volume in The Lord of the Rings\, This title is also available as a film.
+63,4977,The Unvanquished,William Faulkner,2011,Vintage,3.50,Set in Mississippi during the Civil War and Reconstruction\, THE UNVANQUISHED focuses on the Sartoris family\, who\, with their code of personal responsibility and courage\, stand for the best of the Old South's traditions.
+64,4382,The Wolves of Witchmaker,Carole Guinane,2001,iUniverse,5.00,Polly Lavender is mysteriously lured onto Witchmaker's grounds along with her best friends Tony Rico\, Gracie Reene\, and Zeus\, the wolf they rescued as a pup. The three must quickly learn to master the art of magic because they have been chosen to lead Witchmaker Prep against a threat that has grim consequences.
+65,7912,The Word For World is Forest,Ursula K. Le Guin,2015,Gollancz,4.22,When the inhabitants of a peaceful world are conquered by the bloodthirsty yumens\, their existence is irrevocably altered. Forced into servitude\, the Athsheans find themselves at the mercy of their brutal masters. Desperation causes the Athsheans\, led by Selver\, to retaliate against their captors\, abandoning their strictures against violence. But in defending their lives\, they have endangered the very foundations of their society. For every blow against the invaders is a blow to the humanity of the Athsheans. And once the killing starts\, there is no turning back.
+66,1211,The brothers Karamazov,Fyodor Dostoevsky,2003,Bantam Classics,1.00,In 1880 Dostoevsky completed The Brothers Karamazov\, the literary effort for which he had been preparing all his life. Compelling\, profound\, complex\, it is the story of a patricide and of the four sons who each had a motive for murder: Dmitry\, the sensualist\, Ivan\, the intellectual\, Alyosha\, the mystic\, and twisted\, cunning Smerdyakov\, the bastard child. Frequently lurid\, nightmarish\, always brilliant\, the novel plunges the reader into a sordid love triangle\, a pathological obsession\, and a gripping courtroom drama. But throughout the whole\, Dostoevsky searhes for the truth--about man\, about life\, about the existence of God. A terrifying answer to man's eternal questions\, this monumental work remains the crowning achievement of perhaps the finest novelist of all time. From the Paperback edition.
+67,8086,The grand inquisitor (Milestones of thought),Fyodor Dostoevsky,1981,A&C Black,4.09,Dostoevsky's portrayal of the Catholic Church during the Inquisition is a plea for the power of pure faith\, and a critique of the tyrannies of institutionalized religion. This is an except from the Brothers Karamazov which stands alone as a statement of philiosophy and a warning about the surrender of freedom for the sake of comfort.
+68,8077,The unvanquished,William Faulkner,2011,Vintage,4.00,Set in Mississippi during the Civil War and Reconstruction\, THE UNVANQUISHED focuses on the Sartoris family\, who\, with their code of personal responsibility and courage\, stand for the best of the Old South's traditions.
+69,8480,The wind's twelve quarters: Short stories,Ursula K. Le Guin,2017,HarperCollins,5.00,The recipient of numerous literary prizes\, including the National Book Award\, the Kafka Award\, and the Pushcart Prize\, Ursula K. Le Guin is renowned for her lyrical writing\, rich characters\, and diverse worlds. The Wind's Twelve Quarters collects seventeen powerful stories\, each with an introduction by the author\, ranging from fantasy to intriguing scientific concepts\, from medieval settings to the future. Including an insightful foreword by Le Guin\, describing her experience\, her inspirations\, and her approach to writing\, this stunning collection explores human values\, relationships\, and survival\, and showcases the myriad talents of one of the most provocative writers of our time.
+70,2847,To Love A Dark Stranger (Lovegram Historical Romance),Colleen Faulkner,1997,Zebra Books,5.00,Bestselling author Colleen Faulkner's tumultuous saga of royal intrigue and forbidden desire sweeps from the magnificent estates of the aristocracy to the shadowy streets of London to King Charles II's glittering Restoration court.
+71,3293,Universe by Design,Danny Faulkner,2004,New Leaf Publishing Group,4.25,Views the stars and planets from a creationist standpoint\, addresses common misconceptions and difficulties about relativity and cosmology\, and discusses problems with the big bang theory with many analogies\, examples\, diagrams\, and illustrations. Original.
+72,5327,War and Peace,Leo Tolstoy,2016,Lulu.com,3.84,Covering the period from the French invasion under Napoleon into Russia. Although not covering solely the war itself\, the serialized novel does cover the effects the war had on Russian society from the common person right up to the Tsar himself. The book starts to move more to a philosophical consideration on war and peace near the end making the book as a whole an important piece of literature.
+73,4536,War and Peace (Signet Classics),[Leo Tolstoy, Pat Conroy, John Hockenberry],2012,Signet Classics,4.75,Presents the classical epic of the Napoleonic Wars and their effects on four Russian families.
+74,9032,War and Peace: A Novel (6 Volumes),Tolstoy Leo,2013,Hardpress Publishing,3.81,Unlike some other reproductions of classic texts (1) We have not used OCR(Optical Character Recognition)\, as this leads to bad quality books with introduced typos. (2) In books where there are images such as portraits\, maps\, sketches etc We have endeavoured to keep the quality of these images\, so they represent accurately the original artefact. Although occasionally there may be certain imperfections with these old texts\, we feel they deserve to be made available for future generations to enjoy.
+75,5119,William Faulkner,William Faulkner,2011,Vintage,4.00,This invaluable volume\, which has been republished to commemorate the one-hundredth anniversary of Faulkner's birth\, contains some of the greatest short fiction by a writer who defined the course of American literature. Its forty-five stories fall into three categories: those not included in Faulkner's earlier collections\, previously unpublished short fiction\, and stories that were later expanded into such novels as The Unvanquished\, The Hamlet\, and Go Down\, Moses. With its Introduction and extensive notes by the biographer Joseph Blotner\, Uncollected Stories of William Faulkner is an essential addition to its author's canon--as well as a book of some of the most haunting\, harrowing\, and atmospheric short fiction written in the twentieth century.
+76,8615,Winter notes on summer impressions,Fyodor Dostoevsky,2018,Alma Books,4.75,In June 1862\, Dostoevsky left Petersburg on his first excursion to Western Europe. Ostensibly making the trip to consult Western specialists about his epilepsy\, he also wished to see first-hand the source of the Western ideas he believed were corrupting Russia. Over the course of his journey he visited a number of major cities\, including Berlin\, Paris\, London\, Florence\, Milan and Vienna.His record of the trip\, Winter Notes on Summer Impressions - first published in the February 1863 issue of Vremya\, the periodical he edited - is the chrysalis out of which many elements of his later masterpieces developed.
+77,6478,Woman-The Full Story: A Dynamic Celebration of Freedoms,Michele Guinness,2003,Zondervan,5.00,What does it mean to be a woman today? What have women inherited from their radical\, risk-taking sisters of the past? And how does God view this half of humanity? Michele Guinness invites us on an adventure of discovery\, exploring the biblical texts\, the annals of history and the experiences of women today in search of the challenges and achievements\, failures and joys\, of women throughout the ages.
+78,8678,Worlds of Exile and Illusion: Three Complete Novels of the Hainish Series in One Volume--Rocannon's World\, Planet of Exile\, City of Illusions,Ursula K. Le Guin,2016,Orb Books,4.41,Worlds of Exile and Illusion contains three novels in the Hainish Series from Ursula K. Le Guin\, one of the greatest science fiction writers and many times the winner of the Hugo and Nebula Awards. Her career as a novelist was launched by the three novels contained here. These books\, Rocannon's World\, Planet of Exile\, and City of Illusions\, are set in the same universe as Le Guin's groundbreaking classic\, The Left Hand of Darkness. At the Publisher's request\, this title is being sold without Digital Rights Management Software (DRM) applied.
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/cartesian_multipolygons.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/cartesian_multipolygons.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/cartesian_multipolygons.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/cartesian_multipolygons.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/client_cidr.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/client_cidr.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/client_cidr.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/client_cidr.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/clientips.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/clientips.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/clientips.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/clientips.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/countries_bbox.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/countries_bbox.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/countries_bbox.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/countries_bbox.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/countries_bbox_web.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/countries_bbox_web.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/countries_bbox_web.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/countries_bbox_web.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/decades.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/decades.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/decades.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/decades.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/distances.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/distances.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/distances.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/distances.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/employees.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/employees.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/employees_incompatible.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/employees_incompatible.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/heights.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/heights.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/heights.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/heights.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hosts.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/hosts.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/hosts.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/hosts.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/k8s.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/k8s.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/languages.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/languages.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/languages_nested_fields.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/languages_nested_fields.csv
new file mode 100644
index 0000000000000..154125cf49304
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/languages_nested_fields.csv
@@ -0,0 +1,5 @@
+_id:integer,language.id:integer,language.name:text,language.code:keyword
+1,1,English,EN
+2,2,French,FR
+3,3,Spanish,ES
+4,4,German,DE
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_non_unique_key.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/languages_non_unique_key.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_non_unique_key.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/languages_non_unique_key.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/message_types.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/message_types.csv
similarity index 70%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/message_types.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/message_types.csv
index 8e00485771445..bb4b58046b843 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/message_types.csv
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/message_types.csv
@@ -4,3 +4,5 @@ Disconnected,Disconnected
 Connected to 10.1.0.1,Success
 Connected to 10.1.0.2,Success
 Connected to 10.1.0.3,Success
+Production environment,Production
+Development environment,Development
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/missing_ip_sample_data.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/missing_ip_sample_data.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/missing_ip_sample_data.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/missing_ip_sample_data.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_geometries.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/multivalue_geometries.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_geometries.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/multivalue_geometries.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/multivalue_points.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/multivalue_points.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_sample_data.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_sample_data.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_sample_data.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_sample_data.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/sample_data.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/sample_data.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data_ts_long.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/sample_data_ts_long.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data_ts_long.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/sample_data_ts_long.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data_ts_nanos.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/sample_data_ts_nanos.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data_ts_nanos.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/sample_data_ts_nanos.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/semantic_text.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/semantic_text.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ul_logs.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/ul_logs.csv
similarity index 100%
rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/ul_logs.csv
rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/ul_logs.csv
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec
index 734e2ef5e475e..bb46c9d31f745 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec
@@ -494,6 +494,66 @@ b:datetime
 null
 ;
 
+evalDateParseWithTimezone
+required_capability: date_parse_tz
+row s = "12/Jul/2022:10:24:10 +0900" | eval d = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s);
+warningRegex:Date format \[dd/MMM/yyyy:HH:mm:ss Z\] contains textual field specifiers that could change in JDK 23.*
+
+s:keyword                  | d:datetime
+12/Jul/2022:10:24:10 +0900 | 2022-07-12T01:24:10.000Z
+;
+
+evalDateParseWithTimezoneCrossingDayBoundary
+required_capability: date_parse_tz
+row s = "12/Jul/2022:08:24:10 +0900" | eval d = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s);
+warningRegex:Date format \[dd/MMM/yyyy:HH:mm:ss Z\] contains textual field specifiers that could change in JDK 23.*
+
+s:keyword                  | d:datetime
+12/Jul/2022:08:24:10 +0900 | 2022-07-11T23:24:10.000Z
+;
+
+evalDateParseWithTimezone2
+required_capability: date_parse_tz
+row s1 = "12/Jul/2022:10:24:10 +0900", s2 = "2022/12/07 09:24:10 +0800"
+| eval d1 = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s1), d2 = date_parse("yyyy/dd/MM HH:mm:ss Z", s2)
+| eval eq = d1 == d2
+| keep d1, eq
+;
+warningRegex:Date format \[dd/MMM/yyyy:HH:mm:ss Z\] contains textual field specifiers that could change in JDK 23.*
+warningRegex:Date format \[yyyy/dd/MM HH:mm:ss Z\] contains textual field specifiers that could change in JDK 23.*
+
+d1:datetime              | eq:boolean
+2022-07-12T01:24:10.000Z | true
+;
+
+evalDateParseWithAndWithoutTimezone
+required_capability: date_parse_tz
+row s = "2022/12/07 09:24:10", format="yyyy/dd/MM HH:mm:ss"
+| eval no_tz = date_parse(format, s)
+| eval with_tz = date_parse(concat(format, " Z"), concat(s, " +0900"))
+| keep s, no_tz, with_tz
+;
+warningRegex:Date format \[yyyy/dd/MM HH:mm:ss Z\] contains textual field specifiers that could change in JDK 23.*
+
+s:keyword           | no_tz:datetime           | with_tz:datetime
+2022/12/07 09:24:10 | 2022-07-12T09:24:10.000Z | 2022-07-12T00:24:10.000Z
+;
+
+evalDateParseWithOtherTimezoneSpecifiers
+required_capability: date_parse_tz
+row s = "2022/12/07 09:24:10", format="yyyy/dd/MM HH:mm:ss"
+| eval with_tz1 = date_parse(concat(format, " Z"), concat(s, " +0900"))
+| eval with_tz2 = date_parse(concat(format, " x"), concat(s, " +09"))
+| eval with_tz3 = date_parse(concat(format, " X"), concat(s, " +0900"))
+| eval with_tz4 = date_parse(concat(format, " O"), concat(s, " GMT+9"))
+| keep s, with_tz*
+;
+warningRegex:Date format \[yyyy/dd/MM HH:mm:ss .\] contains textual field specifiers that could change in JDK 23.*
+
+s:keyword           | with_tz1:datetime        | with_tz2:datetime        | with_tz3:datetime        | with_tz4:datetime        
+2022/12/07 09:24:10 | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z
+;
+
 evalDateParseDynamic
 from employees | where emp_no == 10039 or emp_no == 10040 | sort emp_no 
 | eval birth_date_string = date_format("yyyy-MM-dd", birth_date)
@@ -1390,3 +1450,47 @@ FROM employees
 cnt:long
 19
 ;
+
+implicit casting strings to dates for IN operator
+FROM employees
+| WHERE birth_date IN ("1953-04-20", "1958-10-31")
+| KEEP emp_no, first_name;
+ignoreOrder:true
+
+emp_no:integer | first_name:keyword
+10006          | Anneke 
+10025          | Prasadram 
+;
+
+IN operator with null in list, finds match
+
+FROM employees
+| EVAL x = NULL
+| WHERE birth_date IN (TO_DATETIME("1958-02-19T00:00:00Z"), x)
+| KEEP birth_date, first_name;
+
+birth_date:datetime      | first_name:keyword
+1958-02-19T00:00:00.000Z | Saniya     
+;
+
+IN operator with null in list, doesn't find match
+
+FROM employees
+| EVAL x = NULL
+| WHERE birth_date IN (TO_DATETIME("1900-02-19T00:00:00Z"), x)
+| KEEP birth_date, first_name;
+
+birth_date:datetime      | first_name:keyword
+;
+
+IN operator with null in list, doesn't find match, EVAL to check value
+
+FROM employees
+| EVAL x = NULL
+| EVAL result = birth_date IN (TO_DATETIME("1900-02-19T00:00:00Z"), x)
+| LIMIT 1
+| KEEP result;
+
+result:boolean
+null
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec
index f4b5c98d596ae..71a65f059528a 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec
@@ -216,6 +216,137 @@ millis:date              | nanos:date_nanos               | num:long
 2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000
 ;
 
+implicit casting to nanos, date only
+required_capability: date_nanos_type
+required_capability: date_nanos_implicit_casting
+
+FROM date_nanos 
+| WHERE MV_MIN(nanos) > "2023-10-23" 
+| SORT nanos DESC 
+| KEEP millis, nanos;
+
+millis:date             | nanos:date_nanos              
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
+2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z
+2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z
+2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z
+2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
+;
+
+implicit casting to nanos, date only, equality test
+required_capability: date_nanos_type
+required_capability: date_nanos_implicit_casting
+
+FROM date_nanos 
+| WHERE MV_MIN(nanos) == "2023-10-23" 
+| SORT nanos DESC 
+| KEEP millis, nanos;
+
+millis:date             | nanos:date_nanos              
+;
+
+
+implicit casting to nanos, date plus time to seconds
+required_capability: date_nanos_type
+required_capability: date_nanos_implicit_casting
+
+FROM date_nanos 
+| WHERE MV_MIN(nanos) > "2023-10-23T00:00:00" 
+| SORT nanos DESC 
+| KEEP millis, nanos;
+
+millis:date             | nanos:date_nanos              
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
+2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z
+2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z
+2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z
+2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
+;
+
+implicit casting to nanos, date plus time to seconds, equality test
+required_capability: date_nanos_type
+required_capability: date_nanos_implicit_casting
+
+FROM date_nanos 
+| WHERE MV_MIN(nanos) == "2023-10-23T12:27:28" 
+| SORT nanos DESC 
+| KEEP millis, nanos;
+
+millis:date             | nanos:date_nanos              
+;
+
+implicit casting to nanos, date plus time to millis
+required_capability: date_nanos_type
+required_capability: date_nanos_implicit_casting
+
+FROM date_nanos 
+| WHERE MV_MIN(nanos) > "2023-10-23T00:00:00.000" 
+| SORT nanos DESC 
+| KEEP millis, nanos;
+
+millis:date             | nanos:date_nanos              
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
+2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z
+2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z
+2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z
+2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
+;
+
+implicit casting to nanos, date plus time to millis, equality test
+required_capability: date_nanos_type
+required_capability: date_nanos_implicit_casting
+
+FROM date_nanos 
+| WHERE MV_MIN(nanos) == "2023-10-23T12:27:28.948" 
+| SORT nanos DESC 
+| KEEP millis, nanos;
+
+millis:date             | nanos:date_nanos              
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
+;
+
+implicit casting to nanos, date plus time to nanos
+required_capability: date_nanos_type
+required_capability: date_nanos_implicit_casting
+
+FROM date_nanos 
+| WHERE MV_MIN(nanos) > "2023-10-23T00:00:00.000000000" 
+| SORT nanos DESC 
+| KEEP millis, nanos;
+
+millis:date             | nanos:date_nanos              
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
+2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z
+2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z
+2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z
+2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
+;
+
+implicit casting to nanos, date plus time to nanos, equality test
+required_capability: date_nanos_type
+required_capability: date_nanos_implicit_casting
+
+FROM date_nanos 
+| WHERE MV_MIN(nanos) == "2023-10-23T12:27:28.948000000" 
+| SORT nanos DESC 
+| KEEP millis, nanos;
+
+millis:date             | nanos:date_nanos              
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
+;
+
 date nanos greater than millis
 required_capability: date_nanos_type
 required_capability: date_nanos_compare_to_millis
@@ -555,7 +686,8 @@ required_capability: date_nanos_bucket
 
 FROM date_nanos 
 | WHERE millis > "2020-01-01" 
-| STATS ct = count(*) BY yr = BUCKET(nanos, 1 year);
+| STATS ct = count(*) BY yr = BUCKET(nanos, 1 year)
+| SORT yr DESC;
 
 ct:long | yr:date_nanos
 8       | 2023-01-01T00:00:00.000000000Z 
@@ -567,7 +699,8 @@ required_capability: date_nanos_bucket
 
 FROM date_nanos 
 | WHERE millis > "2020-01-01" 
-| STATS ct = count(*) BY yr = BUCKET(nanos, 5, "1999-01-01", NOW());
+| STATS ct = count(*) BY yr = BUCKET(nanos, 5, "1999-01-01", NOW())
+| SORT yr DESC;
 
 ct:long | yr:date_nanos
 8       | 2023-01-01T00:00:00.000000000Z 
@@ -579,7 +712,8 @@ required_capability: date_nanos_bucket
 
 FROM date_nanos 
 | WHERE millis > "2020-01-01" 
-| STATS ct = count(*) BY mo = BUCKET(nanos, 1 month);
+| STATS ct = count(*) BY mo = BUCKET(nanos, 1 month)
+| SORT mo DESC;
 
 ct:long | mo:date_nanos
 8       | 2023-10-01T00:00:00.000000000Z 
@@ -591,7 +725,8 @@ required_capability: date_nanos_bucket
 
 FROM date_nanos 
 | WHERE millis > "2020-01-01" 
-| STATS ct = count(*) BY mo = BUCKET(nanos, 20, "2023-01-01", "2023-12-31");
+| STATS ct = count(*) BY mo = BUCKET(nanos, 20, "2023-01-01", "2023-12-31")
+| SORT mo DESC;
 
 ct:long | mo:date_nanos
 8       | 2023-10-01T00:00:00.000000000Z 
@@ -603,18 +738,21 @@ required_capability: date_nanos_bucket
 
 FROM date_nanos 
 | WHERE millis > "2020-01-01" 
-| STATS ct = count(*) BY mo = BUCKET(nanos, 55, "2023-01-01", "2023-12-31");
+| STATS ct = count(*) BY mo = BUCKET(nanos, 55, "2023-01-01", "2023-12-31")
+| SORT mo DESC;
 
 ct:long | mo:date_nanos
 8       | 2023-10-23T00:00:00.000000000Z 
 ;
+
 Bucket Date nanos by 10 minutes
 required_capability: date_trunc_date_nanos
 required_capability: date_nanos_bucket
 
 FROM date_nanos 
 | WHERE millis > "2020-01-01" 
-| STATS ct = count(*) BY mn = BUCKET(nanos, 10 minutes);
+| STATS ct = count(*) BY mn = BUCKET(nanos, 10 minutes)
+| SORT mn DESC;
 
 ct:long | mn:date_nanos                 
 4       | 2023-10-23T13:50:00.000000000Z
@@ -1023,3 +1161,68 @@ row dt = to_date_nanos(0::long)
 plus:date_nanos
 1970-01-01T00:00:01.000Z
 ;
+
+Date Nanos IN constant date nanos
+required_capability: date_nanos_in_operator
+required_capability: to_date_nanos
+
+FROM date_nanos 
+| WHERE MV_FIRST(nanos) IN (TO_DATE_NANOS("2023-10-23T13:55:01.543123456Z"), TO_DATE_NANOS("2023-10-23T12:27:28.948Z"), TO_DATE_NANOS("2017-10-23T13:53:55.832987654Z"));
+ignoreOrder:true
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
+;
+
+Date Nanos IN constant date nanos, implicit casting
+required_capability: date_nanos_in_operator
+required_capability: to_date_nanos
+required_capability: date_nanos_implicit_casting
+
+FROM date_nanos 
+| WHERE MV_FIRST(nanos) IN ("2023-10-23T13:55:01.543123456Z", "2023-10-23T12:27:28.948Z", "2017-10-23T13:53:55.832987654Z");
+ignoreOrder:true
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
+;
+
+Date Nanos IN date nanos field, implicit casting
+required_capability: date_nanos_in_operator
+required_capability: to_date_nanos
+required_capability: date_nanos_implicit_casting
+
+FROM date_nanos 
+| WHERE "2023-10-23T13:55:01.543123456Z" IN (MV_FIRST(nanos));
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
+;
+
+Date nanos IN millisecond date field
+required_capability: date_nanos_in_operator
+required_capability: to_date_nanos
+required_capability: date_nanos_implicit_casting
+
+# Note: It is important to have two dates in the IN so the optimizer doesn't turn this into an ==
+FROM date_nanos
+| WHERE MV_FIRST(nanos) IN (TO_DATETIME("2023-10-23T12:27:28.948"), TO_DATETIME("2020-02-02"));
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
+;
+
+Millisecond field IN date nanos constants
+required_capability: date_nanos_in_operator
+required_capability: to_date_nanos
+required_capability: date_nanos_implicit_casting
+
+# Note: It is important to have two dates in the IN so the optimizer doesn't turn this into an ==
+FROM date_nanos
+| WHERE MV_FIRST(millis) IN (TO_DATE_NANOS("2023-10-23T12:27:28.948"), TO_DATE_NANOS("2020-02-02"));
+
+millis:date              | nanos:date_nanos               | num:long
+2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec
new file mode 100644
index 0000000000000..2614ff09fed06
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec
@@ -0,0 +1,172 @@
+hash
+required_capability: hash_function
+
+// tag::hash[]
+FROM sample_data 
+| WHERE message != "Connection error"
+| EVAL md5 = hash("md5", message), sha256 = hash("sha256", message) 
+| KEEP message, md5, sha256;
+// end::hash[]
+ignoreOrder:true
+
+// tag::hash-result[]
+message:keyword       | md5:keyword                      | sha256:keyword
+Connected to 10.1.0.1 | abd7d1ce2bb636842a29246b3512dcae | 6d8372129ad78770f7185554dd39864749a62690216460752d6c075fa38ad85c
+Connected to 10.1.0.2 | 8f8f1cb60832d153f5b9ec6dc828b93f | b0db24720f15857091b3c99f4c4833586d0ea3229911b8777efb8d917cf27e9a
+Connected to 10.1.0.3 | 912b6dc13503165a15de43304bb77c78 | 75b0480188db8acc4d5cc666a51227eb2bc5b989cd8ca912609f33e0846eff57
+Disconnected          | ef70e46fd3bbc21e3e1f0b6815e750c0 | 04dfac3671b494ad53fcd152f7a14511bfb35747278aad8ce254a0d6e4ba4718
+;
+// end::hash-result[]
+
+
+hashOfConvertedType
+required_capability: hash_function
+
+FROM sample_data 
+| WHERE message != "Connection error"
+| EVAL input = event_duration::STRING, md5 = hash("md5", input), sha256 = hash("sha256", input) 
+| KEEP message, input, md5, sha256;
+ignoreOrder:true
+
+message:keyword       | input:keyword | md5:keyword                      | sha256:keyword
+Connected to 10.1.0.1 | 1756467       | c4fc1c57ee9b1d2b2023b70c8c167b54 | 8376a50a7ba7e6bd1bf9ad0c32d27d2f49fd0fa422573f98f239e21048b078f3
+Connected to 10.1.0.2 | 2764889       | 8e8cf005e11a7b5df1d9478a4715a444 | 1031f2bef8eaecbf47319505422300b27ea1f7c38b6717d41332325062f9a56a
+Connected to 10.1.0.3 | 3450233       | 09f2c64f5a55e9edf8ffbad336b561d8 | f77d7545769c4ecc85092f4f0b7ec8c20f467e4beb15fe67ca29f9aa8e9a6900
+Disconnected          | 1232382       | 6beac1485638d51e13c2c53990a2f611 | 9a03c1274a3ebb6c1cb85d170ce0a6fdb9d2232724e06b9f5e7cb9274af3cad6
+;
+
+
+hashOfEmptyInput
+required_capability: hash_function
+
+ROW input="" | EVAL md5 = hash("md5", input), sha256 = hash("sha256", input);
+
+input:keyword | md5:keyword                      | sha256:keyword
+              | d41d8cd98f00b204e9800998ecf8427e | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+;
+
+hashOfNullInput
+required_capability: hash_function
+
+ROW input=null::STRING | EVAL md5 = hash("md5", input), sha256 = hash("sha256", input);
+
+input:keyword | md5:keyword | sha256:keyword
+null          | null        | null
+;
+
+
+hashWithNullAlgorithm
+required_capability: hash_function
+
+ROW input="input" | EVAL hash = hash(null, input);
+
+input:keyword | hash:keyword
+input         | null
+;
+
+
+hashWithMv
+required_capability: hash_function
+
+ROW input=["foo", "bar"] | mv_expand input | EVAL md5 = hash("md5", input), sha256 = hash("sha256", input);
+
+input:keyword  | md5:keyword                      | sha256:keyword
+foo            | acbd18db4cc2f85cedef654fccc4a4d8 | 2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae
+bar            | 37b51d194a7513e45b56f6524f2d51f2 | fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9
+;
+
+
+hashWithNestedFunctions
+required_capability: hash_function
+
+ROW input=["foo", "bar"] | EVAL hash = concat(hash("md5", mv_concat(input, "-")), "-", hash("sha256", mv_concat(input, "-")));
+
+input:keyword  | hash:keyword
+["foo", "bar"] | e5f9ec048d1dbe19c70f720e002f9cb1-7d89c4f517e3bd4b5e8e76687937005b602ea00c5cba3e25ef1fc6575a55103e
+;
+
+
+hashWithConvertedTypes
+required_capability: hash_function
+
+ROW input=42 | EVAL md5 = hash("md5", input::STRING), sha256 = hash("sha256", to_string(input));
+
+input:integer | md5:keyword                      | sha256:keyword
+42            | a1d0c6e83f027327d8461063f4ac58a6 | 73475cb40a568e8da8a045ced110137e159f890ac4da883b6b17dc651b3a8049
+;
+
+
+hashWithStats
+required_capability: hash_function
+required_capability: hash_function_aliases_v1
+
+FROM sample_data 
+| EVAL md5="md5"
+| STATS count = count(*) by hash(md5, message), md5(message), sha1(message), sha256(message)
+| WHERE count > 1;
+
+count:long | hash(md5, message):keyword        | md5(message):keyword             | sha1(message):keyword                    | sha256(message):keyword
+3          | 2e92ae79ff32b37fee4368a594792183  | 2e92ae79ff32b37fee4368a594792183 | 1dbb3521876a899f82d6b0ff10eb32a01e03aba8 | 8d137af9c64fba09bbb003aba93f0b029898fe19e7927cd696f4c3e2b69f538d
+;
+
+
+md5Hash
+required_capability: hash_function_aliases_v1
+
+// tag::md5[]
+FROM sample_data 
+| WHERE message != "Connection error"
+| EVAL md5 = md5(message)
+| KEEP message, md5;
+// end::md5[]
+ignoreOrder:true
+
+// tag::md5-result[]
+message:keyword       | md5:keyword
+Connected to 10.1.0.1 | abd7d1ce2bb636842a29246b3512dcae
+Connected to 10.1.0.2 | 8f8f1cb60832d153f5b9ec6dc828b93f
+Connected to 10.1.0.3 | 912b6dc13503165a15de43304bb77c78
+Disconnected          | ef70e46fd3bbc21e3e1f0b6815e750c0
+;
+// end::md5-result[]
+
+
+sha1Hash
+required_capability: hash_function_aliases_v1
+
+// tag::sha1[]
+FROM sample_data 
+| WHERE message != "Connection error"
+| EVAL sha1 = sha1(message)
+| KEEP message, sha1;
+// end::sha1[]
+ignoreOrder:true
+
+// tag::sha1-result[]
+message:keyword       | sha1:keyword
+Connected to 10.1.0.1 | 42b85531a79088036a17759db7d2de292b92f57f
+Connected to 10.1.0.2 | d30db445da2e9237c9718d0c7e4fb7cbbe9c2cb4
+Connected to 10.1.0.3 | 2733848d943809f0b10cad3e980763e88afb9853
+Disconnected          | 771e05f27b99fd59f638f41a7a4e977b1d4691fe
+;
+// end::sha1-result[]
+
+sha256Hash
+required_capability: hash_function_aliases_v1
+
+// tag::sha256[]
+FROM sample_data 
+| WHERE message != "Connection error"
+| EVAL sha256 = sha256(message)
+| KEEP message, sha256;
+// end::sha256[]
+ignoreOrder:true
+
+// tag::sha256-result[]
+message:keyword       | sha256:keyword
+Connected to 10.1.0.1 | 6d8372129ad78770f7185554dd39864749a62690216460752d6c075fa38ad85c
+Connected to 10.1.0.2 | b0db24720f15857091b3c99f4c4833586d0ea3229911b8777efb8d917cf27e9a
+Connected to 10.1.0.3 | 75b0480188db8acc4d5cc666a51227eb2bc5b989cd8ca912609f33e0846eff57
+Disconnected          | 04dfac3671b494ad53fcd152f7a14511bfb35747278aad8ce254a0d6e4ba4718
+;
+// end::sha256-result[]
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec
index 7fed4f377096f..95119cae95590 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec
@@ -3,8 +3,12 @@
 // Reuses the sample dataset and commands from enrich.csv-spec
 //
 
+###############################################
+# Tests with languages_lookup index
+###############################################
+
 basicOnTheDataNode
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM employees
 | EVAL language_code = languages
@@ -21,7 +25,7 @@ emp_no:integer | language_code:integer | language_name:keyword
 ;
 
 basicRow
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 ROW language_code = 1
 | LOOKUP JOIN languages_lookup ON language_code
@@ -32,7 +36,7 @@ language_code:integer  | language_name:keyword
 ;
 
 basicOnTheCoordinator
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM employees
 | SORT emp_no
@@ -49,7 +53,7 @@ emp_no:integer | language_code:integer | language_name:keyword
 ;
 
 subsequentEvalOnTheDataNode
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM employees
 | EVAL language_code = languages
@@ -67,7 +71,7 @@ emp_no:integer | language_code:integer | language_name:keyword | language_code_x
 ;
 
 subsequentEvalOnTheCoordinator
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM employees
 | SORT emp_no
@@ -85,7 +89,7 @@ emp_no:integer | language_code:integer | language_name:keyword | language_code_x
 ;
 
 sortEvalBeforeLookup
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM employees
 | SORT emp_no
@@ -102,7 +106,7 @@ emp_no:integer | language_code:integer | language_name:keyword
 ;
 
 nonUniqueLeftKeyOnTheDataNode
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM employees
 | WHERE emp_no <= 10030
@@ -126,7 +130,7 @@ emp_no:integer | language_code:integer | language_name:keyword
 ;
 
 nonUniqueRightKeyOnTheDataNode
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM employees
 | EVAL language_code = emp_no % 10
@@ -146,7 +150,7 @@ emp_no:integer | language_code:integer | language_name:keyword       | country:k
 ;
 
 nonUniqueRightKeyOnTheCoordinator
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM employees
 | SORT emp_no
@@ -166,7 +170,7 @@ emp_no:integer | language_code:integer | language_name:keyword       | country:k
 ;
 
 nonUniqueRightKeyFromRow
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 ROW language_code = 2
 | LOOKUP JOIN languages_lookup_non_unique_key ON language_code
@@ -178,8 +182,220 @@ language_code:integer | language_name:keyword       | country:keyword
 2                     | [German, German, German]    | [Austria, Germany, Switzerland]
 ;
 
+repeatedIndexOnFrom
+required_capability: join_lookup_v10
+
+FROM languages_lookup
+| LOOKUP JOIN languages_lookup ON language_code
+| SORT language_code
+;
+
+language_code:integer | language_name:keyword
+1                     | English
+2                     | French
+3                     | Spanish
+4                     | German
+;
+
+dropAllLookedUpFieldsOnTheDataNode-Ignore
+// Depends on
+// https://github.com/elastic/elasticsearch/issues/118778
+// https://github.com/elastic/elasticsearch/issues/118781
+required_capability: join_lookup_v10
+
+FROM employees
+| EVAL language_code = emp_no % 10
+| LOOKUP JOIN languages_lookup_non_unique_key ON language_code
+| WHERE emp_no == 10001
+| SORT emp_no
+| DROP language*
+;
+
+emp_no:integer
+10001
+10001
+10001
+10001
+;
+
+dropAllLookedUpFieldsOnTheCoordinator-Ignore
+// Depends on
+// https://github.com/elastic/elasticsearch/issues/118778
+// https://github.com/elastic/elasticsearch/issues/118781
+required_capability: join_lookup_v10
+
+FROM employees
+| SORT emp_no
+| LIMIT 2
+| EVAL language_code = emp_no % 10
+| LOOKUP JOIN languages_lookup_non_unique_key ON language_code
+| DROP language*
+;
+
+emp_no:integer
+10001
+10001
+10001
+10001
+10002
+10002
+10002
+;
+
+###############################################
+# Filtering tests with languages_lookup index
+###############################################
+
+filterOnLeftSide
+required_capability: join_lookup_v10
+
+FROM employees
+| EVAL language_code = languages
+| LOOKUP JOIN languages_lookup ON language_code
+| SORT emp_no
+| KEEP emp_no, language_code, language_name
+| WHERE emp_no >= 10091 AND emp_no < 10094
+;
+
+emp_no:integer | language_code:integer | language_name:keyword
+10091          | 3                     | Spanish
+10092          | 1                     | English
+10093          | 3                     | Spanish
+;
+
+filterOnRightSide
+required_capability: join_lookup_v10
+
+FROM sample_data
+| LOOKUP JOIN message_types_lookup ON message
+| WHERE type == "Error"
+| KEEP @timestamp, client_ip, event_duration, message, type
+| SORT @timestamp DESC
+;
+
+@timestamp:date          | client_ip:ip | event_duration:long | message:keyword       | type:keyword
+2023-10-23T13:53:55.832Z | 172.21.3.15  | 5033755             | Connection error      | Error
+2023-10-23T13:52:55.015Z | 172.21.3.15  | 8268153             | Connection error      | Error
+2023-10-23T13:51:54.732Z | 172.21.3.15  | 725448              | Connection error      | Error
+;
+
+filterOnRightSideAfterStats
+required_capability: join_lookup_v10
+
+FROM sample_data
+| LOOKUP JOIN message_types_lookup ON message
+| STATS count = count(message) BY type
+| WHERE type == "Error"
+;
+
+count:long | type:keyword
+3          | Error
+;
+
+filterOnJoinKey
+required_capability: join_lookup_v10
+
+FROM employees
+| EVAL language_code = languages
+| WHERE emp_no >= 10091 AND emp_no < 10094
+| LOOKUP JOIN languages_lookup ON language_code
+| WHERE language_code == 1
+| KEEP emp_no, language_code, language_name
+;
+
+emp_no:integer | language_code:integer | language_name:keyword
+10092          | 1                     | English
+;
+
+filterOnJoinKeyAndRightSide
+required_capability: join_lookup_v10
+
+FROM employees
+| WHERE emp_no < 10006
+| EVAL language_code = languages
+| LOOKUP JOIN languages_lookup ON language_code
+| WHERE language_code > 1 AND language_name IS NOT NULL
+| KEEP emp_no, language_code, language_name
+;
+ignoreOrder:true
+
+emp_no:integer | language_code:integer | language_name:keyword
+10001          | 2                     | French
+10003          | 4                     | German
+;
+
+filterOnRightSideOnTheCoordinator
+required_capability: join_lookup_v10
+
+FROM employees
+| SORT emp_no
+| LIMIT 5
+| EVAL language_code = languages
+| LOOKUP JOIN languages_lookup ON language_code
+| WHERE language_name == "English"
+| KEEP emp_no, language_code, language_name
+;
+
+emp_no:integer | language_code:integer | language_name:keyword
+10005          | 1                     | English
+;
+
+filterOnJoinKeyOnTheCoordinator
+required_capability: join_lookup_v10
+
+FROM employees
+| SORT emp_no
+| LIMIT 5
+| EVAL language_code = languages
+| LOOKUP JOIN languages_lookup ON language_code
+| WHERE language_code == 1
+| KEEP emp_no, language_code, language_name
+;
+
+emp_no:integer | language_code:integer | language_name:keyword
+10005          | 1                     | English
+;
+
+filterOnJoinKeyAndRightSideOnTheCoordinator
+required_capability: join_lookup_v10
+
+FROM employees
+| SORT emp_no
+| LIMIT 5
+| EVAL language_code = languages
+| LOOKUP JOIN languages_lookup ON language_code
+| WHERE language_code > 1 AND language_name IS NOT NULL
+| KEEP emp_no, language_code, language_name
+;
+
+emp_no:integer | language_code:integer | language_name:keyword
+10001          | 2                     | French
+10003          | 4                     | German
+;
+
+filterOnTheDataNodeThenFilterOnTheCoordinator
+required_capability: join_lookup_v10
+
+FROM employees
+| EVAL language_code = languages
+| WHERE emp_no >= 10091 AND emp_no < 10094
+| LOOKUP JOIN languages_lookup ON language_code
+| WHERE language_name == "English"
+| KEEP emp_no, language_code, language_name
+| SORT emp_no
+| WHERE language_code == 1
+;
+
+emp_no:integer | language_code:integer | language_name:keyword
+10092          | 1                     | English
+;
+
+###########################################################################
+# null and multi-value behavior with languages_lookup_non_unique_key index
+###########################################################################
+
 nullJoinKeyOnTheDataNode
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM employees
 | WHERE emp_no < 10004
@@ -195,15 +411,15 @@ emp_no:integer | language_code:integer | language_name:keyword
 10003          | null                  | null
 ;
 
-
 mvJoinKeyOnTheDataNode
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM employees
 | WHERE 10003 < emp_no AND emp_no < 10008
 | EVAL language_code = emp_no % 10
 | LOOKUP JOIN languages_lookup_non_unique_key ON language_code
 | SORT emp_no
+| EVAL language_name = MV_SORT(language_name)
 | KEEP emp_no, language_code, language_name
 ;
 
@@ -215,7 +431,7 @@ emp_no:integer | language_code:integer | language_name:keyword
 ;
 
 mvJoinKeyFromRow
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 ROW language_code = [4, 5, 6, 7]
 | LOOKUP JOIN languages_lookup_non_unique_key ON language_code
@@ -228,7 +444,7 @@ language_code:integer | language_name:keyword       | country:keyword
 ;
 
 mvJoinKeyFromRowExpanded
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 ROW language_code = [4, 5, 6, 7, 8]
 | MV_EXPAND language_code
@@ -245,10 +461,75 @@ language_code:integer | language_name:keyword       | country:keyword
 8                     | Mv-Lang2                    | Mv-Land2
 ;
 
+###########################################################################
+# nested filed join behavior with languages_nested_fields index
+###########################################################################
+
+joinOnNestedField
+required_capability: join_lookup_v10
+
+FROM employees
+| WHERE 10000 < emp_no AND emp_no < 10006
+| EVAL language.id = emp_no % 10
+| LOOKUP JOIN languages_nested_fields ON language.id
+| SORT emp_no
+| KEEP emp_no, language.id, language.name
+;
+
+emp_no:integer | language.id:integer | language.name:text
+10001          | 1                   | English
+10002          | 2                   | French
+10003          | 3                   | Spanish
+10004          | 4                   | German
+10005          | 5                   | null
+;
+
+
+joinOnNestedFieldRow
+required_capability: join_lookup_v10
+
+ROW language.code = "EN"
+| LOOKUP JOIN languages_nested_fields ON language.code
+| KEEP language.id, language.code, language.name.keyword
+;
+
+language.id:integer | language.code:keyword | language.name.keyword:keyword
+1                   | EN                    | English
+;
+
+
+joinOnNestedNestedFieldRow
+required_capability: join_lookup_v10
+
+ROW language.name.keyword = "English"
+| LOOKUP JOIN languages_nested_fields ON language.name.keyword
+| KEEP language.id, language.name, language.name.keyword
+;
+
+language.id:integer | language.name:text | language.name.keyword:keyword
+1                   | English            | English
+;
+
+###############################################
+# Tests with clientips_lookup index
+###############################################
+
 lookupIPFromRow
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
+
+ROW left = "left", client_ip = "172.21.0.5", right = "right"
+| LOOKUP JOIN clientips_lookup ON client_ip
+;
+
+left:keyword | client_ip:keyword | right:keyword | env:keyword
+left         | 172.21.0.5        | right         | Development
+;
+
+lookupIPFromKeepRow
+required_capability: join_lookup_v10
 
 ROW left = "left", client_ip = "172.21.0.5", right = "right"
+| KEEP left, client_ip, right
 | LOOKUP JOIN clientips_lookup ON client_ip
 ;
 
@@ -257,7 +538,7 @@ left         | 172.21.0.5        | right         | Development
 ;
 
 lookupIPFromRowWithShadowing
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 ROW left = "left", client_ip = "172.21.0.5", env = "env", right = "right"
 | LOOKUP JOIN clientips_lookup ON client_ip
@@ -268,7 +549,7 @@ left         | 172.21.0.5        | right         | Development
 ;
 
 lookupIPFromRowWithShadowingKeep
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 ROW left = "left", client_ip = "172.21.0.5", env = "env", right = "right"
 | EVAL client_ip = client_ip::keyword
@@ -281,7 +562,7 @@ left         | 172.21.0.5        | right         | Development
 ;
 
 lookupIPFromRowWithShadowingKeepReordered
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 ROW left = "left", client_ip = "172.21.0.5", env = "env", right = "right"
 | EVAL client_ip = client_ip::keyword
@@ -294,7 +575,7 @@ right         | Development | 172.21.0.5
 ;
 
 lookupIPFromIndex
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM sample_data
 | EVAL client_ip = client_ip::keyword
@@ -313,7 +594,7 @@ ignoreOrder:true
 ;
 
 lookupIPFromIndexKeep
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM sample_data
 | EVAL client_ip = client_ip::keyword
@@ -332,8 +613,30 @@ ignoreOrder:true
 2023-10-23T12:15:03.360Z | 172.21.2.162      | 3450233             | Connected to 10.1.0.3 | QA
 ;
 
+lookupIPFromIndexKeepKeep
+required_capability: join_lookup_v10
+
+FROM sample_data
+| KEEP client_ip, event_duration, @timestamp, message
+| RENAME @timestamp AS timestamp, message AS msg
+| EVAL client_ip = client_ip::keyword
+| LOOKUP JOIN clientips_lookup ON client_ip
+| KEEP timestamp, client_ip, event_duration, msg, env
+;
+ignoreOrder:true
+
+timestamp:date           | client_ip:keyword | event_duration:long | msg:keyword           | env:keyword
+2023-10-23T13:55:01.543Z | 172.21.3.15       | 1756467             | Connected to 10.1.0.1 | Production
+2023-10-23T13:53:55.832Z | 172.21.3.15       | 5033755             | Connection error      | Production
+2023-10-23T13:52:55.015Z | 172.21.3.15       | 8268153             | Connection error      | Production
+2023-10-23T13:51:54.732Z | 172.21.3.15       | 725448              | Connection error      | Production
+2023-10-23T13:33:34.937Z | 172.21.0.5        | 1232382             | Disconnected          | Development
+2023-10-23T12:27:28.948Z | 172.21.2.113      | 2764889             | Connected to 10.1.0.2 | QA
+2023-10-23T12:15:03.360Z | 172.21.2.162      | 3450233             | Connected to 10.1.0.3 | QA
+;
+
 lookupIPFromIndexStats
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM sample_data
 | EVAL client_ip = client_ip::keyword
@@ -349,7 +652,7 @@ count:long | env:keyword
 ;
 
 lookupIPFromIndexStatsKeep
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM sample_data
 | EVAL client_ip = client_ip::keyword
@@ -365,8 +668,29 @@ count:long | env:keyword
 1          | Development
 ;
 
+statsAndLookupIPFromIndex
+required_capability: join_lookup_v10
+
+FROM sample_data
+| EVAL client_ip = client_ip::keyword
+| STATS count = count(client_ip) BY client_ip
+| LOOKUP JOIN clientips_lookup ON client_ip
+| SORT count DESC, client_ip ASC, env ASC
+;
+
+count:long | client_ip:keyword | env:keyword
+4          | 172.21.3.15       | Production
+1          | 172.21.0.5        | Development
+1          | 172.21.2.113      | QA
+1          | 172.21.2.162      | QA
+;
+
+###############################################
+# Tests with message_types_lookup index
+###############################################
+
 lookupMessageFromRow
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 ROW left = "left", message = "Connected to 10.1.0.1", right = "right"
 | LOOKUP JOIN message_types_lookup ON message
@@ -376,8 +700,20 @@ left:keyword | message:keyword       | right:keyword | type:keyword
 left         | Connected to 10.1.0.1 | right         | Success
 ;
 
+lookupMessageFromKeepRow
+required_capability: join_lookup_v10
+
+ROW left = "left", message = "Connected to 10.1.0.1", right = "right"
+| KEEP left, message, right
+| LOOKUP JOIN message_types_lookup ON message
+;
+
+left:keyword | message:keyword       | right:keyword | type:keyword
+left         | Connected to 10.1.0.1 | right         | Success
+;
+
 lookupMessageFromRowWithShadowing
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 ROW left = "left", message = "Connected to 10.1.0.1", type = "unknown", right = "right"
 | LOOKUP JOIN message_types_lookup ON message
@@ -388,7 +724,7 @@ left         | Connected to 10.1.0.1 | right         | Success
 ;
 
 lookupMessageFromRowWithShadowingKeep
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 ROW left = "left", message = "Connected to 10.1.0.1", type = "unknown", right = "right"
 | LOOKUP JOIN message_types_lookup ON message
@@ -400,7 +736,7 @@ left         | Connected to 10.1.0.1 | right         | Success
 ;
 
 lookupMessageFromIndex
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM sample_data
 | LOOKUP JOIN message_types_lookup ON message
@@ -418,9 +754,29 @@ ignoreOrder:true
 ;
 
 lookupMessageFromIndexKeep
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
+
+FROM sample_data
+| LOOKUP JOIN message_types_lookup ON message
+| KEEP @timestamp, client_ip, event_duration, message, type
+;
+ignoreOrder:true
+
+@timestamp:date          | client_ip:ip | event_duration:long | message:keyword       | type:keyword
+2023-10-23T13:55:01.543Z | 172.21.3.15  | 1756467             | Connected to 10.1.0.1 | Success
+2023-10-23T13:53:55.832Z | 172.21.3.15  | 5033755             | Connection error      | Error
+2023-10-23T13:52:55.015Z | 172.21.3.15  | 8268153             | Connection error      | Error
+2023-10-23T13:51:54.732Z | 172.21.3.15  | 725448              | Connection error      | Error
+2023-10-23T13:33:34.937Z | 172.21.0.5   | 1232382             | Disconnected          | Disconnected
+2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889             | Connected to 10.1.0.2 | Success
+2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233             | Connected to 10.1.0.3 | Success
+;
+
+lookupMessageFromIndexKeepKeep
+required_capability: join_lookup_v10
 
 FROM sample_data
+| KEEP client_ip, event_duration, @timestamp, message
 | LOOKUP JOIN message_types_lookup ON message
 | KEEP @timestamp, client_ip, event_duration, message, type
 ;
@@ -437,7 +793,7 @@ ignoreOrder:true
 ;
 
 lookupMessageFromIndexKeepReordered
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM sample_data
 | LOOKUP JOIN message_types_lookup ON message
@@ -456,7 +812,7 @@ Success      | 172.21.2.162 | 3450233             | Connected to 10.1.0.3
 ;
 
 lookupMessageFromIndexStats
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM sample_data
 | LOOKUP JOIN message_types_lookup ON message
@@ -471,7 +827,7 @@ count:long | type:keyword
 ;
 
 lookupMessageFromIndexStatsKeep
-required_capability: join_lookup_v5
+required_capability: join_lookup_v10
 
 FROM sample_data
 | LOOKUP JOIN message_types_lookup ON message
@@ -486,67 +842,368 @@ count:long | type:keyword
 1          | Disconnected
 ;
 
-// 
-// Filtering tests
-// 
+statsAndLookupMessageFromIndex
+required_capability: join_lookup_v10
 
-lookupWithFilterOnLeftSideField
-required_capability: join_lookup_v5
+FROM sample_data
+| STATS count = count(message) BY message
+| LOOKUP JOIN message_types_lookup ON message
+| KEEP count, type, message
+| SORT count DESC, message ASC
+;
 
-FROM employees
-| EVAL language_code = languages
-| LOOKUP JOIN languages_lookup ON language_code
-| SORT emp_no
-| KEEP emp_no, language_code, language_name
-| WHERE emp_no >= 10091 AND emp_no < 10094
+count:long | type:keyword | message:keyword
+3          | Error        | Connection error
+1          | Success      | Connected to 10.1.0.1
+1          | Success      | Connected to 10.1.0.2
+1          | Success      | Connected to 10.1.0.3
+1          | Disconnected | Disconnected
 ;
 
-emp_no:integer | language_code:integer | language_name:keyword
-10091          | 3                     | Spanish
-10092          | 1                     | English
-10093          | 3                     | Spanish
+lookupMessageFromIndexTwice
+required_capability: join_lookup_v10
+
+FROM sample_data
+| LOOKUP JOIN message_types_lookup ON message
+| RENAME message AS message1, type AS type1
+| EVAL message = client_ip::keyword
+| LOOKUP JOIN message_types_lookup ON message
+| RENAME message AS message2, type AS type2
+;
+ignoreOrder:true
+
+@timestamp:date          | client_ip:ip | event_duration:long | message1:keyword      | type1:keyword | message2:keyword | type2:keyword 
+2023-10-23T13:55:01.543Z | 172.21.3.15  | 1756467             | Connected to 10.1.0.1 | Success       | 172.21.3.15      | null
+2023-10-23T13:53:55.832Z | 172.21.3.15  | 5033755             | Connection error      | Error         | 172.21.3.15      | null
+2023-10-23T13:52:55.015Z | 172.21.3.15  | 8268153             | Connection error      | Error         | 172.21.3.15      | null
+2023-10-23T13:51:54.732Z | 172.21.3.15  | 725448              | Connection error      | Error         | 172.21.3.15      | null
+2023-10-23T13:33:34.937Z | 172.21.0.5   | 1232382             | Disconnected          | Disconnected  | 172.21.0.5       | null
+2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889             | Connected to 10.1.0.2 | Success       | 172.21.2.113     | null
+2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233             | Connected to 10.1.0.3 | Success       | 172.21.2.162     | null
 ;
 
-lookupMessageWithFilterOnRightSideField-Ignore
-required_capability: join_lookup_v5
+lookupMessageFromIndexTwiceKeep
+required_capability: join_lookup_v10
 
 FROM sample_data
 | LOOKUP JOIN message_types_lookup ON message
-| WHERE type == "Error"
+| RENAME message AS message1, type AS type1
+| EVAL message = client_ip::keyword
+| LOOKUP JOIN message_types_lookup ON message
+| RENAME message AS message2, type AS type2
+| KEEP @timestamp, client_ip, event_duration, message1, type1, message2, type2
+;
+ignoreOrder:true
+
+@timestamp:date          | client_ip:ip | event_duration:long | message1:keyword      | type1:keyword | message2:keyword | type2:keyword 
+2023-10-23T13:55:01.543Z | 172.21.3.15  | 1756467             | Connected to 10.1.0.1 | Success       | 172.21.3.15      | null
+2023-10-23T13:53:55.832Z | 172.21.3.15  | 5033755             | Connection error      | Error         | 172.21.3.15      | null
+2023-10-23T13:52:55.015Z | 172.21.3.15  | 8268153             | Connection error      | Error         | 172.21.3.15      | null
+2023-10-23T13:51:54.732Z | 172.21.3.15  | 725448              | Connection error      | Error         | 172.21.3.15      | null
+2023-10-23T13:33:34.937Z | 172.21.0.5   | 1232382             | Disconnected          | Disconnected  | 172.21.0.5       | null
+2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889             | Connected to 10.1.0.2 | Success       | 172.21.2.113     | null
+2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233             | Connected to 10.1.0.3 | Success       | 172.21.2.162     | null
+;
+
+lookupMessageFromIndexTwiceFullyShadowing
+required_capability: join_lookup_v10
+
+FROM sample_data
+| LOOKUP JOIN message_types_lookup ON message
+| LOOKUP JOIN message_types_lookup ON message
 | KEEP @timestamp, client_ip, event_duration, message, type
-| SORT @timestamp DESC
 ;
+ignoreOrder:true
 
 @timestamp:date          | client_ip:ip | event_duration:long | message:keyword       | type:keyword
+2023-10-23T13:55:01.543Z | 172.21.3.15  | 1756467             | Connected to 10.1.0.1 | Success
 2023-10-23T13:53:55.832Z | 172.21.3.15  | 5033755             | Connection error      | Error
 2023-10-23T13:52:55.015Z | 172.21.3.15  | 8268153             | Connection error      | Error
 2023-10-23T13:51:54.732Z | 172.21.3.15  | 725448              | Connection error      | Error
+2023-10-23T13:33:34.937Z | 172.21.0.5   | 1232382             | Disconnected          | Disconnected
+2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889             | Connected to 10.1.0.2 | Success
+2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233             | Connected to 10.1.0.3 | Success
+;
+
+###############################################
+# Tests with clientips_lookup and message_types_lookup indexes
+###############################################
+
+lookupIPAndMessageFromRow
+required_capability: join_lookup_v10
+
+ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", right = "right"
+| LOOKUP JOIN clientips_lookup ON client_ip
+| LOOKUP JOIN message_types_lookup ON message
+;
+
+left:keyword | client_ip:keyword | message:keyword       | right:keyword | env:keyword | type:keyword
+left         | 172.21.0.5        | Connected to 10.1.0.1 | right         | Development | Success
+;
+
+lookupIPAndMessageFromRowKeepBefore
+required_capability: join_lookup_v10
+
+ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", right = "right"
+| KEEP left, client_ip, message, right
+| LOOKUP JOIN clientips_lookup ON client_ip
+| LOOKUP JOIN message_types_lookup ON message
+;
+
+left:keyword | client_ip:keyword | message:keyword       | right:keyword | env:keyword | type:keyword
+left         | 172.21.0.5        | Connected to 10.1.0.1 | right         | Development | Success
+;
+
+lookupIPAndMessageFromRowKeepBetween
+required_capability: join_lookup_v10
+
+ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", right = "right"
+| LOOKUP JOIN clientips_lookup ON client_ip
+| KEEP left, client_ip, message, right, env
+| LOOKUP JOIN message_types_lookup ON message
+;
+
+left:keyword | client_ip:keyword | message:keyword       | right:keyword | env:keyword | type:keyword
+left         | 172.21.0.5        | Connected to 10.1.0.1 | right         | Development | Success
+;
+
+lookupIPAndMessageFromRowKeepAfter
+required_capability: join_lookup_v10
+
+ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", right = "right"
+| LOOKUP JOIN clientips_lookup ON client_ip
+| LOOKUP JOIN message_types_lookup ON message
+| KEEP left, client_ip, message, right, env, type
+;
+
+left:keyword | client_ip:keyword | message:keyword       | right:keyword | env:keyword | type:keyword
+left         | 172.21.0.5        | Connected to 10.1.0.1 | right         | Development | Success
+;
+
+lookupIPAndMessageFromRowWithShadowing
+required_capability: join_lookup_v10
+
+ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", type = "type", right = "right"
+| LOOKUP JOIN clientips_lookup ON client_ip
+| LOOKUP JOIN message_types_lookup ON message
+;
+
+left:keyword | client_ip:keyword | message:keyword       | right:keyword | env:keyword | type:keyword
+left         | 172.21.0.5        | Connected to 10.1.0.1 | right         | Development | Success
+;
+
+lookupIPAndMessageFromRowWithShadowingKeep
+required_capability: join_lookup_v10
+
+ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", right = "right"
+| EVAL client_ip = client_ip::keyword
+| LOOKUP JOIN clientips_lookup ON client_ip
+| LOOKUP JOIN message_types_lookup ON message
+| KEEP left, client_ip, message, right, env, type
+;
+
+left:keyword | client_ip:keyword | message:keyword       | right:keyword | env:keyword | type:keyword
+left         | 172.21.0.5        | Connected to 10.1.0.1 | right         | Development | Success
+;
+
+lookupIPAndMessageFromRowWithShadowingKeepKeep
+required_capability: join_lookup_v10
+
+ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", right = "right"
+| EVAL client_ip = client_ip::keyword
+| LOOKUP JOIN clientips_lookup ON client_ip
+| KEEP left, client_ip, message, right, env
+| LOOKUP JOIN message_types_lookup ON message
+| KEEP left, client_ip, message, right, env, type
+;
+
+left:keyword | client_ip:keyword | message:keyword       | right:keyword | env:keyword | type:keyword
+left         | 172.21.0.5        | Connected to 10.1.0.1 | right         | Development | Success
+;
+
+lookupIPAndMessageFromRowWithShadowingKeepKeepKeep
+required_capability: join_lookup_v10
+
+ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", right = "right"
+| EVAL client_ip = client_ip::keyword
+| KEEP left, client_ip, message, right, env
+| LOOKUP JOIN clientips_lookup ON client_ip
+| KEEP left, client_ip, message, right, env
+| LOOKUP JOIN message_types_lookup ON message
+| KEEP left, client_ip, message, right, env, type
+;
+
+left:keyword | client_ip:keyword | message:keyword       | right:keyword | env:keyword | type:keyword
+left         | 172.21.0.5        | Connected to 10.1.0.1 | right         | Development | Success
 ;
 
-lookupWithFieldAndRightSideAfterStats
-required_capability: join_lookup_v5
+lookupIPAndMessageFromRowWithShadowingKeepReordered
+required_capability: join_lookup_v10
+
+ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", right = "right"
+| EVAL client_ip = client_ip::keyword
+| LOOKUP JOIN clientips_lookup ON client_ip
+| LOOKUP JOIN message_types_lookup ON message
+| KEEP right, env, type, client_ip
+;
+
+right:keyword | env:keyword  | type:keyword | client_ip:keyword
+right         | Development  | Success      | 172.21.0.5
+;
+
+lookupIPAndMessageFromIndex
+required_capability: join_lookup_v10
 
 FROM sample_data
+| EVAL client_ip = client_ip::keyword
+| LOOKUP JOIN clientips_lookup ON client_ip
 | LOOKUP JOIN message_types_lookup ON message
-| STATS count = count(message) BY type
-| WHERE type == "Error"
 ;
+ignoreOrder:true
 
-count:long | type:keyword
-3          | Error
+@timestamp:date          | event_duration:long | message:keyword       | client_ip:keyword | env:keyword | type:keyword
+2023-10-23T13:55:01.543Z | 1756467             | Connected to 10.1.0.1 | 172.21.3.15       | Production  | Success
+2023-10-23T13:53:55.832Z | 5033755             | Connection error      | 172.21.3.15       | Production  | Error
+2023-10-23T13:52:55.015Z | 8268153             | Connection error      | 172.21.3.15       | Production  | Error
+2023-10-23T13:51:54.732Z | 725448              | Connection error      | 172.21.3.15       | Production  | Error
+2023-10-23T13:33:34.937Z | 1232382             | Disconnected          | 172.21.0.5        | Development | Disconnected
+2023-10-23T12:27:28.948Z | 2764889             | Connected to 10.1.0.2 | 172.21.2.113      | QA          | Success
+2023-10-23T12:15:03.360Z | 3450233             | Connected to 10.1.0.3 | 172.21.2.162      | QA          | Success
 ;
 
-lookupWithFieldOnJoinKey-Ignore
-required_capability: join_lookup_v5
+lookupIPAndMessageFromIndexKeep
+required_capability: join_lookup_v10
 
-FROM employees
-| EVAL language_code = languages
+FROM sample_data
+| EVAL client_ip = client_ip::keyword
+| LOOKUP JOIN clientips_lookup ON client_ip
+| LOOKUP JOIN message_types_lookup ON message
+| KEEP @timestamp, client_ip, event_duration, message, env, type
+;
+ignoreOrder:true
+
+@timestamp:date          | client_ip:keyword | event_duration:long | message:keyword       | env:keyword | type:keyword
+2023-10-23T13:55:01.543Z | 172.21.3.15       | 1756467             | Connected to 10.1.0.1 | Production  | Success
+2023-10-23T13:53:55.832Z | 172.21.3.15       | 5033755             | Connection error      | Production  | Error
+2023-10-23T13:52:55.015Z | 172.21.3.15       | 8268153             | Connection error      | Production  | Error
+2023-10-23T13:51:54.732Z | 172.21.3.15       | 725448              | Connection error      | Production  | Error
+2023-10-23T13:33:34.937Z | 172.21.0.5        | 1232382             | Disconnected          | Development | Disconnected
+2023-10-23T12:27:28.948Z | 172.21.2.113      | 2764889             | Connected to 10.1.0.2 | QA          | Success
+2023-10-23T12:15:03.360Z | 172.21.2.162      | 3450233             | Connected to 10.1.0.3 | QA          | Success
+;
+
+lookupIPAndMessageFromIndexStats
+required_capability: join_lookup_v10
+
+FROM sample_data
+| EVAL client_ip = client_ip::keyword
+| LOOKUP JOIN clientips_lookup ON client_ip
+| LOOKUP JOIN message_types_lookup ON message
+| STATS count = count(*) BY env, type
+| SORT count DESC, env ASC, type ASC
+;
+
+count:long | env:keyword | type:keyword
+3          | Production  | Error
+2          | QA          | Success
+1          | Development | Disconnected
+1          | Production  | Success
+;
+
+lookupIPAndMessageFromIndexStatsKeep
+required_capability: join_lookup_v10
+
+FROM sample_data
+| EVAL client_ip = client_ip::keyword
+| LOOKUP JOIN clientips_lookup ON client_ip
+| LOOKUP JOIN message_types_lookup ON message
+| KEEP client_ip, env, type
+| STATS count = count(*) BY env, type
+| SORT count DESC, env ASC, type ASC
+;
+
+count:long | env:keyword | type:keyword
+3          | Production  | Error
+2          | QA          | Success
+1          | Development | Disconnected
+1          | Production  | Success
+;
+
+statsAndLookupIPAndMessageFromIndex
+required_capability: join_lookup_v10
+
+FROM sample_data
+| EVAL client_ip = client_ip::keyword
+| STATS count = count(*) BY client_ip, message
+| LOOKUP JOIN clientips_lookup ON client_ip
+| LOOKUP JOIN message_types_lookup ON message
+| SORT count DESC, client_ip ASC, message ASC
+;
+
+count:long | client_ip:keyword | message:keyword       | env:keyword | type:keyword
+3          | 172.21.3.15       | Connection error      | Production  | Error
+1          | 172.21.0.5        | Disconnected          | Development | Disconnected
+1          | 172.21.2.113      | Connected to 10.1.0.2 | QA          | Success
+1          | 172.21.2.162      | Connected to 10.1.0.3 | QA          | Success
+1          | 172.21.3.15       | Connected to 10.1.0.1 | Production  | Success
+;
+
+lookupIPAndMessageFromIndexChainedEvalKeep
+required_capability: join_lookup_v10
+
+FROM sample_data
+| EVAL client_ip = client_ip::keyword
+| LOOKUP JOIN clientips_lookup ON client_ip
+| EVAL message = CONCAT(env, " environment")
+| LOOKUP JOIN message_types_lookup ON message
+| KEEP @timestamp, client_ip, event_duration, message, type
+;
+ignoreOrder:true
+
+@timestamp:date          | client_ip:keyword | event_duration:long | message:keyword         | type:keyword
+2023-10-23T13:55:01.543Z | 172.21.3.15       | 1756467             | Production environment  | Production
+2023-10-23T13:53:55.832Z | 172.21.3.15       | 5033755             | Production environment  | Production
+2023-10-23T13:52:55.015Z | 172.21.3.15       | 8268153             | Production environment  | Production
+2023-10-23T13:51:54.732Z | 172.21.3.15       | 725448              | Production environment  | Production
+2023-10-23T13:33:34.937Z | 172.21.0.5        | 1232382             | Development environment | Development
+2023-10-23T12:27:28.948Z | 172.21.2.113      | 2764889             | QA environment          | null
+2023-10-23T12:15:03.360Z | 172.21.2.162      | 3450233             | QA environment          | null
+;
+
+lookupIPAndMessageFromIndexChainedRenameKeep
+required_capability: join_lookup_v10
+
+FROM sample_data
+| EVAL client_ip = client_ip::keyword
+| LOOKUP JOIN clientips_lookup ON client_ip
+| RENAME env AS message
+| LOOKUP JOIN message_types_lookup ON message
+| KEEP @timestamp, client_ip, event_duration, message, type
+;
+ignoreOrder:true
+
+@timestamp:date          | client_ip:keyword | event_duration:long | message:keyword | type:keyword
+2023-10-23T13:55:01.543Z | 172.21.3.15       | 1756467             | Production      | null
+2023-10-23T13:53:55.832Z | 172.21.3.15       | 5033755             | Production      | null
+2023-10-23T13:52:55.015Z | 172.21.3.15       | 8268153             | Production      | null
+2023-10-23T13:51:54.732Z | 172.21.3.15       | 725448              | Production      | null
+2023-10-23T13:33:34.937Z | 172.21.0.5        | 1232382             | Development     | null
+2023-10-23T12:27:28.948Z | 172.21.2.113      | 2764889             | QA              | null
+2023-10-23T12:15:03.360Z | 172.21.2.162      | 3450233             | QA              | null
+;
+
+lookupIndexInFromRepeatedRowBug
+required_capability: join_lookup_v10
+FROM languages_lookup_non_unique_key
+| WHERE language_code == 1
 | LOOKUP JOIN languages_lookup ON language_code
-| WHERE language_code > 1 AND language_name IS NOT NULL
-| KEEP emp_no, language_code, language_name
+| KEEP language_code, language_name, country
+| SORT language_code, language_name, country
 ;
 
-emp_no:integer | language_code:integer | language_name:keyword
-10001          | 2                     | French
-10003          | 4                     | German
+language_code:integer | language_name:keyword       | country:text
+1                     | English                     | Canada
+1                     | English                     | United Kingdom
+1                     | English                     | United States of America
+1                     | English                     | null
 ;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-all-types.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-all-types.json
index 04b59f347ebfc..17348adb6af4f 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-all-types.json
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-all-types.json
@@ -59,6 +59,10 @@
         },
         "wildcard": {
             "type": "wildcard"
+        },
+        "semantic_text": {
+            "type": "semantic_text",
+            "inference_id": "foo_inference_id"
         }
     }
 }
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-cartesian_multipolygons_no_doc_values.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-cartesian_multipolygons_no_doc_values.json
new file mode 100644
index 0000000000000..fb271c4975462
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-cartesian_multipolygons_no_doc_values.json
@@ -0,0 +1,15 @@
+{
+  "properties": {
+    "id": {
+      "type": "long"
+    },
+    "name": {
+      "type": "keyword"
+    },
+    "shape": {
+      "type": "shape",
+      "index": true,
+      "doc_values": false
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-languages_nested_fields.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-languages_nested_fields.json
new file mode 100644
index 0000000000000..9b46a85ed8d11
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-languages_nested_fields.json
@@ -0,0 +1,22 @@
+{
+  "properties" : {
+    "language" : {
+      "properties" : {
+        "id": {
+          "type": "integer"
+        },
+        "name": {
+          "type": "text",
+          "fields": {
+            "keyword": {
+              "type": "keyword"
+            }
+          }
+        },
+        "code": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec
index 03b24555dbeff..8a576e841a32e 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec
@@ -115,6 +115,80 @@ book_no:keyword | title:text
 7140            |The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1)     
 ;
 
+matchWithDisjunction
+required_capability: match_function
+required_capability: full_text_functions_disjunctions
+
+from books
+| where match(author, "Vonnegut") or match(author, "Guinane") 
+| keep book_no, author;
+ignoreOrder:true
+
+book_no:keyword | author:text
+2464            | Kurt Vonnegut  
+6970            | Edith Vonnegut 
+8956            | Kurt Vonnegut  
+3950            | Kurt Vonnegut  
+4382            | Carole Guinane 
+;
+
+matchWithDisjunctionAndFiltersConjunction
+required_capability: match_function
+required_capability: full_text_functions_disjunctions
+
+from books
+| where (match(author, "Vonnegut") or match(author, "Guinane")) and year > 1997
+| keep book_no, author, year;
+ignoreOrder:true
+
+book_no:keyword | author:text       | year:integer
+6970            | Edith Vonnegut    | 1998
+4382            | Carole Guinane    | 2001
+;
+
+matchWithDisjunctionAndConjunction
+required_capability: match_function
+required_capability: full_text_functions_disjunctions
+
+from books
+| where (match(author, "Vonnegut") or match(author, "Marquez")) and match(description, "realism")
+| keep book_no;
+
+book_no:keyword
+4814
+;
+
+matchWithMoreComplexDisjunctionAndConjunction
+required_capability: match_function
+required_capability: full_text_functions_disjunctions
+
+from books
+| where (match(author, "Vonnegut") and match(description, "charming")) or (match(author, "Marquez") and match(description, "realism"))
+| keep book_no;
+ignoreOrder:true
+
+book_no:keyword
+6970
+4814
+;
+
+matchWithDisjunctionIncludingConjunction
+required_capability: match_function
+required_capability: full_text_functions_disjunctions
+
+from books
+| where match(author, "Vonnegut") or (match(author, "Marquez") and match(description, "realism"))
+| keep book_no;
+ignoreOrder:true
+
+book_no:keyword
+2464
+6970
+4814
+8956
+3950
+;
+
 matchWithFunctionPushedToLucene
 required_capability: match_function
 
@@ -523,3 +597,79 @@ from employees,employees_incompatible
 
 emp_no_bool:boolean
 ;
+
+testMatchWithSemanticText
+required_capability: match_function
+required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
+
+from semantic_text
+| where match(semantic_text_field, "something")
+| keep semantic_text_field
+| sort semantic_text_field asc
+;
+
+semantic_text_field:text
+all we have to decide is what to do with the time that is given to us
+be excellent to each other
+live long and prosper
+;
+
+testMatchWithSemanticTextAndKeyword
+required_capability: match_function
+required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
+
+from semantic_text
+| where match(semantic_text_field, "something") AND match(host, "host1")
+| keep semantic_text_field, host
+;
+
+semantic_text_field:text | host:keyword
+live long and prosper    | host1
+;
+
+testMatchWithSemanticTextMultiValueField
+required_capability: match_function
+required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
+
+from semantic_text metadata _id
+| where match(st_multi_value, "something") AND match(host, "host1")
+| keep _id, st_multi_value
+;
+
+_id: keyword | st_multi_value:text
+1            | ["Hello there!", "This is a random value", "for testing purposes"]
+;
+
+testMatchWithSemanticTextWithEvalsAndOtherFunctionsAndStats
+required_capability: match_function
+required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
+
+from semantic_text
+| where qstr("description:some*")
+| eval size = mv_count(st_multi_value)
+| where match(semantic_text_field, "something") AND size > 1 AND match(host, "host1")
+| STATS result = count(*)
+;
+
+result:long
+1
+;
+
+testMatchWithSemanticTextAndKql
+required_capability: match_function
+required_capability: semantic_text_type
+required_capability: kql_function
+required_capability: semantic_text_field_caps
+
+from semantic_text
+| where kql("host:host1") AND match(semantic_text_field, "something")
+| KEEP host, semantic_text_field
+;
+
+host:keyword | semantic_text_field:text
+"host1"      | live long and prosper
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec
index 56f7f5ccd8823..6ccf0ea734175 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec
@@ -102,6 +102,81 @@ book_no:keyword | title:text
 7140            |The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1)     
 ;
 
+
+matchWithDisjunction
+required_capability: match_operator_colon
+required_capability: full_text_functions_disjunctions
+
+from books
+| where author : "Vonnegut" or author : "Guinane" 
+| keep book_no, author;
+ignoreOrder:true
+
+book_no:keyword | author:text
+2464            | Kurt Vonnegut  
+6970            | Edith Vonnegut 
+8956            | Kurt Vonnegut  
+3950            | Kurt Vonnegut  
+4382            | Carole Guinane 
+;
+
+matchWithDisjunctionAndFiltersConjunction
+required_capability: match_operator_colon
+required_capability: full_text_functions_disjunctions
+
+from books
+| where (author : "Vonnegut" or author : "Guinane") and year > 1997
+| keep book_no, author, year;
+ignoreOrder:true
+
+book_no:keyword | author:text       | year:integer
+6970            | Edith Vonnegut    | 1998
+4382            | Carole Guinane    | 2001
+;
+
+matchWithDisjunctionAndConjunction
+required_capability: match_operator_colon
+required_capability: full_text_functions_disjunctions
+
+from books
+| where (author : "Vonnegut" or author : "Marquez") and description : "realism"
+| keep book_no;
+
+book_no:keyword
+4814
+;
+
+matchWithMoreComplexDisjunctionAndConjunction
+required_capability: match_function
+required_capability: full_text_functions_disjunctions
+
+from books
+| where (author : "Vonnegut" and description : "charming") or (author : "Marquez" and description : "realism")
+| keep book_no;
+ignoreOrder:true
+
+book_no:keyword
+6970
+4814
+;
+
+matchWithDisjunctionIncludingConjunction
+required_capability: match_operator_colon
+required_capability: full_text_functions_disjunctions
+
+from books
+| where author : "Vonnegut" or (author : "Marquez" and description : "realism")
+| keep book_no;
+ignoreOrder:true
+
+book_no:keyword
+2464
+6970
+4814
+8956
+3950
+;
+
 matchWithFunctionPushedToLucene
 required_capability: match_operator_colon
 
@@ -219,7 +294,7 @@ count(*): long  | author.keyword:keyword
 ;
 
 testMatchBooleanField
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from employees 
@@ -235,7 +310,7 @@ Amabile             | true                  | 2.09
 ;
 
 testMatchIntegerField
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from employees 
@@ -247,7 +322,7 @@ emp_no:integer | first_name:keyword
 ;
 
 testMatchDoubleField
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from employees 
@@ -259,7 +334,7 @@ emp_no:integer | salary_change:double
 ;  
 
 testMatchLongField
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from date_nanos
@@ -271,7 +346,7 @@ num:long
 ;
 
 testMatchUnsignedLongField
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from ul_logs
@@ -283,7 +358,7 @@ bytes_out:unsigned_long
 ;
 
 testMatchIpFieldAsString
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from sample_data
@@ -295,7 +370,7 @@ client_ip:ip   | message:keyword
 ;
 
 testMatchDateFieldAsString
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from date_nanos
@@ -307,7 +382,7 @@ millis:date
 ;
 
 testMatchDateNanosFieldAsString
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from date_nanos
@@ -319,7 +394,7 @@ nanos:date_nanos
 ;
 
 testMatchBooleanFieldAsString
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from employees 
@@ -335,7 +410,7 @@ Amabile             | true                  | 2.09
 ;
 
 testMatchIntegerFieldAsString
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from employees 
@@ -347,7 +422,7 @@ emp_no:integer | first_name:keyword
 ;
 
 testMatchDoubleFieldAsString
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from employees 
@@ -359,7 +434,7 @@ emp_no:integer | salary_change:double
 ;
 
 testMatchLongFieldAsString
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from date_nanos
@@ -371,7 +446,7 @@ num:long
 ;
 
 testMatchUnsignedLongFieldAsString
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from ul_logs
@@ -383,7 +458,7 @@ bytes_out:unsigned_long
 ;
 
 testMatchVersionFieldAsString
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from apps 
@@ -395,7 +470,7 @@ bbbbb        | 2.1
 ;
 
 testMatchIntegerAsDouble
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from employees 
@@ -408,7 +483,7 @@ emp_no:integer | first_name:keyword
 ;
 
 testMatchDoubleAsIntegerField
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from employees
@@ -423,7 +498,7 @@ emp_no:integer | height:double
 ;
 
 testMatchMultipleFieldTypes
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from employees,employees_incompatible 
@@ -440,7 +515,7 @@ emp_as_int:integer | name_as_kw:keyword
 
 
 testMatchMultipleFieldTypesKeywordText
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from employees,employees_incompatible 
@@ -455,7 +530,7 @@ Kazuhito
 ;
 
 testMatchMultipleFieldTypesDoubleFloat
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from employees,employees_incompatible
@@ -474,7 +549,7 @@ emp_no:integer | height_dbl:double
 ;
 
 testMatchMultipleFieldTypesBooleanKeyword
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from employees,employees_incompatible
@@ -491,7 +566,7 @@ true
 ;
 
 testMatchMultipleFieldTypesLongUnsignedLong
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from employees,employees_incompatible  
@@ -506,7 +581,7 @@ avg_worked_seconds_ul:unsigned_long
 ;
 
 testMatchMultipleFieldTypesDateNanosDate
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from employees,employees_incompatible
@@ -521,7 +596,7 @@ hire_date_nanos:date_nanos
 ;
 
 testMatchWithWrongFieldValue
-required_capability: match_function
+required_capability: match_operator_colon
 required_capability: match_additional_types
 
 from employees,employees_incompatible
@@ -533,3 +608,78 @@ from employees,employees_incompatible
 emp_no_bool:boolean
 ;
 
+testMatchWithSemanticText
+required_capability: match_operator_colon
+required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
+
+from semantic_text
+| where semantic_text_field:"something"
+| keep semantic_text_field
+| sort semantic_text_field asc
+;
+
+semantic_text_field:text
+all we have to decide is what to do with the time that is given to us
+be excellent to each other
+live long and prosper
+;
+
+testMatchWithSemanticTextAndKeyword
+required_capability: match_operator_colon
+required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
+
+from semantic_text
+| where semantic_text_field:"something" AND host:"host1"
+| keep semantic_text_field, host
+;
+
+semantic_text_field:text | host:keyword
+live long and prosper    | host1
+;
+
+testMatchWithSemanticTextMultiValueField
+required_capability: match_operator_colon
+required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
+
+from semantic_text metadata _id
+| where st_multi_value:"something" AND match(host, "host1")
+| keep _id, st_multi_value
+;
+
+_id: keyword | st_multi_value:text
+1            | ["Hello there!", "This is a random value", "for testing purposes"]
+;
+
+testMatchWithSemanticTextWithEvalsAndOtherFunctionsAndStats
+required_capability: match_operator_colon
+required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
+
+from semantic_text
+| where qstr("description:some*")
+| eval size = mv_count(st_multi_value)
+| where semantic_text_field:"something" AND size > 1 AND match(host, "host1")
+| STATS result = count(*)
+;
+
+result:long
+1
+;
+
+testMatchWithSemanticTextAndKql
+required_capability: match_operator_colon
+required_capability: semantic_text_type
+required_capability: kql_function
+required_capability: semantic_text_field_caps
+
+from semantic_text
+| where kql("host:host1") AND semantic_text_field:"something"
+| KEEP host, semantic_text_field
+;
+
+host:keyword | semantic_text_field:text
+"host1"      | live long and prosper
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec
index 9914d073a589d..7bf3bc7613e01 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec
@@ -191,3 +191,15 @@ emp_no:integer | languages:integer | height:double | x:double | y:double | z:dou
          10020 | null              | 1.41          | 1.41     | 1.41     | 40031.0  | 40031
          10021 | null              | 1.47          | 1.47     | 1.47     | 60408.0  | 60408
 ;
+
+whereNull
+FROM employees
+| WHERE NULL and emp_no <= 10021
+| SORT first_name, last_name
+| EVAL fullname = CONCAT(first_name, " ", last_name)
+| KEEP fullname, job_positions, salary, salary_change
+| limit 5
+;
+
+fullname:keyword | job_positions:keyword | salary:integer | salary_change:double
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec
index cb38204a71ab0..f1fe4ce5484dc 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec
@@ -13,9 +13,9 @@ from books metadata _score
 | sort c_score desc, book_no asc
 | LIMIT 2;
 
-book_no:keyword | title:text                                                                                  | c_score:double
-2675            | The Lord of the Rings - Boxed Set                                                           | 6.0
-4023            | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | 6.0
+book_no:keyword | title:text                                | c_score:double
+1463            | Realms of Tolkien: Images of Middle-earth | 6.0           
+2675            | The Lord of the Rings - Boxed Set         | 6.0
 ;
 
 singleMatchWithKeywordFieldScoring
@@ -28,15 +28,15 @@ from books metadata _score
 | sort book_no;
 
 book_no:keyword | author:text | _score:double
-2713            | William Faulkner | 2.3142893314361572
-2883            | William Faulkner | 2.3142893314361572
-4724            | William Faulkner | 2.3142893314361572
-4977            | William Faulkner | 2.3142893314361572
-5119            | William Faulkner | 2.3142893314361572
-5404            | William Faulkner | 2.3142893314361572
-5578            | William Faulkner | 2.3142893314361572
-8077            | William Faulkner | 2.3142893314361572
-9896            | William Faulkner | 2.3142893314361572
+2713            | William Faulkner | 1.7589385509490967
+2883            | William Faulkner | 1.7589385509490967
+4724            | William Faulkner | 1.7589385509490967
+4977            | William Faulkner | 2.6145541667938232
+5119            | William Faulkner | 2.513157367706299 
+5404            | William Faulkner | 1.7589385509490967
+5578            | William Faulkner | 2.513157367706299 
+8077            | William Faulkner | 1.7589385509490967
+9896            | William Faulkner | 2.6145541667938232
 ;
 
 qstrWithFieldAndScoringSortedEval
@@ -51,9 +51,9 @@ from books metadata _score
 | limit 3;
 
 book_no:keyword | title:text                                                                 | _score:double 
-2675            | The Lord of the Rings - Boxed Set                                          | 2.7583377361297607
-7140            | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) | 1.9239964485168457
-2714            | Return of the King Being the Third Part of The Lord of the Rings           | 1.9239964485168457
+2675            | The Lord of the Rings - Boxed Set                                          | 2.5619282722473145
+2714            | Return of the King Being the Third Part of The Lord of the Rings           | 1.9245924949645996
+7140            | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) | 1.746896743774414
 ;
 
 qstrWithFieldAndScoringSorted
@@ -67,9 +67,9 @@ from books metadata _score
 | limit 3;
 
 book_no:keyword | title:text                                                                 | _score:double 
-2675            | The Lord of the Rings - Boxed Set                                          | 2.7583377361297607
-7140            | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) | 1.9239964485168457
-2714            | Return of the King Being the Third Part of The Lord of the Rings           | 1.9239964485168457
+2675            | The Lord of the Rings - Boxed Set                                          | 2.5619282722473145
+2714            | Return of the King Being the Third Part of The Lord of the Rings           | 1.9245924949645996
+7140            | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) | 1.746896743774414
 ;
 
 singleQstrScoringManipulated
@@ -84,34 +84,34 @@ from books metadata _score
 | LIMIT 2;
 
 book_no:keyword | author:text                                        | add_score:double
-2378            | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 2.0
-2713            | William Faulkner                                   | 7.0
+2378            | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 3.0
+2713            | William Faulkner                                   | 6.0
 ;
 
 testMultiValuedFieldWithConjunctionWithScore
 required_capability: match_function
 required_capability: metadata_score
 
-from employees metadata _score
-| where match(job_positions, "Data Scientist") and match(job_positions, "Support Engineer")
-| keep emp_no, first_name, last_name, job_positions, _score;
+from books metadata _score
+| where match(author, "Keith Faulkner") and match(author, "Rory Tyger")
+| keep book_no, title, author, _score;
 
-emp_no:integer | first_name:keyword | last_name:keyword | job_positions:keyword | _score:double
-10043          | Yishay             | Tzvieli           | [Data Scientist, Python Developer, Support Engineer] | 5.233309745788574
+book_no:keyword | title:text                                                                            | author:text                  | _score:double
+6151            | Pop! Went Another Balloon: A Magical Counting Storybook (Magical Counting Storybooks) | [Keith Faulkner, Rory Tyger] | 8.5822172164917
 ;
 
 testMatchAndQueryStringFunctionsWithScore
 required_capability: match_function
 required_capability: metadata_score
 
-from employees metadata _score
-| where match(job_positions, "Data Scientist") and qstr("job_positions: (Support Engineer) and gender: F")
-| keep emp_no, first_name, last_name, job_positions, _score;
+from books metadata _score
+| where match(author, "Keith Faulkner") and qstr("author:Rory or author:Beverlie")
+| keep book_no, title, author, _score;
 ignoreOrder:true
 
-emp_no:integer | first_name:keyword | last_name:keyword | job_positions:keyword | _score:double
-10041          | Uri                 | Lenart           | [Data Scientist, Head Human Resources, Internship, Senior Team Lead] | 3.509873867034912
-10043          | Yishay              | Tzvieli          | [Data Scientist, Python Developer, Support Engineer] | 5.233309745788574
+book_no:keyword | title:text                                                                            | author:text                       | _score:double
+3535            | Rainbow's End: A Magical Story and Moneybox                                           | [Beverlie Manson, Keith Faulkner] | 6.5579609870910645
+6151            | Pop! Went Another Balloon: A Magical Counting Storybook (Magical Counting Storybooks) | [Keith Faulkner, Rory Tyger]      | 5.975414276123047 
 ;
 
 multipleWhereWithMatchScoringNoSort
@@ -125,7 +125,7 @@ from books metadata _score
 
 ignoreOrder:true
 book_no:keyword | title:text                                | author:text        | _score:double
-8480            | The wind's twelve quarters: Short stories | Ursula K. Le Guin  | 14.489097595214844
+8480            | The wind's twelve quarters: Short stories | Ursula K. Le Guin  | 11.193471908569336
 ;
 
 multipleWhereWithMatchScoring
@@ -139,7 +139,7 @@ from books metadata _score
 | sort book_no;
 
 book_no:keyword | title:text                                | author:text        | _score:double
-8480            | The wind's twelve quarters: Short stories | Ursula K. Le Guin  | 14.489097595214844
+8480            | The wind's twelve quarters: Short stories | Ursula K. Le Guin  | 11.193471908569336
 ;
 
 combinedMatchWithFunctionsScoring
@@ -153,7 +153,7 @@ from books metadata _score
 | sort book_no;
 
 book_no:keyword | title:text               | author:text    | year:integer | _score:double
-5335            | Letters of J R R Tolkien | J.R.R. Tolkien | 2014         | 5.448054313659668
+5335            | Letters of J R R Tolkien | J.R.R. Tolkien | 2014         | 4.733664035797119
 ;
 
 singleQstrScoring
@@ -167,8 +167,8 @@ from books metadata _score
 | LIMIT 2;
 
 book_no:keyword | author:text                                        | _score:double
-2378            | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 0.9976131916046143
-2713            | William Faulkner                                   | 5.9556169509887695
+2378            | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 1.3697924613952637
+2713            | William Faulkner                                   | 4.631696701049805
 ;
 
 singleQstrScoringGrok
@@ -183,9 +183,9 @@ from books metadata _score
 | LIMIT 3;
 
 book_no:keyword | title:keyword | _score:double
-8875            | The | 2.9505908489227295
-4023            | A   | 2.8327860832214355
-2675            | The | 2.7583377361297607
+8875            | The           | 2.769660472869873 
+1463            | Realms        | 2.6714818477630615
+2675            | The           | 2.5619282722473145
 ;
 
 combinedMatchWithScoringEvalNoSort
@@ -200,7 +200,7 @@ from books metadata _score
 
 ignoreOrder:true
 book_no:keyword | title:text               | author:text    | year:integer | c_score:double
-5335            | Letters of J R R Tolkien | J.R.R. Tolkien | 2014         | 6
+5335            | Letters of J R R Tolkien | J.R.R. Tolkien | 2014         | 5.0
 ;
 
 singleQstrScoringRename
@@ -215,9 +215,9 @@ from books metadata _score
 | LIMIT 3;
 
 book_no:keyword | rank:double
-8875            | 2.9505908489227295
-4023            | 2.8327860832214355
-2675            | 2.7583377361297607
+8875            | 2.769660472869873 
+1463            | 2.6714818477630615
+2675            | 2.5619282722473145
 ;
 
 singleMatchWithTextFieldScoring
@@ -231,11 +231,11 @@ from books metadata _score
 | limit 5;
 
 book_no:keyword | author:text                                        | _score:double
-2378            | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 0.9976131916046143
-2713            | William Faulkner                                   | 4.272439002990723
-2847            | Colleen Faulkner                                   | 1.7401835918426514
-2883            | William Faulkner                                   | 4.272439002990723
-3293            | Danny Faulkner                                     | 1.7401835918426514
+2378            | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 1.3697924613952637
+2713            | William Faulkner                                   | 3.2750158309936523
+2847            | Colleen Faulkner                                   | 1.593343734741211 
+2883            | William Faulkner                                   | 3.2750158309936523
+3293            | Danny Faulkner                                     | 1.593343734741211
 ;
 
 combinedMatchWithFunctionsScoringNoSort
@@ -249,7 +249,7 @@ from books metadata _score
 
 ignoreOrder:true
 book_no:keyword | title:text               | author:text    | year:integer | _score:double
-5335            | Letters of J R R Tolkien | J.R.R. Tolkien | 2014         | 5.448054313659668
+5335            | Letters of J R R Tolkien | J.R.R. Tolkien | 2014         | 4.733664035797119
 ;
 
 combinedMatchWithScoringEval
@@ -264,7 +264,7 @@ from books metadata _score
 | sort book_no;
 
 book_no:keyword | title:text               | author:text    | year:integer | c_score:double
-5335            | Letters of J R R Tolkien | J.R.R. Tolkien | 2014         | 6
+5335            | Letters of J R R Tolkien | J.R.R. Tolkien | 2014         | 5.0
 ;
 
 singleQstrScoringEval
@@ -280,7 +280,7 @@ from books metadata _score
 
 book_no:keyword | c_score:double
 8875            | 3.0
-7350            | 2.0
+7350            | 1.0
 7140            | 3.0
 ;
 
@@ -289,14 +289,16 @@ required_capability: metadata_score
 required_capability: qstr_function
 
 from books metadata _score 
-| where qstr("title:rings") 
+| where qstr("title:gentle") 
 | eval _score = _score + 1 
 | keep book_no, title, _score
-| limit 2;
+| limit 10
+;
+ignoreOrder:true
 
-book_no:keyword | title:text                                                                                  | _score:double
-4023            | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | 2.6404519081115723
-2714            | Return of the King Being the Third Part of The Lord of the Rings                            | 2.9239964485168457
+book_no:keyword | title:text                                                                                                                     | _score:double
+2924            | A Gentle Creature and Other Stories: White Nights, A Gentle Creature, and The Dream of a Ridiculous Man (The World's Classics) | 3.158426523208618
+5948            | That We Are Gentle Creatures                                                                                                   | 3.727346897125244 
 ;
 
 QstrScoreOverride
@@ -304,12 +306,54 @@ required_capability: metadata_score
 required_capability: qstr_function
 
 from books metadata _score 
-| where qstr("title:rings") 
+| where qstr("title:gentle") 
 | eval _score = "foobar"
 | keep book_no, title, _score 
-| limit 2;
+| limit 10
+;
+ignoreOrder:true
+
+book_no:keyword | title:text                                                                                                                     | _score:keyword
+2924            | A Gentle Creature and Other Stories: White Nights, A Gentle Creature, and The Dream of a Ridiculous Man (The World's Classics) | foobar        
+5948            | That We Are Gentle Creatures                                                                                                   | foobar
+;
+
+
+semanticTextMatch
+required_capability: metadata_score
+required_capability: semantic_text_type
+required_capability: match_function
+
+from semantic_text metadata _id, _score
+| where match(semantic_text_field, "something")
+| sort _score desc
+| keep _id
+;
+
+_id:keyword
+2
+3
+1
+;
+
+semanticTextMatchWithAllTheTextFunctions
+
+required_capability: metadata_score
+required_capability: semantic_text_type
+required_capability: match_function
+required_capability: kql_function
+required_capability: qstr_function
+
+from semantic_text metadata _id, _score
+| where match(semantic_text_field, "something")
+        AND match(description, "some")
+        AND kql("description:some*")
+        AND NOT qstr("host:host1")
+| sort _score desc
+| keep _id
+;
 
-book_no:keyword | title:text                                                                                  | _score:keyword
-4023            | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | foobar
-2714            | Return of the King Being the Third Part of The Lord of the Rings                            | foobar
+_id:keyword
+2
+3
 ;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec
index 43dc6e4d4acd2..19b3388af0772 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec
@@ -1,32 +1,32 @@
 simple
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text
 | KEEP semantic_text_field
 | sort semantic_text_field asc;
 
-semantic_text_field:semantic_text
+semantic_text_field:text
 all we have to decide is what to do with the time that is given to us
 be excellent to each other
 live long and prosper
 ;
 
 simpleWithUnicode
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text
 | KEEP st_unicode
 | SORT st_unicode
 ;
 
-st_unicode:semantic_text
+st_unicode:text
 你吃饭了吗
 ["谢谢", "对不起我的中文不好"]
 null
 ;
 
 mvExpand
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | KEEP _id, st_multi_value
@@ -34,7 +34,7 @@ FROM semantic_text METADATA _id
 | SORT st_multi_value
 ;
 
-_id:keyword | st_multi_value:semantic_text
+_id:keyword | st_multi_value:text
 1           | Hello there!
 1           | This is a random value
 2           | bye bye!
@@ -44,7 +44,7 @@ _id:keyword | st_multi_value:semantic_text
 ;
 
 withDropAndKeep
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | KEEP _id, semantic_text_field, st_double
@@ -52,14 +52,14 @@ FROM semantic_text METADATA _id
 | SORT _id
 ;
 
-_id:keyword | semantic_text_field:semantic_text
+_id:keyword | semantic_text_field:text
 1           | live long and prosper
 2           | all we have to decide is what to do with the time that is given to us
 3           | be excellent to each other
 ;
 
 rename
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | RENAME semantic_text_field AS my_field
@@ -67,14 +67,14 @@ FROM semantic_text METADATA _id
 | SORT _id
 ;
 
-_id:keyword | my_field:semantic_text
+_id:keyword | my_field:text
 1           | live long and prosper
 2           | all we have to decide is what to do with the time that is given to us
 3           | be excellent to each other
 ;
 
 eval
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL my_field = semantic_text_field
@@ -82,7 +82,7 @@ FROM semantic_text METADATA _id
 | SORT _id
 ;
 
-_id:keyword | my_field:semantic_text
+_id:keyword | my_field:text
 1           | live long and prosper
 2           | all we have to decide is what to do with the time that is given to us
 3           | be excellent to each other
@@ -163,14 +163,14 @@ FROM semantic_text METADATA _id
 | SORT st_version
 ;
 
-COUNT(*):long | st_version:semantic_text
+COUNT(*):long | st_version:text
 1             | 1.2.3
 1             | 9.0.0
 1             | null
 ;
 
 withDropKeepStatsMvExpandRenameSortLimit
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | KEEP _id, semantic_text_field, st_multi_value
@@ -182,14 +182,14 @@ FROM semantic_text METADATA _id
 | LIMIT 3
 ;
 
-COUNT(*):long | my_field:semantic_text
+COUNT(*):long | my_field:text
 1             | Hello there!
 1             | This is a random value
 1             | bye bye!
 ;
 
 grok
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | GROK st_logs """%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num}"""
@@ -197,14 +197,14 @@ FROM semantic_text METADATA _id
 | SORT st_logs
 ;
 
-st_logs:semantic_text                                     | date:keyword             | ip:keyword | email:keyword       | num:keyword
+st_logs:text                                              | date:keyword             | ip:keyword | email:keyword       | num:keyword
 2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42  | 2023-01-23T12:15:00.000Z | 127.0.0.1  | some.email@foo.com  | 42
 2024-01-23T12:15:00.000Z 1.2.3.4 foo@example.com 42       | 2024-01-23T12:15:00.000Z | 1.2.3.4    | foo@example.com     | 42
 2024-12-23T12:15:00.000Z 1.2.3.4 example@example.com 4553 | 2024-12-23T12:15:00.000Z | 1.2.3.4    | example@example.com | 4553
 ;
 
 dissect
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | DISSECT st_logs """%{date} %{ip} %{email} %{num}"""
@@ -212,56 +212,56 @@ FROM semantic_text METADATA _id
 | SORT st_logs
 ;
 
-st_logs:semantic_text                                     | date:keyword             | ip:keyword | email:keyword       | num:keyword
+st_logs:text                                              | date:keyword             | ip:keyword | email:keyword       | num:keyword
 2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42  | 2023-01-23T12:15:00.000Z | 127.0.0.1  | some.email@foo.com  | 42
 2024-01-23T12:15:00.000Z 1.2.3.4 foo@example.com 42       | 2024-01-23T12:15:00.000Z | 1.2.3.4    | foo@example.com     | 42
 2024-12-23T12:15:00.000Z 1.2.3.4 example@example.com 4553 | 2024-12-23T12:15:00.000Z | 1.2.3.4    | example@example.com | 4553
 ;
 
 simpleWithLongValue
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text
 | KEEP value, semantic_text_field
 | SORT value
 ;
 
-value:long | semantic_text_field:semantic_text
+value:long      | semantic_text_field:text
 1001            | live long and prosper
 1002            | all we have to decide is what to do with the time that is given to us
 1003            | be excellent to each other
 ;
 
 simpleWithText
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | KEEP description, semantic_text_field
 | SORT description
 ;
 
-description:text     | semantic_text_field:semantic_text
+description:text     | semantic_text_field:text
 "some description1"  | live long and prosper
 "some description2"  | all we have to decide is what to do with the time that is given to us
 "some description3"  | be excellent to each other
 ;
 
 simpleWithKeyword
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | KEEP host, semantic_text_field
 | SORT host
 ;
 
-host:keyword | semantic_text_field:semantic_text
+host:keyword | semantic_text_field:text
 "host1"      | live long and prosper
 "host2"      | all we have to decide is what to do with the time that is given to us
 "host3"      | be excellent to each other
 ;
 
 case
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = case(st_ip == "1.1.1.1", "okay", "try again")
@@ -276,7 +276,7 @@ _id:keyword | result:keyword
 ;
 
 coalesce
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = coalesce(st_version, st_ip, semantic_text_field)
@@ -291,7 +291,7 @@ _id:keyword | result:keyword
 ;
 
 greatest
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = greatest(semantic_text_field, st_version)
@@ -306,7 +306,7 @@ _id:keyword | result:keyword
 ;
 
 least
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = least(semantic_text_field, st_version)
@@ -321,7 +321,7 @@ _id:keyword | result:keyword
 ;
 
 convertToBool
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = to_bool(st_bool)
@@ -336,7 +336,7 @@ _id:keyword | result:bool
 ;
 
 convertToCartesianPoint
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = to_cartesianpoint(st_cartesian_point)
@@ -351,7 +351,7 @@ _id:keyword | result:cartesian_point
 ;
 
 convertToCartesianShape
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = to_cartesianshape(st_cartesian_shape)
@@ -366,7 +366,7 @@ _id:keyword | result:cartesian_shape
 ;
 
 convertToDatetime
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = to_datetime(st_datetime)
@@ -381,7 +381,7 @@ _id:keyword|result:datetime
 ;
 
 convertToDouble
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = to_double(st_double)
@@ -396,7 +396,7 @@ _id:keyword|result:double
 ;
 
 convertToGeopoint
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = to_geopoint(st_geopoint)
@@ -411,7 +411,7 @@ _id:keyword | result:geo_point
 ;
 
 convertToGeoshape
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = to_geoshape(st_geoshape)
@@ -426,7 +426,7 @@ _id:keyword | result:geo_shape
 ;
 
 convertToInteger
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = to_integer(st_integer)
@@ -441,7 +441,7 @@ _id:keyword | result:integer
 ;
 
 convertToIp
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = to_ip(st_ip)
@@ -456,7 +456,7 @@ _id:keyword | result:ip
 ;
 
 convertToLong
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = to_long(st_long)
@@ -471,7 +471,7 @@ _id:keyword | result:long
 ;
 
 convertToUnsignedLong
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = to_unsigned_long(st_unsigned_long)
@@ -486,7 +486,7 @@ _id:keyword | result:unsigned_long
 ;
 
 convertToVersion
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = to_version(st_version)
@@ -501,7 +501,7 @@ _id:keyword | result:version
 ;
 
 concat
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text
 | EVAL result = concat("<em>", semantic_text_field, "</em>")
@@ -516,7 +516,7 @@ result:keyword
 ;
 
 endsWith
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text
 | WHERE ends_with(semantic_text_field, "er")
@@ -524,13 +524,13 @@ FROM semantic_text
 | SORT semantic_text_field
 ;
 
-semantic_text_field:semantic_text
+semantic_text_field:text
 be excellent to each other
 live long and prosper
 ;
 
 fromBase64
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 FROM semantic_text
 | EVAL result = from_base64(st_base64)
 | SORT result
@@ -544,7 +544,7 @@ null
 ;
 
 left
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text
 | EVAL result = left(semantic_text_field, 2)
@@ -559,7 +559,7 @@ li
 ;
 
 length
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text
 | EVAL result = length(st_version)
@@ -574,7 +574,7 @@ null
 ;
 
 locate
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = locate(semantic_text_field, "all")
@@ -589,7 +589,7 @@ _id:keyword | result:integer
 ;
 
 ltrim
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text
 | EVAL result = ltrim(semantic_text_field)
@@ -604,7 +604,7 @@ live long and prosper
 ;
 
 repeat
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text
 | EVAL result = repeat(semantic_text_field, 2)
@@ -617,7 +617,7 @@ live long and prosperlive long and prosper
 ;
 
 replace
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text
 | EVAL result = replace(semantic_text_field, "excellent", "good")
@@ -632,7 +632,7 @@ live long and prosper
 ;
 
 right
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text
 | EVAL result = right(semantic_text_field, 2)
@@ -647,7 +647,7 @@ us
 ;
 
 rtrim
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text
 | EVAL result = rtrim(semantic_text_field)
@@ -662,7 +662,7 @@ live long and prosper
 ;
 
 split
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = split(st_version, ".")
@@ -677,7 +677,7 @@ null
 ;
 
 startsWith
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = starts_with(semantic_text_field, "be")
@@ -692,7 +692,7 @@ _id:keyword | result:bool
 ;
 
 substring
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = substring(semantic_text_field, 2, 1)
@@ -707,7 +707,7 @@ _id:keyword | result:keyword
 ;
 
 toBase64
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = to_base64(st_integer)
@@ -722,7 +722,7 @@ _id:keyword | result:keyword
 ;
 
 toLower
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = to_lower(st_cartesian_point)
@@ -737,7 +737,7 @@ _id:keyword | result:keyword
 ;
 
 toUpper
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = to_upper(semantic_text_field)
@@ -752,7 +752,7 @@ _id:keyword | result:keyword
 ;
 
 trim
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text
 | EVAL result = trim(semantic_text_field)
@@ -767,7 +767,7 @@ live long and prosper
 ;
 
 mvAppend
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = mv_append(st_multi_value, st_long)
@@ -782,7 +782,7 @@ _id: keyword | result:keyword
 ;
 
 mvConcat
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = mv_concat(st_multi_value, "; ")
@@ -797,7 +797,7 @@ _id: keyword | result:keyword
 ;
 
 mvCount
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = mv_count(st_multi_value)
@@ -812,7 +812,7 @@ _id: keyword | result:integer
 ;
 
 mvDedupe
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = mv_dedupe(st_multi_value)
@@ -827,7 +827,7 @@ _id: keyword | result:keyword
 ;
 
 mvFirst
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = mv_first(st_multi_value)
@@ -842,7 +842,7 @@ _id: keyword | result:keyword
 ;
 
 mvLast
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = mv_last(st_multi_value)
@@ -857,7 +857,7 @@ _id: keyword | result:keyword
 ;
 
 mvMax
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = mv_max(st_multi_value)
@@ -872,7 +872,7 @@ _id: keyword | result:keyword
 ;
 
 mvMin
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = mv_min(st_multi_value)
@@ -887,7 +887,7 @@ _id: keyword | result:keyword
 ;
 
 mvSlice
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = mv_slice(st_multi_value, 1, 2)
@@ -902,7 +902,7 @@ _id: keyword | result:keyword
 ;
 
 mvSort
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = mv_sort(st_multi_value, "ASC")
@@ -917,7 +917,7 @@ _id: keyword | result:keyword
 ;
 
 mvZip
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = mv_zip(st_multi_value, st_multi_value, " + ")
@@ -932,7 +932,7 @@ _id: keyword | result:keyword
 ;
 
 equalityWithConstant
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = st_ip == "1.1.1.1"
@@ -947,7 +947,7 @@ _id: keyword | result:bool
 ;
 
 equalityBetweenFields
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = st_long == st_unsigned_long
@@ -962,7 +962,7 @@ _id: keyword | result:bool
 ;
 
 inequalityWithConstant
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = st_ip != "1.1.1.1"
@@ -977,7 +977,7 @@ _id: keyword | result:bool
 ;
 
 inequalityBetweenFields
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = st_long != st_unsigned_long
@@ -992,7 +992,7 @@ _id: keyword | result:bool
 ;
 
 lessThanWithConstant
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = semantic_text_field < "bye!"
@@ -1007,7 +1007,7 @@ _id: keyword | result:bool
 ;
 
 lessThanBetweenFields
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = semantic_text_field < st_version
@@ -1023,7 +1023,7 @@ _id: keyword | result:bool
 
 
 lessThanOrEqualToWithConstant
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 
 FROM semantic_text METADATA _id
@@ -1039,7 +1039,7 @@ _id: keyword | result:bool
 ;
 
 lessThanOrEqualToBetweenFields
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = st_integer <= st_long
@@ -1054,7 +1054,7 @@ _id: keyword | result:bool
 ;
 
 greaterThanWithConstant
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = semantic_text_field > "bye!"
@@ -1069,7 +1069,7 @@ _id: keyword | result:bool
 ;
 
 greaterThanBetweenFields
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = semantic_text_field > st_version
@@ -1084,7 +1084,7 @@ _id: keyword | result:bool
 ;
 
 greaterThanOrEqualToWithConstant
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = semantic_text_field >= "be excellent to each other"
@@ -1099,7 +1099,7 @@ _id: keyword | result:bool
 ;
 
 greaterThanOrEqualToBetweenFields
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = st_integer >= st_long
@@ -1114,7 +1114,7 @@ _id: keyword | result:bool
 ;
 
 isNull
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = st_integer IS NULL
@@ -1129,7 +1129,7 @@ _id: keyword | result:bool
 ;
 
 isNotNull
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = st_integer IS NOT NULL
@@ -1144,7 +1144,7 @@ _id: keyword | result:bool
 ;
 
 cast
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = st_bool::BOOL
@@ -1159,7 +1159,7 @@ _id:keyword | result:bool
 ;
 
 in
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = st_integer IN ("123", "23")
@@ -1174,7 +1174,7 @@ _id: keyword | result:bool
 ;
 
 like
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = semantic_text_field LIKE "all*"
@@ -1189,7 +1189,7 @@ _id: keyword | result:bool
 ;
 
 rlike
-required_capability: semantic_text_type
+required_capability: semantic_text_field_caps
 
 FROM semantic_text METADATA _id
 | EVAL result = st_version RLIKE "[0-9].[0-9].[0-9]"
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec
index 8694c973448e9..689c36197ee70 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec
@@ -1797,6 +1797,47 @@ extent:cartesian_shape
 BBOX (0.0, 3.0, 3.0, 0.0)
 ;
 
+stExtentCartesianShapesGrouping
+required_capability: st_extent_agg
+
+FROM cartesian_multipolygons
+| EVAL key = SUBSTRING(name,1,3)
+| STATS extent = ST_EXTENT_AGG(shape), count = COUNT() BY key
+| KEEP count, key, extent
+| SORT count DESC, key ASC
+;
+
+count:long | key:keyword | extent:cartesian_shape
+8          | Bot         | BBOX (0.0, 3.0, 1.0, 0.0)
+8          | Top         | BBOX (0.0, 3.0, 3.0, 2.0)
+4          | Fou         | BBOX (0.0, 3.0, 3.0, 0.0)
+;
+
+stExtentCartesianShapesNoDocValues
+required_capability: st_extent_agg
+FROM cartesian_multipolygons_no_doc_values | STATS extent = ST_EXTENT_AGG(shape)
+;
+
+extent:cartesian_shape
+BBOX (0.0, 3.0, 3.0, 0.0)
+;
+
+stExtentCartesianShapesGroupingNoDocValues
+required_capability: st_extent_agg
+
+FROM cartesian_multipolygons_no_doc_values
+| EVAL key = SUBSTRING(name,1,3)
+| STATS extent = ST_EXTENT_AGG(shape), count = COUNT() BY key
+| KEEP count, key, extent
+| SORT count DESC, key ASC
+;
+
+count:long | key:keyword | extent:cartesian_shape
+8          | Bot         | BBOX (0.0, 3.0, 1.0, 0.0)
+8          | Top         | BBOX (0.0, 3.0, 3.0, 2.0)
+4          | Fou         | BBOX (0.0, 3.0, 3.0, 0.0)
+;
+
 ###############################################
 # Tests for ST_INTERSECTS on CARTESIAN_POINT type
 
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec
index 80586ce9bcb09..b6c48ac0bb4b9 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec
@@ -931,6 +931,15 @@ null           | 1950
 null           | 1960
 ;
 
+countAllOnly
+from employees | stats c = count()
+;
+
+c:l
+100
+;
+
+
 countFieldNoGrouping
 from employees | where emp_no < 10050 | stats c = count(salary);
 
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
index e103168d2e589..091baafe293d2 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
@@ -1231,6 +1231,189 @@ a:keyword           | upper:keyword         | lower:keyword
 π/2 + a + B + Λ ºC  | Π/2 + A + B + Λ ºC    | π/2 + a + b + λ ºc
 ;
 
+equalsToUpperPushedDown#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees
+| where to_upper(first_name) == "GEORGI"
+| keep emp_no, first_name
+;
+
+emp_no:integer  | first_name:keyword
+10001           | Georgi
+;
+
+equalsToUpperNestedPushedDown#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees
+| where to_upper(to_upper(to_lower(first_name))) == "GEORGI"
+| keep emp_no, first_name
+;
+
+emp_no:integer  | first_name:keyword
+10001           | Georgi
+;
+
+negatedEqualsToUpperPushedDown#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees
+| sort emp_no
+| where not(to_upper(first_name) == "GEORGI")
+| keep emp_no, first_name
+| limit 1
+;
+
+emp_no:integer  | first_name:keyword
+10002           | Bezalel
+;
+
+notEqualsToUpperPushedDown#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees
+| sort emp_no
+| where to_upper(first_name) != "GEORGI"
+| keep emp_no, first_name
+| limit 1
+;
+
+emp_no:integer  | first_name:keyword
+10002           | Bezalel
+;
+
+negatedNotEqualsToUpperPushedDown#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees
+| sort emp_no
+| where not(to_upper(first_name) != "GEORGI")
+| keep emp_no, first_name
+| limit 1
+;
+
+emp_no:integer  | first_name:keyword
+10001           | Georgi
+;
+
+equalsToUpperFolded#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees
+| where to_upper(first_name) == "Georgi"
+| keep emp_no, first_name
+;
+
+emp_no:integer  | first_name:keyword
+;
+
+negatedEqualsToUpperFolded#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees
+| where not(to_upper(first_name) == "Georgi")
+| stats c = count()
+;
+
+c:long
+90
+;
+
+equalsToUpperNullFolded#[skip:-8.16.99, reason:function's type corrected in #114334]
+from employees
+| where to_upper(null) == "Georgi"
+| keep emp_no, first_name
+;
+
+emp_no:integer  | first_name:keyword
+;
+
+equalsNullToUpperFolded#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees
+| where to_upper(first_name) == to_string(null)
+| keep emp_no, first_name
+;
+
+emp_no:integer  | first_name:keyword
+;
+
+notEqualsToUpperNullFolded#[skip:-8.16.99, reason:function's type corrected in #114334]
+from employees
+| where to_upper(null) != "Georgi"
+| keep emp_no, first_name
+;
+
+emp_no:integer  | first_name:keyword
+;
+
+notEqualsNullToUpperFolded#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees
+| where to_upper(first_name) != to_string(null)
+| keep emp_no, first_name
+;
+
+emp_no:integer  | first_name:keyword
+;
+
+notEqualsToUpperFolded#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees
+| where to_upper(first_name) != "Georgi"
+| stats c = count()
+;
+
+c:long
+90
+;
+
+negatedNotEqualsToUpperFolded#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees
+| where not(to_upper(first_name) != "Georgi")
+| stats c = count()
+;
+
+c:long
+0
+;
+
+equalsToLowerPushedDown#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees
+| where to_lower(first_name) == "georgi"
+| keep emp_no, first_name
+;
+
+emp_no:integer  | first_name:keyword
+10001           | Georgi
+;
+
+notEqualsToLowerPushedDown#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees
+| sort emp_no
+| where to_lower(first_name) != "georgi"
+| keep emp_no, first_name
+| limit 1
+;
+
+emp_no:integer  | first_name:keyword
+10002           | Bezalel
+;
+
+equalsToLowerFolded#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees
+| where to_lower(first_name) == "Georgi"
+| keep emp_no, first_name
+;
+
+emp_no:integer  | first_name:keyword
+;
+
+notEqualsToLowerFolded#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees
+| where to_lower(first_name) != "Georgi"
+| stats c = count()
+;
+
+c:long
+90
+;
+
+equalsToLowerWithUnico(rn|d)s#[skip:-8.12.99, reason:case insensitive operators implemented in v 8.13]
+from employees
+| where to_lower(concat(first_name, "🦄🦄")) != "georgi🦄🦄"
+| stats c = count()
+;
+
+// 10 null first names
+c:long
+89
+;
+
 reverse
 required_capability: fn_reverse
 from employees | sort emp_no | eval name_reversed = REVERSE(first_name) | keep emp_no, first_name, name_reversed | limit 1;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec
index bf6e2f8ae0893..a2f491e20e3b9 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec
@@ -263,8 +263,8 @@ FROM sample_data, sample_data_str
 | LIMIT 1
 ;
 
-@timestamp:date           |  client_ip:null  |  event_duration:long  |  message:keyword
-2023-10-23T13:33:34.937Z  |  null            |  1232382              |  Disconnected
+@timestamp:date           |  client_ip:unsupported  |  event_duration:long  |  message:keyword
+2023-10-23T13:33:34.937Z  |  null                   |  1232382              |  Disconnected
 ;
 
 multiIndexSortIpStringEval
@@ -278,8 +278,8 @@ FROM sample_data, sample_data_str
 | LIMIT 1
 ;
 
-@timestamp:date           |  client_ip:null  |  event_duration:long  |  message:keyword | client_ip_as_ip:ip
-2023-10-23T13:33:34.937Z  |  null            |  1232382              |  Disconnected    | 172.21.0.5
+@timestamp:date           |  client_ip:unsupported  |  event_duration:long  |  message:keyword | client_ip_as_ip:ip
+2023-10-23T13:33:34.937Z  |  null                   |  1232382              |  Disconnected    | 172.21.0.5
 ;
 
 multiIndexIpStringStats
@@ -445,6 +445,7 @@ count:long  |  message:keyword
 
 multiIndexMissingIpToString
 required_capability: union_types
+required_capability: metadata_fields
 required_capability: union_types_missing_field
 
 FROM sample_data, sample_data_str, missing_ip_sample_data METADATA _index
@@ -479,6 +480,7 @@ sample_data_str        | 2023-10-23T12:15:03.360Z  |  172.21.2.162       |  3450
 
 multiIndexMissingIpToIp
 required_capability: union_types
+required_capability: metadata_fields
 required_capability: union_types_missing_field
 
 FROM sample_data, sample_data_str, missing_ip_sample_data METADATA _index
@@ -1268,11 +1270,11 @@ FROM sample_data* METADATA _index
 | SORT _index ASC, ts DESC
 ;
 
-@timestamp:null | client_ip:null | event_duration:long | message:keyword  | _index:keyword       | ts:date                   | ts_str:keyword                 | ts_l:long           |  ip:ip         |  ip_str:k
-null            | null           |  8268153            | Connection error | sample_data          | 2023-10-23T13:52:55.015Z  | 2023-10-23T13:52:55.015Z       | 1698069175015       |  172.21.3.15   |  172.21.3.15
-null            | null           |  8268153            | Connection error | sample_data_str      | 2023-10-23T13:52:55.015Z  | 2023-10-23T13:52:55.015Z       | 1698069175015       |  172.21.3.15   |  172.21.3.15
-null            | null           |  8268153            | Connection error | sample_data_ts_long  | 2023-10-23T13:52:55.015Z  | 1698069175015                  | 1698069175015       |  172.21.3.15   |  172.21.3.15
-null            | null           |  8268153            | Connection error | sample_data_ts_nanos | 2023-10-23T13:52:55.015Z  | 2023-10-23T13:52:55.015123456Z | 1698069175015123456 |  172.21.3.15   |  172.21.3.15
+@timestamp:unsupported | client_ip:unsupported | event_duration:long | message:keyword  | _index:keyword       | ts:date                   | ts_str:keyword                 | ts_l:long           |  ip:ip         |  ip_str:k
+null                   | null                  |  8268153            | Connection error | sample_data          | 2023-10-23T13:52:55.015Z  | 2023-10-23T13:52:55.015Z       | 1698069175015       |  172.21.3.15   |  172.21.3.15
+null                   | null                  |  8268153            | Connection error | sample_data_str      | 2023-10-23T13:52:55.015Z  | 2023-10-23T13:52:55.015Z       | 1698069175015       |  172.21.3.15   |  172.21.3.15
+null                   | null                  |  8268153            | Connection error | sample_data_ts_long  | 2023-10-23T13:52:55.015Z  | 1698069175015                  | 1698069175015       |  172.21.3.15   |  172.21.3.15
+null                   | null                  |  8268153            | Connection error | sample_data_ts_nanos | 2023-10-23T13:52:55.015Z  | 2023-10-23T13:52:55.015123456Z | 1698069175015123456 |  172.21.3.15   |  172.21.3.15
 ;
 
 multiIndexMultiColumnTypesRenameAndKeep
@@ -1373,21 +1375,17 @@ client_ip:ip | event_duration:long |    message:keyword    |    @timestamp:keywo
 # Once INLINESTATS supports expressions in agg functions and groups, convert the group in the inlinestats
 
 multiIndexIndirectUseOfUnionTypesInSort
-// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
-// make the csv tests work with multiple indices.
 required_capability: union_types
 FROM sample_data, sample_data_ts_long
 | SORT client_ip ASC
 | LIMIT 1
 ;
 
-@timestamp:null | client_ip:ip | event_duration:long | message:keyword
-           null | 172.21.0.5   |             1232382 | Disconnected
+@timestamp:unsupported | client_ip:ip | event_duration:long | message:keyword
+           null        | 172.21.0.5   |             1232382 | Disconnected
 ;
 
 multiIndexIndirectUseOfUnionTypesInEval
-// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
-// make the csv tests work with multiple indices.
 required_capability: union_types
 FROM sample_data, sample_data_ts_long
 | EVAL foo = event_duration > 1232381
@@ -1395,13 +1393,11 @@ FROM sample_data, sample_data_ts_long
 | LIMIT 1
 ;
 
-@timestamp:null | client_ip:ip | event_duration:long | message:keyword | foo:boolean
-           null | 172.21.0.5   |             1232382 | Disconnected    | true
+@timestamp:unsupported | client_ip:ip | event_duration:long | message:keyword | foo:boolean
+           null        | 172.21.0.5   |             1232382 | Disconnected    | true
 ;
 
 multiIndexIndirectUseOfUnionTypesInRename
-// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
-// make the csv tests work with multiple indices.
 required_capability: union_types
 required_capability: union_types_fix_rename_resolution
 FROM sample_data, sample_data_ts_long
@@ -1410,13 +1406,11 @@ FROM sample_data, sample_data_ts_long
 | LIMIT 1
 ;
 
-@timestamp:null | client_ip:ip | event_duration:long | event_message:keyword
-           null | 172.21.0.5   |             1232382 | Disconnected
+@timestamp:unsupported | client_ip:ip | event_duration:long | event_message:keyword
+           null        | 172.21.0.5   |             1232382 | Disconnected
 ;
 
 multiIndexIndirectUseOfUnionTypesInKeep
-// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
-// make the csv tests work with multiple indices.
 required_capability: union_types
 FROM sample_data, sample_data_ts_long
 | KEEP client_ip, event_duration, message
@@ -1429,8 +1423,6 @@ client_ip:ip | event_duration:long | message:keyword
 ;
 
 multiIndexIndirectUseOfUnionTypesInWildcardKeep
-// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
-// make the csv tests work with multiple indices.
 required_capability: union_types
 required_capability: union_types_fix_rename_resolution
 FROM sample_data, sample_data_ts_long
@@ -1439,13 +1431,11 @@ FROM sample_data, sample_data_ts_long
 | LIMIT 1
 ;
 
-@timestamp:null | client_ip:ip | event_duration:long | message:keyword
-           null | 172.21.0.5   |             1232382 | Disconnected
+@timestamp:unsupported | client_ip:ip | event_duration:long | message:keyword
+           null        | 172.21.0.5   |             1232382 | Disconnected
 ;
 
 multiIndexIndirectUseOfUnionTypesInWildcardKeep2
-// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
-// make the csv tests work with multiple indices.
 required_capability: union_types
 required_capability: union_types_fix_rename_resolution
 FROM sample_data, sample_data_ts_long
@@ -1454,29 +1444,23 @@ FROM sample_data, sample_data_ts_long
 | LIMIT 1
 ;
 
-@timestamp:null | client_ip:ip | event_duration:long | message:keyword
-           null | 172.21.0.5   |             1232382 | Disconnected
+@timestamp:unsupported | client_ip:ip | event_duration:long | message:keyword
+           null        | 172.21.0.5   |             1232382 | Disconnected
 ;
 
 
 multiIndexUseOfUnionTypesInKeep
-// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
-// make the csv tests work with multiple indices.
-required_capability: union_types
 required_capability: union_types_fix_rename_resolution
 FROM sample_data, sample_data_ts_long
 | KEEP @timestamp
 | LIMIT 1
 ;
 
-@timestamp:null
+@timestamp:unsupported
 null
 ;
 
 multiIndexUseOfUnionTypesInDrop
-// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
-// make the csv tests work with multiple indices.
-required_capability: union_types
 required_capability: union_types_fix_rename_resolution
 FROM sample_data, sample_data_ts_long
 | DROP @timestamp
@@ -1489,8 +1473,6 @@ client_ip:ip | event_duration:long | message:keyword
 ;
 
 multiIndexIndirectUseOfUnionTypesInWildcardDrop
-// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
-// make the csv tests work with multiple indices.
 required_capability: union_types
 required_capability: union_types_fix_rename_resolution
 FROM sample_data, sample_data_ts_long
@@ -1504,21 +1486,17 @@ client_ip:ip | event_duration:long | message:keyword
 ;
 
 multiIndexIndirectUseOfUnionTypesInWhere
-// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
-// make the csv tests work with multiple indices.
 required_capability: union_types
 FROM sample_data, sample_data_ts_long
 | WHERE message == "Disconnected"
 ;
 
-@timestamp:null | client_ip:ip | event_duration:long | message:keyword
-           null | 172.21.0.5   |             1232382 | Disconnected
-           null | 172.21.0.5   |             1232382 | Disconnected
+@timestamp:unsupported | client_ip:ip | event_duration:long | message:keyword
+           null        | 172.21.0.5   |             1232382 | Disconnected
+           null        | 172.21.0.5   |             1232382 | Disconnected
 ;
 
 multiIndexIndirectUseOfUnionTypesInDissect
-// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
-// make the csv tests work with multiple indices.
 required_capability: union_types
 FROM sample_data, sample_data_ts_long
 | DISSECT message "%{foo}"
@@ -1526,13 +1504,11 @@ FROM sample_data, sample_data_ts_long
 | LIMIT 1
 ;
 
-@timestamp:null | client_ip:ip | event_duration:long | message:keyword | foo:keyword
-           null | 172.21.0.5   |             1232382 | Disconnected    | Disconnected
+@timestamp:unsupported | client_ip:ip | event_duration:long | message:keyword | foo:keyword
+           null        | 172.21.0.5   |             1232382 | Disconnected    | Disconnected
 ;
 
 multiIndexIndirectUseOfUnionTypesInGrok
-// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
-// make the csv tests work with multiple indices.
 required_capability: union_types
 FROM sample_data, sample_data_ts_long
 | GROK message "%{WORD:foo}"
@@ -1540,13 +1516,11 @@ FROM sample_data, sample_data_ts_long
 | LIMIT 1
 ;
 
-@timestamp:null | client_ip:ip | event_duration:long | message:keyword | foo:keyword
-           null | 172.21.0.5   |             1232382 | Disconnected    | Disconnected
+@timestamp:unsupported | client_ip:ip | event_duration:long | message:keyword | foo:keyword
+           null        | 172.21.0.5   |             1232382 | Disconnected    | Disconnected
 ;
 
 multiIndexIndirectUseOfUnionTypesInEnrich
-// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
-// make the csv tests work with multiple indices.
 required_capability: union_types
 required_capability: enrich_load
 FROM sample_data, sample_data_ts_long
@@ -1556,13 +1530,11 @@ FROM sample_data, sample_data_ts_long
 | LIMIT 1
 ;
 
-@timestamp:null | event_duration:long | message:keyword | client_ip:keyword | env:keyword
-           null |             1232382 | Disconnected    | 172.21.0.5        | Development
+@timestamp:unsupported | event_duration:long | message:keyword | client_ip:keyword | env:keyword
+           null        |             1232382 | Disconnected    | 172.21.0.5        | Development
 ;
 
 multiIndexIndirectUseOfUnionTypesInStats
-// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
-// make the csv tests work with multiple indices.
 required_capability: union_types
 FROM sample_data, sample_data_ts_long
 | STATS foo = max(event_duration) BY client_ip
@@ -1577,8 +1549,6 @@ foo:long | client_ip:ip
 ;
 
 multiIndexIndirectUseOfUnionTypesInInlineStats-Ignore
-// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
-// make the csv tests work with multiple indices.
 required_capability: union_types
 required_capability: inlinestats
 FROM sample_data, sample_data_ts_long
@@ -1587,13 +1557,11 @@ FROM sample_data, sample_data_ts_long
 | LIMIT 1
 ;
 
-@timestamp:null | client_ip:ip | event_duration:long | message:keyword | foo:long
-           null | 172.21.0.5   |             1232382 | Disconnected    |  8268153
+@timestamp:unsupported | client_ip:ip | event_duration:long | message:keyword | foo:long
+           null        | 172.21.0.5   |             1232382 | Disconnected    |  8268153
 ;
 
 multiIndexIndirectUseOfUnionTypesInLookup-Ignore
-// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
-// make the csv tests work with multiple indices.
 required_capability: union_types
 required_capability: lookup_v4
 FROM sample_data, sample_data_ts_long
@@ -1603,13 +1571,11 @@ FROM sample_data, sample_data_ts_long
 | LOOKUP_🐔 int_number_names ON int
 ;
 
-@timestamp:null | client_ip:ip | event_duration:long | message:keyword | int:integer | name:keyword
-           null | 172.21.0.5   |             1232382 | Disconnected    |           2 | two
+@timestamp:unsupported | client_ip:ip | event_duration:long | message:keyword | int:integer | name:keyword
+           null        | 172.21.0.5   |             1232382 | Disconnected    |           2 | two
 ;
 
 multiIndexIndirectUseOfUnionTypesInMvExpand
-// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
-// make the csv tests work with multiple indices.
 required_capability: union_types
 FROM sample_data, sample_data_ts_long
 | EVAL foo = MV_APPEND(message, message)
@@ -1618,9 +1584,9 @@ FROM sample_data, sample_data_ts_long
 | MV_EXPAND foo
 ;
 
-@timestamp:null | client_ip:ip | event_duration:long | message:keyword | foo:keyword
-           null | 172.21.0.5   |             1232382 | Disconnected    | Disconnected
-           null | 172.21.0.5   |             1232382 | Disconnected    | Disconnected
+@timestamp:unsupported | client_ip:ip | event_duration:long | message:keyword | foo:keyword
+           null        | 172.21.0.5   |             1232382 | Disconnected    | Disconnected
+           null        | 172.21.0.5   |             1232382 | Disconnected    | Disconnected
 ;
 
 shortIntegerWidening
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractCrossClustersUsageTelemetryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractCrossClustersUsageTelemetryIT.java
new file mode 100644
index 0000000000000..ffbddd52b2551
--- /dev/null
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractCrossClustersUsageTelemetryIT.java
@@ -0,0 +1,205 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.admin.cluster.stats.CCSTelemetrySnapshot;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.test.AbstractMultiClustersTestCase;
+import org.elasticsearch.test.SkipUnavailableRule;
+import org.elasticsearch.usage.UsageService;
+import org.elasticsearch.xpack.core.async.GetAsyncResultRequest;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.elasticsearch.core.TimeValue.timeValueMillis;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
+
+public class AbstractCrossClustersUsageTelemetryIT extends AbstractMultiClustersTestCase {
+    private static final Logger LOGGER = LogManager.getLogger(AbstractCrossClustersUsageTelemetryIT.class);
+    protected static final String REMOTE1 = "cluster-a";
+    protected static final String REMOTE2 = "cluster-b";
+    protected static final String LOCAL_INDEX = "logs-1";
+    protected static final String REMOTE_INDEX = "logs-2";
+    // We want to send search to a specific node (we don't care which one) so that we could
+    // collect the CCS telemetry from it later
+    protected String queryNode;
+
+    @Before
+    public void setupQueryNode() {
+        // The tests are set up in a way that all queries within a single test are sent to the same node,
+        // thus enabling incremental collection of telemetry data, but the node is random for each test.
+        queryNode = cluster(LOCAL_CLUSTER).getRandomNodeName();
+    }
+
+    protected CCSTelemetrySnapshot getTelemetryFromQuery(String query, String client) throws ExecutionException, InterruptedException {
+        EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest();
+        request.query(query);
+        request.pragmas(AbstractEsqlIntegTestCase.randomPragmas());
+        request.columnar(randomBoolean());
+        request.includeCCSMetadata(randomBoolean());
+        return getTelemetryFromQuery(request, client);
+    }
+
+    protected CCSTelemetrySnapshot getTelemetryFromQuery(EsqlQueryRequest request, String client) throws ExecutionException,
+        InterruptedException {
+        // We don't care here too much about the response, we just want to trigger the telemetry collection.
+        // So we check it's not null and leave the rest to other tests.
+        if (client != null) {
+            assertResponse(
+                cluster(LOCAL_CLUSTER).client(queryNode)
+                    .filterWithHeader(Map.of(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER, client))
+                    .execute(EsqlQueryAction.INSTANCE, request),
+                Assert::assertNotNull
+            );
+
+        } else {
+            assertResponse(cluster(LOCAL_CLUSTER).client(queryNode).execute(EsqlQueryAction.INSTANCE, request), Assert::assertNotNull);
+        }
+        return getTelemetrySnapshot(queryNode);
+    }
+
+    protected CCSTelemetrySnapshot getTelemetryFromAsyncQuery(String query) throws Exception {
+        EsqlQueryRequest request = EsqlQueryRequest.asyncEsqlQueryRequest();
+        request.query(query);
+        request.pragmas(AbstractEsqlIntegTestCase.randomPragmas());
+        request.columnar(randomBoolean());
+        request.includeCCSMetadata(randomBoolean());
+        request.waitForCompletionTimeout(TimeValue.timeValueMillis(100));
+        request.keepOnCompletion(false);
+        return getTelemetryFromAsyncQuery(request);
+    }
+
+    protected CCSTelemetrySnapshot getTelemetryFromAsyncQuery(EsqlQueryRequest request) throws Exception {
+        AtomicReference<String> asyncExecutionId = new AtomicReference<>();
+        assertResponse(cluster(LOCAL_CLUSTER).client(queryNode).execute(EsqlQueryAction.INSTANCE, request), resp -> {
+            if (resp.isRunning()) {
+                assertNotNull("async execution id is null", resp.asyncExecutionId());
+                asyncExecutionId.set(resp.asyncExecutionId().get());
+            }
+        });
+        if (asyncExecutionId.get() != null) {
+            assertBusy(() -> {
+                var getResultsRequest = new GetAsyncResultRequest(asyncExecutionId.get()).setWaitForCompletionTimeout(timeValueMillis(1));
+                try (
+                    var resp = cluster(LOCAL_CLUSTER).client(queryNode)
+                        .execute(EsqlAsyncGetResultAction.INSTANCE, getResultsRequest)
+                        .actionGet(30, TimeUnit.SECONDS)
+                ) {
+                    assertFalse(resp.isRunning());
+                }
+            });
+        }
+        return getTelemetrySnapshot(queryNode);
+    }
+
+    protected CCSTelemetrySnapshot getTelemetryFromFailedQuery(String query) throws Exception {
+        EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest();
+        request.query(query);
+        request.pragmas(AbstractEsqlIntegTestCase.randomPragmas());
+        request.columnar(randomBoolean());
+        request.includeCCSMetadata(randomBoolean());
+
+        ExecutionException ee = expectThrows(
+            ExecutionException.class,
+            cluster(LOCAL_CLUSTER).client(queryNode).execute(EsqlQueryAction.INSTANCE, request)::get
+        );
+        assertNotNull(ee.getCause());
+
+        return getTelemetrySnapshot(queryNode);
+    }
+
+    private CCSTelemetrySnapshot getTelemetrySnapshot(String nodeName) {
+        var usage = cluster(LOCAL_CLUSTER).getInstance(UsageService.class, nodeName);
+        return usage.getEsqlUsageHolder().getCCSTelemetrySnapshot();
+    }
+
+    @Override
+    protected boolean reuseClusters() {
+        return false;
+    }
+
+    @Override
+    protected List<String> remoteClusterAlias() {
+        return List.of(REMOTE1, REMOTE2);
+    }
+
+    @Rule
+    public SkipUnavailableRule skipOverride = new SkipUnavailableRule(REMOTE1, REMOTE2);
+
+    protected Map<String, Object> setupClusters() {
+        int numShardsLocal = randomIntBetween(1, 5);
+        populateLocalIndices(LOCAL_INDEX, numShardsLocal);
+
+        int numShardsRemote = randomIntBetween(1, 5);
+        populateRemoteIndices(REMOTE1, REMOTE_INDEX, numShardsRemote);
+
+        Map<String, Object> clusterInfo = new HashMap<>();
+        clusterInfo.put("local.num_shards", numShardsLocal);
+        clusterInfo.put("local.index", LOCAL_INDEX);
+        clusterInfo.put("remote.num_shards", numShardsRemote);
+        clusterInfo.put("remote.index", REMOTE_INDEX);
+
+        int numShardsRemote2 = randomIntBetween(1, 5);
+        populateRemoteIndices(REMOTE2, REMOTE_INDEX, numShardsRemote2);
+        clusterInfo.put("remote2.index", REMOTE_INDEX);
+        clusterInfo.put("remote2.num_shards", numShardsRemote2);
+
+        return clusterInfo;
+    }
+
+    void populateLocalIndices(String indexName, int numShards) {
+        Client localClient = client(LOCAL_CLUSTER);
+        assertAcked(
+            localClient.admin()
+                .indices()
+                .prepareCreate(indexName)
+                .setSettings(Settings.builder().put("index.number_of_shards", numShards))
+                .setMapping("id", "type=keyword", "tag", "type=keyword", "v", "type=long")
+        );
+        for (int i = 0; i < 10; i++) {
+            localClient.prepareIndex(indexName).setSource("id", "local-" + i, "tag", "local", "v", i).get();
+        }
+        localClient.admin().indices().prepareRefresh(indexName).get();
+    }
+
+    void populateRemoteIndices(String clusterAlias, String indexName, int numShards) {
+        Client remoteClient = client(clusterAlias);
+        assertAcked(
+            remoteClient.admin()
+                .indices()
+                .prepareCreate(indexName)
+                .setSettings(Settings.builder().put("index.number_of_shards", numShards))
+                .setMapping("id", "type=keyword", "tag", "type=keyword", "v", "type=long")
+        );
+        for (int i = 0; i < 10; i++) {
+            remoteClient.prepareIndex(indexName).setSource("id", "remote-" + i, "tag", "remote", "v", i * i).get();
+        }
+        remoteClient.admin().indices().prepareRefresh(indexName).get();
+    }
+
+    @Override
+    protected Map<String, Boolean> skipUnavailableForRemoteClusters() {
+        var map = skipOverride.getMap();
+        LOGGER.info("Using skip_unavailable map: [{}]", map);
+        return map;
+    }
+}
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEnrichBasedCrossClusterTestCase.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEnrichBasedCrossClusterTestCase.java
index 66ac32b33cd4d..4d1f908702152 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEnrichBasedCrossClusterTestCase.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEnrichBasedCrossClusterTestCase.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.ingest.common.IngestCommonPlugin;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseService;
@@ -244,15 +243,6 @@ protected EsqlQueryResponse runQuery(String query, Boolean ccsMetadataInResponse
         return client(LOCAL_CLUSTER).execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS);
     }
 
-    public static Tuple<Boolean, Boolean> randomIncludeCCSMetadata() {
-        return switch (randomIntBetween(1, 3)) {
-            case 1 -> new Tuple<>(Boolean.TRUE, Boolean.TRUE);
-            case 2 -> new Tuple<>(Boolean.FALSE, Boolean.FALSE);
-            case 3 -> new Tuple<>(null, Boolean.FALSE);
-            default -> throw new AssertionError("should not get here");
-        };
-    }
-
     public static class LocalStateEnrich extends LocalStateCompositeXPackPlugin {
         public LocalStateEnrich(final Settings settings, final Path configPath) throws Exception {
             super(settings, configPath);
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java
index 7ae45497f7297..90bf34b499390 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.operator.exchange.ExchangeService;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.health.node.selection.HealthNode;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
@@ -233,4 +234,18 @@ protected static void assertValuesInAnyOrder(Iterator<Iterator<Object>> actualVa
         }
         assertThat(getValuesList(actualValues), containsInAnyOrder(items.toArray()));
     }
+
+    /**
+    * v1: value to send to runQuery (can be null; null means use default value)
+    * v2: whether to expect CCS Metadata in the response (cannot be null)
+    * @return
+    */
+    public static Tuple<Boolean, Boolean> randomIncludeCCSMetadata() {
+        return switch (randomIntBetween(1, 3)) {
+            case 1 -> new Tuple<>(Boolean.TRUE, Boolean.TRUE);
+            case 2 -> new Tuple<>(Boolean.FALSE, Boolean.FALSE);
+            case 3 -> new Tuple<>(null, Boolean.FALSE);
+            default -> throw new AssertionError("should not get here");
+        };
+    }
 }
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractPausableIntegTestCase.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractPausableIntegTestCase.java
index 8de65847c3f85..8054b260f0060 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractPausableIntegTestCase.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractPausableIntegTestCase.java
@@ -10,26 +10,15 @@
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.index.engine.SegmentsStats;
-import org.elasticsearch.index.mapper.OnScriptError;
-import org.elasticsearch.logging.LogManager;
-import org.elasticsearch.logging.Logger;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.ScriptPlugin;
-import org.elasticsearch.script.LongFieldScript;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptEngine;
-import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.util.Collection;
-import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 
@@ -40,8 +29,6 @@
  */
 public abstract class AbstractPausableIntegTestCase extends AbstractEsqlIntegTestCase {
 
-    private static final Logger LOGGER = LogManager.getLogger(AbstractPausableIntegTestCase.class);
-
     protected static final Semaphore scriptPermits = new Semaphore(0);
 
     protected int pageSize = -1;
@@ -108,53 +95,10 @@ public void setupIndex() throws IOException {
         }
     }
 
-    public static class PausableFieldPlugin extends Plugin implements ScriptPlugin {
-
+    public static class PausableFieldPlugin extends AbstractPauseFieldPlugin {
         @Override
-        public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<?>> contexts) {
-            return new ScriptEngine() {
-                @Override
-                public String getType() {
-                    return "pause";
-                }
-
-                @Override
-                @SuppressWarnings("unchecked")
-                public <FactoryType> FactoryType compile(
-                    String name,
-                    String code,
-                    ScriptContext<FactoryType> context,
-                    Map<String, String> params
-                ) {
-                    return (FactoryType) new LongFieldScript.Factory() {
-                        @Override
-                        public LongFieldScript.LeafFactory newFactory(
-                            String fieldName,
-                            Map<String, Object> params,
-                            SearchLookup searchLookup,
-                            OnScriptError onScriptError
-                        ) {
-                            return ctx -> new LongFieldScript(fieldName, params, searchLookup, onScriptError, ctx) {
-                                @Override
-                                public void execute() {
-                                    try {
-                                        assertTrue(scriptPermits.tryAcquire(1, TimeUnit.MINUTES));
-                                    } catch (Exception e) {
-                                        throw new AssertionError(e);
-                                    }
-                                    LOGGER.debug("--> emitting value");
-                                    emit(1);
-                                }
-                            };
-                        }
-                    };
-                }
-
-                @Override
-                public Set<ScriptContext<?>> getSupportedContexts() {
-                    return Set.of(LongFieldScript.CONTEXT);
-                }
-            };
+        protected boolean onWait() throws InterruptedException {
+            return scriptPermits.tryAcquire(1, TimeUnit.MINUTES);
         }
     }
 }
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractPauseFieldPlugin.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractPauseFieldPlugin.java
new file mode 100644
index 0000000000000..5554f7e571dfb
--- /dev/null
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractPauseFieldPlugin.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.action;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.OnScriptError;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
+import org.elasticsearch.script.LongFieldScript;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.ScriptEngine;
+import org.elasticsearch.search.lookup.SearchLookup;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * A plugin that provides a script language "pause" that can be used to simulate slow running queries.
+ * See also {@link AbstractPausableIntegTestCase}.
+ */
+public abstract class AbstractPauseFieldPlugin extends Plugin implements ScriptPlugin {
+
+    // Called when the engine enters the execute() method.
+    protected void onStartExecute() {}
+
+    // Called when the engine needs to wait for further execution to be allowed.
+    protected abstract boolean onWait() throws InterruptedException;
+
+    @Override
+    public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<?>> contexts) {
+        return new ScriptEngine() {
+            @Override
+            public String getType() {
+                return "pause";
+            }
+
+            @Override
+            @SuppressWarnings("unchecked")
+            public <FactoryType> FactoryType compile(
+                String name,
+                String code,
+                ScriptContext<FactoryType> context,
+                Map<String, String> params
+            ) {
+                if (context == LongFieldScript.CONTEXT) {
+                    return (FactoryType) new LongFieldScript.Factory() {
+                        @Override
+                        public LongFieldScript.LeafFactory newFactory(
+                            String fieldName,
+                            Map<String, Object> params,
+                            SearchLookup searchLookup,
+                            OnScriptError onScriptError
+                        ) {
+                            return ctx -> new LongFieldScript(fieldName, params, searchLookup, onScriptError, ctx) {
+                                @Override
+                                public void execute() {
+                                    onStartExecute();
+                                    try {
+                                        assertTrue(onWait());
+                                    } catch (InterruptedException e) {
+                                        throw new AssertionError(e);
+                                    }
+                                    emit(1);
+                                }
+                            };
+                        }
+                    };
+                }
+                throw new IllegalStateException("unsupported type " + context);
+            }
+
+            @Override
+            public Set<ScriptContext<?>> getSupportedContexts() {
+                return Set.of(LongFieldScript.CONTEXT);
+            }
+        };
+    }
+}
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AsyncEsqlQueryActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AsyncEsqlQueryActionIT.java
index f85de51101af5..163cbc8491298 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AsyncEsqlQueryActionIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AsyncEsqlQueryActionIT.java
@@ -10,9 +10,11 @@
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.compute.operator.DriverTaskRunner;
 import org.elasticsearch.compute.operator.exchange.ExchangeService;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 import org.elasticsearch.xpack.core.async.DeleteAsyncResultRequest;
@@ -34,8 +36,11 @@
 import static org.elasticsearch.test.hamcrest.OptionalMatchers.isEmpty;
 import static org.elasticsearch.test.hamcrest.OptionalMatchers.isPresent;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 
 /**
@@ -112,6 +117,59 @@ public void testBasicAsyncExecution() throws Exception {
         }
     }
 
+    public void testGetAsyncWhileQueryTaskIsBeingCancelled() throws Exception {
+        try (var initialResponse = sendAsyncQuery()) {
+            assertThat(initialResponse.asyncExecutionId(), isPresent());
+            assertThat(initialResponse.isRunning(), is(true));
+            String id = initialResponse.asyncExecutionId().get();
+            // ensure we have started Lucene operators
+            assertBusy(() -> {
+                var tasks = client().admin()
+                    .cluster()
+                    .prepareListTasks()
+                    .setActions(DriverTaskRunner.ACTION_NAME)
+                    .setDetailed(true)
+                    .get()
+                    .getTasks()
+                    .stream()
+                    .filter(t -> t.description().contains("_LuceneSourceOperator"))
+                    .toList();
+                assertThat(tasks.size(), greaterThanOrEqualTo(1));
+            });
+            client().admin().cluster().prepareCancelTasks().setActions(EsqlQueryAction.NAME + "[a]").get();
+            assertBusy(() -> {
+                List<TaskInfo> tasks = getEsqlQueryTasks().stream().filter(TaskInfo::cancelled).toList();
+                assertThat(tasks, not(empty()));
+            });
+            // get the result while the query is being cancelled
+            {
+                var getResultsRequest = new GetAsyncResultRequest(id);
+                getResultsRequest.setWaitForCompletionTimeout(timeValueMillis(10));
+                getResultsRequest.setKeepAlive(randomKeepAlive());
+                var future = client().execute(EsqlAsyncGetResultAction.INSTANCE, getResultsRequest);
+                try (var resp = future.get()) {
+                    assertThat(initialResponse.asyncExecutionId(), isPresent());
+                    assertThat(resp.asyncExecutionId().get(), equalTo(id));
+                    assertThat(resp.isRunning(), is(true));
+                }
+            }
+            // release the permits to allow the query to proceed
+            scriptPermits.release(numberOfDocs());
+            // get the result after the cancellation is done
+            {
+                var getResultsRequest = new GetAsyncResultRequest(id);
+                getResultsRequest.setWaitForCompletionTimeout(timeValueSeconds(10));
+                getResultsRequest.setKeepAlive(randomKeepAlive());
+                var future = client().execute(EsqlAsyncGetResultAction.INSTANCE, getResultsRequest);
+                TaskCancelledException error = expectThrows(TaskCancelledException.class, future::actionGet);
+                assertThat(error.getMessage(), equalTo("by user request"));
+            }
+            assertTrue(deleteAsyncId(id).isAcknowledged());
+        } finally {
+            scriptPermits.drainPermits();
+        }
+    }
+
     public void testAsyncCancellation() throws Exception {
         try (var initialResponse = sendAsyncQuery()) {
             assertThat(initialResponse.asyncExecutionId(), isPresent());
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java
index a2bba19db50fc..79ac8816a0039 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java
@@ -19,14 +19,8 @@
 import org.elasticsearch.compute.operator.exchange.ExchangeService;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.ScriptPlugin;
-import org.elasticsearch.script.LongFieldScript;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptEngine;
-import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.AbstractMultiClustersTestCase;
 import org.elasticsearch.test.XContentTestUtils;
 import org.elasticsearch.transport.RemoteClusterAware;
@@ -44,12 +38,12 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.core.TimeValue.timeValueMillis;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase.randomIncludeCCSMetadata;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
@@ -80,7 +74,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins(String clusterAlias) {
         plugins.add(EsqlPluginWithEnterpriseOrTrialLicense.class);
         plugins.add(EsqlAsyncActionIT.LocalStateEsqlAsync.class); // allows the async_search DELETE action
         plugins.add(InternalExchangePlugin.class);
-        plugins.add(PauseFieldPlugin.class);
+        plugins.add(SimplePauseFieldPlugin.class);
         return plugins;
     }
 
@@ -99,64 +93,7 @@ public List<Setting<?>> getSettings() {
 
     @Before
     public void resetPlugin() {
-        PauseFieldPlugin.allowEmitting = new CountDownLatch(1);
-        PauseFieldPlugin.startEmitting = new CountDownLatch(1);
-    }
-
-    public static class PauseFieldPlugin extends Plugin implements ScriptPlugin {
-        public static CountDownLatch startEmitting = new CountDownLatch(1);
-        public static CountDownLatch allowEmitting = new CountDownLatch(1);
-
-        @Override
-        public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<?>> contexts) {
-            return new ScriptEngine() {
-                @Override
-
-                public String getType() {
-                    return "pause";
-                }
-
-                @Override
-                @SuppressWarnings("unchecked")
-                public <FactoryType> FactoryType compile(
-                    String name,
-                    String code,
-                    ScriptContext<FactoryType> context,
-                    Map<String, String> params
-                ) {
-                    if (context == LongFieldScript.CONTEXT) {
-                        return (FactoryType) new LongFieldScript.Factory() {
-                            @Override
-                            public LongFieldScript.LeafFactory newFactory(
-                                String fieldName,
-                                Map<String, Object> params,
-                                SearchLookup searchLookup,
-                                OnScriptError onScriptError
-                            ) {
-                                return ctx -> new LongFieldScript(fieldName, params, searchLookup, onScriptError, ctx) {
-                                    @Override
-                                    public void execute() {
-                                        startEmitting.countDown();
-                                        try {
-                                            assertTrue(allowEmitting.await(30, TimeUnit.SECONDS));
-                                        } catch (InterruptedException e) {
-                                            throw new AssertionError(e);
-                                        }
-                                        emit(1);
-                                    }
-                                };
-                            }
-                        };
-                    }
-                    throw new IllegalStateException("unsupported type " + context);
-                }
-
-                @Override
-                public Set<ScriptContext<?>> getSupportedContexts() {
-                    return Set.of(LongFieldScript.CONTEXT);
-                }
-            };
-        }
+        SimplePauseFieldPlugin.resetPlugin();
     }
 
     /**
@@ -184,7 +121,7 @@ public void testSuccessfulPathways() throws Exception {
         }
 
         // wait until we know that the query against 'remote-b:blocking' has started
-        PauseFieldPlugin.startEmitting.await(30, TimeUnit.SECONDS);
+        SimplePauseFieldPlugin.startEmitting.await(30, TimeUnit.SECONDS);
 
         // wait until the query of 'cluster-a:logs-*' has finished (it is not blocked since we are not searching the 'blocking' index on it)
         assertBusy(() -> {
@@ -234,7 +171,7 @@ public void testSuccessfulPathways() throws Exception {
         }
 
         // allow remoteB query to proceed
-        PauseFieldPlugin.allowEmitting.countDown();
+        SimplePauseFieldPlugin.allowEmitting.countDown();
 
         // wait until both remoteB and local queries have finished
         assertBusy(() -> {
@@ -419,20 +356,6 @@ private static void assertClusterMetadataInResponse(EsqlQueryResponse resp, bool
         }
     }
 
-    /**
-     * v1: value to send to runQuery (can be null; null means use default value)
-     * v2: whether to expect CCS Metadata in the response (cannot be null)
-     * @return
-     */
-    public static Tuple<Boolean, Boolean> randomIncludeCCSMetadata() {
-        return switch (randomIntBetween(1, 3)) {
-            case 1 -> new Tuple<>(Boolean.TRUE, Boolean.TRUE);
-            case 2 -> new Tuple<>(Boolean.FALSE, Boolean.FALSE);
-            case 3 -> new Tuple<>(null, Boolean.FALSE);
-            default -> throw new AssertionError("should not get here");
-        };
-    }
-
     Map<String, Object> setupClusters(int numClusters) throws IOException {
         assert numClusters == 2 || numClusters == 3 : "2 or 3 clusters supported not: " + numClusters;
         int numShardsLocal = randomIntBetween(1, 5);
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java
index 09ad97b08f357..9c0447affc754 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java
@@ -23,6 +23,7 @@
 import java.util.Set;
 
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
+import static org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase.randomIncludeCCSMetadata;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -60,7 +61,7 @@ public void testEnrichWithHostsPolicyAndDisconnectedRemotesWithSkipUnavailableTr
             // close remote-cluster-1 so that it is unavailable
             cluster(REMOTE_CLUSTER_1).close();
 
-            Tuple<Boolean, Boolean> includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata();
+            Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
             Boolean requestIncludeMeta = includeCCSMetadata.v1();
             boolean responseExpectMeta = includeCCSMetadata.v2();
 
@@ -142,7 +143,7 @@ public void testEnrichWithHostsPolicyAndDisconnectedRemotesWithSkipUnavailableFa
             // close remote-cluster-1 so that it is unavailable
             cluster(REMOTE_CLUSTER_1).close();
 
-            Tuple<Boolean, Boolean> includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata();
+            Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
             Boolean requestIncludeMeta = includeCCSMetadata.v1();
             boolean responseExpectMeta = includeCCSMetadata.v2();
 
@@ -188,7 +189,7 @@ public void testEnrichWithHostsPolicyAndDisconnectedRemotesWithSkipUnavailableFa
     }
 
     public void testEnrichTwiceThenAggsWithUnavailableRemotes() throws IOException {
-        Tuple<Boolean, Boolean> includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata();
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
         Boolean requestIncludeMeta = includeCCSMetadata.v1();
         boolean responseExpectMeta = includeCCSMetadata.v2();
 
@@ -292,7 +293,7 @@ public void testEnrichTwiceThenAggsWithUnavailableRemotes() throws IOException {
     }
 
     public void testEnrichCoordinatorThenAnyWithSingleUnavailableRemoteAndLocal() throws IOException {
-        Tuple<Boolean, Boolean> includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata();
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
         Boolean requestIncludeMeta = includeCCSMetadata.v1();
         boolean responseExpectMeta = includeCCSMetadata.v2();
 
@@ -345,7 +346,7 @@ public void testEnrichCoordinatorThenAnyWithSingleUnavailableRemoteAndLocal() th
     }
 
     public void testEnrichCoordinatorThenAnyWithSingleUnavailableRemoteAndNotLocal() throws IOException {
-        Tuple<Boolean, Boolean> includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata();
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
         Boolean requestIncludeMeta = includeCCSMetadata.v1();
         boolean responseExpectMeta = includeCCSMetadata.v2();
 
@@ -394,7 +395,7 @@ public void testEnrichCoordinatorThenAnyWithSingleUnavailableRemoteAndNotLocal()
     }
 
     public void testEnrichRemoteWithVendor() throws IOException {
-        Tuple<Boolean, Boolean> includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata();
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
         Boolean requestIncludeMeta = includeCCSMetadata.v1();
         boolean responseExpectMeta = includeCCSMetadata.v2();
 
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java
index f65764daafb8a..eb728895cd00c 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java
@@ -30,6 +30,7 @@
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
+import static org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase.randomIncludeCCSMetadata;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasSize;
@@ -452,20 +453,6 @@ protected EsqlQueryResponse runQuery(EsqlQueryRequest request) {
         return client(LOCAL_CLUSTER).execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS);
     }
 
-    /**
-     * v1: value to send to runQuery (can be null; null means use default value)
-     * v2: whether to expect CCS Metadata in the response (cannot be null)
-     * @return
-     */
-    public static Tuple<Boolean, Boolean> randomIncludeCCSMetadata() {
-        return switch (randomIntBetween(1, 3)) {
-            case 1 -> new Tuple<>(Boolean.TRUE, Boolean.TRUE);
-            case 2 -> new Tuple<>(Boolean.FALSE, Boolean.FALSE);
-            case 3 -> new Tuple<>(null, Boolean.FALSE);
-            default -> throw new AssertionError("should not get here");
-        };
-    }
-
     Map<String, Object> setupClusters(int numClusters) {
         assert numClusters == 2 || numClusters == 3 : "2 or 3 clusters supported not: " + numClusters;
         String localIndex = "logs-1";
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersCancellationIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersCancellationIT.java
index 17f5f81486651..cfe6fdeccb190 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersCancellationIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersCancellationIT.java
@@ -15,18 +15,11 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.compute.operator.DriverTaskRunner;
 import org.elasticsearch.compute.operator.exchange.ExchangeService;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.ScriptPlugin;
-import org.elasticsearch.script.LongFieldScript;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptEngine;
-import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.test.AbstractMultiClustersTestCase;
 import org.elasticsearch.transport.TransportService;
@@ -38,9 +31,6 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
@@ -63,7 +53,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins(String clusterAlias) {
         List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins(clusterAlias));
         plugins.add(EsqlPluginWithEnterpriseOrTrialLicense.class);
         plugins.add(InternalExchangePlugin.class);
-        plugins.add(PauseFieldPlugin.class);
+        plugins.add(SimplePauseFieldPlugin.class);
         return plugins;
     }
 
@@ -82,63 +72,7 @@ public List<Setting<?>> getSettings() {
 
     @Before
     public void resetPlugin() {
-        PauseFieldPlugin.allowEmitting = new CountDownLatch(1);
-        PauseFieldPlugin.startEmitting = new CountDownLatch(1);
-    }
-
-    public static class PauseFieldPlugin extends Plugin implements ScriptPlugin {
-        public static CountDownLatch startEmitting = new CountDownLatch(1);
-        public static CountDownLatch allowEmitting = new CountDownLatch(1);
-
-        @Override
-        public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<?>> contexts) {
-            return new ScriptEngine() {
-                @Override
-                public String getType() {
-                    return "pause";
-                }
-
-                @Override
-                @SuppressWarnings("unchecked")
-                public <FactoryType> FactoryType compile(
-                    String name,
-                    String code,
-                    ScriptContext<FactoryType> context,
-                    Map<String, String> params
-                ) {
-                    if (context == LongFieldScript.CONTEXT) {
-                        return (FactoryType) new LongFieldScript.Factory() {
-                            @Override
-                            public LongFieldScript.LeafFactory newFactory(
-                                String fieldName,
-                                Map<String, Object> params,
-                                SearchLookup searchLookup,
-                                OnScriptError onScriptError
-                            ) {
-                                return ctx -> new LongFieldScript(fieldName, params, searchLookup, onScriptError, ctx) {
-                                    @Override
-                                    public void execute() {
-                                        startEmitting.countDown();
-                                        try {
-                                            assertTrue(allowEmitting.await(30, TimeUnit.SECONDS));
-                                        } catch (InterruptedException e) {
-                                            throw new AssertionError(e);
-                                        }
-                                        emit(1);
-                                    }
-                                };
-                            }
-                        };
-                    }
-                    throw new IllegalStateException("unsupported type " + context);
-                }
-
-                @Override
-                public Set<ScriptContext<?>> getSupportedContexts() {
-                    return Set.of(LongFieldScript.CONTEXT);
-                }
-            };
-        }
+        SimplePauseFieldPlugin.resetPlugin();
     }
 
     private void createRemoteIndex(int numDocs) throws Exception {
@@ -169,7 +103,7 @@ public void testCancel() throws Exception {
         request.pragmas(randomPragmas());
         PlainActionFuture<EsqlQueryResponse> requestFuture = new PlainActionFuture<>();
         client().execute(EsqlQueryAction.INSTANCE, request, requestFuture);
-        assertTrue(PauseFieldPlugin.startEmitting.await(30, TimeUnit.SECONDS));
+        assertTrue(SimplePauseFieldPlugin.startEmitting.await(30, TimeUnit.SECONDS));
         List<TaskInfo> rootTasks = new ArrayList<>();
         assertBusy(() -> {
             List<TaskInfo> tasks = client().admin().cluster().prepareListTasks().setActions(EsqlQueryAction.NAME).get().getTasks();
@@ -192,7 +126,7 @@ public void testCancel() throws Exception {
                 }
             });
         } finally {
-            PauseFieldPlugin.allowEmitting.countDown();
+            SimplePauseFieldPlugin.allowEmitting.countDown();
         }
         Exception error = expectThrows(Exception.class, requestFuture::actionGet);
         assertThat(error.getMessage(), containsString("proxy timeout"));
@@ -223,7 +157,7 @@ public void testSameRemoteClusters() throws Exception {
                     assertThat(tasks, hasSize(moreClusters + 1));
                 });
             } finally {
-                PauseFieldPlugin.allowEmitting.countDown();
+                SimplePauseFieldPlugin.allowEmitting.countDown();
             }
             try (EsqlQueryResponse resp = future.actionGet(30, TimeUnit.SECONDS)) {
                 // TODO: This produces incorrect results because data on the remote cluster is processed multiple times.
@@ -244,7 +178,7 @@ public void testTasks() throws Exception {
         request.query("FROM *:test | STATS total=sum(const) | LIMIT 1");
         request.pragmas(randomPragmas());
         ActionFuture<EsqlQueryResponse> requestFuture = client().execute(EsqlQueryAction.INSTANCE, request);
-        assertTrue(PauseFieldPlugin.startEmitting.await(30, TimeUnit.SECONDS));
+        assertTrue(SimplePauseFieldPlugin.startEmitting.await(30, TimeUnit.SECONDS));
         try {
             assertBusy(() -> {
                 List<TaskInfo> clusterTasks = client(REMOTE_CLUSTER).admin()
@@ -270,7 +204,7 @@ public void testTasks() throws Exception {
                     \\_ExchangeSinkOperator"""));
             });
         } finally {
-            PauseFieldPlugin.allowEmitting.countDown();
+            SimplePauseFieldPlugin.allowEmitting.countDown();
         }
         requestFuture.actionGet(30, TimeUnit.SECONDS).close();
     }
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java
index 4e6be6cc2bf74..51ad4a0d2053f 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java
@@ -22,6 +22,7 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
+import static org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase.randomIncludeCCSMetadata;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueriesWithInvalidLicenseIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueriesWithInvalidLicenseIT.java
index 1ed42b696d65e..a9c8190976a02 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueriesWithInvalidLicenseIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueriesWithInvalidLicenseIT.java
@@ -20,6 +20,7 @@
 import java.util.Set;
 
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
+import static org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase.randomIncludeCCSMetadata;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java
index 347ef419cab9b..6cfc42523007e 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java
@@ -48,6 +48,7 @@
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
+import static org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase.randomIncludeCCSMetadata;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -1251,20 +1252,6 @@ protected EsqlQueryResponse runQuery(EsqlQueryRequest request) {
         return client(LOCAL_CLUSTER).execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS);
     }
 
-    /**
-     * v1: value to send to runQuery (can be null; null means use default value)
-     * v2: whether to expect CCS Metadata in the response (cannot be null)
-     * @return
-     */
-    public static Tuple<Boolean, Boolean> randomIncludeCCSMetadata() {
-        return switch (randomIntBetween(1, 3)) {
-            case 1 -> new Tuple<>(Boolean.TRUE, Boolean.TRUE);
-            case 2 -> new Tuple<>(Boolean.FALSE, Boolean.FALSE);
-            case 3 -> new Tuple<>(null, Boolean.FALSE);
-            default -> throw new AssertionError("should not get here");
-        };
-    }
-
     void waitForNoInitializingShards(Client client, TimeValue timeout, String... indices) {
         ClusterHealthResponse resp = client.admin()
             .cluster()
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersUsageTelemetryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersUsageTelemetryIT.java
new file mode 100644
index 0000000000000..33d868e7a69eb
--- /dev/null
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersUsageTelemetryIT.java
@@ -0,0 +1,231 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.action;
+
+import org.elasticsearch.action.admin.cluster.stats.CCSTelemetrySnapshot;
+import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.SkipUnavailableRule;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.ASYNC_FEATURE;
+import static org.hamcrest.Matchers.equalTo;
+
+public class CrossClustersUsageTelemetryIT extends AbstractCrossClustersUsageTelemetryIT {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins(String clusterAlias) {
+        List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins(clusterAlias));
+        plugins.add(EsqlPluginWithEnterpriseOrTrialLicense.class);
+        plugins.add(CrossClustersQueryIT.InternalExchangePlugin.class);
+        return plugins;
+    }
+
+    public void assertPerClusterCount(CCSTelemetrySnapshot.PerClusterCCSTelemetry perCluster, long count) {
+        assertThat(perCluster.getCount(), equalTo(count));
+        assertThat(perCluster.getSkippedCount(), equalTo(0L));
+        assertThat(perCluster.getTook().count(), equalTo(count));
+    }
+
+    public void testLocalRemote() throws Exception {
+        setupClusters();
+        var telemetry = getTelemetryFromQuery("from logs-*,c*:logs-* | stats sum (v)", "kibana");
+
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(1L));
+        assertThat(telemetry.getFailureReasons().size(), equalTo(0));
+        assertThat(telemetry.getTook().count(), equalTo(1L));
+        assertThat(telemetry.getTookMrtFalse().count(), equalTo(0L));
+        assertThat(telemetry.getTookMrtTrue().count(), equalTo(0L));
+        assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0));
+        assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L));
+        assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L));
+        assertThat(telemetry.getClientCounts().size(), equalTo(1));
+        assertThat(telemetry.getClientCounts().get("kibana"), equalTo(1L));
+        assertThat(telemetry.getFeatureCounts().get(ASYNC_FEATURE), equalTo(null));
+
+        var perCluster = telemetry.getByRemoteCluster();
+        assertThat(perCluster.size(), equalTo(3));
+        for (String clusterAlias : remoteClusterAlias()) {
+            assertPerClusterCount(perCluster.get(clusterAlias), 1L);
+        }
+        assertPerClusterCount(perCluster.get(LOCAL_CLUSTER), 1L);
+
+        telemetry = getTelemetryFromQuery("from logs-*,c*:logs-* | stats sum (v)", "kibana");
+        assertThat(telemetry.getTotalCount(), equalTo(2L));
+        assertThat(telemetry.getClientCounts().get("kibana"), equalTo(2L));
+        perCluster = telemetry.getByRemoteCluster();
+        assertThat(perCluster.size(), equalTo(3));
+        for (String clusterAlias : remoteClusterAlias()) {
+            assertPerClusterCount(perCluster.get(clusterAlias), 2L);
+        }
+        assertPerClusterCount(perCluster.get(LOCAL_CLUSTER), 2L);
+    }
+
+    public void testLocalOnly() throws Exception {
+        setupClusters();
+        // Should not produce any usage info since it's a local search
+        var telemetry = getTelemetryFromQuery("from logs-* | stats sum (v)", "kibana");
+
+        assertThat(telemetry.getTotalCount(), equalTo(0L));
+        assertThat(telemetry.getSuccessCount(), equalTo(0L));
+        assertThat(telemetry.getByRemoteCluster().size(), equalTo(0));
+    }
+
+    @SkipUnavailableRule.NotSkipped(aliases = REMOTE1)
+    public void testFailed() throws Exception {
+        setupClusters();
+        // Should not produce any usage info since it's a local search
+        var telemetry = getTelemetryFromFailedQuery("from no_such_index | stats sum (v)");
+
+        assertThat(telemetry.getTotalCount(), equalTo(0L));
+        assertThat(telemetry.getSuccessCount(), equalTo(0L));
+        assertThat(telemetry.getByRemoteCluster().size(), equalTo(0));
+
+        // One remote is skipped, one is not
+        telemetry = getTelemetryFromFailedQuery("from logs-*,c*:no_such_index | stats sum (v)");
+
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(0L));
+        assertThat(telemetry.getByRemoteCluster().size(), equalTo(1));
+        assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0));
+        assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L));
+        assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L));
+        Map<String, Long> expectedFailure = Map.of(CCSUsageTelemetry.Result.NOT_FOUND.getName(), 1L);
+        assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure));
+        // cluster-b should be skipped
+        assertThat(telemetry.getByRemoteCluster().get(REMOTE2).getCount(), equalTo(0L));
+        assertThat(telemetry.getByRemoteCluster().get(REMOTE2).getSkippedCount(), equalTo(1L));
+
+        // this is only for cluster-a so no skipped remotes
+        telemetry = getTelemetryFromFailedQuery("from logs-*,cluster-a:no_such_index | stats sum (v)");
+        assertThat(telemetry.getTotalCount(), equalTo(2L));
+        assertThat(telemetry.getSuccessCount(), equalTo(0L));
+        assertThat(telemetry.getByRemoteCluster().size(), equalTo(1));
+        assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0));
+        assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L));
+        assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L));
+        expectedFailure = Map.of(CCSUsageTelemetry.Result.NOT_FOUND.getName(), 2L);
+        assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure));
+        assertThat(telemetry.getByRemoteCluster().size(), equalTo(1));
+    }
+
+    // TODO: enable when skip-up patch is merged
+    // public void testSkipAllRemotes() throws Exception {
+    // var telemetry = getTelemetryFromQuery("from logs-*,c*:no_such_index | stats sum (v)", "unknown");
+    //
+    // assertThat(telemetry.getTotalCount(), equalTo(1L));
+    // assertThat(telemetry.getSuccessCount(), equalTo(1L));
+    // assertThat(telemetry.getFailureReasons().size(), equalTo(0));
+    // assertThat(telemetry.getTook().count(), equalTo(1L));
+    // assertThat(telemetry.getTookMrtFalse().count(), equalTo(0L));
+    // assertThat(telemetry.getTookMrtTrue().count(), equalTo(0L));
+    // assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0));
+    // assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L));
+    // assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L));
+    // assertThat(telemetry.getClientCounts().size(), equalTo(0));
+    //
+    // var perCluster = telemetry.getByRemoteCluster();
+    // assertThat(perCluster.size(), equalTo(3));
+    // for (String clusterAlias : remoteClusterAlias()) {
+    // var clusterData = perCluster.get(clusterAlias);
+    // assertThat(clusterData.getCount(), equalTo(0L));
+    // assertThat(clusterData.getSkippedCount(), equalTo(1L));
+    // assertThat(clusterData.getTook().count(), equalTo(0L));
+    // }
+    // assertPerClusterCount(perCluster.get(LOCAL_CLUSTER), 1L);
+    // }
+
+    public void testRemoteOnly() throws Exception {
+        setupClusters();
+        var telemetry = getTelemetryFromQuery("from c*:logs-* | stats sum (v)", "kibana");
+
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(1L));
+        assertThat(telemetry.getFailureReasons().size(), equalTo(0));
+        assertThat(telemetry.getTook().count(), equalTo(1L));
+        assertThat(telemetry.getTookMrtFalse().count(), equalTo(0L));
+        assertThat(telemetry.getTookMrtTrue().count(), equalTo(0L));
+        assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0));
+        assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L));
+        assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L));
+        assertThat(telemetry.getClientCounts().size(), equalTo(1));
+        assertThat(telemetry.getClientCounts().get("kibana"), equalTo(1L));
+        assertThat(telemetry.getFeatureCounts().get(ASYNC_FEATURE), equalTo(null));
+
+        var perCluster = telemetry.getByRemoteCluster();
+        assertThat(perCluster.size(), equalTo(2));
+        for (String clusterAlias : remoteClusterAlias()) {
+            assertPerClusterCount(perCluster.get(clusterAlias), 1L);
+        }
+        assertThat(telemetry.getByRemoteCluster().size(), equalTo(2));
+    }
+
+    public void testAsync() throws Exception {
+        setupClusters();
+        var telemetry = getTelemetryFromAsyncQuery("from logs-*,c*:logs-* | stats sum (v)");
+
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(1L));
+        assertThat(telemetry.getFailureReasons().size(), equalTo(0));
+        assertThat(telemetry.getTook().count(), equalTo(1L));
+        assertThat(telemetry.getTookMrtFalse().count(), equalTo(0L));
+        assertThat(telemetry.getTookMrtTrue().count(), equalTo(0L));
+        assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0));
+        assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L));
+        assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L));
+        assertThat(telemetry.getClientCounts().size(), equalTo(0));
+        assertThat(telemetry.getFeatureCounts().get(ASYNC_FEATURE), equalTo(1L));
+
+        var perCluster = telemetry.getByRemoteCluster();
+        assertThat(perCluster.size(), equalTo(3));
+        for (String clusterAlias : remoteClusterAlias()) {
+            assertPerClusterCount(perCluster.get(clusterAlias), 1L);
+        }
+        assertPerClusterCount(perCluster.get(LOCAL_CLUSTER), 1L);
+
+        // do it again
+        telemetry = getTelemetryFromAsyncQuery("from logs-*,c*:logs-* |  stats sum (v)");
+        assertThat(telemetry.getTotalCount(), equalTo(2L));
+        assertThat(telemetry.getFeatureCounts().get(ASYNC_FEATURE), equalTo(2L));
+        perCluster = telemetry.getByRemoteCluster();
+        assertThat(perCluster.size(), equalTo(3));
+        for (String clusterAlias : remoteClusterAlias()) {
+            assertPerClusterCount(perCluster.get(clusterAlias), 2L);
+        }
+        assertPerClusterCount(perCluster.get(LOCAL_CLUSTER), 2L);
+    }
+
+    public void testNoSuchCluster() throws Exception {
+        setupClusters();
+        // This is not recognized as a cross-cluster search
+        var telemetry = getTelemetryFromFailedQuery("from c*:logs*, nocluster:nomatch | stats sum (v)");
+
+        assertThat(telemetry.getTotalCount(), equalTo(0L));
+        assertThat(telemetry.getSuccessCount(), equalTo(0L));
+        assertThat(telemetry.getByRemoteCluster().size(), equalTo(0));
+    }
+
+    @SkipUnavailableRule.NotSkipped(aliases = REMOTE1)
+    public void testDisconnect() throws Exception {
+        setupClusters();
+        // Disconnect remote1
+        cluster(REMOTE1).close();
+        var telemetry = getTelemetryFromFailedQuery("from logs-*,cluster-a:logs-* | stats sum (v)");
+
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(0L));
+        Map<String, Long> expectedFailure = Map.of(CCSUsageTelemetry.Result.REMOTES_UNAVAILABLE.getName(), 1L);
+        assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure));
+    }
+
+}
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersUsageTelemetryNoLicenseIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersUsageTelemetryNoLicenseIT.java
new file mode 100644
index 0000000000000..2b993e9474062
--- /dev/null
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersUsageTelemetryNoLicenseIT.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.action;
+
+import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry;
+import org.elasticsearch.plugins.Plugin;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class CrossClustersUsageTelemetryNoLicenseIT extends AbstractCrossClustersUsageTelemetryIT {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins(String clusterAlias) {
+        List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins(clusterAlias));
+        plugins.add(EsqlPluginWithNonEnterpriseOrExpiredLicense.class);
+        plugins.add(CrossClustersQueryIT.InternalExchangePlugin.class);
+        return plugins;
+    }
+
+    public void testLicenseFailure() throws Exception {
+        setupClusters();
+        var telemetry = getTelemetryFromFailedQuery("from logs-*,c*:logs-* | stats sum (v)");
+
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(0L));
+        assertThat(telemetry.getTook().count(), equalTo(0L));
+        assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0));
+        assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L));
+        Map<String, Long> expectedFailure = Map.of(CCSUsageTelemetry.Result.LICENSE.getName(), 1L);
+        assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure));
+    }
+}
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java
index abd4f6b49d7b4..34991c3958f2a 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java
@@ -240,13 +240,13 @@ private ActionFuture<EsqlQueryResponse> startEsql(String query) {
             // Report the status after every action
             .put("status_interval", "0ms");
 
-        if (nodeLevelReduction == false) {
-            // explicitly set the default (false) or don't
+        if (nodeLevelReduction) {
+            // explicitly set the default (true) or don't
             if (randomBoolean()) {
-                settingsBuilder.put("node_level_reduction", nodeLevelReduction);
+                settingsBuilder.put("node_level_reduction", true);
             }
         } else {
-            settingsBuilder.put("node_level_reduction", nodeLevelReduction);
+            settingsBuilder.put("node_level_reduction", false);
         }
 
         var pragmas = new QueryPragmas(settingsBuilder.build());
@@ -273,14 +273,7 @@ private void cancelTask(TaskId taskId) {
     private List<TaskInfo> getTasksStarting() throws Exception {
         List<TaskInfo> foundTasks = new ArrayList<>();
         assertBusy(() -> {
-            List<TaskInfo> tasks = client().admin()
-                .cluster()
-                .prepareListTasks()
-                .setActions(DriverTaskRunner.ACTION_NAME)
-                .setDetailed(true)
-                .get()
-                .getTasks();
-            assertThat(tasks, hasSize(equalTo(3)));
+            List<TaskInfo> tasks = getDriverTasks();
             for (TaskInfo task : tasks) {
                 assertThat(task.action(), equalTo(DriverTaskRunner.ACTION_NAME));
                 DriverStatus status = (DriverStatus) task.status();
@@ -305,14 +298,7 @@ private List<TaskInfo> getTasksStarting() throws Exception {
     private List<TaskInfo> getTasksRunning() throws Exception {
         List<TaskInfo> foundTasks = new ArrayList<>();
         assertBusy(() -> {
-            List<TaskInfo> tasks = client().admin()
-                .cluster()
-                .prepareListTasks()
-                .setActions(DriverTaskRunner.ACTION_NAME)
-                .setDetailed(true)
-                .get()
-                .getTasks();
-            assertThat(tasks, hasSize(equalTo(3)));
+            List<TaskInfo> tasks = getDriverTasks();
             for (TaskInfo task : tasks) {
                 assertThat(task.action(), equalTo(DriverTaskRunner.ACTION_NAME));
                 DriverStatus status = (DriverStatus) task.status();
@@ -328,6 +314,37 @@ private List<TaskInfo> getTasksRunning() throws Exception {
         return foundTasks;
     }
 
+    /**
+     * Fetches tasks until all three driver tasks have started
+     */
+    private List<TaskInfo> getDriverTasks() throws Exception {
+        List<TaskInfo> foundTasks = new ArrayList<>();
+        assertBusy(() -> {
+            List<TaskInfo> tasks = client().admin()
+                .cluster()
+                .prepareListTasks()
+                .setActions(DriverTaskRunner.ACTION_NAME)
+                .setDetailed(true)
+                .get()
+                .getTasks();
+            assertThat(tasks, hasSize(equalTo(3)));
+            List<TaskInfo> readTasks = tasks.stream().filter(t -> t.description().equals(READ_DESCRIPTION)).toList();
+            List<TaskInfo> mergeTasks = tasks.stream().filter(t -> t.description().equals(MERGE_DESCRIPTION)).toList();
+            assertThat(readTasks, hasSize(1));
+            assertThat(mergeTasks, hasSize(1));
+            // node-level reduction is disabled when the target data node is also the coordinator
+            if (readTasks.get(0).node().equals(mergeTasks.get(0).node())) {
+                REDUCE_DESCRIPTION = """
+                    \\_ExchangeSourceOperator[]
+                    \\_ExchangeSinkOperator""";
+            }
+            List<TaskInfo> reduceTasks = tasks.stream().filter(t -> t.description().equals(REDUCE_DESCRIPTION)).toList();
+            assertThat(reduceTasks, hasSize(1));
+            foundTasks.addAll(tasks);
+        });
+        return foundTasks;
+    }
+
     private void assertCancelled(ActionFuture<EsqlQueryResponse> response) throws Exception {
         Exception e = expectThrows(Exception.class, response);
         Throwable cancelException = ExceptionsHelper.unwrap(e, TaskCancelledException.class);
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlNodeFailureIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlNodeFailureIT.java
index 3a69983a0d86e..1e34421097aac 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlNodeFailureIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlNodeFailureIT.java
@@ -7,18 +7,11 @@
 
 package org.elasticsearch.xpack.esql.action;
 
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
-import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.ScriptPlugin;
-import org.elasticsearch.script.LongFieldScript;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptEngine;
-import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.FailingFieldPlugin;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
 
@@ -26,8 +19,6 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
 
 import static org.hamcrest.Matchers.equalTo;
 
@@ -51,7 +42,7 @@ public void testFailureLoadingFields() throws IOException {
             mapping.startObject("fail_me");
             {
                 mapping.field("type", "long");
-                mapping.startObject("script").field("source", "").field("lang", "fail").endObject();
+                mapping.startObject("script").field("source", "").field("lang", "failing_field").endObject();
             }
             mapping.endObject();
         }
@@ -66,51 +57,7 @@ public void testFailureLoadingFields() throws IOException {
         docs.add(client().prepareIndex("fail").setSource("foo", 0));
         indexRandom(true, docs);
 
-        ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> run("FROM fail,ok | LIMIT 100").close());
-        assertThat(e.getMessage(), equalTo("test failure"));
-    }
-
-    public static class FailingFieldPlugin extends Plugin implements ScriptPlugin {
-
-        @Override
-        public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<?>> contexts) {
-            return new ScriptEngine() {
-                @Override
-                public String getType() {
-                    return "fail";
-                }
-
-                @Override
-                @SuppressWarnings("unchecked")
-                public <FactoryType> FactoryType compile(
-                    String name,
-                    String code,
-                    ScriptContext<FactoryType> context,
-                    Map<String, String> params
-                ) {
-                    return (FactoryType) new LongFieldScript.Factory() {
-                        @Override
-                        public LongFieldScript.LeafFactory newFactory(
-                            String fieldName,
-                            Map<String, Object> params,
-                            SearchLookup searchLookup,
-                            OnScriptError onScriptError
-                        ) {
-                            return ctx -> new LongFieldScript(fieldName, params, searchLookup, onScriptError, ctx) {
-                                @Override
-                                public void execute() {
-                                    throw new ElasticsearchException("test failure");
-                                }
-                            };
-                        }
-                    };
-                }
-
-                @Override
-                public Set<ScriptContext<?>> getSupportedContexts() {
-                    return Set.of(LongFieldScript.CONTEXT);
-                }
-            };
-        }
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> run("FROM fail,ok | LIMIT 100").close());
+        assertThat(e.getMessage(), equalTo("Accessing failing field"));
     }
 }
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java
index 3b9359fe66d40..f31eabea9d616 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java
@@ -69,13 +69,56 @@
 import static org.hamcrest.Matchers.empty;
 
 public class LookupFromIndexIT extends AbstractEsqlIntegTestCase {
+    // TODO should we remove this now that this is integrated into ESQL proper?
     /**
      * Quick and dirty test for looking up data from a lookup index.
      */
     public void testLookupIndex() throws IOException {
-        // TODO this should *fail* if the target index isn't a lookup type index - it doesn't now.
-        int docCount = between(10, 1000);
-        List<String> expected = new ArrayList<>(docCount);
+        runLookup(new UsingSingleLookupTable(new String[] { "aa", "bb", "cc", "dd" }));
+    }
+
+    /**
+     * Tests when multiple results match.
+     */
+    @AwaitsFix(bugUrl = "fixing real soon now")
+    public void testLookupIndexMultiResults() throws IOException {
+        runLookup(new UsingSingleLookupTable(new Object[] { "aa", new String[] { "bb", "ff" }, "cc", "dd" }));
+    }
+
+    interface PopulateIndices {
+        void populate(int docCount, List<String> expected) throws IOException;
+    }
+
+    class UsingSingleLookupTable implements PopulateIndices {
+        private final Object[] lookupData;
+
+        UsingSingleLookupTable(Object[] lookupData) {
+            this.lookupData = lookupData;
+        }
+
+        @Override
+        public void populate(int docCount, List<String> expected) throws IOException {
+            List<IndexRequestBuilder> docs = new ArrayList<>();
+            for (int i = 0; i < docCount; i++) {
+                docs.add(client().prepareIndex("source").setSource(Map.of("data", lookupData[i % lookupData.length])));
+                Object d = lookupData[i % lookupData.length];
+                if (d instanceof String s) {
+                    expected.add(s + ":" + (i % lookupData.length));
+                } else if (d instanceof String[] ss) {
+                    for (String s : ss) {
+                        expected.add(s + ":" + (i % lookupData.length));
+                    }
+                }
+            }
+            for (int i = 0; i < lookupData.length; i++) {
+                docs.add(client().prepareIndex("lookup").setSource(Map.of("data", lookupData[i], "l", i)));
+            }
+            Collections.sort(expected);
+            indexRandom(true, true, docs);
+        }
+    }
+
+    private void runLookup(PopulateIndices populateIndices) throws IOException {
         client().admin()
             .indices()
             .prepareCreate("source")
@@ -95,17 +138,9 @@ public void testLookupIndex() throws IOException {
             .get();
         client().admin().cluster().prepareHealth(TEST_REQUEST_TIMEOUT).setWaitForGreenStatus().get();
 
-        String[] data = new String[] { "aa", "bb", "cc", "dd" };
-        List<IndexRequestBuilder> docs = new ArrayList<>();
-        for (int i = 0; i < docCount; i++) {
-            docs.add(client().prepareIndex("source").setSource(Map.of("data", data[i % data.length])));
-            expected.add(data[i % data.length] + ":" + (i % data.length));
-        }
-        for (int i = 0; i < data.length; i++) {
-            docs.add(client().prepareIndex("lookup").setSource(Map.of("data", data[i], "l", i)));
-        }
-        Collections.sort(expected);
-        indexRandom(true, true, docs);
+        int docCount = between(10, 1000);
+        List<String> expected = new ArrayList<>(docCount);
+        populateIndices.populate(docCount, expected);
 
         /*
          * Find the data node hosting the only shard of the source index.
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/SimplePauseFieldPlugin.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/SimplePauseFieldPlugin.java
new file mode 100644
index 0000000000000..3ba73dd9a402e
--- /dev/null
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/SimplePauseFieldPlugin.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.action;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A plugin that provides a script language "pause" that can be used to simulate slow running queries.
+ * This implementation allows to know when it arrives at execute() via startEmitting and to allow the execution to proceed
+ * via allowEmitting.
+ */
+public class SimplePauseFieldPlugin extends AbstractPauseFieldPlugin {
+    public static CountDownLatch startEmitting = new CountDownLatch(1);
+    public static CountDownLatch allowEmitting = new CountDownLatch(1);
+
+    public static void resetPlugin() {
+        allowEmitting = new CountDownLatch(1);
+        startEmitting = new CountDownLatch(1);
+    }
+
+    @Override
+    public void onStartExecute() {
+        startEmitting.countDown();
+    }
+
+    @Override
+    public boolean onWait() throws InterruptedException {
+        return allowEmitting.await(30, TimeUnit.SECONDS);
+    }
+}
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java
index d58637ab52c86..39c963e938ac5 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java
@@ -10,25 +10,21 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xpack.esql.VerificationException;
 import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase;
-import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
+import org.elasticsearch.xpack.kql.KqlPlugin;
 import org.junit.Before;
-import org.junit.BeforeClass;
 
+import java.util.Collection;
 import java.util.List;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.CoreMatchers.containsString;
 
 public class KqlFunctionIT extends AbstractEsqlIntegTestCase {
-
-    @BeforeClass
-    protected static void ensureKqlFunctionEnabled() {
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-    }
-
     @Before
     public void setupIndex() {
         createAndPopulateIndex();
@@ -141,4 +137,9 @@ private void createAndPopulateIndex() {
             .get();
         ensureYellow(indexName);
     }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return CollectionUtils.appendToCopy(super.nodePlugins(), KqlPlugin.class);
+    }
 }
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java
index 58b1652653ca3..ad90bbf6ae9db 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java
@@ -168,7 +168,7 @@ public void testWhereMatchWithScoringNoSort() {
         var query = """
             FROM test
             METADATA _score
-            | WHERE content:"fox"
+            | WHERE match(content, "fox")
             | KEEP id, _score
             """;
 
@@ -182,7 +182,7 @@ public void testWhereMatchWithScoringNoSort() {
     public void testNonExistingColumn() {
         var query = """
             FROM test
-            | WHERE something:"fox"
+            | WHERE match(something, "fox")
             """;
 
         var error = expectThrows(VerificationException.class, () -> run(query));
@@ -193,14 +193,14 @@ public void testWhereMatchEvalColumn() {
         var query = """
             FROM test
             | EVAL upper_content = to_upper(content)
-            | WHERE upper_content:"FOX"
+            | WHERE match(upper_content, "FOX")
             | KEEP id
             """;
 
         var error = expectThrows(VerificationException.class, () -> run(query));
         assertThat(
             error.getMessage(),
-            containsString("[:] operator cannot operate on [upper_content], which is not a field from an index mapping")
+            containsString("[MATCH] function cannot operate on [upper_content], which is not a field from an index mapping")
         );
     }
 
@@ -209,13 +209,13 @@ public void testWhereMatchOverWrittenColumn() {
             FROM test
             | DROP content
             | EVAL content = CONCAT("document with ID ", to_str(id))
-            | WHERE content:"document"
+            | WHERE match(content, "document")
             """;
 
         var error = expectThrows(VerificationException.class, () -> run(query));
         assertThat(
             error.getMessage(),
-            containsString("[:] operator cannot operate on [content], which is not a field from an index mapping")
+            containsString("[MATCH] function cannot operate on [content], which is not a field from an index mapping")
         );
     }
 
@@ -223,7 +223,7 @@ public void testWhereMatchAfterStats() {
         var query = """
             FROM test
             | STATS count(*)
-            | WHERE content:"fox"
+            | WHERE match(content, "fox")
             """;
 
         var error = expectThrows(VerificationException.class, () -> run(query));
@@ -233,14 +233,15 @@ public void testWhereMatchAfterStats() {
     public void testWhereMatchWithFunctions() {
         var query = """
             FROM test
-            | WHERE content:"fox" OR to_upper(content) == "FOX"
+            | WHERE match(content, "fox") OR to_upper(content) == "FOX"
             """;
         var error = expectThrows(ElasticsearchException.class, () -> run(query));
         assertThat(
             error.getMessage(),
             containsString(
-                "Invalid condition [content:\"fox\" OR to_upper(content) == \"FOX\"]. "
-                    + "[:] operator can't be used as part of an or condition"
+                "Invalid condition [match(content, \"fox\") OR to_upper(content) == \"FOX\"]. "
+                    + "Full text functions can be used in an OR condition,"
+                    + " but only if just full text functions are used in the OR condition"
             )
         );
     }
@@ -248,24 +249,24 @@ public void testWhereMatchWithFunctions() {
     public void testWhereMatchWithRow() {
         var query = """
             ROW content = "a brown fox"
-            | WHERE content:"fox"
+            | WHERE match(content, "fox")
             """;
 
         var error = expectThrows(ElasticsearchException.class, () -> run(query));
         assertThat(
             error.getMessage(),
-            containsString("[:] operator cannot operate on [\"a brown fox\"], which is not a field from an index mapping")
+            containsString("[MATCH] function cannot operate on [\"a brown fox\"], which is not a field from an index mapping")
         );
     }
 
     public void testMatchWithinEval() {
         var query = """
             FROM test
-            | EVAL matches_query = content:"fox"
+            | EVAL matches_query = match(content, "fox")
             """;
 
         var error = expectThrows(VerificationException.class, () -> run(query));
-        assertThat(error.getMessage(), containsString("[:] operator is only supported in WHERE commands"));
+        assertThat(error.getMessage(), containsString("[MATCH] function is only supported in WHERE commands"));
     }
 
     private void createAndPopulateIndex() {
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java
index d0a641f086fe4..758878b46d51f 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java
@@ -216,7 +216,8 @@ public void testWhereMatchWithFunctions() {
             error.getMessage(),
             containsString(
                 "Invalid condition [content:\"fox\" OR to_upper(content) == \"FOX\"]. "
-                    + "[:] operator can't be used as part of an or condition"
+                    + "Full text functions can be used in an OR condition, "
+                    + "but only if just full text functions are used in the OR condition"
             )
         );
     }
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
index efc2e36609902..4d4cb7d2caac4 100644
--- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
+++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
@@ -143,18 +143,9 @@ indexString
     ;
 
 metadata
-    : metadataOption
-    | deprecated_metadata
-    ;
-
-metadataOption
     : METADATA UNQUOTED_SOURCE (COMMA UNQUOTED_SOURCE)*
     ;
 
-deprecated_metadata
-    : OPENING_BRACKET metadataOption CLOSING_BRACKET
-    ;
-
 metricsCommand
     : DEV_METRICS indexPattern (COMMA indexPattern)* aggregates=aggFields? (BY grouping=fields)?
     ;
diff --git a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InMillisNanosEvaluator.java b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InMillisNanosEvaluator.java
new file mode 100644
index 0000000000000..14ca1e9a768fb
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InMillisNanosEvaluator.java
@@ -0,0 +1,189 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison;
+
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BooleanBlock;
+import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.data.LongVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.compute.operator.Warnings;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.Arrays;
+import java.util.BitSet;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link In}.
+ * This class is generated. Edit {@code X-InEvaluator.java.st} instead.
+ */
+public class InMillisNanosEvaluator implements EvalOperator.ExpressionEvaluator {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator lhs;
+
+    private final EvalOperator.ExpressionEvaluator[] rhs;
+
+    private final DriverContext driverContext;
+
+    private Warnings warnings;
+
+    public InMillisNanosEvaluator(
+        Source source,
+        EvalOperator.ExpressionEvaluator lhs,
+        EvalOperator.ExpressionEvaluator[] rhs,
+        DriverContext driverContext
+    ) {
+        this.source = source;
+        this.lhs = lhs;
+        this.rhs = rhs;
+        this.driverContext = driverContext;
+    }
+
+    @Override
+    public Block eval(Page page) {
+        try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) {
+            LongBlock[] rhsBlocks = new LongBlock[rhs.length];
+            try (Releasable rhsRelease = Releasables.wrap(rhsBlocks)) {
+                for (int i = 0; i < rhsBlocks.length; i++) {
+                    rhsBlocks[i] = (LongBlock) rhs[i].eval(page);
+                }
+                LongVector lhsVector = lhsBlock.asVector();
+                if (lhsVector == null) {
+                    return eval(page.getPositionCount(), lhsBlock, rhsBlocks);
+                }
+                LongVector[] rhsVectors = new LongVector[rhs.length];
+                for (int i = 0; i < rhsBlocks.length; i++) {
+                    rhsVectors[i] = rhsBlocks[i].asVector();
+                    if (rhsVectors[i] == null) {
+                        return eval(page.getPositionCount(), lhsBlock, rhsBlocks);
+                    }
+                }
+                return eval(page.getPositionCount(), lhsVector, rhsVectors);
+            }
+        }
+    }
+
+    private BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock[] rhsBlocks) {
+        try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
+            long[] rhsValues = new long[rhs.length];
+            BitSet nulls = new BitSet(rhs.length);
+            BitSet mvs = new BitSet(rhs.length);
+            boolean foundMatch;
+            for (int p = 0; p < positionCount; p++) {
+                if (lhsBlock.isNull(p)) {
+                    result.appendNull();
+                    continue;
+                }
+                if (lhsBlock.getValueCount(p) != 1) {
+                    if (lhsBlock.getValueCount(p) > 1) {
+                        warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+                    }
+                    result.appendNull();
+                    continue;
+                }
+                // unpack rhsBlocks into rhsValues
+                nulls.clear();
+                mvs.clear();
+                for (int i = 0; i < rhsBlocks.length; i++) {
+                    if (rhsBlocks[i].isNull(p)) {
+                        nulls.set(i);
+                        continue;
+                    }
+                    if (rhsBlocks[i].getValueCount(p) > 1) {
+                        mvs.set(i);
+                        warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+                        continue;
+                    }
+                    int o = rhsBlocks[i].getFirstValueIndex(p);
+                    rhsValues[i] = rhsBlocks[i].getLong(o);
+                }
+                if (nulls.cardinality() == rhsBlocks.length || mvs.cardinality() == rhsBlocks.length) {
+                    result.appendNull();
+                    continue;
+                }
+                foundMatch = In.process(nulls, mvs, lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsValues);
+                if (foundMatch) {
+                    result.appendBoolean(true);
+                } else {
+                    if (nulls.cardinality() > 0) {
+                        result.appendNull();
+                    } else {
+                        result.appendBoolean(false);
+                    }
+                }
+            }
+            return result.build();
+        }
+    }
+
+    private BooleanBlock eval(int positionCount, LongVector lhsVector, LongVector[] rhsVectors) {
+        try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
+            long[] rhsValues = new long[rhs.length];
+            for (int p = 0; p < positionCount; p++) {
+                // unpack rhsVectors into rhsValues
+                for (int i = 0; i < rhsVectors.length; i++) {
+                    rhsValues[i] = rhsVectors[i].getLong(p);
+                }
+                result.appendBoolean(In.processMillisNanos(null, null, lhsVector.getLong(p), rhsValues));
+            }
+            return result.build();
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "InMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]";
+    }
+
+    @Override
+    public void close() {
+        Releasables.closeExpectNoException(lhs, () -> Releasables.close(rhs));
+    }
+
+    private Warnings warnings() {
+        if (warnings == null) {
+            this.warnings = Warnings.createWarnings(
+                driverContext.warningsMode(),
+                source.source().getLineNumber(),
+                source.source().getColumnNumber(),
+                source.text()
+            );
+        }
+        return warnings;
+    }
+
+    static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+        private final Source source;
+        private final EvalOperator.ExpressionEvaluator.Factory lhs;
+        private final EvalOperator.ExpressionEvaluator.Factory[] rhs;
+
+        Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, EvalOperator.ExpressionEvaluator.Factory[] rhs) {
+            this.source = source;
+            this.lhs = lhs;
+            this.rhs = rhs;
+        }
+
+        @Override
+        public InMillisNanosEvaluator get(DriverContext context) {
+            EvalOperator.ExpressionEvaluator[] rhs = Arrays.stream(this.rhs)
+                .map(a -> a.get(context))
+                .toArray(EvalOperator.ExpressionEvaluator[]::new);
+            return new InMillisNanosEvaluator(source, lhs.get(context), rhs, context);
+        }
+
+        @Override
+        public String toString() {
+            return "InMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]";
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InNanosMillisEvaluator.java b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InNanosMillisEvaluator.java
new file mode 100644
index 0000000000000..278ecdba7d9e5
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InNanosMillisEvaluator.java
@@ -0,0 +1,189 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison;
+
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BooleanBlock;
+import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.data.LongVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.compute.operator.Warnings;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.Arrays;
+import java.util.BitSet;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link In}.
+ * This class is generated. Edit {@code X-InEvaluator.java.st} instead.
+ */
+public class InNanosMillisEvaluator implements EvalOperator.ExpressionEvaluator {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator lhs;
+
+    private final EvalOperator.ExpressionEvaluator[] rhs;
+
+    private final DriverContext driverContext;
+
+    private Warnings warnings;
+
+    public InNanosMillisEvaluator(
+        Source source,
+        EvalOperator.ExpressionEvaluator lhs,
+        EvalOperator.ExpressionEvaluator[] rhs,
+        DriverContext driverContext
+    ) {
+        this.source = source;
+        this.lhs = lhs;
+        this.rhs = rhs;
+        this.driverContext = driverContext;
+    }
+
+    @Override
+    public Block eval(Page page) {
+        try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) {
+            LongBlock[] rhsBlocks = new LongBlock[rhs.length];
+            try (Releasable rhsRelease = Releasables.wrap(rhsBlocks)) {
+                for (int i = 0; i < rhsBlocks.length; i++) {
+                    rhsBlocks[i] = (LongBlock) rhs[i].eval(page);
+                }
+                LongVector lhsVector = lhsBlock.asVector();
+                if (lhsVector == null) {
+                    return eval(page.getPositionCount(), lhsBlock, rhsBlocks);
+                }
+                LongVector[] rhsVectors = new LongVector[rhs.length];
+                for (int i = 0; i < rhsBlocks.length; i++) {
+                    rhsVectors[i] = rhsBlocks[i].asVector();
+                    if (rhsVectors[i] == null) {
+                        return eval(page.getPositionCount(), lhsBlock, rhsBlocks);
+                    }
+                }
+                return eval(page.getPositionCount(), lhsVector, rhsVectors);
+            }
+        }
+    }
+
+    private BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock[] rhsBlocks) {
+        try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
+            long[] rhsValues = new long[rhs.length];
+            BitSet nulls = new BitSet(rhs.length);
+            BitSet mvs = new BitSet(rhs.length);
+            boolean foundMatch;
+            for (int p = 0; p < positionCount; p++) {
+                if (lhsBlock.isNull(p)) {
+                    result.appendNull();
+                    continue;
+                }
+                if (lhsBlock.getValueCount(p) != 1) {
+                    if (lhsBlock.getValueCount(p) > 1) {
+                        warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+                    }
+                    result.appendNull();
+                    continue;
+                }
+                // unpack rhsBlocks into rhsValues
+                nulls.clear();
+                mvs.clear();
+                for (int i = 0; i < rhsBlocks.length; i++) {
+                    if (rhsBlocks[i].isNull(p)) {
+                        nulls.set(i);
+                        continue;
+                    }
+                    if (rhsBlocks[i].getValueCount(p) > 1) {
+                        mvs.set(i);
+                        warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+                        continue;
+                    }
+                    int o = rhsBlocks[i].getFirstValueIndex(p);
+                    rhsValues[i] = rhsBlocks[i].getLong(o);
+                }
+                if (nulls.cardinality() == rhsBlocks.length || mvs.cardinality() == rhsBlocks.length) {
+                    result.appendNull();
+                    continue;
+                }
+                foundMatch = In.process(nulls, mvs, lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsValues);
+                if (foundMatch) {
+                    result.appendBoolean(true);
+                } else {
+                    if (nulls.cardinality() > 0) {
+                        result.appendNull();
+                    } else {
+                        result.appendBoolean(false);
+                    }
+                }
+            }
+            return result.build();
+        }
+    }
+
+    private BooleanBlock eval(int positionCount, LongVector lhsVector, LongVector[] rhsVectors) {
+        try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
+            long[] rhsValues = new long[rhs.length];
+            for (int p = 0; p < positionCount; p++) {
+                // unpack rhsVectors into rhsValues
+                for (int i = 0; i < rhsVectors.length; i++) {
+                    rhsValues[i] = rhsVectors[i].getLong(p);
+                }
+                result.appendBoolean(In.processNanosMillis(null, null, lhsVector.getLong(p), rhsValues));
+            }
+            return result.build();
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "InNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]";
+    }
+
+    @Override
+    public void close() {
+        Releasables.closeExpectNoException(lhs, () -> Releasables.close(rhs));
+    }
+
+    private Warnings warnings() {
+        if (warnings == null) {
+            this.warnings = Warnings.createWarnings(
+                driverContext.warningsMode(),
+                source.source().getLineNumber(),
+                source.source().getColumnNumber(),
+                source.text()
+            );
+        }
+        return warnings;
+    }
+
+    static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+        private final Source source;
+        private final EvalOperator.ExpressionEvaluator.Factory lhs;
+        private final EvalOperator.ExpressionEvaluator.Factory[] rhs;
+
+        Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, EvalOperator.ExpressionEvaluator.Factory[] rhs) {
+            this.source = source;
+            this.lhs = lhs;
+            this.rhs = rhs;
+        }
+
+        @Override
+        public InNanosMillisEvaluator get(DriverContext context) {
+            EvalOperator.ExpressionEvaluator[] rhs = Arrays.stream(this.rhs)
+                .map(a -> a.get(context))
+                .toArray(EvalOperator.ExpressionEvaluator[]::new);
+            return new InNanosMillisEvaluator(source, lhs.get(context), rhs, context);
+        }
+
+        @Override
+        public String toString() {
+            return "InNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]";
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java
index 3ea782931a0a3..6c432855e38fb 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java
@@ -7,7 +7,6 @@
 import java.lang.IllegalArgumentException;
 import java.lang.Override;
 import java.lang.String;
-import java.time.ZoneId;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BytesRefBlock;
@@ -31,18 +30,15 @@ public final class DateParseEvaluator implements EvalOperator.ExpressionEvaluato
 
   private final EvalOperator.ExpressionEvaluator formatter;
 
-  private final ZoneId zoneId;
-
   private final DriverContext driverContext;
 
   private Warnings warnings;
 
   public DateParseEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
-      EvalOperator.ExpressionEvaluator formatter, ZoneId zoneId, DriverContext driverContext) {
+      EvalOperator.ExpressionEvaluator formatter, DriverContext driverContext) {
     this.source = source;
     this.val = val;
     this.formatter = formatter;
-    this.zoneId = zoneId;
     this.driverContext = driverContext;
   }
 
@@ -91,7 +87,7 @@ public LongBlock eval(int positionCount, BytesRefBlock valBlock, BytesRefBlock f
           continue position;
         }
         try {
-          result.appendLong(DateParse.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch), formatterBlock.getBytesRef(formatterBlock.getFirstValueIndex(p), formatterScratch), this.zoneId));
+          result.appendLong(DateParse.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch), formatterBlock.getBytesRef(formatterBlock.getFirstValueIndex(p), formatterScratch)));
         } catch (IllegalArgumentException e) {
           warnings().registerException(e);
           result.appendNull();
@@ -108,7 +104,7 @@ public LongBlock eval(int positionCount, BytesRefVector valVector,
       BytesRef formatterScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendLong(DateParse.process(valVector.getBytesRef(p, valScratch), formatterVector.getBytesRef(p, formatterScratch), this.zoneId));
+          result.appendLong(DateParse.process(valVector.getBytesRef(p, valScratch), formatterVector.getBytesRef(p, formatterScratch)));
         } catch (IllegalArgumentException e) {
           warnings().registerException(e);
           result.appendNull();
@@ -120,7 +116,7 @@ public LongBlock eval(int positionCount, BytesRefVector valVector,
 
   @Override
   public String toString() {
-    return "DateParseEvaluator[" + "val=" + val + ", formatter=" + formatter + ", zoneId=" + zoneId + "]";
+    return "DateParseEvaluator[" + "val=" + val + ", formatter=" + formatter + "]";
   }
 
   @Override
@@ -147,24 +143,21 @@ static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
 
     private final EvalOperator.ExpressionEvaluator.Factory formatter;
 
-    private final ZoneId zoneId;
-
     public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val,
-        EvalOperator.ExpressionEvaluator.Factory formatter, ZoneId zoneId) {
+        EvalOperator.ExpressionEvaluator.Factory formatter) {
       this.source = source;
       this.val = val;
       this.formatter = formatter;
-      this.zoneId = zoneId;
     }
 
     @Override
     public DateParseEvaluator get(DriverContext context) {
-      return new DateParseEvaluator(source, val.get(context), formatter.get(context), zoneId, context);
+      return new DateParseEvaluator(source, val.get(context), formatter.get(context), context);
     }
 
     @Override
     public String toString() {
-      return "DateParseEvaluator[" + "val=" + val + ", formatter=" + formatter + ", zoneId=" + zoneId + "]";
+      return "DateParseEvaluator[" + "val=" + val + ", formatter=" + formatter + "]";
     }
   }
 }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ChangeCaseEvaluator.java
similarity index 76%
rename from x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerEvaluator.java
rename to x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ChangeCaseEvaluator.java
index 61e7c5c3042df..02d1b1c86ea32 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ChangeCaseEvaluator.java
@@ -20,25 +20,28 @@
 import org.elasticsearch.xpack.esql.core.tree.Source;
 
 /**
- * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToLower}.
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ChangeCase}.
  * This class is generated. Do not edit it.
  */
-public final class ToLowerEvaluator implements EvalOperator.ExpressionEvaluator {
+public final class ChangeCaseEvaluator implements EvalOperator.ExpressionEvaluator {
   private final Source source;
 
   private final EvalOperator.ExpressionEvaluator val;
 
   private final Locale locale;
 
+  private final ChangeCase.Case caseType;
+
   private final DriverContext driverContext;
 
   private Warnings warnings;
 
-  public ToLowerEvaluator(Source source, EvalOperator.ExpressionEvaluator val, Locale locale,
-      DriverContext driverContext) {
+  public ChangeCaseEvaluator(Source source, EvalOperator.ExpressionEvaluator val, Locale locale,
+      ChangeCase.Case caseType, DriverContext driverContext) {
     this.source = source;
     this.val = val;
     this.locale = locale;
+    this.caseType = caseType;
     this.driverContext = driverContext;
   }
 
@@ -68,7 +71,7 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock valBlock) {
           result.appendNull();
           continue position;
         }
-        result.appendBytesRef(ToLower.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch), this.locale));
+        result.appendBytesRef(ChangeCase.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch), this.locale, this.caseType));
       }
       return result.build();
     }
@@ -78,7 +81,7 @@ public BytesRefVector eval(int positionCount, BytesRefVector valVector) {
     try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
       BytesRef valScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        result.appendBytesRef(ToLower.process(valVector.getBytesRef(p, valScratch), this.locale));
+        result.appendBytesRef(ChangeCase.process(valVector.getBytesRef(p, valScratch), this.locale, this.caseType));
       }
       return result.build();
     }
@@ -86,7 +89,7 @@ public BytesRefVector eval(int positionCount, BytesRefVector valVector) {
 
   @Override
   public String toString() {
-    return "ToLowerEvaluator[" + "val=" + val + ", locale=" + locale + "]";
+    return "ChangeCaseEvaluator[" + "val=" + val + ", locale=" + locale + ", caseType=" + caseType + "]";
   }
 
   @Override
@@ -113,20 +116,24 @@ static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
 
     private final Locale locale;
 
-    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val, Locale locale) {
+    private final ChangeCase.Case caseType;
+
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val, Locale locale,
+        ChangeCase.Case caseType) {
       this.source = source;
       this.val = val;
       this.locale = locale;
+      this.caseType = caseType;
     }
 
     @Override
-    public ToLowerEvaluator get(DriverContext context) {
-      return new ToLowerEvaluator(source, val.get(context), locale, context);
+    public ChangeCaseEvaluator get(DriverContext context) {
+      return new ChangeCaseEvaluator(source, val.get(context), locale, caseType, context);
     }
 
     @Override
     public String toString() {
-      return "ToLowerEvaluator[" + "val=" + val + ", locale=" + locale + "]";
+      return "ChangeCaseEvaluator[" + "val=" + val + ", locale=" + locale + ", caseType=" + caseType + "]";
     }
   }
 }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashConstantEvaluator.java
similarity index 51%
rename from x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperEvaluator.java
rename to x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashConstantEvaluator.java
index 82412dde53941..34cff73018634 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashConstantEvaluator.java
@@ -7,12 +7,13 @@
 import java.lang.IllegalArgumentException;
 import java.lang.Override;
 import java.lang.String;
-import java.util.Locale;
+import java.util.function.Function;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BytesRefBlock;
 import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.compute.operator.Warnings;
@@ -20,65 +21,69 @@
 import org.elasticsearch.xpack.esql.core.tree.Source;
 
 /**
- * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToUpper}.
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Hash}.
  * This class is generated. Do not edit it.
  */
-public final class ToUpperEvaluator implements EvalOperator.ExpressionEvaluator {
+public final class HashConstantEvaluator implements EvalOperator.ExpressionEvaluator {
   private final Source source;
 
-  private final EvalOperator.ExpressionEvaluator val;
+  private final BreakingBytesRefBuilder scratch;
 
-  private final Locale locale;
+  private final Hash.HashFunction algorithm;
+
+  private final EvalOperator.ExpressionEvaluator input;
 
   private final DriverContext driverContext;
 
   private Warnings warnings;
 
-  public ToUpperEvaluator(Source source, EvalOperator.ExpressionEvaluator val, Locale locale,
+  public HashConstantEvaluator(Source source, BreakingBytesRefBuilder scratch,
+      Hash.HashFunction algorithm, EvalOperator.ExpressionEvaluator input,
       DriverContext driverContext) {
     this.source = source;
-    this.val = val;
-    this.locale = locale;
+    this.scratch = scratch;
+    this.algorithm = algorithm;
+    this.input = input;
     this.driverContext = driverContext;
   }
 
   @Override
   public Block eval(Page page) {
-    try (BytesRefBlock valBlock = (BytesRefBlock) val.eval(page)) {
-      BytesRefVector valVector = valBlock.asVector();
-      if (valVector == null) {
-        return eval(page.getPositionCount(), valBlock);
+    try (BytesRefBlock inputBlock = (BytesRefBlock) input.eval(page)) {
+      BytesRefVector inputVector = inputBlock.asVector();
+      if (inputVector == null) {
+        return eval(page.getPositionCount(), inputBlock);
       }
-      return eval(page.getPositionCount(), valVector).asBlock();
+      return eval(page.getPositionCount(), inputVector).asBlock();
     }
   }
 
-  public BytesRefBlock eval(int positionCount, BytesRefBlock valBlock) {
+  public BytesRefBlock eval(int positionCount, BytesRefBlock inputBlock) {
     try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) {
-      BytesRef valScratch = new BytesRef();
+      BytesRef inputScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        if (valBlock.isNull(p)) {
+        if (inputBlock.isNull(p)) {
           result.appendNull();
           continue position;
         }
-        if (valBlock.getValueCount(p) != 1) {
-          if (valBlock.getValueCount(p) > 1) {
+        if (inputBlock.getValueCount(p) != 1) {
+          if (inputBlock.getValueCount(p) > 1) {
             warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
           }
           result.appendNull();
           continue position;
         }
-        result.appendBytesRef(ToUpper.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch), this.locale));
+        result.appendBytesRef(Hash.processConstant(this.scratch, this.algorithm, inputBlock.getBytesRef(inputBlock.getFirstValueIndex(p), inputScratch)));
       }
       return result.build();
     }
   }
 
-  public BytesRefVector eval(int positionCount, BytesRefVector valVector) {
+  public BytesRefVector eval(int positionCount, BytesRefVector inputVector) {
     try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
-      BytesRef valScratch = new BytesRef();
+      BytesRef inputScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        result.appendBytesRef(ToUpper.process(valVector.getBytesRef(p, valScratch), this.locale));
+        result.appendBytesRef(Hash.processConstant(this.scratch, this.algorithm, inputVector.getBytesRef(p, inputScratch)));
       }
       return result.build();
     }
@@ -86,12 +91,12 @@ public BytesRefVector eval(int positionCount, BytesRefVector valVector) {
 
   @Override
   public String toString() {
-    return "ToUpperEvaluator[" + "val=" + val + ", locale=" + locale + "]";
+    return "HashConstantEvaluator[" + "algorithm=" + algorithm + ", input=" + input + "]";
   }
 
   @Override
   public void close() {
-    Releasables.closeExpectNoException(val);
+    Releasables.closeExpectNoException(scratch, input);
   }
 
   private Warnings warnings() {
@@ -109,24 +114,29 @@ private Warnings warnings() {
   static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
     private final Source source;
 
-    private final EvalOperator.ExpressionEvaluator.Factory val;
+    private final Function<DriverContext, BreakingBytesRefBuilder> scratch;
+
+    private final Function<DriverContext, Hash.HashFunction> algorithm;
 
-    private final Locale locale;
+    private final EvalOperator.ExpressionEvaluator.Factory input;
 
-    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val, Locale locale) {
+    public Factory(Source source, Function<DriverContext, BreakingBytesRefBuilder> scratch,
+        Function<DriverContext, Hash.HashFunction> algorithm,
+        EvalOperator.ExpressionEvaluator.Factory input) {
       this.source = source;
-      this.val = val;
-      this.locale = locale;
+      this.scratch = scratch;
+      this.algorithm = algorithm;
+      this.input = input;
     }
 
     @Override
-    public ToUpperEvaluator get(DriverContext context) {
-      return new ToUpperEvaluator(source, val.get(context), locale, context);
+    public HashConstantEvaluator get(DriverContext context) {
+      return new HashConstantEvaluator(source, scratch.apply(context), algorithm.apply(context), input.get(context), context);
     }
 
     @Override
     public String toString() {
-      return "ToUpperEvaluator[" + "val=" + val + ", locale=" + locale + "]";
+      return "HashConstantEvaluator[" + "algorithm=" + algorithm + ", input=" + input + "]";
     }
   }
 }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashEvaluator.java
new file mode 100644
index 0000000000000..8b01cc0330142
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashEvaluator.java
@@ -0,0 +1,174 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import java.lang.IllegalArgumentException;
+import java.lang.Override;
+import java.lang.String;
+import java.security.NoSuchAlgorithmException;
+import java.util.function.Function;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.compute.operator.Warnings;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Hash}.
+ * This class is generated. Do not edit it.
+ */
+public final class HashEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Source source;
+
+  private final BreakingBytesRefBuilder scratch;
+
+  private final EvalOperator.ExpressionEvaluator algorithm;
+
+  private final EvalOperator.ExpressionEvaluator input;
+
+  private final DriverContext driverContext;
+
+  private Warnings warnings;
+
+  public HashEvaluator(Source source, BreakingBytesRefBuilder scratch,
+      EvalOperator.ExpressionEvaluator algorithm, EvalOperator.ExpressionEvaluator input,
+      DriverContext driverContext) {
+    this.source = source;
+    this.scratch = scratch;
+    this.algorithm = algorithm;
+    this.input = input;
+    this.driverContext = driverContext;
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (BytesRefBlock algorithmBlock = (BytesRefBlock) algorithm.eval(page)) {
+      try (BytesRefBlock inputBlock = (BytesRefBlock) input.eval(page)) {
+        BytesRefVector algorithmVector = algorithmBlock.asVector();
+        if (algorithmVector == null) {
+          return eval(page.getPositionCount(), algorithmBlock, inputBlock);
+        }
+        BytesRefVector inputVector = inputBlock.asVector();
+        if (inputVector == null) {
+          return eval(page.getPositionCount(), algorithmBlock, inputBlock);
+        }
+        return eval(page.getPositionCount(), algorithmVector, inputVector);
+      }
+    }
+  }
+
+  public BytesRefBlock eval(int positionCount, BytesRefBlock algorithmBlock,
+      BytesRefBlock inputBlock) {
+    try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) {
+      BytesRef algorithmScratch = new BytesRef();
+      BytesRef inputScratch = new BytesRef();
+      position: for (int p = 0; p < positionCount; p++) {
+        if (algorithmBlock.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (algorithmBlock.getValueCount(p) != 1) {
+          if (algorithmBlock.getValueCount(p) > 1) {
+            warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        if (inputBlock.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (inputBlock.getValueCount(p) != 1) {
+          if (inputBlock.getValueCount(p) > 1) {
+            warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        try {
+          result.appendBytesRef(Hash.process(this.scratch, algorithmBlock.getBytesRef(algorithmBlock.getFirstValueIndex(p), algorithmScratch), inputBlock.getBytesRef(inputBlock.getFirstValueIndex(p), inputScratch)));
+        } catch (NoSuchAlgorithmException e) {
+          warnings().registerException(e);
+          result.appendNull();
+        }
+      }
+      return result.build();
+    }
+  }
+
+  public BytesRefBlock eval(int positionCount, BytesRefVector algorithmVector,
+      BytesRefVector inputVector) {
+    try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) {
+      BytesRef algorithmScratch = new BytesRef();
+      BytesRef inputScratch = new BytesRef();
+      position: for (int p = 0; p < positionCount; p++) {
+        try {
+          result.appendBytesRef(Hash.process(this.scratch, algorithmVector.getBytesRef(p, algorithmScratch), inputVector.getBytesRef(p, inputScratch)));
+        } catch (NoSuchAlgorithmException e) {
+          warnings().registerException(e);
+          result.appendNull();
+        }
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "HashEvaluator[" + "algorithm=" + algorithm + ", input=" + input + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(scratch, algorithm, input);
+  }
+
+  private Warnings warnings() {
+    if (warnings == null) {
+      this.warnings = Warnings.createWarnings(
+              driverContext.warningsMode(),
+              source.source().getLineNumber(),
+              source.source().getColumnNumber(),
+              source.text()
+          );
+    }
+    return warnings;
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final Function<DriverContext, BreakingBytesRefBuilder> scratch;
+
+    private final EvalOperator.ExpressionEvaluator.Factory algorithm;
+
+    private final EvalOperator.ExpressionEvaluator.Factory input;
+
+    public Factory(Source source, Function<DriverContext, BreakingBytesRefBuilder> scratch,
+        EvalOperator.ExpressionEvaluator.Factory algorithm,
+        EvalOperator.ExpressionEvaluator.Factory input) {
+      this.source = source;
+      this.scratch = scratch;
+      this.algorithm = algorithm;
+      this.input = input;
+    }
+
+    @Override
+    public HashEvaluator get(DriverContext context) {
+      return new HashEvaluator(source, scratch.apply(context), algorithm.get(context), input.get(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "HashEvaluator[" + "algorithm=" + algorithm + ", input=" + input + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/LicenseAware.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/LicenseAware.java
new file mode 100644
index 0000000000000..04fcdb8a7c8e1
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/LicenseAware.java
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql;
+
+import org.elasticsearch.license.XPackLicenseState;
+
+public interface LicenseAware {
+    /** Return true if the implementer can be executed under the provided {@link XPackLicenseState}, otherwise false.*/
+    boolean licenseCheck(XPackLicenseState state);
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
index e9a0f89e4f448..1acc72d4ea33e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
@@ -27,6 +27,113 @@
  */
 public class EsqlCapabilities {
     public enum Cap {
+        /**
+         * Introduction of {@code MV_SORT}, {@code MV_SLICE}, and {@code MV_ZIP}.
+         * Added in #106095.
+         */
+        MV_SORT,
+
+        /**
+         * When we disabled some broken optimizations around {@code nullable}.
+         * Fixed in #105691.
+         */
+        DISABLE_NULLABLE_OPTS,
+
+        /**
+         * Introduction of {@code ST_X} and {@code ST_Y}. Added in #105768.
+         */
+        ST_X_Y,
+
+        /**
+         * Changed precision of {@code geo_point} and {@code cartesian_point} fields, by loading from source into WKB. Done in #103691.
+         */
+        SPATIAL_POINTS_FROM_SOURCE,
+
+        /**
+         * Support for loading {@code geo_shape} and {@code cartesian_shape} fields. Done in #104269.
+         */
+        SPATIAL_SHAPES,
+
+        /**
+         * Support for spatial aggregation {@code ST_CENTROID}. Done in #104269.
+         */
+        ST_CENTROID_AGG,
+
+        /**
+         * Support for spatial aggregation {@code ST_INTERSECTS}. Done in #104907.
+         */
+        ST_INTERSECTS,
+
+        /**
+         * Support for spatial aggregation {@code ST_CONTAINS} and {@code ST_WITHIN}. Done in #106503.
+         */
+        ST_CONTAINS_WITHIN,
+
+        /**
+         * Support for spatial aggregation {@code ST_DISJOINT}. Done in #107007.
+         */
+        ST_DISJOINT,
+
+        /**
+         * The introduction of the {@code VALUES} agg.
+         */
+        AGG_VALUES,
+
+        /**
+         * Does ESQL support async queries.
+         */
+        ASYNC_QUERY,
+
+        /**
+         * Does ESQL support FROM OPTIONS?
+         */
+        @Deprecated
+        FROM_OPTIONS,
+
+        /**
+         * Cast string literals to a desired data type.
+         */
+        STRING_LITERAL_AUTO_CASTING,
+
+        /**
+         * Base64 encoding and decoding functions.
+         */
+        BASE64_DECODE_ENCODE,
+
+        /**
+         * Support for the :: casting operator
+         */
+        CASTING_OPERATOR,
+
+        /**
+         * Blocks can be labelled with {@link org.elasticsearch.compute.data.Block.MvOrdering#SORTED_ASCENDING} for optimizations.
+         */
+        MV_ORDERING_SORTED_ASCENDING,
+
+        /**
+         * Support for metrics counter fields
+         */
+        METRICS_COUNTER_FIELDS,
+
+        /**
+         * Cast string literals to a desired data type for IN predicate and more types for BinaryComparison.
+         */
+        STRING_LITERAL_AUTO_CASTING_EXTENDED,
+
+        /**
+         * Support for metadata fields.
+         */
+        METADATA_FIELDS,
+
+        /**
+         * Support for timespan units abbreviations
+         */
+        TIMESPAN_ABBREVIATIONS,
+
+        /**
+         * Support metrics counter types
+         */
+        COUNTER_TYPES,
 
         /**
          * Support for function {@code BIT_LENGTH}. Done in #115792
@@ -352,7 +459,10 @@ public enum Cap {
          * Support for mixed comparisons between nanosecond and millisecond dates
          */
         DATE_NANOS_COMPARE_TO_MILLIS(),
-
+        /**
+         * Support implicit casting of strings to date nanos
+         */
+        DATE_NANOS_IMPLICIT_CASTING(),
         /**
          * Support Least and Greatest functions on Date Nanos type
          */
@@ -377,6 +487,16 @@ public enum Cap {
          */
         DATE_NANOS_AGGREGATIONS(),
 
+        /**
+         * Support the {@link org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In} operator for date nanos
+         */
+        DATE_NANOS_IN_OPERATOR(),
+
+        /**
+         * DATE_PARSE supports reading timezones
+         */
+        DATE_PARSE_TZ(),
+
         /**
          * Support for datetime in least and greatest functions
          */
@@ -444,7 +564,16 @@ public enum Cap {
         /**
          * KQL function
          */
-        KQL_FUNCTION(Build.current().isSnapshot()),
+        KQL_FUNCTION,
+
+        /**
+         * Hash function
+         */
+        HASH_FUNCTION,
+        /**
+         * Hash function aliases such as MD5
+         */
+        HASH_FUNCTION_ALIASES_V1,
 
         /**
          * Don't optimize CASE IS NOT NULL function by not requiring the fields to be not null as well.
@@ -547,7 +676,7 @@ public enum Cap {
         /**
          * LOOKUP JOIN
          */
-        JOIN_LOOKUP_V5(Build.current().isSnapshot()),
+        JOIN_LOOKUP_V10(Build.current().isSnapshot()),
 
         /**
          * Fix for https://github.com/elastic/elasticsearch/issues/117054
@@ -582,7 +711,17 @@ public enum Cap {
         /**
          * Fix for regex folding with case-insensitive pattern https://github.com/elastic/elasticsearch/issues/118371
          */
-        FIXED_REGEX_FOLD;
+        FIXED_REGEX_FOLD,
+
+        /**
+         * Full text functions can be used in disjunctions
+         */
+        FULL_TEXT_FUNCTIONS_DISJUNCTIONS,
+
+        /**
+         * Change field caps response for semantic_text fields to be reported as text
+         */
+        SEMANTIC_TEXT_FIELD_CAPS;
 
         private final boolean enabled;
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java
index 52170dfb05256..61c7135cef42d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java
@@ -9,11 +9,12 @@
 
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.Predicates;
 import org.elasticsearch.core.TimeValue;
@@ -205,6 +206,10 @@ public Cluster getCluster(String clusterAlias) {
         return clusterInfo.get(clusterAlias);
     }
 
+    public Map<String, Cluster> getClusters() {
+        return clusterInfo;
+    }
+
     /**
      * Utility to swap a Cluster object. Guidelines for the remapping function:
      * <ul>
@@ -233,16 +238,18 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
         if (isCrossClusterSearch() == false || clusterInfo.isEmpty()) {
             return Collections.emptyIterator();
         }
-        return ChunkedToXContent.builder(params).object(b -> {
-            b.field(TOTAL_FIELD.getPreferredName(), clusterInfo.size());
-            b.field(SUCCESSFUL_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.SUCCESSFUL));
-            b.field(RUNNING_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.RUNNING));
-            b.field(SKIPPED_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.SKIPPED));
-            b.field(PARTIAL_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.PARTIAL));
-            b.field(FAILED_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.FAILED));
+        return Iterators.concat(
+            ChunkedToXContentHelper.startObject(),
+            ChunkedToXContentHelper.field(TOTAL_FIELD.getPreferredName(), clusterInfo.size()),
+            ChunkedToXContentHelper.field(SUCCESSFUL_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.SUCCESSFUL)),
+            ChunkedToXContentHelper.field(RUNNING_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.RUNNING)),
+            ChunkedToXContentHelper.field(SKIPPED_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.SKIPPED)),
+            ChunkedToXContentHelper.field(PARTIAL_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.PARTIAL)),
+            ChunkedToXContentHelper.field(FAILED_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.FAILED)),
             // each Cluster object defines its own field object name
-            b.xContentObject("details", clusterInfo.values().iterator());
-        });
+            ChunkedToXContentHelper.object("details", clusterInfo.values().iterator()),
+            ChunkedToXContentHelper.endObject()
+        );
     }
 
     /**
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java
index 4163a222b1a28..8530d9b48da08 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
-import org.elasticsearch.common.xcontent.ChunkedToXContentBuilder;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BlockStreamInput;
@@ -29,6 +29,7 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
@@ -184,38 +185,60 @@ public EsqlExecutionInfo getExecutionInfo() {
         return executionInfo;
     }
 
+    private Iterator<? extends ToXContent> asyncPropertiesOrEmpty() {
+        if (isAsync) {
+            return ChunkedToXContentHelper.chunk((builder, params) -> {
+                if (asyncExecutionId != null) {
+                    builder.field("id", asyncExecutionId);
+                }
+                builder.field("is_running", isRunning);
+                return builder;
+            });
+        } else {
+            return Collections.emptyIterator();
+        }
+    }
+
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).object(b -> {
-            boolean dropNullColumns = b.params().paramAsBoolean(DROP_NULL_COLUMNS_OPTION, false);
-            boolean[] nullColumns = dropNullColumns ? nullColumns() : null;
+        boolean dropNullColumns = params.paramAsBoolean(DROP_NULL_COLUMNS_OPTION, false);
+        boolean[] nullColumns = dropNullColumns ? nullColumns() : null;
+
+        Iterator<ToXContent> tookTime;
+        if (executionInfo != null && executionInfo.overallTook() != null) {
+            tookTime = ChunkedToXContentHelper.chunk((builder, p) -> {
+                builder.field("took", executionInfo.overallTook().millis());
+                return builder;
+            });
+        } else {
+            tookTime = Collections.emptyIterator();
+        }
 
-            if (isAsync) {
-                if (asyncExecutionId != null) {
-                    b.field("id", asyncExecutionId);
-                }
-                b.field("is_running", isRunning);
-            }
-            if (executionInfo != null) {
-                long tookInMillis = executionInfo.overallTook() == null
-                    ? executionInfo.tookSoFar().millis()
-                    : executionInfo.overallTook().millis();
-                b.field("took", tookInMillis);
-            }
-            if (dropNullColumns) {
-                b.append(ResponseXContentUtils.allColumns(columns, "all_columns"))
-                    .append(ResponseXContentUtils.nonNullColumns(columns, nullColumns, "columns"));
-            } else {
-                b.append(ResponseXContentUtils.allColumns(columns, "columns"));
-            }
-            b.array("values", ResponseXContentUtils.columnValues(this.columns, this.pages, columnar, nullColumns));
-            if (executionInfo != null && executionInfo.isCrossClusterSearch() && executionInfo.includeCCSMetadata()) {
-                b.field("_clusters", executionInfo);
-            }
-            if (profile != null) {
-                b.field("profile", profile);
-            }
-        });
+        Iterator<? extends ToXContent> columnHeadings = dropNullColumns
+            ? Iterators.concat(
+                ResponseXContentUtils.allColumns(columns, "all_columns"),
+                ResponseXContentUtils.nonNullColumns(columns, nullColumns, "columns")
+            )
+            : ResponseXContentUtils.allColumns(columns, "columns");
+        Iterator<? extends ToXContent> valuesIt = ResponseXContentUtils.columnValues(this.columns, this.pages, columnar, nullColumns);
+        Iterator<ToXContent> profileRender = profile == null
+            ? Collections.emptyIterator()
+            : ChunkedToXContentHelper.field("profile", profile, params);
+        Iterator<ToXContent> executionInfoRender = executionInfo != null
+            && executionInfo.isCrossClusterSearch()
+            && executionInfo.includeCCSMetadata()
+                ? ChunkedToXContentHelper.field("_clusters", executionInfo, params)
+                : Collections.emptyIterator();
+        return Iterators.concat(
+            ChunkedToXContentHelper.startObject(),
+            asyncPropertiesOrEmpty(),
+            tookTime,
+            columnHeadings,
+            ChunkedToXContentHelper.array("values", valuesIt),
+            executionInfoRender,
+            profileRender,
+            ChunkedToXContentHelper.endObject()
+        );
     }
 
     public boolean[] nullColumns() {
@@ -354,8 +377,11 @@ public int hashCode() {
 
         @Override
         public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-            return ChunkedToXContent.builder(params)
-                .object(ob -> ob.array("drivers", drivers.iterator(), ChunkedToXContentBuilder::append));
+            return Iterators.concat(
+                ChunkedToXContentHelper.startObject(),
+                ChunkedToXContentHelper.array("drivers", drivers.iterator(), params),
+                ChunkedToXContentHelper.endObject()
+            );
         }
 
         List<DriverProfile> drivers() {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseXContentUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseXContentUtils.java
index d7d0d9033d3b9..eb1f4f95db6fe 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseXContentUtils.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseXContentUtils.java
@@ -28,7 +28,7 @@ final class ResponseXContentUtils {
      * Returns the column headings for the given columns.
      */
     static Iterator<? extends ToXContent> allColumns(List<ColumnInfoImpl> columns, String name) {
-        return ChunkedToXContentHelper.singleChunk((builder, params) -> {
+        return ChunkedToXContentHelper.chunk((builder, params) -> {
             builder.startArray(name);
             for (ColumnInfo col : columns) {
                 col.toXContent(builder, params);
@@ -42,7 +42,7 @@ static Iterator<? extends ToXContent> allColumns(List<ColumnInfoImpl> columns, S
      * for always-null columns to a {@code null_columns} section.
      */
     static Iterator<? extends ToXContent> nonNullColumns(List<ColumnInfoImpl> columns, boolean[] nullColumns, String name) {
-        return ChunkedToXContentHelper.singleChunk((builder, params) -> {
+        return ChunkedToXContentHelper.chunk((builder, params) -> {
             builder.startArray(name);
             for (int c = 0; c < columns.size(); c++) {
                 if (nullColumns[c] == false) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
index cf91c7df9a034..a11b511cb83b7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
@@ -118,6 +119,7 @@
 import static org.elasticsearch.xpack.core.enrich.EnrichPolicy.GEO_MATCH_TYPE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT;
@@ -198,12 +200,16 @@ private static class ResolveTable extends ParameterizedAnalyzerRule<UnresolvedRe
 
         @Override
         protected LogicalPlan rule(UnresolvedRelation plan, AnalyzerContext context) {
-            return resolveIndex(plan, plan.indexMode().equals(IndexMode.LOOKUP) ? context.lookupResolution() : context.indexResolution());
+            return resolveIndex(
+                plan,
+                plan.indexMode().equals(IndexMode.LOOKUP) ? context.lookupResolution().get(plan.table().index()) : context.indexResolution()
+            );
         }
 
         private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexResolution) {
-            if (indexResolution.isValid() == false) {
-                return plan.unresolvedMessage().equals(indexResolution.toString())
+            if (indexResolution == null || indexResolution.isValid() == false) {
+                String indexResolutionMessage = indexResolution == null ? "[none specified]" : indexResolution.toString();
+                return plan.unresolvedMessage().equals(indexResolutionMessage)
                     ? plan
                     : new UnresolvedRelation(
                         plan.source(),
@@ -211,7 +217,7 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR
                         plan.frozen(),
                         plan.metadataFields(),
                         plan.indexMode(),
-                        indexResolution.toString(),
+                        indexResolutionMessage,
                         plan.commandName()
                     );
             }
@@ -230,6 +236,37 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR
             }
 
             EsIndex esIndex = indexResolution.get();
+
+            if (plan.indexMode().equals(IndexMode.LOOKUP)) {
+                String indexResolutionMessage = null;
+
+                var indexNameWithModes = esIndex.indexNameWithModes();
+                if (indexNameWithModes.size() != 1) {
+                    indexResolutionMessage = "invalid ["
+                        + table
+                        + "] resolution in lookup mode to ["
+                        + indexNameWithModes.size()
+                        + "] indices";
+                } else if (indexNameWithModes.values().iterator().next() != IndexMode.LOOKUP) {
+                    indexResolutionMessage = "invalid ["
+                        + table
+                        + "] resolution in lookup mode to an index in ["
+                        + indexNameWithModes.values().iterator().next()
+                        + "] mode";
+                }
+
+                if (indexResolutionMessage != null) {
+                    return new UnresolvedRelation(
+                        plan.source(),
+                        plan.table(),
+                        plan.frozen(),
+                        plan.metadataFields(),
+                        plan.indexMode(),
+                        indexResolutionMessage,
+                        plan.commandName()
+                    );
+                }
+            }
             var attributes = mappingAsAttributes(plan.source(), esIndex.mapping());
             attributes.addAll(plan.metadataFields());
             return new EsRelation(plan.source(), esIndex, attributes.isEmpty() ? NO_FIELDS : attributes, plan.indexMode());
@@ -289,7 +326,7 @@ protected LogicalPlan rule(Enrich plan, AnalyzerContext context) {
                 // the policy does not exist
                 return plan;
             }
-            final String policyName = (String) plan.policyName().fold();
+            final String policyName = (String) plan.policyName().fold(FoldContext.small() /* TODO remove me */);
             final var resolved = context.enrichResolution().getResolvedPolicy(policyName, plan.mode());
             if (resolved != null) {
                 var policy = new EnrichPolicy(resolved.matchType(), null, List.of(), resolved.matchField(), resolved.enrichFields());
@@ -655,15 +692,9 @@ private List<Attribute> resolveUsingColumns(List<Attribute> cols, List<Attribute
                         resolvedCol = ucol.withUnresolvedMessage(message.replace(match, match + " in " + side + " side of join"));
                     }
                     resolved.add(resolvedCol);
-                }
-                // columns are expected to be unresolved - if that's not the case return an error
-                else {
-                    return singletonList(
-                        new UnresolvedAttribute(
-                            col.source(),
-                            col.name(),
-                            "Surprised to discover column [ " + col.name() + "] already resolved"
-                        )
+                } else {
+                    throw new IllegalStateException(
+                        "Surprised to discover column [ " + col.name() + "] already resolved when resolving JOIN keys"
                     );
                 }
             }
@@ -1046,21 +1077,23 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) {
     /**
      * Cast string literals in ScalarFunction, EsqlArithmeticOperation, BinaryComparison, In and GroupingFunction to desired data types.
      * For example, the string literals in the following expressions will be cast implicitly to the field data type on the left hand side.
-     * date > "2024-08-21"
-     * date in ("2024-08-21", "2024-08-22", "2024-08-23")
-     * date = "2024-08-21" + 3 days
-     * ip == "127.0.0.1"
-     * version != "1.0"
-     * bucket(dateField, "1 month")
-     * date_trunc("1 minute", dateField)
-     *
+     * <ul>
+     * <li>date > "2024-08-21"</li>
+     * <li>date in ("2024-08-21", "2024-08-22", "2024-08-23")</li>
+     * <li>date = "2024-08-21" + 3 days</li>
+     * <li>ip == "127.0.0.1"</li>
+     * <li>version != "1.0"</li>
+     * <li>bucket(dateField, "1 month")</li>
+     * <li>date_trunc("1 minute", dateField)</li>
+     * </ul>
      * If the inputs to Coalesce are mixed numeric types, cast the rest of the numeric field or value to the first numeric data type if
      * applicable. For example, implicit casting converts:
-     * Coalesce(Long, Int) to Coalesce(Long, Long)
-     * Coalesce(null, Long, Int) to Coalesce(null, Long, Long)
-     * Coalesce(Double, Long, Int) to Coalesce(Double, Double, Double)
-     * Coalesce(null, Double, Long, Int) to Coalesce(null, Double, Double, Double)
-     *
+     * <ul>
+     * <li>Coalesce(Long, Int) to Coalesce(Long, Long)</li>
+     * <li>Coalesce(null, Long, Int) to Coalesce(null, Long, Long)</li>
+     * <li>Coalesce(Double, Long, Int) to Coalesce(Double, Double, Double)</li>
+     * <li>Coalesce(null, Double, Long, Int) to Coalesce(null, Double, Double, Double)</li>
+     * </ul>
      * Coalesce(Int, Long) will NOT be converted to Coalesce(Long, Long) or Coalesce(Int, Int).
      */
     private static class ImplicitCasting extends ParameterizedRule<LogicalPlan, LogicalPlan, AnalyzerContext> {
@@ -1241,22 +1274,22 @@ private static boolean supportsImplicitTemporalCasting(Expression e, BinaryOpera
         }
 
         private static boolean supportsStringImplicitCasting(DataType type) {
-            return type == DATETIME || type == IP || type == VERSION || type == BOOLEAN;
+            return type == DATETIME || type == DATE_NANOS || type == IP || type == VERSION || type == BOOLEAN;
         }
 
         private static UnresolvedAttribute unresolvedAttribute(Expression value, String type, Exception e) {
             String message = format(
                 "Cannot convert string [{}] to [{}], error [{}]",
-                value.fold(),
+                value.fold(FoldContext.small() /* TODO remove me */),
                 type,
                 (e instanceof ParsingException pe) ? pe.getErrorMessage() : e.getMessage()
             );
-            return new UnresolvedAttribute(value.source(), String.valueOf(value.fold()), message);
+            return new UnresolvedAttribute(value.source(), String.valueOf(value.fold(FoldContext.small() /* TODO remove me */)), message);
         }
 
         private static Expression castStringLiteralToTemporalAmount(Expression from) {
             try {
-                TemporalAmount result = maybeParseTemporalAmount(from.fold().toString().strip());
+                TemporalAmount result = maybeParseTemporalAmount(from.fold(FoldContext.small() /* TODO remove me */).toString().strip());
                 if (result == null) {
                     return from;
                 }
@@ -1272,7 +1305,11 @@ private static Expression castStringLiteral(Expression from, DataType target) {
             try {
                 return isTemporalAmount(target)
                     ? castStringLiteralToTemporalAmount(from)
-                    : new Literal(from.source(), EsqlDataTypeConverter.convert(from.fold(), target), target);
+                    : new Literal(
+                        from.source(),
+                        EsqlDataTypeConverter.convert(from.fold(FoldContext.small() /* TODO remove me */), target),
+                        target
+                    );
             } catch (Exception e) {
                 return unresolvedAttribute(from, target.toString(), e);
             }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerContext.java
index ea6e5ed425711..2aad172bd9946 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerContext.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerContext.java
@@ -11,11 +11,13 @@
 import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.session.Configuration;
 
+import java.util.Map;
+
 public record AnalyzerContext(
     Configuration configuration,
     EsqlFunctionRegistry functionRegistry,
     IndexResolution indexResolution,
-    IndexResolution lookupResolution,
+    Map<String, IndexResolution> lookupResolution,
     EnrichResolution enrichResolution
 ) {
     // Currently for tests only, since most do not test lookups
@@ -26,12 +28,6 @@ public AnalyzerContext(
         IndexResolution indexResolution,
         EnrichResolution enrichResolution
     ) {
-        this(
-            configuration,
-            functionRegistry,
-            indexResolution,
-            IndexResolution.invalid("AnalyzerContext constructed without any lookup join resolution"),
-            enrichResolution
-        );
+        this(configuration, functionRegistry, indexResolution, Map.of(), enrichResolution);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
index f01cc265e330b..b59a112b1adb6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
@@ -8,55 +8,30 @@
 package org.elasticsearch.xpack.esql.analysis;
 
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.xpack.esql.LicenseAware;
 import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
+import org.elasticsearch.xpack.esql.capabilities.PostAnalysisPlanVerificationAware;
+import org.elasticsearch.xpack.esql.capabilities.PostAnalysisVerificationAware;
 import org.elasticsearch.xpack.esql.common.Failure;
+import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
-import org.elasticsearch.xpack.esql.core.expression.Attribute;
-import org.elasticsearch.xpack.esql.core.expression.AttributeMap;
-import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.Expressions;
-import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
-import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.expression.function.Function;
 import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
-import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic;
-import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
-import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
+import org.elasticsearch.xpack.esql.core.tree.Node;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
-import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
-import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression;
-import org.elasticsearch.xpack.esql.expression.function.aggregate.Rate;
-import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction;
-import org.elasticsearch.xpack.esql.expression.function.fulltext.Kql;
-import org.elasticsearch.xpack.esql.expression.function.fulltext.Match;
-import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString;
-import org.elasticsearch.xpack.esql.expression.function.fulltext.Term;
-import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize;
-import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
-import org.elasticsearch.xpack.esql.plan.logical.Enrich;
-import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
-import org.elasticsearch.xpack.esql.plan.logical.Eval;
-import org.elasticsearch.xpack.esql.plan.logical.Filter;
-import org.elasticsearch.xpack.esql.plan.logical.Limit;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.plan.logical.Lookup;
-import org.elasticsearch.xpack.esql.plan.logical.OrderBy;
 import org.elasticsearch.xpack.esql.plan.logical.Project;
-import org.elasticsearch.xpack.esql.plan.logical.RegexExtract;
-import org.elasticsearch.xpack.esql.plan.logical.Row;
-import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan;
-import org.elasticsearch.xpack.esql.plan.logical.join.Join;
-import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig;
 import org.elasticsearch.xpack.esql.stats.FeatureMetric;
 import org.elasticsearch.xpack.esql.stats.Metrics;
 
@@ -64,19 +39,14 @@
 import java.util.BitSet;
 import java.util.Collection;
 import java.util.HashSet;
-import java.util.LinkedHashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Set;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
-import java.util.function.Predicate;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.xpack.esql.common.Failure.fail;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
-import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
 
 /**
  * This class is part of the planner. Responsible for failing impossible queries with a human-readable error message.  In particular, this
@@ -101,11 +71,45 @@ public Verifier(Metrics metrics, XPackLicenseState licenseState) {
      */
     Collection<Failure> verify(LogicalPlan plan, BitSet partialMetrics) {
         assert partialMetrics != null;
-        Set<Failure> failures = new LinkedHashSet<>();
-        // alias map, collected during the first iteration for better error messages
-        AttributeMap<Expression> aliases = new AttributeMap<>();
+        Failures failures = new Failures();
 
         // quick verification for unresolved attributes
+        checkUnresolvedAttributes(plan, failures);
+
+        // in case of failures bail-out as all other checks will be redundant
+        if (failures.hasFailures()) {
+            return failures.failures();
+        }
+
+        // collect plan checkers
+        var planCheckers = planCheckers(plan);
+
+        // Concrete verifications
+        plan.forEachDown(p -> {
+            // if the children are unresolved, so will this node; counting it will only add noise
+            if (p.childrenResolved() == false) {
+                return;
+            }
+
+            planCheckers.forEach(c -> c.accept(p, failures));
+
+            checkOperationsOnUnsignedLong(p, failures);
+            checkBinaryComparison(p, failures);
+        });
+
+        if (failures.hasFailures() == false) {
+            licenseCheck(plan, failures);
+        }
+
+        // gather metrics
+        if (failures.hasFailures() == false) {
+            gatherMetrics(plan, partialMetrics);
+        }
+
+        return failures.failures();
+    }
+
+    private static void checkUnresolvedAttributes(LogicalPlan plan, Failures failures) {
         plan.forEachUp(p -> {
             // if the children are unresolved, so will this node; counting it will only add noise
             if (p.childrenResolved() == false) {
@@ -117,7 +121,6 @@ Collection<Failure> verify(LogicalPlan plan, BitSet partialMetrics) {
             }
             // p is resolved, skip
             else if (p.resolved()) {
-                p.forEachExpressionUp(Alias.class, a -> aliases.put(a.toAttribute(), a.child()));
                 return;
             }
 
@@ -179,390 +182,31 @@ else if (p instanceof Lookup lookup) {
                 p.forEachExpression(unresolvedExpressions);
             }
         });
-
-        // in case of failures bail-out as all other checks will be redundant
-        if (failures.isEmpty() == false) {
-            return failures;
-        }
-
-        // Concrete verifications
-        plan.forEachDown(p -> {
-            // if the children are unresolved, so will this node; counting it will only add noise
-            if (p.childrenResolved() == false) {
-                return;
-            }
-            checkFilterConditionType(p, failures);
-            checkAggregate(p, failures);
-            checkRegexExtractOnlyOnStrings(p, failures);
-
-            checkRow(p, failures);
-            checkEvalFields(p, failures);
-
-            checkOperationsOnUnsignedLong(p, failures);
-            checkBinaryComparison(p, failures);
-            checkForSortableDataTypes(p, failures);
-            checkSort(p, failures);
-
-            checkFullTextQueryFunctions(p, failures);
-            checkJoin(p, failures);
-        });
-        checkRemoteEnrich(plan, failures);
-
-        if (failures.isEmpty()) {
-            checkLicense(plan, licenseState, failures);
-        }
-
-        // gather metrics
-        if (failures.isEmpty()) {
-            gatherMetrics(plan, partialMetrics);
-        }
-
-        return failures;
     }
 
-    private void checkSort(LogicalPlan p, Set<Failure> failures) {
-        if (p instanceof OrderBy ob) {
-            ob.order().forEach(o -> {
-                o.forEachDown(Function.class, f -> {
-                    if (f instanceof AggregateFunction) {
-                        failures.add(fail(f, "Aggregate functions are not allowed in SORT [{}]", f.functionName()));
-                    }
-                });
-            });
-        }
-    }
-
-    private static void checkFilterConditionType(LogicalPlan p, Set<Failure> localFailures) {
-        if (p instanceof Filter f) {
-            Expression condition = f.condition();
-            if (condition.dataType() != BOOLEAN) {
-                localFailures.add(fail(condition, "Condition expression needs to be boolean, found [{}]", condition.dataType()));
+    private static List<BiConsumer<LogicalPlan, Failures>> planCheckers(LogicalPlan plan) {
+        List<BiConsumer<LogicalPlan, Failures>> planCheckers = new ArrayList<>();
+        Consumer<? super Node<?>> collectPlanCheckers = p -> {
+            if (p instanceof PostAnalysisPlanVerificationAware pva) {
+                planCheckers.add(pva.postAnalysisPlanVerification());
             }
-        }
-    }
+        };
+        plan.forEachDown(p -> {
+            collectPlanCheckers.accept(p);
+            p.forEachExpression(collectPlanCheckers);
 
-    private static void checkAggregate(LogicalPlan p, Set<Failure> failures) {
-        if (p instanceof Aggregate agg) {
-            List<Expression> groupings = agg.groupings();
-            AttributeSet groupRefs = new AttributeSet();
-            // check grouping
-            // The grouping can not be an aggregate function
-            groupings.forEach(e -> {
-                e.forEachUp(g -> {
-                    if (g instanceof AggregateFunction af) {
-                        failures.add(fail(g, "cannot use an aggregate [{}] for grouping", af));
-                    } else if (g instanceof GroupingFunction gf) {
-                        gf.children()
-                            .forEach(
-                                c -> c.forEachDown(
-                                    GroupingFunction.class,
-                                    inner -> failures.add(
-                                        fail(
-                                            inner,
-                                            "cannot nest grouping functions; found [{}] inside [{}]",
-                                            inner.sourceText(),
-                                            gf.sourceText()
-                                        )
-                                    )
-                                )
-                            );
+            if (p instanceof PostAnalysisVerificationAware va) {
+                planCheckers.add((lp, failures) -> {
+                    if (lp.getClass().equals(va.getClass())) {
+                        va.postAnalysisVerification(failures);
                     }
                 });
-                // keep the grouping attributes (common case)
-                Attribute attr = Expressions.attribute(e);
-                if (attr != null) {
-                    groupRefs.add(attr);
-                }
-                if (e instanceof FieldAttribute f && f.dataType().isCounter()) {
-                    failures.add(fail(e, "cannot group by on [{}] type for grouping [{}]", f.dataType().typeName(), e.sourceText()));
-                }
-            });
-
-            // check aggregates - accept only aggregate functions or expressions over grouping
-            // don't allow the group by itself to avoid duplicates in the output
-            // and since the groups are copied, only look at the declared aggregates
-            List<? extends NamedExpression> aggs = agg.aggregates();
-            aggs.subList(0, aggs.size() - groupings.size()).forEach(e -> {
-                var exp = Alias.unwrap(e);
-                if (exp.foldable()) {
-                    failures.add(fail(exp, "expected an aggregate function but found [{}]", exp.sourceText()));
-                }
-                // traverse the tree to find invalid matches
-                checkInvalidNamedExpressionUsage(exp, groupings, groupRefs, failures, 0);
-            });
-            if (agg.aggregateType() == Aggregate.AggregateType.METRICS) {
-                aggs.forEach(a -> checkRateAggregates(a, 0, failures));
-            } else {
-                agg.forEachExpression(
-                    Rate.class,
-                    r -> failures.add(fail(r, "the rate aggregate[{}] can only be used within the metrics command", r.sourceText()))
-                );
             }
-            checkCategorizeGrouping(agg, failures);
-        } else {
-            p.forEachExpression(
-                GroupingFunction.class,
-                gf -> failures.add(fail(gf, "cannot use grouping function [{}] outside of a STATS command", gf.sourceText()))
-            );
-        }
-    }
-
-    /**
-     * Check CATEGORIZE grouping function usages.
-     * <p>
-     *     Some of those checks are temporary, until the required syntax or engine changes are implemented.
-     * </p>
-     */
-    private static void checkCategorizeGrouping(Aggregate agg, Set<Failure> failures) {
-        // Forbid CATEGORIZE grouping function with other groupings
-        if (agg.groupings().size() > 1) {
-            agg.groupings().subList(1, agg.groupings().size()).forEach(g -> {
-                g.forEachDown(
-                    Categorize.class,
-                    categorize -> failures.add(
-                        fail(
-                            categorize,
-                            "CATEGORIZE grouping function [{}] can only be in the first grouping expression",
-                            categorize.sourceText()
-                        )
-                    )
-                );
-            });
-        }
-
-        // Forbid CATEGORIZE grouping functions not being top level groupings
-        agg.groupings().forEach(g -> {
-            // Check all CATEGORIZE but the top level one
-            Alias.unwrap(g)
-                .children()
-                .forEach(
-                    child -> child.forEachDown(
-                        Categorize.class,
-                        c -> failures.add(
-                            fail(c, "CATEGORIZE grouping function [{}] can't be used within other expressions", c.sourceText())
-                        )
-                    )
-                );
-        });
-
-        // Forbid CATEGORIZE being used in the aggregations, unless it appears as a grouping
-        agg.aggregates()
-            .forEach(
-                a -> a.forEachDown(
-                    AggregateFunction.class,
-                    aggregateFunction -> aggregateFunction.forEachDown(
-                        Categorize.class,
-                        categorize -> failures.add(
-                            fail(categorize, "cannot use CATEGORIZE grouping function [{}] within an aggregation", categorize.sourceText())
-                        )
-                    )
-                )
-            );
-
-        // Forbid CATEGORIZE being referenced as a child of an aggregation function
-        AttributeMap<Categorize> categorizeByAttribute = new AttributeMap<>();
-        agg.groupings().forEach(g -> {
-            g.forEachDown(Alias.class, alias -> {
-                if (alias.child() instanceof Categorize categorize) {
-                    categorizeByAttribute.put(alias.toAttribute(), categorize);
-                }
-            });
         });
-        agg.aggregates()
-            .forEach(a -> a.forEachDown(AggregateFunction.class, aggregate -> aggregate.forEachDown(Attribute.class, attribute -> {
-                var categorize = categorizeByAttribute.get(attribute);
-                if (categorize != null) {
-                    failures.add(
-                        fail(attribute, "cannot reference CATEGORIZE grouping function [{}] within an aggregation", attribute.sourceText())
-                    );
-                }
-            })));
-        agg.aggregates().forEach(a -> a.forEachDown(FilteredExpression.class, fe -> fe.filter().forEachDown(Attribute.class, attribute -> {
-            var categorize = categorizeByAttribute.get(attribute);
-            if (categorize != null) {
-                failures.add(
-                    fail(
-                        attribute,
-                        "cannot reference CATEGORIZE grouping function [{}] within an aggregation filter",
-                        attribute.sourceText()
-                    )
-                );
-            }
-        })));
-    }
-
-    private static void checkRateAggregates(Expression expr, int nestedLevel, Set<Failure> failures) {
-        if (expr instanceof AggregateFunction) {
-            nestedLevel++;
-        }
-        if (expr instanceof Rate r) {
-            if (nestedLevel != 2) {
-                failures.add(
-                    fail(
-                        expr,
-                        "the rate aggregate [{}] can only be used within the metrics command and inside another aggregate",
-                        r.sourceText()
-                    )
-                );
-            }
-        }
-        for (Expression child : expr.children()) {
-            checkRateAggregates(child, nestedLevel, failures);
-        }
+        return planCheckers;
     }
 
-    // traverse the expression and look either for an agg function or a grouping match
-    // stop either when no children are left, the leafs are literals or a reference attribute is given
-    private static void checkInvalidNamedExpressionUsage(
-        Expression e,
-        List<Expression> groups,
-        AttributeSet groupRefs,
-        Set<Failure> failures,
-        int level
-    ) {
-        // unwrap filtered expression
-        if (e instanceof FilteredExpression fe) {
-            e = fe.delegate();
-            // make sure they work on aggregate functions
-            if (e.anyMatch(AggregateFunction.class::isInstance) == false) {
-                Expression filter = fe.filter();
-                failures.add(fail(filter, "WHERE clause allowed only for aggregate functions, none found in [{}]", fe.sourceText()));
-            }
-            Expression f = fe.filter();
-            // check the filter has to be a boolean term, similar as checkFilterConditionType
-            if (f.dataType() != NULL && f.dataType() != BOOLEAN) {
-                failures.add(fail(f, "Condition expression needs to be boolean, found [{}]", f.dataType()));
-            }
-            // but that the filter doesn't use grouping or aggregate functions
-            fe.filter().forEachDown(c -> {
-                if (c instanceof AggregateFunction af) {
-                    failures.add(
-                        fail(af, "cannot use aggregate function [{}] in aggregate WHERE clause [{}]", af.sourceText(), fe.sourceText())
-                    );
-                }
-                // check the grouping function against the group
-                else if (c instanceof GroupingFunction gf) {
-                    if (c instanceof Categorize
-                        || Expressions.anyMatch(groups, ex -> ex instanceof Alias a && a.child().semanticEquals(gf)) == false) {
-                        failures.add(fail(gf, "can only use grouping function [{}] as part of the BY clause", gf.sourceText()));
-                    }
-                }
-            });
-        }
-        // found an aggregate, constant or a group, bail out
-        if (e instanceof AggregateFunction af) {
-            af.field().forEachDown(AggregateFunction.class, f -> {
-                // rate aggregate is allowed to be inside another aggregate
-                if (f instanceof Rate == false) {
-                    failures.add(fail(f, "nested aggregations [{}] not allowed inside other aggregations [{}]", f, af));
-                }
-            });
-        } else if (e instanceof GroupingFunction gf) {
-            // optimizer will later unroll expressions with aggs and non-aggs with a grouping function into an EVAL, but that will no longer
-            // be verified (by check above in checkAggregate()), so do it explicitly here
-            if (Expressions.anyMatch(groups, ex -> ex instanceof Alias a && a.child().semanticEquals(gf)) == false) {
-                failures.add(fail(gf, "can only use grouping function [{}] as part of the BY clause", gf.sourceText()));
-            } else if (level == 0) {
-                addFailureOnGroupingUsedNakedInAggs(failures, gf, "function");
-            }
-        } else if (e.foldable()) {
-            // don't do anything
-        } else if (groups.contains(e) || groupRefs.contains(e)) {
-            if (level == 0) {
-                addFailureOnGroupingUsedNakedInAggs(failures, e, "key");
-            }
-        }
-        // if a reference is found, mark it as an error
-        else if (e instanceof NamedExpression ne) {
-            boolean foundInGrouping = false;
-            for (Expression g : groups) {
-                if (g.anyMatch(se -> se.semanticEquals(ne))) {
-                    foundInGrouping = true;
-                    failures.add(
-                        fail(
-                            e,
-                            "column [{}] cannot be used as an aggregate once declared in the STATS BY grouping key [{}]",
-                            ne.name(),
-                            g.sourceText()
-                        )
-                    );
-                    break;
-                }
-            }
-            if (foundInGrouping == false) {
-                failures.add(fail(e, "column [{}] must appear in the STATS BY clause or be used in an aggregate function", ne.name()));
-            }
-        }
-        // other keep on going
-        else {
-            for (Expression child : e.children()) {
-                checkInvalidNamedExpressionUsage(child, groups, groupRefs, failures, level + 1);
-            }
-        }
-    }
-
-    private static void addFailureOnGroupingUsedNakedInAggs(Set<Failure> failures, Expression e, String element) {
-        failures.add(
-            fail(e, "grouping {} [{}] cannot be used as an aggregate once declared in the STATS BY clause", element, e.sourceText())
-        );
-    }
-
-    private static void checkRegexExtractOnlyOnStrings(LogicalPlan p, Set<Failure> failures) {
-        if (p instanceof RegexExtract re) {
-            Expression expr = re.input();
-            DataType type = expr.dataType();
-            if (DataType.isString(type) == false) {
-                failures.add(
-                    fail(
-                        expr,
-                        "{} only supports KEYWORD or TEXT values, found expression [{}] type [{}]",
-                        re.getClass().getSimpleName(),
-                        expr.sourceText(),
-                        type
-                    )
-                );
-            }
-        }
-    }
-
-    private static void checkRow(LogicalPlan p, Set<Failure> failures) {
-        if (p instanceof Row row) {
-            row.fields().forEach(a -> {
-                if (DataType.isRepresentable(a.dataType()) == false) {
-                    failures.add(fail(a.child(), "cannot use [{}] directly in a row assignment", a.child().sourceText()));
-                }
-            });
-        }
-    }
-
-    private static void checkEvalFields(LogicalPlan p, Set<Failure> failures) {
-        if (p instanceof Eval eval) {
-            eval.fields().forEach(field -> {
-                // check supported types
-                DataType dataType = field.dataType();
-                if (DataType.isRepresentable(dataType) == false) {
-                    failures.add(
-                        fail(
-                            field,
-                            "EVAL does not support type [{}] as the return data type of expression [{}]",
-                            dataType.typeName(),
-                            field.child().sourceText()
-                        )
-                    );
-                }
-                // check no aggregate functions are used
-                field.forEachDown(AggregateFunction.class, af -> {
-                    if (af instanceof Rate) {
-                        failures.add(fail(af, "aggregate function [{}] not allowed outside METRICS command", af.sourceText()));
-                    } else {
-                        failures.add(fail(af, "aggregate function [{}] not allowed outside STATS command", af.sourceText()));
-                    }
-                });
-            });
-        }
-    }
-
-    private static void checkOperationsOnUnsignedLong(LogicalPlan p, Set<Failure> failures) {
+    private static void checkOperationsOnUnsignedLong(LogicalPlan p, Failures failures) {
         p.forEachExpression(e -> {
             Failure f = null;
 
@@ -578,7 +222,7 @@ private static void checkOperationsOnUnsignedLong(LogicalPlan p, Set<Failure> fa
         });
     }
 
-    private static void checkBinaryComparison(LogicalPlan p, Set<Failure> failures) {
+    private static void checkBinaryComparison(LogicalPlan p, Failures failures) {
         p.forEachExpression(BinaryComparison.class, bc -> {
             Failure f = validateBinaryComparison(bc);
             if (f != null) {
@@ -587,11 +231,15 @@ private static void checkBinaryComparison(LogicalPlan p, Set<Failure> failures)
         });
     }
 
-    private void checkLicense(LogicalPlan plan, XPackLicenseState licenseState, Set<Failure> failures) {
-        plan.forEachExpressionDown(Function.class, p -> {
-            if (p.checkLicense(licenseState) == false) {
-                failures.add(new Failure(p, "current license is non-compliant for function [" + p.sourceText() + "]"));
+    private void licenseCheck(LogicalPlan plan, Failures failures) {
+        Consumer<Node<?>> licenseCheck = n -> {
+            if (n instanceof LicenseAware la && la.licenseCheck(licenseState) == false) {
+                failures.add(fail(n, "current license is non-compliant for [{}]", n.sourceText()));
             }
+        };
+        plan.forEachDown(p -> {
+            licenseCheck.accept(p);
+            p.forEachExpression(Expression.class, licenseCheck);
         });
     }
 
@@ -718,242 +366,4 @@ private static Failure validateUnsignedLongNegation(Neg neg) {
         }
         return null;
     }
-
-    /**
-     * Some datatypes are not sortable
-     */
-    private static void checkForSortableDataTypes(LogicalPlan p, Set<Failure> localFailures) {
-        if (p instanceof OrderBy ob) {
-            ob.order().forEach(order -> {
-                if (DataType.isSortable(order.dataType()) == false) {
-                    localFailures.add(fail(order, "cannot sort on " + order.dataType().typeName()));
-                }
-            });
-        }
-    }
-
-    /**
-     * Ensure that no remote enrich is allowed after a reduction or an enrich with coordinator mode.
-     * <p>
-     * TODO:
-     * For Limit and TopN, we can insert the same node after the remote enrich (also needs to move projections around)
-     * to eliminate this limitation. Otherwise, we force users to write queries that might not perform well.
-     * For example, `FROM test | ORDER @timestamp | LIMIT 10 | ENRICH _remote:` doesn't work.
-     * In that case, users have to write it as `FROM test | ENRICH _remote: | ORDER @timestamp | LIMIT 10`,
-     * which is equivalent to bringing all data to the coordinating cluster.
-     * We might consider implementing the actual remote enrich on the coordinating cluster, however, this requires
-     * retaining the originating cluster and restructing pages for routing, which might be complicated.
-     */
-    private static void checkRemoteEnrich(LogicalPlan plan, Set<Failure> failures) {
-        boolean[] agg = { false };
-        boolean[] enrichCoord = { false };
-
-        plan.forEachUp(UnaryPlan.class, u -> {
-            if (u instanceof Aggregate) {
-                agg[0] = true;
-            } else if (u instanceof Enrich enrich && enrich.mode() == Enrich.Mode.COORDINATOR) {
-                enrichCoord[0] = true;
-            }
-            if (u instanceof Enrich enrich && enrich.mode() == Enrich.Mode.REMOTE) {
-                if (agg[0]) {
-                    failures.add(fail(enrich, "ENRICH with remote policy can't be executed after STATS"));
-                }
-                if (enrichCoord[0]) {
-                    failures.add(fail(enrich, "ENRICH with remote policy can't be executed after another ENRICH with coordinator policy"));
-                }
-            }
-        });
-    }
-
-    /**
-     * Checks whether a condition contains a disjunction with the specified typeToken. Adds to failure if it does.
-     *
-     * @param condition        condition to check for disjunctions
-     * @param typeNameProvider provider for the type name to add in the failure message
-     * @param failures         failures collection to add to
-     */
-    private static void checkNotPresentInDisjunctions(
-        Expression condition,
-        java.util.function.Function<FullTextFunction, String> typeNameProvider,
-        Set<Failure> failures
-    ) {
-        condition.forEachUp(Or.class, or -> {
-            checkNotPresentInDisjunctions(or.left(), or, typeNameProvider, failures);
-            checkNotPresentInDisjunctions(or.right(), or, typeNameProvider, failures);
-        });
-    }
-
-    /**
-     * Checks whether a condition contains a disjunction with the specified typeToken. Adds to failure if it does.
-     *
-     * @param parentExpression parent expression to add to the failure message
-     * @param or               disjunction that is being checked
-     * @param failures         failures collection to add to
-     */
-    private static void checkNotPresentInDisjunctions(
-        Expression parentExpression,
-        Or or,
-        java.util.function.Function<FullTextFunction, String> elementName,
-        Set<Failure> failures
-    ) {
-        parentExpression.forEachDown(FullTextFunction.class, ftp -> {
-            failures.add(
-                fail(or, "Invalid condition [{}]. {} can't be used as part of an or condition", or.sourceText(), elementName.apply(ftp))
-            );
-        });
-    }
-
-    /**
-     * Checks Joins for invalid usage.
-     *
-     * @param plan root plan to check
-     * @param failures failures found
-     */
-    private static void checkJoin(LogicalPlan plan, Set<Failure> failures) {
-        if (plan instanceof Join join) {
-            JoinConfig config = join.config();
-            for (int i = 0; i < config.leftFields().size(); i++) {
-                Attribute leftField = config.leftFields().get(i);
-                Attribute rightField = config.rightFields().get(i);
-                if (leftField.dataType() != rightField.dataType()) {
-                    failures.add(
-                        fail(
-                            leftField,
-                            "JOIN left field [{}] of type [{}] is incompatible with right field [{}] of type [{}]",
-                            leftField.name(),
-                            leftField.dataType(),
-                            rightField.name(),
-                            rightField.dataType()
-                        )
-                    );
-                }
-            }
-
-        }
-    }
-
-    /**
-     * Checks full text query functions for invalid usage.
-     *
-     * @param plan root plan to check
-     * @param failures failures found
-     */
-    private static void checkFullTextQueryFunctions(LogicalPlan plan, Set<Failure> failures) {
-        if (plan instanceof Filter f) {
-            Expression condition = f.condition();
-
-            List.of(QueryString.class, Kql.class).forEach(functionClass -> {
-                // Check for limitations of QSTR and KQL function.
-                checkCommandsBeforeExpression(
-                    plan,
-                    condition,
-                    functionClass,
-                    lp -> (lp instanceof Filter || lp instanceof OrderBy || lp instanceof EsRelation),
-                    fullTextFunction -> "[" + fullTextFunction.functionName() + "] " + fullTextFunction.functionType(),
-                    failures
-                );
-            });
-
-            checkCommandsBeforeExpression(
-                plan,
-                condition,
-                Match.class,
-                lp -> (lp instanceof Limit == false) && (lp instanceof Aggregate == false),
-                m -> "[" + m.functionName() + "] " + m.functionType(),
-                failures
-            );
-            checkCommandsBeforeExpression(
-                plan,
-                condition,
-                Term.class,
-                lp -> (lp instanceof Limit == false) && (lp instanceof Aggregate == false),
-                m -> "[" + m.functionName() + "] " + m.functionType(),
-                failures
-            );
-            checkNotPresentInDisjunctions(condition, ftf -> "[" + ftf.functionName() + "] " + ftf.functionType(), failures);
-            checkFullTextFunctionsParents(condition, failures);
-        } else {
-            plan.forEachExpression(FullTextFunction.class, ftf -> {
-                failures.add(fail(ftf, "[{}] {} is only supported in WHERE commands", ftf.functionName(), ftf.functionType()));
-            });
-        }
-    }
-
-    /**
-     * Checks all commands that exist before a specific type satisfy conditions.
-     *
-     * @param plan plan that contains the condition
-     * @param condition condition to check
-     * @param typeToken type to check for. When a type is found in the condition, all plans before the root plan are checked
-     * @param commandCheck check to perform on each command that precedes the plan that contains the typeToken
-     * @param typeErrorMsgProvider provider for the type name in the error message
-     * @param failures failures to add errors to
-     * @param <E> class of the type to look for
-     */
-    private static <E extends Expression> void checkCommandsBeforeExpression(
-        LogicalPlan plan,
-        Expression condition,
-        Class<E> typeToken,
-        Predicate<LogicalPlan> commandCheck,
-        java.util.function.Function<E, String> typeErrorMsgProvider,
-        Set<Failure> failures
-    ) {
-        condition.forEachDown(typeToken, exp -> {
-            plan.forEachDown(LogicalPlan.class, lp -> {
-                if (commandCheck.test(lp) == false) {
-                    failures.add(
-                        fail(
-                            plan,
-                            "{} cannot be used after {}",
-                            typeErrorMsgProvider.apply(exp),
-                            lp.sourceText().split(" ")[0].toUpperCase(Locale.ROOT)
-                        )
-                    );
-                }
-            });
-        });
-    }
-
-    /**
-     * Checks parents of a full text function to ensure they are allowed
-     * @param condition condition that contains the full text function
-     * @param failures failures to add errors to
-     */
-    private static void checkFullTextFunctionsParents(Expression condition, Set<Failure> failures) {
-        forEachFullTextFunctionParent(condition, (ftf, parent) -> {
-            if ((parent instanceof FullTextFunction == false)
-                && (parent instanceof BinaryLogic == false)
-                && (parent instanceof Not == false)) {
-                failures.add(
-                    fail(
-                        condition,
-                        "Invalid condition [{}]. [{}] {} can't be used with {}",
-                        condition.sourceText(),
-                        ftf.functionName(),
-                        ftf.functionType(),
-                        ((Function) parent).functionName()
-                    )
-                );
-            }
-        });
-    }
-
-    /**
-     * Executes the action on every parent of a FullTextFunction in the condition if it is found
-     *
-     * @param action the action to execute for each parent of a FullTextFunction
-     */
-    private static FullTextFunction forEachFullTextFunctionParent(Expression condition, BiConsumer<FullTextFunction, Expression> action) {
-        if (condition instanceof FullTextFunction ftf) {
-            return ftf;
-        }
-        for (Expression child : condition.children()) {
-            FullTextFunction foundMatchingChild = forEachFullTextFunctionParent(child, action);
-            if (foundMatchingChild != null) {
-                action.accept(foundMatchingChild, condition);
-                return foundMatchingChild;
-            }
-        }
-        return null;
-    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/PostAnalysisPlanVerificationAware.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/PostAnalysisPlanVerificationAware.java
new file mode 100644
index 0000000000000..826896c195ea9
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/PostAnalysisPlanVerificationAware.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.capabilities;
+
+import org.elasticsearch.xpack.esql.common.Failures;
+import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction;
+import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
+import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
+
+import java.util.function.BiConsumer;
+
+/**
+ * Interface implemented by expressions or plans that require validation after query plan analysis,
+ * when the indices and references have been resolved, but before the plan is transformed further by optimizations.
+ * The interface is similar to {@link PostAnalysisVerificationAware}, but focused on the tree structure, oftentimes covering semantic
+ * checks.
+ */
+public interface PostAnalysisPlanVerificationAware {
+
+    /**
+     * Allows the implementer to return a consumer that will perform self-validation in the context of the tree structure the implementer
+     * is part of. This usually involves checking the type and configuration of the children or that of the parent.
+     * <p>
+     * It is often more useful to perform the checks as extended as it makes sense, over stopping at the first failure. This will allow the
+     * author to progress faster to a correct query.
+     * </p>
+     * <p>
+     *     Example: a {@link GroupingFunction} instance, which models a function to group documents to aggregate over, can only be used in
+     *     the context of the STATS command, modeled by the {@link Aggregate} class. This is how this verification is performed:
+     *     <pre>
+     *     {@code
+     *      @Override
+     *      public BiConsumer<LogicalPlan, Failures> postAnalysisPlanVerification() {
+     *          return (p, failures) -> {
+     *              if (p instanceof Aggregate == false) {
+     *                  p.forEachExpression(
+     *                      GroupingFunction.class,
+     *                      gf -> failures.add(fail(gf, "cannot use grouping function [{}] outside of a STATS command", gf.sourceText()))
+     *                  );
+     *              }
+     *          };
+     *      }
+     *     }
+     *     </pre>
+     *
+     * @return a consumer that will receive a tree to check and an accumulator of failures found during inspection.
+     */
+    BiConsumer<LogicalPlan, Failures> postAnalysisPlanVerification();
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/PostAnalysisVerificationAware.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/PostAnalysisVerificationAware.java
new file mode 100644
index 0000000000000..b2af53b4e92ed
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/PostAnalysisVerificationAware.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.capabilities;
+
+import org.elasticsearch.xpack.esql.common.Failures;
+import org.elasticsearch.xpack.esql.plan.logical.Filter;
+
+/**
+ * Interface implemented by expressions or plans that require validation after query plan analysis,
+ * when the indices and references have been resolved, but before the plan is transformed further by optimizations.
+ * The interface is similar to {@link PostAnalysisPlanVerificationAware}, but focused on individual expressions or plans, typically
+ * covering syntactic checks.
+ */
+public interface PostAnalysisVerificationAware {
+
+    /**
+     * Allows the implementer to validate itself. This usually involves checking its internal setup, which often means checking the
+     * parameters it received on construction: their data or syntactic type, class, their count, expressions' structure etc.
+     * The discovered failures are added to the given {@link Failures} object.
+     * <p>
+     * It is often more useful to perform the checks as extended as it makes sense, over stopping at the first failure. This will allow the
+     * author to progress faster to a correct query.
+     * </p>
+     * <p>
+     *     Example: the {@link Filter} class, which models the WHERE command, checks that the expression it filters on - {@code condition}
+     *     - is of a Boolean or NULL type:
+     *     <pre>
+     *     {@code
+     *     @Override
+     *     void postAnalysisVerification(Failures failures) {
+     *          if (condition.dataType() != NULL && condition.dataType() != BOOLEAN) {
+     *              failures.add(fail(condition, "Condition expression needs to be boolean, found [{}]", condition.dataType()));
+     *          }
+     *     }
+     *     }
+     *     </pre>
+     *
+     * @param failures the object to add failures to.
+     */
+    void postAnalysisVerification(Failures failures);
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/PostOptimizationVerificationAware.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/PostOptimizationVerificationAware.java
new file mode 100644
index 0000000000000..e3628d2554a78
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/PostOptimizationVerificationAware.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.capabilities;
+
+import org.elasticsearch.xpack.esql.common.Failures;
+import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket;
+
+/**
+ * Interface implemented by expressions that require validation post logical optimization,
+ * when the plan and references have been not just resolved but also replaced.
+ */
+public interface PostOptimizationVerificationAware {
+
+    /**
+     * Validates the implementing expression - discovered failures are reported to the given
+     * {@link Failures} class.
+     *
+     * <p>
+     *     Example: the {@link Bucket} function, which produces buckets over a numerical or date field, based on a number of literal
+     *     arguments needs to check if its arguments are all indeed literals. This is how this verification is performed:
+     *     <pre>
+     *     {@code
+     *
+     *      @Override
+     *      public void postLogicalOptimizationVerification(Failures failures) {
+     *          String operation = sourceText();
+     *
+     *          failures.add(isFoldable(buckets, operation, SECOND))
+     *              .add(from != null ? isFoldable(from, operation, THIRD) : null)
+     *              .add(to != null ? isFoldable(to, operation, FOURTH) : null);
+     *      }
+     *     }
+     *     </pre>
+     *
+     */
+    void postLogicalOptimizationVerification(Failures failures);
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/Validatable.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/Validatable.java
deleted file mode 100644
index f6733fa3f175c..0000000000000
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/Validatable.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.capabilities;
-
-import org.elasticsearch.xpack.esql.common.Failures;
-
-/**
- * Interface implemented by expressions that require validation post logical optimization,
- * when the plan and references have been not just resolved but also replaced.
- */
-public interface Validatable {
-
-    /**
-     * Validates the implementing expression - discovered failures are reported to the given
-     * {@link Failures} class.
-     */
-    default void validate(Failures failures) {}
-}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java
index e52e9ae989a92..74c66c0d1b338 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java
@@ -132,7 +132,6 @@
  */
 abstract class AbstractLookupService<R extends AbstractLookupService.Request, T extends AbstractLookupService.TransportRequest> {
     private final String actionName;
-    private final String privilegeName;
     private final ClusterService clusterService;
     private final SearchService searchService;
     private final TransportService transportService;
@@ -143,7 +142,6 @@ abstract class AbstractLookupService<R extends AbstractLookupService.Request, T
 
     AbstractLookupService(
         String actionName,
-        String privilegeName,
         ClusterService clusterService,
         SearchService searchService,
         TransportService transportService,
@@ -152,7 +150,6 @@ abstract class AbstractLookupService<R extends AbstractLookupService.Request, T
         CheckedBiFunction<StreamInput, BlockFactory, T, IOException> readRequest
     ) {
         this.actionName = actionName;
-        this.privilegeName = privilegeName;
         this.clusterService = clusterService;
         this.searchService = searchService;
         this.transportService = transportService;
@@ -237,9 +234,21 @@ public final void lookupAsync(R request, CancellableTask parentTask, ActionListe
         }));
     }
 
+    /**
+     * Get the privilege required to perform the lookup.
+     * <p>
+     *     If null is returned, no privilege check will be performed.
+     * </p>
+     */
+    @Nullable
+    protected abstract String getRequiredPrivilege();
+
     private void hasPrivilege(ActionListener<Void> outListener) {
         final Settings settings = clusterService.getSettings();
-        if (settings.hasValue(XPackSettings.SECURITY_ENABLED.getKey()) == false || XPackSettings.SECURITY_ENABLED.get(settings) == false) {
+        String privilegeName = getRequiredPrivilege();
+        if (privilegeName == null
+            || settings.hasValue(XPackSettings.SECURITY_ENABLED.getKey()) == false
+            || XPackSettings.SECURITY_ENABLED.get(settings) == false) {
             outListener.onResponse(null);
             return;
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java
index 50a1ffce4841f..7057b586871eb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java
@@ -52,16 +52,7 @@ public EnrichLookupService(
         BigArrays bigArrays,
         BlockFactory blockFactory
     ) {
-        super(
-            LOOKUP_ACTION_NAME,
-            ClusterPrivilegeResolver.MONITOR_ENRICH.name(),
-            clusterService,
-            searchService,
-            transportService,
-            bigArrays,
-            blockFactory,
-            TransportRequest::readFrom
-        );
+        super(LOOKUP_ACTION_NAME, clusterService, searchService, transportService, bigArrays, blockFactory, TransportRequest::readFrom);
     }
 
     @Override
@@ -90,6 +81,11 @@ protected QueryList queryList(TransportRequest request, SearchExecutionContext c
         };
     }
 
+    @Override
+    protected String getRequiredPrivilege() {
+        return ClusterPrivilegeResolver.MONITOR_ENRICH.name();
+    }
+
     private static void validateTypes(DataType inputDataType, MappedFieldType fieldType) {
         if (fieldType instanceof RangeFieldMapper.RangeFieldType rangeType) {
             // For range policy types, the ENRICH index field type will be one of a list of supported range types,
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexService.java
index 4f429c46b9123..0bbfc6dd0ce99 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexService.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexService.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilegeResolver;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.action.EsqlQueryAction;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
@@ -50,16 +49,7 @@ public LookupFromIndexService(
         BigArrays bigArrays,
         BlockFactory blockFactory
     ) {
-        super(
-            LOOKUP_ACTION_NAME,
-            ClusterPrivilegeResolver.MONITOR_ENRICH.name(), // TODO some other privilege
-            clusterService,
-            searchService,
-            transportService,
-            bigArrays,
-            blockFactory,
-            TransportRequest::readFrom
-        );
+        super(LOOKUP_ACTION_NAME, clusterService, searchService, transportService, bigArrays, blockFactory, TransportRequest::readFrom);
     }
 
     @Override
@@ -83,6 +73,11 @@ protected QueryList queryList(TransportRequest request, SearchExecutionContext c
         return termQueryList(fieldType, context, inputBlock, inputDataType);
     }
 
+    @Override
+    protected String getRequiredPrivilege() {
+        return null;
+    }
+
     private static void validateTypes(DataType inputDataType, MappedFieldType fieldType) {
         // TODO: consider supporting implicit type conversion as done in ENRICH for some types
         if (fieldType.typeName().equals(inputDataType.typeName()) == false) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java
index 9a2e9398f52fd..b9c2b92ea72dd 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
@@ -50,13 +51,23 @@ public final class EvalMapper {
     private EvalMapper() {}
 
     @SuppressWarnings({ "rawtypes", "unchecked" })
-    public static ExpressionEvaluator.Factory toEvaluator(Expression exp, Layout layout) {
+    public static ExpressionEvaluator.Factory toEvaluator(FoldContext foldCtx, Expression exp, Layout layout) {
         if (exp instanceof EvaluatorMapper m) {
-            return m.toEvaluator(e -> toEvaluator(e, layout));
+            return m.toEvaluator(new EvaluatorMapper.ToEvaluator() {
+                @Override
+                public ExpressionEvaluator.Factory apply(Expression expression) {
+                    return toEvaluator(foldCtx, expression, layout);
+                }
+
+                @Override
+                public FoldContext foldCtx() {
+                    return foldCtx;
+                }
+            });
         }
         for (ExpressionMapper em : MAPPERS) {
             if (em.typeToken.isInstance(exp)) {
-                return em.map(exp, layout);
+                return em.map(foldCtx, exp, layout);
             }
         }
         throw new QlIllegalArgumentException("Unsupported expression [{}]", exp);
@@ -64,9 +75,9 @@ public static ExpressionEvaluator.Factory toEvaluator(Expression exp, Layout lay
 
     static class BooleanLogic extends ExpressionMapper<BinaryLogic> {
         @Override
-        public ExpressionEvaluator.Factory map(BinaryLogic bc, Layout layout) {
-            var leftEval = toEvaluator(bc.left(), layout);
-            var rightEval = toEvaluator(bc.right(), layout);
+        public ExpressionEvaluator.Factory map(FoldContext foldCtx, BinaryLogic bc, Layout layout) {
+            var leftEval = toEvaluator(foldCtx, bc.left(), layout);
+            var rightEval = toEvaluator(foldCtx, bc.right(), layout);
             /**
              * Evaluator for the <href a="https://en.wikipedia.org/wiki/Three-valued_logic">three-valued boolean expressions</href>.
              * We can't generate these with the {@link Evaluator} annotation because that
@@ -142,8 +153,8 @@ public void close() {
 
     static class Nots extends ExpressionMapper<Not> {
         @Override
-        public ExpressionEvaluator.Factory map(Not not, Layout layout) {
-            var expEval = toEvaluator(not.field(), layout);
+        public ExpressionEvaluator.Factory map(FoldContext foldCtx, Not not, Layout layout) {
+            var expEval = toEvaluator(foldCtx, not.field(), layout);
             return dvrCtx -> new org.elasticsearch.xpack.esql.evaluator.predicate.operator.logical.NotEvaluator(
                 not.source(),
                 expEval.get(dvrCtx),
@@ -154,7 +165,7 @@ public ExpressionEvaluator.Factory map(Not not, Layout layout) {
 
     static class Attributes extends ExpressionMapper<Attribute> {
         @Override
-        public ExpressionEvaluator.Factory map(Attribute attr, Layout layout) {
+        public ExpressionEvaluator.Factory map(FoldContext foldCtx, Attribute attr, Layout layout) {
             record Attribute(int channel) implements ExpressionEvaluator {
                 @Override
                 public Block eval(Page page) {
@@ -189,7 +200,7 @@ public boolean eagerEvalSafeInLazy() {
     static class Literals extends ExpressionMapper<Literal> {
 
         @Override
-        public ExpressionEvaluator.Factory map(Literal lit, Layout layout) {
+        public ExpressionEvaluator.Factory map(FoldContext foldCtx, Literal lit, Layout layout) {
             record LiteralsEvaluator(DriverContext context, Literal lit) implements ExpressionEvaluator {
                 @Override
                 public Block eval(Page page) {
@@ -246,8 +257,8 @@ private static Block block(Literal lit, BlockFactory blockFactory, int positions
     static class IsNulls extends ExpressionMapper<IsNull> {
 
         @Override
-        public ExpressionEvaluator.Factory map(IsNull isNull, Layout layout) {
-            var field = toEvaluator(isNull.field(), layout);
+        public ExpressionEvaluator.Factory map(FoldContext foldCtx, IsNull isNull, Layout layout) {
+            var field = toEvaluator(foldCtx, isNull.field(), layout);
             return new IsNullEvaluatorFactory(field);
         }
 
@@ -294,8 +305,8 @@ public String toString() {
     static class IsNotNulls extends ExpressionMapper<IsNotNull> {
 
         @Override
-        public ExpressionEvaluator.Factory map(IsNotNull isNotNull, Layout layout) {
-            return new IsNotNullEvaluatorFactory(toEvaluator(isNotNull.field(), layout));
+        public ExpressionEvaluator.Factory map(FoldContext foldCtx, IsNotNull isNotNull, Layout layout) {
+            return new IsNotNullEvaluatorFactory(toEvaluator(foldCtx, isNotNull.field(), layout));
         }
 
         record IsNotNullEvaluatorFactory(EvalOperator.ExpressionEvaluator.Factory field) implements ExpressionEvaluator.Factory {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java
index d8692faef5290..5a8b3d32e7db0 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java
@@ -7,11 +7,20 @@
 
 package org.elasticsearch.xpack.esql.evaluator.mapper;
 
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
+import org.elasticsearch.indices.breaker.AllCircuitBreakerStats;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.indices.breaker.CircuitBreakerStats;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
 import org.elasticsearch.xpack.esql.planner.Layout;
 
 import static org.elasticsearch.compute.data.BlockUtils.fromArrayRow;
@@ -23,9 +32,12 @@
 public interface EvaluatorMapper {
     interface ToEvaluator {
         ExpressionEvaluator.Factory apply(Expression expression);
+
+        FoldContext foldCtx();
     }
 
     /**
+     * Convert this into an {@link ExpressionEvaluator}.
      * <p>
      * Note for implementors:
      * If you are implementing this function, you should call the passed-in
@@ -35,8 +47,8 @@ interface ToEvaluator {
      * <p>
      * Note for Callers:
      * If you are attempting to call this method, and you have an
-     * {@link Expression} and a {@link org.elasticsearch.xpack.esql.planner.Layout},
-     * you likely want to call {@link org.elasticsearch.xpack.esql.evaluator.EvalMapper#toEvaluator(Expression, Layout)}
+     * {@link Expression} and a {@link Layout},
+     * you likely want to call {@link EvalMapper#toEvaluator}
      * instead.  On the other hand, if you already have something that
      * looks like the parameter for this method, you should call this method
      * with that function.
@@ -56,19 +68,89 @@ interface ToEvaluator {
 
     /**
      * Fold using {@link #toEvaluator} so you don't need a "by hand"
-     * implementation of fold. The evaluator that it makes is "funny"
-     * in that it'll always call {@link Expression#fold}, but that's
-     * good enough.
+     * implementation of {@link Expression#fold}.
      */
-    default Object fold() {
-        return toJavaObject(toEvaluator(e -> driverContext -> new ExpressionEvaluator() {
+    default Object fold(Source source, FoldContext ctx) {
+        /*
+         * OK! So! We're going to build a bunch of *stuff* that so that we can
+         * call toEvaluator and use it without standing up an entire compute
+         * engine.
+         *
+         * Step 1 is creation of a `toEvaluator` which we'll soon use to turn
+         * the *children* of this Expression into ExpressionEvaluators. They
+         * have to be foldable or else we wouldn't have ended up here. So!
+         * We just call `fold` on them and turn the result of that into a
+         * Block.
+         *
+         * If the tree of expressions is pretty deep that `fold` call will
+         * likely end up being implemented by calling this method for the
+         * child. That's fine. Recursion is how you process trees.
+         */
+        ToEvaluator foldChildren = new ToEvaluator() {
             @Override
-            public Block eval(Page page) {
-                return fromArrayRow(driverContext.blockFactory(), e.fold())[0];
+            public ExpressionEvaluator.Factory apply(Expression expression) {
+                return driverContext -> new ExpressionEvaluator() {
+                    @Override
+                    public Block eval(Page page) {
+                        return fromArrayRow(driverContext.blockFactory(), expression.fold(ctx))[0];
+                    }
+
+                    @Override
+                    public void close() {}
+                };
             }
 
             @Override
-            public void close() {}
-        }).get(DriverContext.getLocalDriver()).eval(new Page(1)), 0);
+            public FoldContext foldCtx() {
+                return ctx;
+            }
+        };
+
+        /*
+         * Step 2 is to create a DriverContext that we can pass to the above.
+         * This DriverContext is mostly about delegating to the FoldContext.
+         * That'll cause us to break if we attempt to allocate a huge amount
+         * of memory. Neat.
+         *
+         * Specifically, we make a CircuitBreaker view of the FoldContext, then
+         * we wrap it in a CircuitBreakerService so we can feed it to a BigArray
+         * so we can feed *that* into a DriverContext. It's a bit hacky, but
+         * that's what's going on here.
+         */
+        CircuitBreaker breaker = ctx.circuitBreakerView(source);
+        BigArrays bigArrays = new BigArrays(null, new CircuitBreakerService() {
+            @Override
+            public CircuitBreaker getBreaker(String name) {
+                if (name.equals(CircuitBreaker.REQUEST) == false) {
+                    throw new UnsupportedOperationException();
+                }
+                return breaker;
+            }
+
+            @Override
+            public AllCircuitBreakerStats stats() {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public CircuitBreakerStats stats(String name) {
+                throw new UnsupportedOperationException();
+            }
+        }, CircuitBreaker.REQUEST).withCircuitBreaking();
+        DriverContext driverCtx = new DriverContext(bigArrays, new BlockFactory(breaker, bigArrays));
+
+        /*
+         * Finally we can call toEvaluator on ourselves! It'll fold our children,
+         * convert the result into Blocks, and then we'll run that with the memory
+         * breaking DriverContext.
+         *
+         * Then, finally finally, we turn the result into a java object to be compatible
+         * with the signature of `fold`.
+         */
+        Block block = toEvaluator(foldChildren).get(driverCtx).eval(new Page(1));
+        if (block.getPositionCount() != 1) {
+            throw new IllegalStateException("generated odd block from fold [" + block + "]");
+        }
+        return toJavaObject(block, 0);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/ExpressionMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/ExpressionMapper.java
index 5cd830058573f..5a76080e7995c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/ExpressionMapper.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/ExpressionMapper.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.util.ReflectionUtils;
 import org.elasticsearch.xpack.esql.planner.Layout;
 
@@ -19,5 +20,5 @@ public ExpressionMapper() {
         typeToken = ReflectionUtils.detectSuperTypeForRuleLike(getClass());
     }
 
-    public abstract ExpressionEvaluator.Factory map(E expression, Layout layout);
+    public abstract ExpressionEvaluator.Factory map(FoldContext foldCtx, E expression, Layout layout);
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java
index c1269009c6a41..94913581f696d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.xpack.esql.action.EsqlQueryRequest;
 import org.elasticsearch.xpack.esql.analysis.PreAnalyzer;
 import org.elasticsearch.xpack.esql.analysis.Verifier;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolver;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
 import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
@@ -23,6 +24,7 @@
 import org.elasticsearch.xpack.esql.session.Configuration;
 import org.elasticsearch.xpack.esql.session.EsqlSession;
 import org.elasticsearch.xpack.esql.session.IndexResolver;
+import org.elasticsearch.xpack.esql.session.QueryBuilderResolver;
 import org.elasticsearch.xpack.esql.session.Result;
 import org.elasticsearch.xpack.esql.stats.Metrics;
 import org.elasticsearch.xpack.esql.stats.PlanningMetrics;
@@ -55,10 +57,12 @@ public void esql(
         EsqlQueryRequest request,
         String sessionId,
         Configuration cfg,
+        FoldContext foldContext,
         EnrichPolicyResolver enrichPolicyResolver,
         EsqlExecutionInfo executionInfo,
         IndicesExpressionGrouper indicesExpressionGrouper,
         EsqlSession.PlanRunner planRunner,
+        QueryBuilderResolver queryBuilderResolver,
         ActionListener<Result> listener
     ) {
         final PlanningMetrics planningMetrics = new PlanningMetrics();
@@ -69,15 +73,17 @@ public void esql(
             enrichPolicyResolver,
             preAnalyzer,
             functionRegistry,
-            new LogicalPlanOptimizer(new LogicalOptimizerContext(cfg)),
+            new LogicalPlanOptimizer(new LogicalOptimizerContext(cfg, foldContext)),
             mapper,
             verifier,
             planningMetrics,
-            indicesExpressionGrouper
+            indicesExpressionGrouper,
+            queryBuilderResolver
         );
         QueryMetric clientId = QueryMetric.fromString("rest");
         metrics.total(clientId);
-        session.execute(request, executionInfo, planRunner, wrap(x -> {
+
+        ActionListener<Result> executeListener = wrap(x -> {
             planningMetricsManager.publish(planningMetrics, true);
             listener.onResponse(x);
         }, ex -> {
@@ -85,7 +91,10 @@ public void esql(
             metrics.failed(clientId);
             planningMetricsManager.publish(planningMetrics, false);
             listener.onFailure(ex);
-        }));
+        });
+        // Wrap it in a listener so that if we have any exceptions during execution, the listener picks it up
+        // and all the metrics are properly updated
+        ActionListener.run(executeListener, l -> session.execute(request, executionInfo, planRunner, l));
     }
 
     public IndexResolver indexResolver() {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
index 1ccc22eb3a6a4..bb170f546b54e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.util.FeatureFlag;
-import org.elasticsearch.xpack.esql.core.ParsingException;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.function.Function;
@@ -130,15 +129,19 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.ByteLength;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.EndsWith;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.Hash;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Locate;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.Md5;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.RTrim;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Repeat;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Reverse;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Right;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.Sha1;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.Sha256;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Space;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith;
@@ -147,6 +150,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Trim;
 import org.elasticsearch.xpack.esql.expression.function.scalar.util.Delay;
+import org.elasticsearch.xpack.esql.parser.ParsingException;
 import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.lang.reflect.Constructor;
@@ -327,15 +331,19 @@ private static FunctionDefinition[][] functions() {
                 def(ByteLength.class, ByteLength::new, "byte_length"),
                 def(Concat.class, Concat::new, "concat"),
                 def(EndsWith.class, EndsWith::new, "ends_with"),
+                def(Hash.class, Hash::new, "hash"),
                 def(LTrim.class, LTrim::new, "ltrim"),
                 def(Left.class, Left::new, "left"),
                 def(Length.class, Length::new, "length"),
                 def(Locate.class, Locate::new, "locate"),
+                def(Md5.class, Md5::new, "md5"),
                 def(RTrim.class, RTrim::new, "rtrim"),
                 def(Repeat.class, Repeat::new, "repeat"),
                 def(Replace.class, Replace::new, "replace"),
                 def(Reverse.class, Reverse::new, "reverse"),
                 def(Right.class, Right::new, "right"),
+                def(Sha1.class, Sha1::new, "sha1"),
+                def(Sha256.class, Sha256::new, "sha256"),
                 def(Space.class, Space::new, "space"),
                 def(StartsWith.class, StartsWith::new, "starts_with"),
                 def(Substring.class, Substring::new, "substring"),
@@ -416,7 +424,10 @@ private static FunctionDefinition[][] functions() {
                 def(MvSum.class, MvSum::new, "mv_sum"),
                 def(Split.class, Split::new, "split") },
             // fulltext functions
-            new FunctionDefinition[] { def(Match.class, bi(Match::new), "match"), def(QueryString.class, uni(QueryString::new), "qstr") } };
+            new FunctionDefinition[] {
+                def(Kql.class, uni(Kql::new), "kql"),
+                def(Match.class, bi(Match::new), "match"),
+                def(QueryString.class, uni(QueryString::new), "qstr") } };
 
     }
 
@@ -426,7 +437,6 @@ private static FunctionDefinition[][] snapshotFunctions() {
                 // The delay() function is for debug/snapshot environments only and should never be enabled in a non-snapshot build.
                 // This is an experimental function and can be removed without notice.
                 def(Delay.class, Delay::new, "delay"),
-                def(Kql.class, uni(Kql::new), "kql"),
                 def(Rate.class, Rate::withUnresolvedTimestamp, "rate"),
                 def(Term.class, bi(Term::new), "term") } };
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java
index 265b08de5556d..8aa7f697489c6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java
@@ -9,26 +9,33 @@
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.esql.capabilities.PostAnalysisPlanVerificationAware;
+import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.expression.function.Function;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.plan.logical.OrderBy;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
+import java.util.function.BiConsumer;
 
 import static java.util.Arrays.asList;
 import static java.util.Collections.emptyList;
+import static org.elasticsearch.xpack.esql.common.Failure.fail;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
 
 /**
  * A type of {@code Function} that takes multiple values and extracts a single value out of them. For example, {@code AVG()}.
  */
-public abstract class AggregateFunction extends Function {
+public abstract class AggregateFunction extends Function implements PostAnalysisPlanVerificationAware {
 
     private final Expression field;
     private final List<? extends Expression> parameters;
@@ -86,7 +93,8 @@ public List<? extends Expression> parameters() {
     }
 
     public boolean hasFilter() {
-        return filter != null && (filter.foldable() == false || Boolean.TRUE.equals(filter.fold()) == false);
+        return filter != null
+            && (filter.foldable() == false || Boolean.TRUE.equals(filter.fold(FoldContext.small() /* TODO remove me */)) == false);
     }
 
     public Expression filter() {
@@ -127,4 +135,19 @@ public boolean equals(Object obj) {
         }
         return false;
     }
+
+    @Override
+    public BiConsumer<LogicalPlan, Failures> postAnalysisPlanVerification() {
+        return (p, failures) -> {
+            if (p instanceof OrderBy order) {
+                order.order().forEach(o -> {
+                    o.forEachDown(Function.class, f -> {
+                        if (f instanceof AggregateFunction) {
+                            failures.add(fail(f, "Aggregate functions are not allowed in SORT [{}]", f.functionName()));
+                        }
+                    });
+                });
+            }
+        };
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java
index 7436db9e00dd2..3170ae8f132c2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.compute.aggregation.CountDistinctLongAggregatorFunctionSupplier;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -210,7 +211,9 @@ protected TypeResolution resolveType() {
     @Override
     public AggregatorFunctionSupplier supplier(List<Integer> inputChannels) {
         DataType type = field().dataType();
-        int precision = this.precision == null ? DEFAULT_PRECISION : ((Number) this.precision.fold()).intValue();
+        int precision = this.precision == null
+            ? DEFAULT_PRECISION
+            : ((Number) this.precision.fold(FoldContext.small() /* TODO remove me */)).intValue();
         if (SUPPLIERS.containsKey(type) == false) {
             // If the type checking did its job, this should never happen
             throw EsqlIllegalArgumentException.illegalDataType(type);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java
index 0d57267da1e29..8c943c991d501 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.compute.aggregation.PercentileIntAggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.PercentileLongAggregatorFunctionSupplier;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -170,7 +171,7 @@ protected AggregatorFunctionSupplier doubleSupplier(List<Integer> inputChannels)
     }
 
     private int percentileValue() {
-        return ((Number) percentile.fold()).intValue();
+        return ((Number) percentile.fold(FoldContext.small() /* TODO remove me */)).intValue();
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java
index 87ac9b77a6826..85ae65b6c5dc3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
@@ -156,7 +157,7 @@ long unitInMillis() {
         }
         final Object foldValue;
         try {
-            foldValue = unit.fold();
+            foldValue = unit.fold(FoldContext.small() /* TODO remove me */);
         } catch (Exception e) {
             throw new IllegalArgumentException("function [" + sourceText() + "] has invalid unit [" + unit.sourceText() + "]");
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java
index 35f99e4b648df..f68f9f2487884 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference;
 import org.elasticsearch.license.License;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.xpack.esql.LicenseAware;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 
@@ -24,7 +25,7 @@
  * The AggregateMapper class will generate multiple aggregation functions for each combination, allowing the planner to
  * select the best one.
  */
-public abstract class SpatialAggregateFunction extends AggregateFunction {
+public abstract class SpatialAggregateFunction extends AggregateFunction implements LicenseAware {
     protected final FieldExtractPreference fieldExtractPreference;
 
     protected SpatialAggregateFunction(Source source, Expression field, Expression filter, FieldExtractPreference fieldExtractPreference) {
@@ -41,7 +42,7 @@ protected SpatialAggregateFunction(StreamInput in, FieldExtractPreference fieldE
     public abstract SpatialAggregateFunction withDocValues();
 
     @Override
-    public boolean checkLicense(XPackLicenseState state) {
+    public boolean licenseCheck(XPackLicenseState state) {
         return switch (field().dataType()) {
             case GEO_SHAPE, CARTESIAN_SHAPE -> state.isAllowedByLicense(License.OperationMode.PLATINUM);
             default -> true;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java
index 84915d024ea82..54c05cf1bad52 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java
@@ -103,11 +103,11 @@ public AggregatorFunctionSupplier supplier(List<Integer> inputChannels) {
         return switch (type) {
             case DataType.GEO_POINT -> switch (fieldExtractPreference) {
                 case DOC_VALUES -> new SpatialCentroidGeoPointDocValuesAggregatorFunctionSupplier(inputChannels);
-                case NONE -> new SpatialCentroidGeoPointSourceValuesAggregatorFunctionSupplier(inputChannels);
+                case NONE, EXTRACT_SPATIAL_BOUNDS -> new SpatialCentroidGeoPointSourceValuesAggregatorFunctionSupplier(inputChannels);
             };
             case DataType.CARTESIAN_POINT -> switch (fieldExtractPreference) {
                 case DOC_VALUES -> new SpatialCentroidCartesianPointDocValuesAggregatorFunctionSupplier(inputChannels);
-                case NONE -> new SpatialCentroidCartesianPointSourceValuesAggregatorFunctionSupplier(inputChannels);
+                case NONE, EXTRACT_SPATIAL_BOUNDS -> new SpatialCentroidCartesianPointSourceValuesAggregatorFunctionSupplier(inputChannels);
             };
             default -> throw EsqlIllegalArgumentException.illegalDataType(type);
         };
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtent.java
index 5cc1701faf13a..34e5c9d68fc86 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtent.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtent.java
@@ -104,11 +104,11 @@ public AggregatorFunctionSupplier supplier(List<Integer> inputChannels) {
         return switch (field().dataType()) {
             case DataType.GEO_POINT -> switch (fieldExtractPreference) {
                 case DOC_VALUES -> new SpatialExtentGeoPointDocValuesAggregatorFunctionSupplier(inputChannels);
-                case NONE -> new SpatialExtentGeoPointSourceValuesAggregatorFunctionSupplier(inputChannels);
+                case NONE, EXTRACT_SPATIAL_BOUNDS -> new SpatialExtentGeoPointSourceValuesAggregatorFunctionSupplier(inputChannels);
             };
             case DataType.CARTESIAN_POINT -> switch (fieldExtractPreference) {
                 case DOC_VALUES -> new SpatialExtentCartesianPointDocValuesAggregatorFunctionSupplier(inputChannels);
-                case NONE -> new SpatialExtentCartesianPointSourceValuesAggregatorFunctionSupplier(inputChannels);
+                case NONE, EXTRACT_SPATIAL_BOUNDS -> new SpatialExtentCartesianPointSourceValuesAggregatorFunctionSupplier(inputChannels);
             };
             // Shapes don't differentiate between source and doc values.
             case DataType.GEO_SHAPE -> new SpatialExtentGeoShapeAggregatorFunctionSupplier(inputChannels);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java
index 40777b4d78dc2..9be8c94266ee8 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.compute.aggregation.TopLongAggregatorFunctionSupplier;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -115,11 +116,11 @@ Expression orderField() {
     }
 
     private int limitValue() {
-        return (int) limitField().fold();
+        return (int) limitField().fold(FoldContext.small() /* TODO remove me */);
     }
 
     private String orderRawValue() {
-        return BytesRefs.toString(orderField().fold());
+        return BytesRefs.toString(orderField().fold(FoldContext.small() /* TODO remove me */));
     }
 
     private boolean orderValue() {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java
index 49c68d002440f..bab65653ba576 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xpack.esql.capabilities.Validatable;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -34,7 +34,7 @@
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
 
-public class WeightedAvg extends AggregateFunction implements SurrogateExpression, Validatable {
+public class WeightedAvg extends AggregateFunction implements SurrogateExpression {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
         Expression.class,
         "WeightedAvg",
@@ -115,9 +115,15 @@ protected Expression.TypeResolution resolveType() {
             return resolution;
         }
 
-        if (weight.dataType() == DataType.NULL
-            || (weight.foldable() && (weight.fold() == null || weight.fold().equals(0) || weight.fold().equals(0.0)))) {
-            return new TypeResolution(format(null, invalidWeightError, SECOND, sourceText(), weight.foldable() ? weight.fold() : null));
+        if (weight.dataType() == DataType.NULL) {
+            return new TypeResolution(format(null, invalidWeightError, SECOND, sourceText(), null));
+        }
+        if (weight.foldable() == false) {
+            return TypeResolution.TYPE_RESOLVED;
+        }
+        Object weightVal = weight.fold(FoldContext.small()/* TODO remove me*/);
+        if (weightVal == null || weightVal.equals(0) || weightVal.equals(0.0)) {
+            return new TypeResolution(format(null, invalidWeightError, SECOND, sourceText(), weightVal));
         }
 
         return TypeResolution.TYPE_RESOLVED;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java
index 432d2d5f07429..4da7c01139c24 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java
@@ -9,21 +9,38 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.xpack.esql.capabilities.PostAnalysisPlanVerificationAware;
+import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.expression.TranslationAware;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.expression.function.Function;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslator;
 import org.elasticsearch.xpack.esql.core.planner.TranslatorHandler;
 import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.querydsl.query.TranslationAwareExpressionQuery;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.util.Holder;
+import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
+import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
+import org.elasticsearch.xpack.esql.plan.logical.Filter;
+import org.elasticsearch.xpack.esql.plan.logical.Limit;
+import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.plan.logical.OrderBy;
 
 import java.util.List;
+import java.util.Locale;
 import java.util.Objects;
+import java.util.function.BiConsumer;
+import java.util.function.Predicate;
 
+import static org.elasticsearch.xpack.esql.common.Failure.fail;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNullAndFoldable;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
@@ -33,7 +50,7 @@
  * These functions needs to be pushed down to Lucene queries to be executed - there's no Evaluator for them, but depend on
  * {@link org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizer} to rewrite them into Lucene queries.
  */
-public abstract class FullTextFunction extends Function implements TranslationAware {
+public abstract class FullTextFunction extends Function implements TranslationAware, PostAnalysisPlanVerificationAware {
 
     private final Expression query;
     private final QueryBuilder queryBuilder;
@@ -95,7 +112,7 @@ public Expression query() {
      * @return query expression as an object
      */
     public Object queryAsObject() {
-        Object queryAsObject = query().fold();
+        Object queryAsObject = query().fold(FoldContext.small() /* TODO remove me */);
         if (queryAsObject instanceof BytesRef bytesRef) {
             return bytesRef.utf8ToString();
         }
@@ -158,4 +175,210 @@ public QueryBuilder queryBuilder() {
     protected abstract ExpressionTranslator<? extends FullTextFunction> translator();
 
     public abstract Expression replaceQueryBuilder(QueryBuilder queryBuilder);
+
+    @Override
+    public BiConsumer<LogicalPlan, Failures> postAnalysisPlanVerification() {
+        return FullTextFunction::checkFullTextQueryFunctions;
+    }
+
+    /**
+     * Checks full text query functions for invalid usage.
+     *
+     * @param plan root plan to check
+     * @param failures failures found
+     */
+    private static void checkFullTextQueryFunctions(LogicalPlan plan, Failures failures) {
+        if (plan instanceof Filter f) {
+            Expression condition = f.condition();
+
+            List.of(QueryString.class, Kql.class).forEach(functionClass -> {
+                // Check for limitations of QSTR and KQL function.
+                checkCommandsBeforeExpression(
+                    plan,
+                    condition,
+                    functionClass,
+                    lp -> (lp instanceof Filter || lp instanceof OrderBy || lp instanceof EsRelation),
+                    fullTextFunction -> "[" + fullTextFunction.functionName() + "] " + fullTextFunction.functionType(),
+                    failures
+                );
+            });
+
+            checkCommandsBeforeExpression(
+                plan,
+                condition,
+                Match.class,
+                lp -> (lp instanceof Limit == false) && (lp instanceof Aggregate == false),
+                m -> "[" + m.functionName() + "] " + m.functionType(),
+                failures
+            );
+            checkCommandsBeforeExpression(
+                plan,
+                condition,
+                Term.class,
+                lp -> (lp instanceof Limit == false) && (lp instanceof Aggregate == false),
+                m -> "[" + m.functionName() + "] " + m.functionType(),
+                failures
+            );
+            checkFullTextSearchDisjunctions(condition, ftf -> "[" + ftf.functionName() + "] " + ftf.functionType(), failures);
+            checkFullTextFunctionsParents(condition, failures);
+        } else {
+            plan.forEachExpression(FullTextFunction.class, ftf -> {
+                failures.add(fail(ftf, "[{}] {} is only supported in WHERE commands", ftf.functionName(), ftf.functionType()));
+            });
+        }
+    }
+
+    /**
+     * Checks whether a condition contains a disjunction with a full text search.
+     * If it does, check that every element of the disjunction is a full text search or combinations (AND, OR, NOT) of them.
+     * If not, add a failure to the failures collection.
+     *
+     * @param condition        condition to check for disjunctions of full text searches
+     * @param typeNameProvider provider for the type name to add in the failure message
+     * @param failures         failures collection to add to
+     */
+    private static void checkFullTextSearchDisjunctions(
+        Expression condition,
+        java.util.function.Function<FullTextFunction, String> typeNameProvider,
+        Failures failures
+    ) {
+        Holder<Boolean> isInvalid = new Holder<>(false);
+        condition.forEachDown(Or.class, or -> {
+            if (isInvalid.get()) {
+                // Exit early if we already have a failures
+                return;
+            }
+            boolean hasFullText = or.anyMatch(FullTextFunction.class::isInstance);
+            if (hasFullText) {
+                boolean hasOnlyFullText = onlyFullTextFunctionsInExpression(or);
+                if (hasOnlyFullText == false) {
+                    isInvalid.set(true);
+                    failures.add(
+                        fail(
+                            or,
+                            "Invalid condition [{}]. Full text functions can be used in an OR condition, "
+                                + "but only if just full text functions are used in the OR condition",
+                            or.sourceText()
+                        )
+                    );
+                }
+            }
+        });
+    }
+
+    /**
+     * Checks whether an expression contains just full text functions or negations (NOT) and combinations (AND, OR) of full text functions
+     *
+     * @param expression expression to check
+     * @return true if all children are full text functions or negations of full text functions, false otherwise
+     */
+    private static boolean onlyFullTextFunctionsInExpression(Expression expression) {
+        if (expression instanceof FullTextFunction) {
+            return true;
+        } else if (expression instanceof Not) {
+            return onlyFullTextFunctionsInExpression(expression.children().get(0));
+        } else if (expression instanceof BinaryLogic binaryLogic) {
+            return onlyFullTextFunctionsInExpression(binaryLogic.left()) && onlyFullTextFunctionsInExpression(binaryLogic.right());
+        }
+
+        return false;
+    }
+
+    /**
+     * Checks whether an expression contains a full text function as part of it
+     *
+     * @param expression expression to check
+     * @return true if the expression or any of its children is a full text function, false otherwise
+     */
+    private static boolean anyFullTextFunctionsInExpression(Expression expression) {
+        if (expression instanceof FullTextFunction) {
+            return true;
+        }
+
+        for (Expression child : expression.children()) {
+            if (anyFullTextFunctionsInExpression(child)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Checks all commands that exist before a specific type satisfy conditions.
+     *
+     * @param plan plan that contains the condition
+     * @param condition condition to check
+     * @param typeToken type to check for. When a type is found in the condition, all plans before the root plan are checked
+     * @param commandCheck check to perform on each command that precedes the plan that contains the typeToken
+     * @param typeErrorMsgProvider provider for the type name in the error message
+     * @param failures failures to add errors to
+     * @param <E> class of the type to look for
+     */
+    private static <E extends Expression> void checkCommandsBeforeExpression(
+        LogicalPlan plan,
+        Expression condition,
+        Class<E> typeToken,
+        Predicate<LogicalPlan> commandCheck,
+        java.util.function.Function<E, String> typeErrorMsgProvider,
+        Failures failures
+    ) {
+        condition.forEachDown(typeToken, exp -> {
+            plan.forEachDown(LogicalPlan.class, lp -> {
+                if (commandCheck.test(lp) == false) {
+                    failures.add(
+                        fail(
+                            plan,
+                            "{} cannot be used after {}",
+                            typeErrorMsgProvider.apply(exp),
+                            lp.sourceText().split(" ")[0].toUpperCase(Locale.ROOT)
+                        )
+                    );
+                }
+            });
+        });
+    }
+
+    /**
+     * Checks parents of a full text function to ensure they are allowed
+     * @param condition condition that contains the full text function
+     * @param failures failures to add errors to
+     */
+    private static void checkFullTextFunctionsParents(Expression condition, Failures failures) {
+        forEachFullTextFunctionParent(condition, (ftf, parent) -> {
+            if ((parent instanceof FullTextFunction == false)
+                && (parent instanceof BinaryLogic == false)
+                && (parent instanceof Not == false)) {
+                failures.add(
+                    fail(
+                        condition,
+                        "Invalid condition [{}]. [{}] {} can't be used with {}",
+                        condition.sourceText(),
+                        ftf.functionName(),
+                        ftf.functionType(),
+                        ((Function) parent).functionName()
+                    )
+                );
+            }
+        });
+    }
+
+    /**
+     * Executes the action on every parent of a FullTextFunction in the condition if it is found
+     *
+     * @param action the action to execute for each parent of a FullTextFunction
+     */
+    private static FullTextFunction forEachFullTextFunctionParent(Expression condition, BiConsumer<FullTextFunction, Expression> action) {
+        if (condition instanceof FullTextFunction ftf) {
+            return ftf;
+        }
+        for (Expression child : condition.children()) {
+            FullTextFunction foundMatchingChild = forEachFullTextFunctionParent(child, action);
+            if (foundMatchingChild != null) {
+                action.accept(foundMatchingChild, condition);
+                return foundMatchingChild;
+            }
+        }
+        return null;
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java
index d6b79d16b74f6..a3a14004b1c89 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java
@@ -9,8 +9,8 @@
 
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
-import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate;
-import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MultiMatchQueryPredicate;
+import org.elasticsearch.xpack.esql.expression.predicate.fulltext.MatchQueryPredicate;
+import org.elasticsearch.xpack.esql.expression.predicate.fulltext.MultiMatchQueryPredicate;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -25,10 +25,8 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         entries.add(MultiMatchQueryPredicate.ENTRY);
         entries.add(QueryString.ENTRY);
         entries.add(Match.ENTRY);
+        entries.add(Kql.ENTRY);
 
-        if (EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()) {
-            entries.add(Kql.ENTRY);
-        }
         if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) {
             entries.add(Term.ENTRY);
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java
index 0b2268fe1b022..93f23d2f7ad08 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java
@@ -13,11 +13,12 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.xpack.esql.capabilities.Validatable;
+import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware;
 import org.elasticsearch.xpack.esql.common.Failure;
 import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslator;
 import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery;
@@ -51,6 +52,7 @@
 import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
 import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
 import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT;
 import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
 import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
@@ -59,7 +61,7 @@
 /**
  * Full text function that performs a {@link QueryStringQuery} .
  */
-public class Match extends FullTextFunction implements Validatable {
+public class Match extends FullTextFunction implements PostOptimizationVerificationAware {
 
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Match", Match::readFrom);
 
@@ -70,6 +72,7 @@ public class Match extends FullTextFunction implements Validatable {
     public static final Set<DataType> FIELD_DATA_TYPES = Set.of(
         KEYWORD,
         TEXT,
+        SEMANTIC_TEXT,
         BOOLEAN,
         DATETIME,
         DATE_NANOS,
@@ -96,8 +99,16 @@ public class Match extends FullTextFunction implements Validatable {
     @FunctionInfo(
         returnType = "boolean",
         preview = true,
-        description = "Performs a <<query-dsl-match-query,match query>> on the specified field. "
-            + "Returns true if the provided query matches the row.",
+        description = """
+            Use `MATCH` to perform a <<query-dsl-match-query,match query>> on the specified field.
+            Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL.
+
+            Match can be used on fields from the text family like <<text, text>> and <<semantic-text, semantic_text>>,
+            as well as other field types like keyword, boolean, dates, and numeric types.
+
+            For a simplified syntax, you can use the <<esql-search-operators,match operator>> `:` operator instead of `MATCH`.
+
+            `MATCH` returns true if the provided query matches the row.""",
         examples = { @Example(file = "match-function", tag = "match-with-field") }
     )
     public Match(
@@ -192,7 +203,7 @@ protected TypeResolution checkParamCompatibility() {
     }
 
     @Override
-    public void validate(Failures failures) {
+    public void postLogicalOptimizationVerification(Failures failures) {
         Expression fieldExpression = field();
         // Field may be converted to other data type (field_name :: data_type), so we need to check the original field
         if (fieldExpression instanceof AbstractConvertFunction convertFunction) {
@@ -213,7 +224,7 @@ public void validate(Failures failures) {
 
     @Override
     public Object queryAsObject() {
-        Object queryAsObject = query().fold();
+        Object queryAsObject = query().fold(FoldContext.small() /* TODO remove me */);
 
         // Convert BytesRef to string for string-based values
         if (queryAsObject instanceof BytesRef bytesRef) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java
index ff8085cd1b44b..e77f95073050f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.xpack.esql.capabilities.Validatable;
+import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware;
 import org.elasticsearch.xpack.esql.common.Failure;
 import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
@@ -39,7 +39,7 @@
 /**
  * Full text function that performs a {@link TermQuery} .
  */
-public class Term extends FullTextFunction implements Validatable {
+public class Term extends FullTextFunction implements PostOptimizationVerificationAware {
 
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Term", Term::readFrom);
 
@@ -100,7 +100,7 @@ protected TypeResolution resolveNonQueryParamTypes() {
     }
 
     @Override
-    public void validate(Failures failures) {
+    public void postLogicalOptimizationVerification(Failures failures) {
         if (field instanceof FieldAttribute == false) {
             failures.add(
                 Failure.fail(
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java
index 347d542f5212d..7a3e080f5c830 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java
@@ -15,9 +15,10 @@
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
-import org.elasticsearch.xpack.esql.capabilities.Validatable;
+import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware;
 import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Foldables;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
@@ -56,7 +57,7 @@
  * from a number of desired buckets (as a hint) and a range (auto mode).
  * In the former case, two parameters will be provided, in the latter four.
  */
-public class Bucket extends GroupingFunction implements Validatable, TwoOptionalArguments {
+public class Bucket extends GroupingFunction implements PostOptimizationVerificationAware, TwoOptionalArguments {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Bucket", Bucket::new);
 
     // TODO maybe we should just cover the whole of representable dates here - like ten years, 100 years, 1000 years, all the way up.
@@ -163,6 +164,17 @@ another in which the bucket size is provided directly (two parameters).
                     grouping part, or that it is invoked with the exact same expression:""",
                 file = "bucket",
                 tag = "reuseGroupingFunctionWithExpression"
+            ),
+            @Example(
+                description = """
+                    Sometimes you need to change the start value of each bucket by a given duration (similar to date histogram
+                    aggregation's <<search-aggregations-bucket-histogram-aggregation,`offset`>> parameter). To do so, you will need to
+                    take into account how the language handles expressions within the `STATS` command: if these contain functions or
+                    arithmetic operators, a virtual `EVAL` is inserted before and/or after the `STATS` command. Consequently, a double
+                    compensation is needed to adjust the bucketed date value before the aggregation and then again after. For instance,
+                    inserting a negative offset of `1 hour` to buckets of `1 year` looks like this:""",
+                file = "bucket",
+                tag = "bucketWithOffset"
             ) }
     )
     public Bucket(
@@ -244,25 +256,25 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
         if (field.dataType() == DataType.DATETIME || field.dataType() == DataType.DATE_NANOS) {
             Rounding.Prepared preparedRounding;
             if (buckets.dataType().isWholeNumber()) {
-                int b = ((Number) buckets.fold()).intValue();
-                long f = foldToLong(from);
-                long t = foldToLong(to);
+                int b = ((Number) buckets.fold(toEvaluator.foldCtx())).intValue();
+                long f = foldToLong(toEvaluator.foldCtx(), from);
+                long t = foldToLong(toEvaluator.foldCtx(), to);
                 preparedRounding = new DateRoundingPicker(b, f, t).pickRounding().prepareForUnknown();
             } else {
                 assert DataType.isTemporalAmount(buckets.dataType()) : "Unexpected span data type [" + buckets.dataType() + "]";
-                preparedRounding = DateTrunc.createRounding(buckets.fold(), DEFAULT_TZ);
+                preparedRounding = DateTrunc.createRounding(buckets.fold(toEvaluator.foldCtx()), DEFAULT_TZ);
             }
             return DateTrunc.evaluator(field.dataType(), source(), toEvaluator.apply(field), preparedRounding);
         }
         if (field.dataType().isNumeric()) {
             double roundTo;
             if (from != null) {
-                int b = ((Number) buckets.fold()).intValue();
-                double f = ((Number) from.fold()).doubleValue();
-                double t = ((Number) to.fold()).doubleValue();
+                int b = ((Number) buckets.fold(toEvaluator.foldCtx())).intValue();
+                double f = ((Number) from.fold(toEvaluator.foldCtx())).doubleValue();
+                double t = ((Number) to.fold(toEvaluator.foldCtx())).doubleValue();
                 roundTo = pickRounding(b, f, t);
             } else {
-                roundTo = ((Number) buckets.fold()).doubleValue();
+                roundTo = ((Number) buckets.fold(toEvaluator.foldCtx())).doubleValue();
             }
             Literal rounding = new Literal(source(), roundTo, DataType.DOUBLE);
 
@@ -397,7 +409,7 @@ private static TypeResolution isStringOrDate(Expression e, String operationName,
     }
 
     @Override
-    public void validate(Failures failures) {
+    public void postLogicalOptimizationVerification(Failures failures) {
         String operation = sourceText();
 
         failures.add(isFoldable(buckets, operation, SECOND))
@@ -405,8 +417,8 @@ public void validate(Failures failures) {
             .add(to != null ? isFoldable(to, operation, FOURTH) : null);
     }
 
-    private long foldToLong(Expression e) {
-        Object value = Foldables.valueOf(e);
+    private long foldToLong(FoldContext ctx, Expression e) {
+        Object value = Foldables.valueOf(ctx, e);
         return DataType.isDateTime(e.dataType()) ? ((Number) value).longValue() : dateTimeToLong(((BytesRef) value).utf8ToString());
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java
index a100dd64915f1..570ce7a96dd68 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
-import org.elasticsearch.xpack.esql.capabilities.Validatable;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
@@ -37,7 +36,7 @@
  *     For the implementation, see {@link org.elasticsearch.compute.aggregation.blockhash.CategorizeBlockHash}
  * </p>
  */
-public class Categorize extends GroupingFunction implements Validatable {
+public class Categorize extends GroupingFunction {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
         Expression.class,
         "Categorize",
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/GroupingFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/GroupingFunction.java
index c66d499776c12..fd025e5e67a7c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/GroupingFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/GroupingFunction.java
@@ -7,22 +7,42 @@
 
 package org.elasticsearch.xpack.esql.expression.function.grouping;
 
+import org.elasticsearch.xpack.esql.capabilities.PostAnalysisPlanVerificationAware;
+import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.function.Function;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
+import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
+import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 
 import java.util.List;
+import java.util.function.BiConsumer;
 
-public abstract class GroupingFunction extends Function implements EvaluatorMapper {
+import static org.elasticsearch.xpack.esql.common.Failure.fail;
+
+public abstract class GroupingFunction extends Function implements EvaluatorMapper, PostAnalysisPlanVerificationAware {
 
     protected GroupingFunction(Source source, List<Expression> fields) {
         super(source, fields);
     }
 
     @Override
-    public Object fold() {
-        return EvaluatorMapper.super.fold();
+    public Object fold(FoldContext ctx) {
+        return EvaluatorMapper.super.fold(source(), ctx);
+    }
+
+    @Override
+    public BiConsumer<LogicalPlan, Failures> postAnalysisPlanVerification() {
+        return (p, failures) -> {
+            if (p instanceof Aggregate == false) {
+                p.forEachExpression(
+                    GroupingFunction.class,
+                    gf -> failures.add(fail(gf, "cannot use grouping function [{}] outside of a STATS command", gf.sourceText()))
+                );
+            }
+        };
     }
 
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java
index 404ce7e3900c9..85d15f82f458a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.esql.expression.function.scalar;
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
@@ -34,7 +35,7 @@ protected EsqlScalarFunction(Source source, List<Expression> fields) {
     }
 
     @Override
-    public Object fold() {
-        return EvaluatorMapper.super.fold();
+    public Object fold(FoldContext ctx) {
+        return EvaluatorMapper.super.fold(source(), ctx);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java
index 192ca6c43e57d..c4b9f6885e617 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java
@@ -34,12 +34,16 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.BitLength;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.EndsWith;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.Hash;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Locate;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.Md5;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Repeat;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Reverse;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Right;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.Sha1;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.Sha256;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring;
@@ -64,6 +68,7 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         entries.add(E.ENTRY);
         entries.add(EndsWith.ENTRY);
         entries.add(Greatest.ENTRY);
+        entries.add(Hash.ENTRY);
         entries.add(Hypot.ENTRY);
         entries.add(In.ENTRY);
         entries.add(InsensitiveEquals.ENTRY);
@@ -77,6 +82,7 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         entries.add(Left.ENTRY);
         entries.add(Locate.ENTRY);
         entries.add(Log.ENTRY);
+        entries.add(Md5.ENTRY);
         entries.add(Now.ENTRY);
         entries.add(Or.ENTRY);
         entries.add(Pi.ENTRY);
@@ -86,6 +92,8 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         entries.add(Replace.ENTRY);
         entries.add(Reverse.ENTRY);
         entries.add(Round.ENTRY);
+        entries.add(Sha1.ENTRY);
+        entries.add(Sha256.ENTRY);
         entries.add(Split.ENTRY);
         entries.add(Substring.ENTRY);
         entries.add(StartsWith.ENTRY);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java
index 824f02ca7ccbb..236e625f7abe1 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
@@ -227,7 +228,7 @@ public boolean foldable() {
             if (condition.condition.foldable() == false) {
                 return false;
             }
-            if (Boolean.TRUE.equals(condition.condition.fold())) {
+            if (Boolean.TRUE.equals(condition.condition.fold(FoldContext.small() /* TODO remove me - use literal true?*/))) {
                 /*
                  * `fold` can make four things here:
                  * 1. `TRUE`
@@ -264,7 +265,8 @@ public boolean foldable() {
      * And those two combine so {@code EVAL c=CASE(false, foo, b, bar, true, bort, el)} becomes
      * {@code EVAL c=CASE(b, bar, bort)}.
      */
-    public Expression partiallyFold() {
+    public Expression partiallyFold(FoldContext ctx) {
+        // TODO don't throw away the results of any `fold`. That might mean looking for literal TRUE on the conditions.
         List<Expression> newChildren = new ArrayList<>(children().size());
         boolean modified = false;
         for (Condition condition : conditions) {
@@ -274,7 +276,7 @@ public Expression partiallyFold() {
                 continue;
             }
             modified = true;
-            if (Boolean.TRUE.equals(condition.condition.fold())) {
+            if (Boolean.TRUE.equals(condition.condition.fold(ctx))) {
                 /*
                  * `fold` can make four things here:
                  * 1. `TRUE`
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FoldablesConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FoldablesConvertFunction.java
index 8f43a6481db07..57f362f86ff4c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FoldablesConvertFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FoldablesConvertFunction.java
@@ -8,9 +8,10 @@
 package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
 
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xpack.esql.capabilities.Validatable;
+import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware;
 import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 
@@ -26,7 +27,7 @@
  * Base class for functions that converts a constant into an interval type - DATE_PERIOD or TIME_DURATION.
  * The functions will be folded at the end of LogicalPlanOptimizer by the coordinator, it does not reach data node.
  */
-public abstract class FoldablesConvertFunction extends AbstractConvertFunction implements Validatable {
+public abstract class FoldablesConvertFunction extends AbstractConvertFunction implements PostOptimizationVerificationAware {
 
     protected FoldablesConvertFunction(Source source, Expression field) {
         super(source, field);
@@ -65,12 +66,12 @@ protected final Map<DataType, BuildFactory> factories() {
     }
 
     @Override
-    public final Object fold() {
-        return foldToTemporalAmount(field(), sourceText(), dataType());
+    public final Object fold(FoldContext ctx) {
+        return foldToTemporalAmount(ctx, field(), sourceText(), dataType());
     }
 
     @Override
-    public final void validate(Failures failures) {
+    public final void postLogicalOptimizationVerification(Failures failures) {
         failures.add(isFoldable(field(), sourceText(), null));
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64.java
index 7f9d0d3f2e647..832c511a2dc50 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64.java
@@ -30,6 +30,7 @@
 import java.util.Base64;
 import java.util.List;
 
+import static org.elasticsearch.compute.ann.Fixed.Scope.THREAD_LOCAL;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
 import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
 
@@ -85,7 +86,7 @@ protected NodeInfo<? extends Expression> info() {
     }
 
     @Evaluator()
-    static BytesRef process(BytesRef field, @Fixed(includeInToString = false, build = true) BytesRefBuilder oScratch) {
+    static BytesRef process(BytesRef field, @Fixed(includeInToString = false, scope = THREAD_LOCAL) BytesRefBuilder oScratch) {
         byte[] bytes = new byte[field.length];
         System.arraycopy(field.bytes, field.offset, bytes, 0, field.length);
         oScratch.grow(field.length);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64.java
index c23cef31f32f5..e78968bb209b6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64.java
@@ -30,6 +30,7 @@
 import java.util.Base64;
 import java.util.List;
 
+import static org.elasticsearch.compute.ann.Fixed.Scope.THREAD_LOCAL;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
 import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
 
@@ -78,7 +79,7 @@ protected NodeInfo<? extends Expression> info() {
     }
 
     @Evaluator(warnExceptions = { ArithmeticException.class })
-    static BytesRef process(BytesRef field, @Fixed(includeInToString = false, build = true) BytesRefBuilder oScratch) {
+    static BytesRef process(BytesRef field, @Fixed(includeInToString = false, scope = THREAD_LOCAL) BytesRefBuilder oScratch) {
         int outLength = Math.multiplyExact(4, (Math.addExact(field.length, 2) / 3));
         byte[] bytes = new byte[field.length];
         System.arraycopy(field.bytes, field.offset, bytes, 0, field.length);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java
index 8c4375b424cdc..837475fca7e74 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java
@@ -63,9 +63,9 @@ public class ToDateNanos extends AbstractConvertFunction {
     @FunctionInfo(
         returnType = "date_nanos",
         description = "Converts an input to a nanosecond-resolution date value (aka date_nanos).",
-        note = "The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z.  Additionally, integers "
-            + "cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.",
-        preview = true
+        note = "The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z, attepting to convert"
+            + "values outside of that range will result in null with a warning..  Additionally, integers cannot be converted into date "
+            + "nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch."
     )
     public ToDateNanos(
         Source source,
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java
index f6a23a5d5962e..b588832aba4cb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java
@@ -232,7 +232,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
 
         if (unit.foldable()) {
             try {
-                Part datePartField = Part.resolve(((BytesRef) unit.fold()).utf8ToString());
+                Part datePartField = Part.resolve(((BytesRef) unit.fold(toEvaluator.foldCtx())).utf8ToString());
                 return new DateDiffConstantEvaluator.Factory(source(), datePartField, startTimestampEvaluator, endTimestampEvaluator);
             } catch (IllegalArgumentException e) {
                 throw new InvalidArgumentException("invalid unit format for [{}]: {}", sourceText(), e.getMessage());
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java
index 501dfd431f106..7fc5d82441802 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -110,9 +111,9 @@ public String getWriteableName() {
     public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
         var fieldEvaluator = toEvaluator.apply(children().get(1));
         if (children().get(0).foldable()) {
-            ChronoField chrono = chronoField();
+            ChronoField chrono = chronoField(toEvaluator.foldCtx());
             if (chrono == null) {
-                BytesRef field = (BytesRef) children().get(0).fold();
+                BytesRef field = (BytesRef) children().get(0).fold(toEvaluator.foldCtx());
                 throw new InvalidArgumentException("invalid date field for [{}]: {}", sourceText(), field.utf8ToString());
             }
             return new DateExtractConstantEvaluator.Factory(source(), fieldEvaluator, chrono, configuration().zoneId());
@@ -121,14 +122,14 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
         return new DateExtractEvaluator.Factory(source(), fieldEvaluator, chronoEvaluator, configuration().zoneId());
     }
 
-    private ChronoField chronoField() {
+    private ChronoField chronoField(FoldContext ctx) {
         // chronoField's never checked (the return is). The foldability test is done twice and type is checked in resolveType() already.
         // TODO: move the slimmed down code here to toEvaluator?
         if (chronoField == null) {
             Expression field = children().get(0);
             try {
                 if (field.foldable() && DataType.isString(field.dataType())) {
-                    chronoField = (ChronoField) STRING_TO_CHRONO_FIELD.convert(field.fold());
+                    chronoField = (ChronoField) STRING_TO_CHRONO_FIELD.convert(field.fold(ctx));
                 }
             } catch (Exception e) {
                 return null;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java
index 60bc014ccbeec..29648d55cadd8 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java
@@ -55,7 +55,7 @@ public class DateFormat extends EsqlConfigurationFunction implements OptionalArg
     )
     public DateFormat(
         Source source,
-        @Param(optional = true, name = "dateFormat", type = { "keyword", "text" }, description = """
+        @Param(optional = true, name = "dateFormat", type = { "keyword", "text", "date" }, description = """
             Date format (optional).  If no format is specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used.
             If `null`, the function returns `null`.""") Expression format,
         @Param(name = "date", type = { "date" }, description = "Date expression. If `null`, the function returns `null`.") Expression date,
@@ -147,7 +147,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
             throw new IllegalArgumentException("unsupported data type for format [" + format.dataType() + "]");
         }
         if (format.foldable()) {
-            DateFormatter formatter = toFormatter(format.fold(), configuration().locale());
+            DateFormatter formatter = toFormatter(format.fold(toEvaluator.foldCtx()), configuration().locale());
             return new DateFormatConstantEvaluator.Factory(source(), fieldEvaluator, formatter);
         }
         var formatEvaluator = toEvaluator.apply(format);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java
index 1aaa227c3846e..7c38b54ed232b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java
@@ -28,14 +28,12 @@
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
 import java.io.IOException;
-import java.time.ZoneId;
 import java.util.List;
 
 import static org.elasticsearch.common.time.DateFormatter.forPattern;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
-import static org.elasticsearch.xpack.esql.core.util.DateUtils.UTC;
 import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong;
@@ -130,13 +128,12 @@ public static long process(BytesRef val, @Fixed DateFormatter formatter) throws
     }
 
     @Evaluator(warnExceptions = { IllegalArgumentException.class })
-    static long process(BytesRef val, BytesRef formatter, @Fixed ZoneId zoneId) throws IllegalArgumentException {
-        return dateTimeToLong(val.utf8ToString(), toFormatter(formatter, zoneId));
+    static long process(BytesRef val, BytesRef formatter) throws IllegalArgumentException {
+        return dateTimeToLong(val.utf8ToString(), toFormatter(formatter));
     }
 
     @Override
     public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
-        ZoneId zone = UTC; // TODO session timezone?
         ExpressionEvaluator.Factory fieldEvaluator = toEvaluator.apply(field);
         if (format == null) {
             return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, DEFAULT_DATE_TIME_FORMATTER);
@@ -146,18 +143,18 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
         }
         if (format.foldable()) {
             try {
-                DateFormatter formatter = toFormatter(format.fold(), zone);
+                DateFormatter formatter = toFormatter(format.fold(toEvaluator.foldCtx()));
                 return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, formatter);
             } catch (IllegalArgumentException e) {
                 throw new InvalidArgumentException(e, "invalid date pattern for [{}]: {}", sourceText(), e.getMessage());
             }
         }
         ExpressionEvaluator.Factory formatEvaluator = toEvaluator.apply(format);
-        return new DateParseEvaluator.Factory(source(), fieldEvaluator, formatEvaluator, zone);
+        return new DateParseEvaluator.Factory(source(), fieldEvaluator, formatEvaluator);
     }
 
-    private static DateFormatter toFormatter(Object format, ZoneId zone) {
-        return forPattern(((BytesRef) format).utf8ToString()).withZone(zone);
+    private static DateFormatter toFormatter(Object format) {
+        return forPattern(((BytesRef) format).utf8ToString());
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java
index a35b67d7ac3fd..7983c38cc4288 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java
@@ -225,7 +225,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
         }
         Object foldedInterval;
         try {
-            foldedInterval = interval.fold();
+            foldedInterval = interval.fold(toEvaluator.foldCtx());
             if (foldedInterval == null) {
                 throw new IllegalArgumentException("Interval cannot not be null");
             }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java
index d259fc6ae57ce..74c2da450995c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.compute.ann.Fixed;
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -59,7 +60,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Object fold() {
+    public Object fold(FoldContext ctx) {
         return now;
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java
index 26e75e752f681..5fc61c5c07b58 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java
@@ -30,6 +30,7 @@
 import java.util.Arrays;
 import java.util.List;
 
+import static org.elasticsearch.compute.ann.Fixed.Scope.THREAD_LOCAL;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.THIRD;
@@ -138,7 +139,7 @@ static BytesRef process(
         BytesRef ip,
         int prefixLengthV4,
         int prefixLengthV6,
-        @Fixed(includeInToString = false, build = true) BytesRef scratch
+        @Fixed(includeInToString = false, scope = THREAD_LOCAL) BytesRef scratch
     ) {
         if (prefixLengthV4 < 0 || prefixLengthV4 > 32) {
             throw new IllegalArgumentException("Prefix length v4 must be in range [0, 32], found " + prefixLengthV4);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/E.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/E.java
index 757b67b47ce72..e1eceef7ed1f5 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/E.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/E.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
@@ -49,7 +50,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Object fold() {
+    public Object fold(FoldContext ctx) {
         return Math.E;
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pi.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pi.java
index 90a4f1f091e91..32b7a0ab88b4e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pi.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pi.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
@@ -49,7 +50,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Object fold() {
+    public Object fold(FoldContext ctx) {
         return Math.PI;
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Tau.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Tau.java
index 17e5b027270d1..1a7669b7391e1 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Tau.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Tau.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
@@ -51,7 +52,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Object fold() {
+    public Object fold(FoldContext ctx) {
         return TAU;
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java
index 1996744a76567..26211258e6ca6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
@@ -91,8 +92,8 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
     }
 
     @Override
-    public Object fold() {
-        return EvaluatorMapper.super.fold();
+    public Object fold(FoldContext ctx) {
+        return EvaluatorMapper.super.fold(source(), ctx);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java
index cf49607893aae..d5093964145b7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java
@@ -33,6 +33,7 @@
 import java.util.Arrays;
 import java.util.List;
 
+import static org.elasticsearch.compute.ann.Fixed.Scope.THREAD_LOCAL;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isFoldable;
@@ -114,7 +115,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua
                 source(),
                 toEvaluator.apply(field),
                 ctx -> new CompensatedSum(),
-                (Double) p.fold()
+                (Double) p.fold(toEvaluator.foldCtx())
             );
             case NULL -> EvalOperator.CONSTANT_NULL_FACTORY;
             default -> throw EsqlIllegalArgumentException.illegalDataType(field.dataType());
@@ -144,7 +145,7 @@ static void process(
         DoubleBlock.Builder builder,
         int position,
         DoubleBlock block,
-        @Fixed(includeInToString = false, build = true) CompensatedSum sum,
+        @Fixed(includeInToString = false, scope = THREAD_LOCAL) CompensatedSum sum,
         @Fixed double p
     ) {
         sum.reset(0, 0);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java
index f3a63c835bd34..4e4aee307f1c7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java
@@ -35,6 +35,7 @@
 import java.util.Arrays;
 import java.util.List;
 
+import static org.elasticsearch.compute.ann.Fixed.Scope.THREAD_LOCAL;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
@@ -167,7 +168,7 @@ static void process(
         int position,
         DoubleBlock values,
         double percentile,
-        @Fixed(includeInToString = false, build = true) DoubleSortingScratch scratch
+        @Fixed(includeInToString = false, scope = THREAD_LOCAL) DoubleSortingScratch scratch
     ) {
         int valueCount = values.getValueCount(position);
         int firstValueIndex = values.getFirstValueIndex(position);
@@ -190,7 +191,7 @@ static void process(
         int position,
         IntBlock values,
         double percentile,
-        @Fixed(includeInToString = false, build = true) IntSortingScratch scratch
+        @Fixed(includeInToString = false, scope = THREAD_LOCAL) IntSortingScratch scratch
     ) {
         int valueCount = values.getValueCount(position);
         int firstValueIndex = values.getFirstValueIndex(position);
@@ -213,7 +214,7 @@ static void process(
         int position,
         LongBlock values,
         double percentile,
-        @Fixed(includeInToString = false, build = true) LongSortingScratch scratch
+        @Fixed(includeInToString = false, scope = THREAD_LOCAL) LongSortingScratch scratch
     ) {
         int valueCount = values.getValueCount(position);
         int firstValueIndex = values.getFirstValueIndex(position);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java
index f4f9679dc3704..4a04524d1b23d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java
@@ -187,8 +187,8 @@ public boolean foldable() {
     @Override
     public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
         if (start.foldable() && end.foldable()) {
-            int startOffset = stringToInt(String.valueOf(start.fold()));
-            int endOffset = stringToInt(String.valueOf(end.fold()));
+            int startOffset = stringToInt(String.valueOf(start.fold(toEvaluator.foldCtx())));
+            int endOffset = stringToInt(String.valueOf(end.fold(toEvaluator.foldCtx())));
             checkStartEnd(startOffset, endOffset);
         }
         return switch (PlannerUtils.toElementType(field.dataType())) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java
index 2286a1357ced8..b68718acfcd0a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java
@@ -29,10 +29,11 @@
 import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeDouble;
 import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeInt;
 import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeLong;
-import org.elasticsearch.xpack.esql.capabilities.Validatable;
+import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware;
 import org.elasticsearch.xpack.esql.common.Failure;
 import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -58,7 +59,7 @@
 /**
  * Sorts a multivalued field in lexicographical order.
  */
-public class MvSort extends EsqlScalarFunction implements OptionalArgument, Validatable {
+public class MvSort extends EsqlScalarFunction implements OptionalArgument, PostOptimizationVerificationAware {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvSort", MvSort::new);
 
     private final Expression field, order;
@@ -155,12 +156,12 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua
                     sourceText(),
                     ASC.value(),
                     DESC.value(),
-                    ((BytesRef) order.fold()).utf8ToString()
+                    ((BytesRef) order.fold(toEvaluator.foldCtx())).utf8ToString()
                 )
             );
         }
         if (order != null && order.foldable()) {
-            ordering = ((BytesRef) order.fold()).utf8ToString().equalsIgnoreCase((String) ASC.value());
+            ordering = ((BytesRef) order.fold(toEvaluator.foldCtx())).utf8ToString().equalsIgnoreCase((String) ASC.value());
         }
 
         return switch (PlannerUtils.toElementType(field.dataType())) {
@@ -230,7 +231,7 @@ public DataType dataType() {
     }
 
     @Override
-    public void validate(Failures failures) {
+    public void postLogicalOptimizationVerification(Failures failures) {
         if (order == null) {
             return;
         }
@@ -238,7 +239,14 @@ public void validate(Failures failures) {
         failures.add(isFoldable(order, operation, SECOND));
         if (isValidOrder() == false) {
             failures.add(
-                Failure.fail(order, INVALID_ORDER_ERROR, sourceText(), ASC.value(), DESC.value(), ((BytesRef) order.fold()).utf8ToString())
+                Failure.fail(
+                    order,
+                    INVALID_ORDER_ERROR,
+                    sourceText(),
+                    ASC.value(),
+                    DESC.value(),
+                    ((BytesRef) order.fold(FoldContext.small() /* TODO remove me */)).utf8ToString()
+                )
             );
         }
     }
@@ -246,7 +254,7 @@ public void validate(Failures failures) {
     private boolean isValidOrder() {
         boolean isValidOrder = true;
         if (order != null && order.foldable()) {
-            Object obj = order.fold();
+            Object obj = order.fold(FoldContext.small() /* TODO remove me */);
             String o = null;
             if (obj instanceof BytesRef ob) {
                 o = ob.utf8ToString();
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java
index eccc7ee4672c9..d0b646a622b60 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java
@@ -131,6 +131,7 @@
  *         the moment, but it's good to extend {@code AbstractScalarFunctionTestCase}. All of
  *         these tests are parameterized and expect to spend some time finding good parameters.
  *         Also add serialization tests that extend {@code AbstractExpressionSerializationTests<>}.
+ *         And also add type error tests that extends {@code ErrorsForCasesWithoutExamplesTestCase}.
  *     </li>
  *     <li>
  *         Once you are happy with the tests run the auto formatter:
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java
index 9189c6a7b8f70..b15d04aa792d9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.lucene.spatial.CoordinateEncoder;
 import org.elasticsearch.lucene.spatial.GeometryDocValueReader;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -214,10 +215,10 @@ protected NodeInfo<? extends Expression> info() {
     }
 
     @Override
-    public Object fold() {
+    public Object fold(FoldContext ctx) {
         try {
-            GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType(), left());
-            Geometry rightGeom = makeGeometryFromLiteral(right());
+            GeometryDocValueReader docValueReader = asGeometryDocValueReader(ctx, crsType(), left());
+            Geometry rightGeom = makeGeometryFromLiteral(ctx, right());
             Component2D[] components = asLuceneComponent2Ds(crsType(), rightGeom);
             return (crsType() == SpatialCrsType.GEO)
                 ? GEO.geometryRelatesGeometries(docValueReader, components)
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java
index ee78f50c4d6bd..3e16fa163fcd6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.lucene.spatial.CoordinateEncoder;
 import org.elasticsearch.lucene.spatial.GeometryDocValueReader;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -129,10 +130,10 @@ protected NodeInfo<? extends Expression> info() {
     }
 
     @Override
-    public Object fold() {
+    public Object fold(FoldContext ctx) {
         try {
-            GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType(), left());
-            Component2D component2D = asLuceneComponent2D(crsType(), right());
+            GeometryDocValueReader docValueReader = asGeometryDocValueReader(ctx, crsType(), left());
+            Component2D component2D = asLuceneComponent2D(ctx, crsType(), right());
             return (crsType() == SpatialCrsType.GEO)
                 ? GEO.geometryRelatesGeometry(docValueReader, component2D)
                 : CARTESIAN.geometryRelatesGeometry(docValueReader, component2D);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java
index 1a51af8dfeeb4..dcd53075cf69c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java
@@ -171,7 +171,11 @@ protected static class SpatialEvaluatorWithConstantFactory extends SpatialEvalua
 
         @Override
         public EvalOperator.ExpressionEvaluator.Factory get(SpatialSourceSupplier s, EvaluatorMapper.ToEvaluator toEvaluator) {
-            return factoryCreator.apply(s.source(), toEvaluator.apply(s.left()), asLuceneComponent2D(s.crsType(), s.right()));
+            return factoryCreator.apply(
+                s.source(),
+                toEvaluator.apply(s.left()),
+                asLuceneComponent2D(toEvaluator.foldCtx(), s.crsType(), s.right())
+            );
         }
     }
 
@@ -197,7 +201,11 @@ protected static class SpatialEvaluatorWithConstantArrayFactory extends SpatialE
 
         @Override
         public EvalOperator.ExpressionEvaluator.Factory get(SpatialSourceSupplier s, EvaluatorMapper.ToEvaluator toEvaluator) {
-            return factoryCreator.apply(s.source(), toEvaluator.apply(s.left()), asLuceneComponent2Ds(s.crsType(), s.right()));
+            return factoryCreator.apply(
+                s.source(),
+                toEvaluator.apply(s.left()),
+                asLuceneComponent2Ds(toEvaluator.foldCtx(), s.crsType(), s.right())
+            );
         }
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java
index 8d54e5ee443c2..601550cd173bb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.lucene.spatial.CoordinateEncoder;
 import org.elasticsearch.lucene.spatial.GeometryDocValueReader;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -127,10 +128,10 @@ protected NodeInfo<? extends Expression> info() {
     }
 
     @Override
-    public Object fold() {
+    public Object fold(FoldContext ctx) {
         try {
-            GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType(), left());
-            Component2D component2D = asLuceneComponent2D(crsType(), right());
+            GeometryDocValueReader docValueReader = asGeometryDocValueReader(ctx, crsType(), left());
+            Component2D component2D = asLuceneComponent2D(ctx, crsType(), right());
             return (crsType() == SpatialCrsType.GEO)
                 ? GEO.geometryRelatesGeometry(docValueReader, component2D)
                 : CARTESIAN.geometryRelatesGeometry(docValueReader, component2D);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java
index 6ae99ea8165cd..1b06c6dfd3dd5 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.lucene.spatial.GeometryDocValueReader;
 import org.elasticsearch.lucene.spatial.GeometryDocValueWriter;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes;
 
@@ -42,8 +43,8 @@
 public class SpatialRelatesUtils {
 
     /** Converts a {@link Expression} into a {@link Component2D}. */
-    static Component2D asLuceneComponent2D(BinarySpatialFunction.SpatialCrsType crsType, Expression expression) {
-        return asLuceneComponent2D(crsType, makeGeometryFromLiteral(expression));
+    static Component2D asLuceneComponent2D(FoldContext ctx, BinarySpatialFunction.SpatialCrsType crsType, Expression expression) {
+        return asLuceneComponent2D(crsType, makeGeometryFromLiteral(ctx, expression));
     }
 
     /** Converts a {@link Geometry} into a {@link Component2D}. */
@@ -66,8 +67,8 @@ static Component2D asLuceneComponent2D(BinarySpatialFunction.SpatialCrsType type
      * Converts a {@link Expression} at a given {@code position} into a {@link Component2D} array.
      * The reason for generating an array instead of a single component is for multi-shape support with ST_CONTAINS.
      */
-    static Component2D[] asLuceneComponent2Ds(BinarySpatialFunction.SpatialCrsType crsType, Expression expression) {
-        return asLuceneComponent2Ds(crsType, makeGeometryFromLiteral(expression));
+    static Component2D[] asLuceneComponent2Ds(FoldContext ctx, BinarySpatialFunction.SpatialCrsType crsType, Expression expression) {
+        return asLuceneComponent2Ds(crsType, makeGeometryFromLiteral(ctx, expression));
     }
 
     /**
@@ -90,9 +91,12 @@ static Component2D[] asLuceneComponent2Ds(BinarySpatialFunction.SpatialCrsType t
     }
 
     /** Converts a {@link Expression} into a {@link GeometryDocValueReader} */
-    static GeometryDocValueReader asGeometryDocValueReader(BinarySpatialFunction.SpatialCrsType crsType, Expression expression)
-        throws IOException {
-        Geometry geometry = makeGeometryFromLiteral(expression);
+    static GeometryDocValueReader asGeometryDocValueReader(
+        FoldContext ctx,
+        BinarySpatialFunction.SpatialCrsType crsType,
+        Expression expression
+    ) throws IOException {
+        Geometry geometry = makeGeometryFromLiteral(ctx, expression);
         if (crsType == BinarySpatialFunction.SpatialCrsType.GEO) {
             return asGeometryDocValueReader(
                 CoordinateEncoder.GEO,
@@ -167,8 +171,8 @@ private static Geometry asGeometry(BytesRefBlock valueBlock, int position) {
      * This function is used in two places, when evaluating a spatial constant in the SpatialRelatesFunction, as well as when
      * we do lucene-pushdown of spatial functions.
      */
-    public static Geometry makeGeometryFromLiteral(Expression expr) {
-        return makeGeometryFromLiteralValue(valueOf(expr), expr.dataType());
+    public static Geometry makeGeometryFromLiteral(FoldContext ctx, Expression expr) {
+        return makeGeometryFromLiteralValue(valueOf(ctx, expr), expr.dataType());
     }
 
     private static Geometry makeGeometryFromLiteralValue(Object value, DataType dataType) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java
index 2005709cd37e9..9fcece1ce65bc 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.lucene.spatial.CoordinateEncoder;
 import org.elasticsearch.lucene.spatial.GeometryDocValueReader;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -129,10 +130,10 @@ protected NodeInfo<? extends Expression> info() {
     }
 
     @Override
-    public Object fold() {
+    public Object fold(FoldContext ctx) {
         try {
-            GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType(), left());
-            Component2D component2D = asLuceneComponent2D(crsType(), right());
+            GeometryDocValueReader docValueReader = asGeometryDocValueReader(ctx, crsType(), left());
+            Component2D component2D = asLuceneComponent2D(ctx, crsType(), right());
             return (crsType() == SpatialCrsType.GEO)
                 ? GEO.geometryRelatesGeometry(docValueReader, component2D)
                 : CARTESIAN.geometryRelatesGeometry(docValueReader, component2D);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java
index 3cf042a2db828..f0c25e3289cc1 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.lucene.spatial.CoordinateEncoder;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -280,18 +281,18 @@ protected NodeInfo<? extends Expression> info() {
     }
 
     @Override
-    public Object fold() {
-        var leftGeom = makeGeometryFromLiteral(left());
-        var rightGeom = makeGeometryFromLiteral(right());
+    public Object fold(FoldContext ctx) {
+        var leftGeom = makeGeometryFromLiteral(ctx, left());
+        var rightGeom = makeGeometryFromLiteral(ctx, right());
         return (crsType() == SpatialCrsType.GEO) ? GEO.distance(leftGeom, rightGeom) : CARTESIAN.distance(leftGeom, rightGeom);
     }
 
     @Override
     public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
         if (right().foldable()) {
-            return toEvaluator(toEvaluator, left(), makeGeometryFromLiteral(right()), leftDocValues);
+            return toEvaluator(toEvaluator, left(), makeGeometryFromLiteral(toEvaluator.foldCtx(), right()), leftDocValues);
         } else if (left().foldable()) {
-            return toEvaluator(toEvaluator, right(), makeGeometryFromLiteral(left()), rightDocValues);
+            return toEvaluator(toEvaluator, right(), makeGeometryFromLiteral(toEvaluator.foldCtx(), left()), rightDocValues);
         } else {
             EvalOperator.ExpressionEvaluator.Factory leftE = toEvaluator.apply(left());
             EvalOperator.ExpressionEvaluator.Factory rightE = toEvaluator.apply(right());
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractHashFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractHashFunction.java
new file mode 100644
index 0000000000000..a39f0a4f32db2
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractHashFunction.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.Hash.HashFunction;
+
+import java.io.IOException;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
+
+public abstract class AbstractHashFunction extends UnaryScalarFunction {
+
+    protected AbstractHashFunction(Source source, Expression field) {
+        super(source, field);
+    }
+
+    protected AbstractHashFunction(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    protected abstract HashFunction getHashFunction();
+
+    @Override
+    public DataType dataType() {
+        return DataType.KEYWORD;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (childrenResolved() == false) {
+            return new TypeResolution("Unresolved children");
+        }
+        return isString(field, sourceText(), DEFAULT);
+    }
+
+    @Override
+    public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
+        return new HashConstantEvaluator.Factory(
+            source(),
+            context -> new BreakingBytesRefBuilder(context.breaker(), "hash"),
+            new Function<>() {
+                @Override
+                public HashFunction apply(DriverContext context) {
+                    return getHashFunction().copy();
+                }
+
+                @Override
+                public String toString() {
+                    return getHashFunction().toString();
+                }
+            },
+            toEvaluator.apply(field)
+        );
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ChangeCase.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ChangeCase.java
new file mode 100644
index 0000000000000..fe9a2d5beb3cf
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ChangeCase.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.compute.ann.Evaluator;
+import org.elasticsearch.compute.ann.Fixed;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction;
+import org.elasticsearch.xpack.esql.session.Configuration;
+
+import java.util.List;
+import java.util.Locale;
+
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
+
+public abstract class ChangeCase extends EsqlConfigurationFunction {
+
+    public enum Case {
+        UPPER {
+            @Override
+            String process(String value, Locale locale) {
+                return value.toUpperCase(locale);
+            }
+
+            @Override
+            public boolean matchesCase(String value) {
+                return value.codePoints().allMatch(cp -> Character.getType(cp) != Character.LOWERCASE_LETTER);
+            }
+        },
+        LOWER {
+            @Override
+            String process(String value, Locale locale) {
+                return value.toLowerCase(locale);
+            }
+
+            @Override
+            public boolean matchesCase(String value) {
+                return value.codePoints().allMatch(cp -> Character.getType(cp) != Character.UPPERCASE_LETTER);
+            }
+        };
+
+        abstract String process(String value, Locale locale);
+
+        public abstract boolean matchesCase(String value);
+    }
+
+    private final Expression field;
+    private final Case caseType;
+
+    protected ChangeCase(Source source, Expression field, Configuration configuration, Case caseType) {
+        super(source, List.of(field), configuration);
+        this.field = field;
+        this.caseType = caseType;
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataType.KEYWORD;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (childrenResolved() == false) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        return isString(field, sourceText(), DEFAULT);
+    }
+
+    @Override
+    public boolean foldable() {
+        return field.foldable();
+    }
+
+    public Expression field() {
+        return field;
+    }
+
+    public Case caseType() {
+        return caseType;
+    }
+
+    public abstract Expression replaceChild(Expression child);
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        assert newChildren.size() == 1;
+        return replaceChild(newChildren.get(0));
+    }
+
+    @Evaluator
+    static BytesRef process(BytesRef val, @Fixed Locale locale, @Fixed Case caseType) {
+        return BytesRefs.toBytesRef(caseType.process(val.utf8ToString(), locale));
+    }
+
+    @Override
+    public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
+        var fieldEvaluator = toEvaluator.apply(field);
+        return new ChangeCaseEvaluator.Factory(source(), fieldEvaluator, configuration().locale(), caseType);
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java
index 46ecc9e026d3d..eb173029876d3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java
@@ -32,6 +32,7 @@
 import java.util.stream.Stream;
 
 import static org.elasticsearch.common.unit.ByteSizeUnit.MB;
+import static org.elasticsearch.compute.ann.Fixed.Scope.THREAD_LOCAL;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
 
@@ -111,7 +112,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
     }
 
     @Evaluator
-    static BytesRef process(@Fixed(includeInToString = false, build = true) BreakingBytesRefBuilder scratch, BytesRef[] values) {
+    static BytesRef process(@Fixed(includeInToString = false, scope = THREAD_LOCAL) BreakingBytesRefBuilder scratch, BytesRef[] values) {
         scratch.grow(checkedTotalLength(values));
         scratch.clear();
         for (int i = 0; i < values.length; i++) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java
new file mode 100644
index 0000000000000..be0a7b2fe27b2
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java
@@ -0,0 +1,227 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.compute.ann.Evaluator;
+import org.elasticsearch.compute.ann.Fixed;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.Example;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
+import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+
+import java.io.IOException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.List;
+import java.util.function.Function;
+
+import static org.elasticsearch.compute.ann.Fixed.Scope.THREAD_LOCAL;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
+
+public class Hash extends EsqlScalarFunction {
+
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Hash", Hash::new);
+
+    private final Expression algorithm;
+    private final Expression input;
+
+    @FunctionInfo(
+        returnType = "keyword",
+        description = "Computes the hash of the input using various algorithms such as MD5, SHA, SHA-224, SHA-256, SHA-384, SHA-512.",
+        examples = { @Example(file = "hash", tag = "hash") }
+    )
+    public Hash(
+        Source source,
+        @Param(name = "algorithm", type = { "keyword", "text" }, description = "Hash algorithm to use.") Expression algorithm,
+        @Param(name = "input", type = { "keyword", "text" }, description = "Input to hash.") Expression input
+    ) {
+        super(source, List.of(algorithm, input));
+        this.algorithm = algorithm;
+        this.input = input;
+    }
+
+    private Hash(StreamInput in) throws IOException {
+        this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        source().writeTo(out);
+        out.writeNamedWriteable(algorithm);
+        out.writeNamedWriteable(input);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataType.KEYWORD;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (childrenResolved() == false) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution resolution = isString(algorithm, sourceText(), FIRST);
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+
+        return isString(input, sourceText(), SECOND);
+    }
+
+    @Override
+    public boolean foldable() {
+        return algorithm.foldable() && input.foldable();
+    }
+
+    @Evaluator(warnExceptions = NoSuchAlgorithmException.class)
+    static BytesRef process(
+        @Fixed(includeInToString = false, scope = THREAD_LOCAL) BreakingBytesRefBuilder scratch,
+        BytesRef algorithm,
+        BytesRef input
+    ) throws NoSuchAlgorithmException {
+        return hash(scratch, MessageDigest.getInstance(algorithm.utf8ToString()), input);
+    }
+
+    @Evaluator(extraName = "Constant")
+    static BytesRef processConstant(
+        @Fixed(includeInToString = false, scope = THREAD_LOCAL) BreakingBytesRefBuilder scratch,
+        @Fixed(scope = THREAD_LOCAL) HashFunction algorithm,
+        BytesRef input
+    ) {
+        return hash(scratch, algorithm.digest, input);
+    }
+
+    private static BytesRef hash(BreakingBytesRefBuilder scratch, MessageDigest algorithm, BytesRef input) {
+        algorithm.reset();
+        algorithm.update(input.bytes, input.offset, input.length);
+        var digest = algorithm.digest();
+        scratch.clear();
+        scratch.grow(digest.length * 2);
+        appendUtf8HexDigest(scratch, digest);
+        return scratch.bytesRefView();
+    }
+
+    private static final byte[] ASCII_HEX_BYTES = new byte[] { 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 97, 98, 99, 100, 101, 102 };
+
+    /**
+     * This function allows to append hex bytes dirrectly to the {@link BreakingBytesRefBuilder}
+     * bypassing unnecessary array allocations and byte array copying.
+     */
+    private static void appendUtf8HexDigest(BreakingBytesRefBuilder scratch, byte[] bytes) {
+        for (byte b : bytes) {
+            scratch.append(ASCII_HEX_BYTES[b >> 4 & 0xf]);
+            scratch.append(ASCII_HEX_BYTES[b & 0xf]);
+        }
+    }
+
+    @Override
+    public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
+        if (algorithm.foldable()) {
+            try {
+                // hash function is created here in order to validate the algorithm is valid before evaluator is created
+                var hf = HashFunction.create((BytesRef) algorithm.fold(toEvaluator.foldCtx()));
+                return new HashConstantEvaluator.Factory(
+                    source(),
+                    context -> new BreakingBytesRefBuilder(context.breaker(), "hash"),
+                    new Function<>() {
+                        @Override
+                        public HashFunction apply(DriverContext context) {
+                            return hf.copy();
+                        }
+
+                        @Override
+                        public String toString() {
+                            return hf.toString();
+                        }
+                    },
+                    toEvaluator.apply(input)
+                );
+            } catch (NoSuchAlgorithmException e) {
+                throw new InvalidArgumentException(e, "invalid algorithm for [{}]: {}", sourceText(), e.getMessage());
+            }
+        } else {
+            return new HashEvaluator.Factory(
+                source(),
+                context -> new BreakingBytesRefBuilder(context.breaker(), "hash"),
+                toEvaluator.apply(algorithm),
+                toEvaluator.apply(input)
+            );
+        }
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new Hash(source(), newChildren.get(0), newChildren.get(1));
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Hash::new, children().get(0), children().get(1));
+    }
+
+    public record HashFunction(String algorithm, MessageDigest digest) {
+
+        public static HashFunction create(String algorithm) {
+            try {
+                return new HashFunction(algorithm, MessageDigest.getInstance(algorithm));
+            } catch (NoSuchAlgorithmException e) {
+                assert false : "Expected to create a valid hashing algorithm";
+                throw new IllegalStateException(e);
+            }
+        }
+
+        public static HashFunction create(BytesRef literal) throws NoSuchAlgorithmException {
+            var algorithm = literal.utf8ToString();
+            return new HashFunction(algorithm, MessageDigest.getInstance(algorithm));
+        }
+
+        public HashFunction copy() {
+            try {
+                return new HashFunction(algorithm, MessageDigest.getInstance(algorithm));
+            } catch (NoSuchAlgorithmException e) {
+                assert false : "Algorithm should be valid at this point";
+                throw new IllegalStateException(e);
+            }
+        }
+
+        @Override
+        public String toString() {
+            return algorithm;
+        }
+    }
+
+    Expression algorithm() {
+        return algorithm;
+    }
+
+    Expression input() {
+        return input;
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java
index e7572caafd8f5..0d885e3f3c341 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java
@@ -30,6 +30,7 @@
 import java.util.Arrays;
 import java.util.List;
 
+import static org.elasticsearch.compute.ann.Fixed.Scope.THREAD_LOCAL;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
@@ -77,8 +78,8 @@ public String getWriteableName() {
 
     @Evaluator
     static BytesRef process(
-        @Fixed(includeInToString = false, build = true) BytesRef out,
-        @Fixed(includeInToString = false, build = true) UnicodeUtil.UTF8CodePoint cp,
+        @Fixed(includeInToString = false, scope = THREAD_LOCAL) BytesRef out,
+        @Fixed(includeInToString = false, scope = THREAD_LOCAL) UnicodeUtil.UTF8CodePoint cp,
         BytesRef str,
         int length
     ) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java
new file mode 100644
index 0000000000000..b42ec1036cb5b
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Example;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.Hash.HashFunction;
+
+import java.io.IOException;
+import java.util.List;
+
+public class Md5 extends AbstractHashFunction {
+
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MD5", Md5::new);
+
+    private static final HashFunction MD5 = HashFunction.create("MD5");
+
+    @FunctionInfo(
+        returnType = "keyword",
+        description = "Computes the MD5 hash of the input.",
+        examples = { @Example(file = "hash", tag = "md5") }
+    )
+    public Md5(Source source, @Param(name = "input", type = { "keyword", "text" }, description = "Input to hash.") Expression input) {
+        super(source, input);
+    }
+
+    private Md5(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    protected HashFunction getHashFunction() {
+        return MD5;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new Md5(source(), newChildren.get(0));
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Md5::new, field);
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java
index 996c90a8e40bc..fb0aac0c85b38 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -94,8 +95,8 @@ protected TypeResolution resolveType() {
     }
 
     @Override
-    public Boolean fold() {
-        return (Boolean) EvaluatorMapper.super.fold();
+    public Boolean fold(FoldContext ctx) {
+        return (Boolean) EvaluatorMapper.super.fold(source(), ctx);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java
index 2cc14399df2ae..363991d1556f1 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java
@@ -31,6 +31,7 @@
 import java.util.List;
 
 import static org.elasticsearch.common.unit.ByteSizeUnit.MB;
+import static org.elasticsearch.compute.ann.Fixed.Scope.THREAD_LOCAL;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
@@ -101,7 +102,7 @@ public boolean foldable() {
 
     @Evaluator(extraName = "Constant", warnExceptions = { IllegalArgumentException.class })
     static BytesRef processConstantNumber(
-        @Fixed(includeInToString = false, build = true) BreakingBytesRefBuilder scratch,
+        @Fixed(includeInToString = false, scope = THREAD_LOCAL) BreakingBytesRefBuilder scratch,
         BytesRef str,
         @Fixed int number
     ) {
@@ -109,7 +110,11 @@ static BytesRef processConstantNumber(
     }
 
     @Evaluator(warnExceptions = { IllegalArgumentException.class })
-    static BytesRef process(@Fixed(includeInToString = false, build = true) BreakingBytesRefBuilder scratch, BytesRef str, int number) {
+    static BytesRef process(
+        @Fixed(includeInToString = false, scope = THREAD_LOCAL) BreakingBytesRefBuilder scratch,
+        BytesRef str,
+        int number
+    ) {
         if (number < 0) {
             throw new IllegalArgumentException("Number parameter cannot be negative, found [" + number + "]");
         }
@@ -146,7 +151,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
         ExpressionEvaluator.Factory strExpr = toEvaluator.apply(str);
 
         if (number.foldable()) {
-            int num = (int) number.fold();
+            int num = (int) number.fold(toEvaluator.foldCtx());
             if (num < 0) {
                 throw new IllegalArgumentException("Number parameter cannot be negative, found [" + number + "]");
             }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java
index 4fa191244cb42..4b963b794aef0 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java
@@ -152,7 +152,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
         if (regex.foldable() && regex.dataType() == DataType.KEYWORD) {
             Pattern regexPattern;
             try {
-                regexPattern = Pattern.compile(((BytesRef) regex.fold()).utf8ToString());
+                regexPattern = Pattern.compile(((BytesRef) regex.fold(toEvaluator.foldCtx())).utf8ToString());
             } catch (PatternSyntaxException pse) {
                 // TODO this is not right (inconsistent). See also https://github.com/elastic/elasticsearch/issues/100038
                 // this should generate a header warning and return null (as do the rest of this functionality in evaluators),
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java
index b069b984ea81e..e0ebed29cca72 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java
@@ -30,6 +30,7 @@
 import java.util.Arrays;
 import java.util.List;
 
+import static org.elasticsearch.compute.ann.Fixed.Scope.THREAD_LOCAL;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
@@ -77,8 +78,8 @@ public String getWriteableName() {
 
     @Evaluator
     static BytesRef process(
-        @Fixed(includeInToString = false, build = true) BytesRef out,
-        @Fixed(includeInToString = false, build = true) UnicodeUtil.UTF8CodePoint cp,
+        @Fixed(includeInToString = false, scope = THREAD_LOCAL) BytesRef out,
+        @Fixed(includeInToString = false, scope = THREAD_LOCAL) UnicodeUtil.UTF8CodePoint cp,
         BytesRef str,
         int length
     ) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1.java
new file mode 100644
index 0000000000000..ba1f62562c03a
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Example;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
+
+import java.io.IOException;
+import java.util.List;
+
+public class Sha1 extends AbstractHashFunction {
+
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "SHA1", Sha1::new);
+
+    private static final Hash.HashFunction SHA1 = Hash.HashFunction.create("SHA1");
+
+    @FunctionInfo(
+        returnType = "keyword",
+        description = "Computes the SHA1 hash of the input.",
+        examples = { @Example(file = "hash", tag = "sha1") }
+    )
+    public Sha1(Source source, @Param(name = "input", type = { "keyword", "text" }, description = "Input to hash.") Expression input) {
+        super(source, input);
+    }
+
+    private Sha1(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    protected Hash.HashFunction getHashFunction() {
+        return SHA1;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new Sha1(source(), newChildren.get(0));
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Sha1::new, field);
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256.java
new file mode 100644
index 0000000000000..b16767a3f7948
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Example;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
+
+import java.io.IOException;
+import java.util.List;
+
+public class Sha256 extends AbstractHashFunction {
+
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "SHA256", Sha256::new);
+
+    private static final Hash.HashFunction SHA256 = Hash.HashFunction.create("SHA256");
+
+    @FunctionInfo(
+        returnType = "keyword",
+        description = "Computes the SHA256 hash of the input.",
+        examples = { @Example(file = "hash", tag = "sha256") }
+    )
+    public Sha256(Source source, @Param(name = "input", type = { "keyword", "text" }, description = "Input to hash.") Expression input) {
+        super(source, input);
+    }
+
+    private Sha256(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    protected Hash.HashFunction getHashFunction() {
+        return SHA256;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new Sha256(source(), newChildren.get(0));
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Sha256::new, field);
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java
index 6481ce5764e1f..e46c0a730431d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java
@@ -31,6 +31,7 @@
 import java.util.List;
 
 import static org.elasticsearch.common.unit.ByteSizeUnit.MB;
+import static org.elasticsearch.compute.ann.Fixed.Scope.THREAD_LOCAL;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
 import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
@@ -82,7 +83,7 @@ protected TypeResolution resolveType() {
     }
 
     @Evaluator(warnExceptions = { IllegalArgumentException.class })
-    static BytesRef process(@Fixed(includeInToString = false, build = true) BreakingBytesRefBuilder scratch, int number) {
+    static BytesRef process(@Fixed(includeInToString = false, scope = THREAD_LOCAL) BreakingBytesRefBuilder scratch, int number) {
         checkNumber(number);
         scratch.grow(number);
         scratch.setLength(number);
@@ -112,7 +113,7 @@ protected NodeInfo<? extends Expression> info() {
     @Override
     public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
         if (field.foldable()) {
-            Object folded = field.fold();
+            Object folded = field.fold(toEvaluator.foldCtx());
             if (folded instanceof Integer num) {
                 checkNumber(num);
                 return toEvaluator.apply(new Literal(source(), " ".repeat(num), KEYWORD));
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java
index b1f5da56d011b..d0c1035978ff3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -29,6 +30,7 @@
 
 import java.io.IOException;
 
+import static org.elasticsearch.compute.ann.Fixed.Scope.THREAD_LOCAL;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact;
@@ -101,8 +103,8 @@ public boolean foldable() {
     }
 
     @Override
-    public Object fold() {
-        return EvaluatorMapper.super.fold();
+    public Object fold(FoldContext ctx) {
+        return EvaluatorMapper.super.fold(source(), ctx);
     }
 
     @Evaluator(extraName = "SingleByte")
@@ -110,7 +112,7 @@ static void process(
         BytesRefBlock.Builder builder,
         BytesRef str,
         @Fixed byte delim,
-        @Fixed(includeInToString = false, build = true) BytesRef scratch
+        @Fixed(includeInToString = false, scope = THREAD_LOCAL) BytesRef scratch
     ) {
         scratch.bytes = str.bytes;
         scratch.offset = str.offset;
@@ -140,7 +142,7 @@ static void process(
         BytesRefBlock.Builder builder,
         BytesRef str,
         BytesRef delim,
-        @Fixed(includeInToString = false, build = true) BytesRef scratch
+        @Fixed(includeInToString = false, scope = THREAD_LOCAL) BytesRef scratch
     ) {
         checkDelimiter(delim);
         process(builder, str, delim.bytes[delim.offset], scratch);
@@ -162,7 +164,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
         if (right().foldable() == false) {
             return new SplitVariableEvaluator.Factory(source(), str, toEvaluator.apply(right()), context -> new BytesRef());
         }
-        BytesRef delim = (BytesRef) right().fold();
+        BytesRef delim = (BytesRef) right().fold(toEvaluator.foldCtx());
         checkDelimiter(delim);
         return new SplitSingleByteEvaluator.Factory(source(), str, delim.bytes[delim.offset], context -> new BytesRef());
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java
index 5f2bbcde52166..084afb1b69996 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java
@@ -7,37 +7,23 @@
 
 package org.elasticsearch.xpack.esql.expression.function.scalar.string;
 
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.lucene.BytesRefs;
-import org.elasticsearch.compute.ann.Evaluator;
-import org.elasticsearch.compute.ann.Fixed;
-import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
-import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.io.IOException;
-import java.util.List;
-import java.util.Locale;
 
-import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
-import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
-
-public class ToLower extends EsqlConfigurationFunction {
+public class ToLower extends ChangeCase {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "ToLower", ToLower::new);
 
-    private final Expression field;
-
     @FunctionInfo(
         returnType = { "keyword" },
         description = "Returns a new string representing the input string converted to lower case.",
@@ -52,8 +38,7 @@ public ToLower(
         ) Expression field,
         Configuration configuration
     ) {
-        super(source, List.of(field), configuration);
-        this.field = field;
+        super(source, field, configuration, Case.LOWER);
     }
 
     private ToLower(StreamInput in) throws IOException {
@@ -70,52 +55,12 @@ public String getWriteableName() {
         return ENTRY.name;
     }
 
-    @Override
-    public DataType dataType() {
-        return DataType.KEYWORD;
-    }
-
-    @Override
-    protected TypeResolution resolveType() {
-        if (childrenResolved() == false) {
-            return new TypeResolution("Unresolved children");
-        }
-
-        return isString(field, sourceText(), DEFAULT);
-    }
-
-    @Override
-    public boolean foldable() {
-        return field.foldable();
-    }
-
-    @Evaluator
-    static BytesRef process(BytesRef val, @Fixed Locale locale) {
-        return BytesRefs.toBytesRef(val.utf8ToString().toLowerCase(locale));
-    }
-
-    @Override
-    public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
-        var fieldEvaluator = toEvaluator.apply(field);
-        return new ToLowerEvaluator.Factory(source(), fieldEvaluator, configuration().locale());
-    }
-
-    public Expression field() {
-        return field;
-    }
-
     public ToLower replaceChild(Expression child) {
         return new ToLower(source(), child, configuration());
     }
 
-    @Override
-    public Expression replaceChildren(List<Expression> newChildren) {
-        assert newChildren.size() == 1;
-        return replaceChild(newChildren.get(0));
-    }
-
     @Override
     protected NodeInfo<? extends Expression> info() {
-        return NodeInfo.create(this, ToLower::new, field, configuration());
+        return NodeInfo.create(this, ToLower::new, field(), configuration());
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java
index 7fdd5e39f96f3..4509404754f36 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java
@@ -7,37 +7,23 @@
 
 package org.elasticsearch.xpack.esql.expression.function.scalar.string;
 
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.lucene.BytesRefs;
-import org.elasticsearch.compute.ann.Evaluator;
-import org.elasticsearch.compute.ann.Fixed;
-import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
-import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.io.IOException;
-import java.util.List;
-import java.util.Locale;
 
-import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
-import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
-
-public class ToUpper extends EsqlConfigurationFunction {
+public class ToUpper extends ChangeCase {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "ToUpper", ToUpper::new);
 
-    private final Expression field;
-
     @FunctionInfo(
         returnType = { "keyword" },
         description = "Returns a new string representing the input string converted to upper case.",
@@ -52,8 +38,7 @@ public ToUpper(
         ) Expression field,
         Configuration configuration
     ) {
-        super(source, List.of(field), configuration);
-        this.field = field;
+        super(source, field, configuration, Case.UPPER);
     }
 
     private ToUpper(StreamInput in) throws IOException {
@@ -62,7 +47,7 @@ private ToUpper(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeNamedWriteable(field);
+        out.writeNamedWriteable(field());
     }
 
     @Override
@@ -70,52 +55,12 @@ public String getWriteableName() {
         return ENTRY.name;
     }
 
-    @Override
-    public DataType dataType() {
-        return DataType.KEYWORD;
-    }
-
-    @Override
-    protected TypeResolution resolveType() {
-        if (childrenResolved() == false) {
-            return new TypeResolution("Unresolved children");
-        }
-
-        return isString(field, sourceText(), DEFAULT);
-    }
-
-    @Override
-    public boolean foldable() {
-        return field.foldable();
-    }
-
-    @Evaluator
-    static BytesRef process(BytesRef val, @Fixed Locale locale) {
-        return BytesRefs.toBytesRef(val.utf8ToString().toUpperCase(locale));
-    }
-
-    @Override
-    public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
-        var fieldEvaluator = toEvaluator.apply(field);
-        return new ToUpperEvaluator.Factory(source(), fieldEvaluator, configuration().locale());
-    }
-
-    public Expression field() {
-        return field;
-    }
-
     public ToUpper replaceChild(Expression child) {
         return new ToUpper(source(), child, configuration());
     }
 
-    @Override
-    public Expression replaceChildren(List<Expression> newChildren) {
-        assert newChildren.size() == 1;
-        return replaceChild(newChildren.get(0));
-    }
-
     @Override
     protected NodeInfo<? extends Expression> info() {
-        return NodeInfo.create(this, ToUpper::new, field, configuration());
+        return NodeInfo.create(this, ToUpper::new, field(), configuration());
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java
index d2edb0f92e8f2..65455c708cc9b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -100,8 +101,8 @@ protected TypeResolution resolveType() {
     }
 
     @Override
-    public Boolean fold() {
-        return (Boolean) EvaluatorMapper.super.fold();
+    public Boolean fold(FoldContext ctx) {
+        return (Boolean) EvaluatorMapper.super.fold(source(), ctx);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/util/Delay.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/util/Delay.java
index 1d03f09c86409..3b17133bf4974 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/util/Delay.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/util/Delay.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.compute.ann.Fixed;
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -84,15 +85,15 @@ public boolean foldable() {
     }
 
     @Override
-    public Object fold() {
+    public Object fold(FoldContext ctx) {
         return null;
     }
 
-    private long msValue() {
+    private long msValue(FoldContext ctx) {
         if (field().foldable() == false) {
             throw new IllegalArgumentException("function [" + sourceText() + "] has invalid argument [" + field().sourceText() + "]");
         }
-        var ms = field().fold();
+        var ms = field().fold(ctx);
         if (ms instanceof Duration duration) {
             return duration.toMillis();
         }
@@ -101,7 +102,7 @@ private long msValue() {
 
     @Override
     public ExpressionEvaluator.Factory toEvaluator(EvaluatorMapper.ToEvaluator toEvaluator) {
-        return new DelayEvaluator.Factory(source(), msValue());
+        return new DelayEvaluator.Factory(source(), msValue(toEvaluator.foldCtx()));
     }
 
     @Evaluator
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/FullTextPredicate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/FullTextPredicate.java
similarity index 97%
rename from x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/FullTextPredicate.java
rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/FullTextPredicate.java
index b23593804f8fe..1dd6f650828c3 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/FullTextPredicate.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/FullTextPredicate.java
@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.core.expression.predicate.fulltext;
+package org.elasticsearch.xpack.esql.expression.predicate.fulltext;
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/FullTextUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/FullTextUtils.java
similarity index 90%
rename from x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/FullTextUtils.java
rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/FullTextUtils.java
index 6ba2650314d04..32c8e70a0fde6 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/FullTextUtils.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/FullTextUtils.java
@@ -4,13 +4,13 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.core.expression.predicate.fulltext;
+package org.elasticsearch.xpack.esql.expression.predicate.fulltext;
 
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.xpack.esql.core.ParsingException;
-import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.FullTextPredicate.Operator;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.predicate.fulltext.FullTextPredicate.Operator;
+import org.elasticsearch.xpack.esql.parser.ParsingException;
 
 import java.util.LinkedHashMap;
 import java.util.Locale;
@@ -86,7 +86,7 @@ private static String[] splitInTwo(String string, String delimiter) {
         return split;
     }
 
-    static FullTextPredicate.Operator operator(Map<String, String> options, String key) {
+    static Operator operator(Map<String, String> options, String key) {
         String value = options.get(key);
         return value != null ? Operator.valueOf(value.toUpperCase(Locale.ROOT)) : null;
     }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/MatchQueryPredicate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/MatchQueryPredicate.java
similarity index 96%
rename from x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/MatchQueryPredicate.java
rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/MatchQueryPredicate.java
index f2e6088167ba5..66c6d8995b24e 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/MatchQueryPredicate.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/MatchQueryPredicate.java
@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.core.expression.predicate.fulltext;
+package org.elasticsearch.xpack.esql.expression.predicate.fulltext;
 
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/MultiMatchQueryPredicate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/MultiMatchQueryPredicate.java
similarity index 97%
rename from x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/MultiMatchQueryPredicate.java
rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/MultiMatchQueryPredicate.java
index 2d66023a1407d..5d165d9ea01f7 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/MultiMatchQueryPredicate.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/MultiMatchQueryPredicate.java
@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.core.expression.predicate.fulltext;
+package org.elasticsearch.xpack.esql.expression.predicate.fulltext;
 
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
index 8bb166fac60bb..424c080c905e3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.ExceptionUtils;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -111,7 +112,7 @@ protected TypeResolution checkCompatibility() {
 
     /**
      * Override this to allow processing literals of type {@link DataType#DATE_PERIOD} when folding constants.
-     * Used in {@link DateTimeArithmeticOperation#fold()}.
+     * Used in {@link DateTimeArithmeticOperation#fold}.
      * @param left the left period
      * @param right the right period
      * @return the result of the evaluation
@@ -120,7 +121,7 @@ protected TypeResolution checkCompatibility() {
 
     /**
      * Override this to allow processing literals of type {@link DataType#TIME_DURATION} when folding constants.
-     * Used in {@link DateTimeArithmeticOperation#fold()}.
+     * Used in {@link DateTimeArithmeticOperation#fold}.
      * @param left the left duration
      * @param right the right duration
      * @return the result of the evaluation
@@ -128,13 +129,13 @@ protected TypeResolution checkCompatibility() {
     abstract Duration fold(Duration left, Duration right);
 
     @Override
-    public final Object fold() {
+    public final Object fold(FoldContext ctx) {
         DataType leftDataType = left().dataType();
         DataType rightDataType = right().dataType();
         if (leftDataType == DATE_PERIOD && rightDataType == DATE_PERIOD) {
             // Both left and right expressions are temporal amounts; we can assume they are both foldable.
-            var l = left().fold();
-            var r = right().fold();
+            var l = left().fold(ctx);
+            var r = right().fold(ctx);
             if (l instanceof Collection<?> || r instanceof Collection<?>) {
                 return null;
             }
@@ -148,8 +149,8 @@ public final Object fold() {
         }
         if (leftDataType == TIME_DURATION && rightDataType == TIME_DURATION) {
             // Both left and right expressions are temporal amounts; we can assume they are both foldable.
-            Duration l = (Duration) left().fold();
-            Duration r = (Duration) right().fold();
+            Duration l = (Duration) left().fold(ctx);
+            Duration r = (Duration) right().fold(ctx);
             try {
                 return fold(l, r);
             } catch (ArithmeticException e) {
@@ -161,7 +162,7 @@ public final Object fold() {
         if (isNull(leftDataType) || isNull(rightDataType)) {
             return null;
         }
-        return super.fold();
+        return super.fold(ctx);
     }
 
     @Override
@@ -178,7 +179,11 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
                 temporalAmountArgument = left();
             }
 
-            return millisEvaluator.apply(source(), toEvaluator.apply(datetimeArgument), (TemporalAmount) temporalAmountArgument.fold());
+            return millisEvaluator.apply(
+                source(),
+                toEvaluator.apply(datetimeArgument),
+                (TemporalAmount) temporalAmountArgument.fold(toEvaluator.foldCtx())
+            );
         } else if (dataType() == DATE_NANOS) {
             // One of the arguments has to be a date_nanos and the other a temporal amount.
             Expression dateNanosArgument;
@@ -191,7 +196,11 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
                 temporalAmountArgument = left();
             }
 
-            return nanosEvaluator.apply(source(), toEvaluator.apply(dateNanosArgument), (TemporalAmount) temporalAmountArgument.fold());
+            return nanosEvaluator.apply(
+                source(),
+                toEvaluator.apply(dateNanosArgument),
+                (TemporalAmount) temporalAmountArgument.fold(toEvaluator.foldCtx())
+            );
         } else {
             return super.toEvaluator(toEvaluator);
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java
index 74394d796855f..e3248665ad486 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryArithmeticOperation;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -120,8 +121,8 @@ public interface BinaryEvaluator {
     }
 
     @Override
-    public Object fold() {
-        return EvaluatorMapper.super.fold();
+    public Object fold(FoldContext ctx) {
+        return EvaluatorMapper.super.fold(source(), ctx);
     }
 
     public DataType dataType() {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java
index fb32282005f02..6663ccf0ef7b6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.ExceptionUtils;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -87,12 +88,12 @@ else if (type == DataType.LONG) {
     }
 
     @Override
-    public final Object fold() {
+    public final Object fold(FoldContext ctx) {
         DataType dataType = field().dataType();
         // For date periods and time durations, we need to treat folding differently. These types are unrepresentable, so there is no
         // evaluator for them - but the default folding requires an evaluator.
         if (dataType == DATE_PERIOD) {
-            Period fieldValue = (Period) field().fold();
+            Period fieldValue = (Period) field().fold(ctx);
             try {
                 return fieldValue.negated();
             } catch (ArithmeticException e) {
@@ -102,7 +103,7 @@ public final Object fold() {
             }
         }
         if (dataType == TIME_DURATION) {
-            Duration fieldValue = (Duration) field().fold();
+            Duration fieldValue = (Duration) field().fold(ctx);
             try {
                 return fieldValue.negated();
             } catch (ArithmeticException e) {
@@ -111,7 +112,7 @@ public final Object fold() {
                 throw ExceptionUtils.math(source(), e);
             }
         }
-        return super.fold();
+        return super.fold(ctx);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java
index 3e2a21664aa7e..e56c19b26a902 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -204,8 +205,8 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua
     }
 
     @Override
-    public Boolean fold() {
-        return (Boolean) EvaluatorMapper.super.fold();
+    public Boolean fold(FoldContext ctx) {
+        return (Boolean) EvaluatorMapper.super.fold(source(), ctx);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java
index f6c23304c189b..6a35fa8e8b957 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java
@@ -11,10 +11,14 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.time.DateUtils;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.Vector;
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.Comparisons;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -38,6 +42,7 @@
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
 import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
@@ -50,6 +55,52 @@
 import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
 import static org.elasticsearch.xpack.esql.core.util.StringUtils.ordinal;
 
+/**
+ * The {@code IN} operator.
+ * <p>
+ *     This function has quite "unique" null handling rules around {@code null} and multivalued
+ *     fields. The {@code null} rules are inspired by PostgreSQL, and, presumably, every other
+ *     SQL implementation. The multivalue rules are pretty much an extension of the "multivalued
+ *     fields are like null in scalars" rule. Here's some examples:
+ * </p>
+ * <ul>
+ *     <li>{@code 'x' IN ('a', 'b', 'c')} => @{code false}</li>
+ *     <li>{@code 'x' IN ('a', 'x', 'c')} => @{code true}</li>
+ *     <li>{@code null IN ('a', 'b', 'c')} => @{code null}</li>
+ *     <li>{@code ['x', 'y'] IN ('a', 'b', 'c')} => @{code null} and a warning</li>
+ *     <li>{@code 'x' IN ('a', null, 'c')} => @{code null}</li>
+ *     <li>{@code 'x' IN ('x', null, 'c')} => @{code true}</li>
+ *     <li>{@code 'x' IN ('x', ['a', 'b'], 'c')} => @{code true} and a warning</li>
+ *     <li>{@code 'x' IN ('a', ['a', 'b'], 'c')} => @{code false} and a warning</li>
+ * </ul>
+ * <p>
+ *     And here's the decision tree for {@code WHERE x IN (a, b, c)}:
+ * </p>
+ * <ol>
+ *     <li>{@code x IS NULL} => return {@code null}</li>
+ *     <li>{@code MV_COUNT(x) > 1} => emit a warning and return {@code null}</li>
+ *     <li>{@code a IS NULL AND b IS NULL AND c IS NULL} => return {@code null}</li>
+ *     <li>{@code MV_COUNT(a) > 1 OR MV_COUNT(b) > 1 OR MV_COUNT(c) > 1} => emit a warning and continue</li>
+ *     <li>{@code MV_COUNT(a) > 1 AND MV_COUNT(b) > 1 AND MV_COUNT(c) > 1} => return {@code null}</li>
+ *     <li>{@code x == a OR x == b OR x == c} => return {@code true}</li>
+ *     <li>{@code a IS NULL OR b IS NULL OR c IS NULL} => return {@code null}</li>
+ *     <li>{@code else} => {@code false}</li>
+ * </ol>
+ * <p>
+ *     I believe the first five entries are *mostly* optimizations and making the
+ *     <a href="https://en.wikipedia.org/wiki/Three-valued_logic">Three-valued logic</a> of SQL
+ *     explicit and integrated with our multivalue field rules. And make all that work with the
+ *     actual evaluator code. You could probably shorten this to the last three points, but lots
+ *     of folks aren't familiar with SQL's three-valued logic anyway, so let's be explicit.
+ * </p>
+ * <p>
+ *     Because of this chain of logic we don't use the standard evaluator generators. They'd just
+ *     require too many special cases and nothing else quite works like this. I mean, everything
+ *     works just like this in that "three-valued logic" sort of way, but not in the "java code"
+ *     sort of way. So! Instead of using the standard evaluator generators we use the
+ *     String Template generators that we use for things like {@link Block} and {@link Vector}.
+ * </p>
+ */
 public class In extends EsqlScalarFunction {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "In", In::new);
 
@@ -157,11 +208,11 @@ public boolean foldable() {
     }
 
     @Override
-    public Object fold() {
+    public Object fold(FoldContext ctx) {
         if (Expressions.isGuaranteedNull(value) || list.stream().allMatch(Expressions::isGuaranteedNull)) {
             return null;
         }
-        return super.fold();
+        return super.fold(ctx);
     }
 
     protected boolean areCompatible(DataType left, DataType right) {
@@ -183,20 +234,47 @@ protected TypeResolution resolveType() { // TODO: move the foldability check fro
         }
 
         DataType dt = value.dataType();
-        for (int i = 0; i < list.size(); i++) {
-            Expression listValue = list.get(i);
-            if (areCompatible(dt, listValue.dataType()) == false) {
-                return new TypeResolution(
-                    format(
-                        null,
-                        "{} argument of [{}] must be [{}], found value [{}] type [{}]",
-                        ordinal(i + 1),
-                        sourceText(),
-                        dt.typeName(),
-                        Expressions.name(listValue),
-                        listValue.dataType().typeName()
-                    )
-                );
+        if (dt.isDate()) {
+            // If value is a date (nanos or millis), list cannot contain both nanos and millis
+            DataType seenDateType = null;
+            for (int i = 0; i < list.size(); i++) {
+                Expression listValue = list.get(i);
+                if (seenDateType == null && listValue.dataType().isDate()) {
+                    seenDateType = listValue.dataType();
+                }
+                // The areCompatible test is still necessary to account for nulls.
+                if ((listValue.dataType().isDate() && listValue.dataType() != seenDateType)
+                    || (listValue.dataType().isDate() == false && areCompatible(dt, listValue.dataType()) == false)) {
+                    return new TypeResolution(
+                        format(
+                            null,
+                            "{} argument of [{}] must be [{}], found value [{}] type [{}]",
+                            ordinal(i + 1),
+                            sourceText(),
+                            dt.typeName(),
+                            Expressions.name(listValue),
+                            listValue.dataType().typeName()
+                        )
+                    );
+                }
+            }
+
+        } else {
+            for (int i = 0; i < list.size(); i++) {
+                Expression listValue = list.get(i);
+                if (areCompatible(dt, listValue.dataType()) == false) {
+                    return new TypeResolution(
+                        format(
+                            null,
+                            "{} argument of [{}] must be [{}], found value [{}] type [{}]",
+                            ordinal(i + 1),
+                            sourceText(),
+                            dt.typeName(),
+                            Expressions.name(listValue),
+                            listValue.dataType().typeName()
+                        )
+                    );
+                }
             }
         }
 
@@ -213,9 +291,19 @@ protected Expression canonicalize() {
 
     @Override
     public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
-        var commonType = commonType();
         EvalOperator.ExpressionEvaluator.Factory lhs;
         EvalOperator.ExpressionEvaluator.Factory[] factories;
+        if (value.dataType() == DATE_NANOS && list.getFirst().dataType() == DATETIME) {
+            lhs = toEvaluator.apply(value);
+            factories = list.stream().map(toEvaluator::apply).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new);
+            return new InNanosMillisEvaluator.Factory(source(), lhs, factories);
+        }
+        if (value.dataType() == DATETIME && list.getFirst().dataType() == DATE_NANOS) {
+            lhs = toEvaluator.apply(value);
+            factories = list.stream().map(toEvaluator::apply).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new);
+            return new InMillisNanosEvaluator.Factory(source(), lhs, factories);
+        }
+        var commonType = commonType();
         if (commonType.isNumeric()) {
             lhs = Cast.cast(source(), value.dataType(), commonType, toEvaluator.apply(value));
             factories = list.stream()
@@ -235,7 +323,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua
         if (commonType == INTEGER) {
             return new InIntEvaluator.Factory(source(), lhs, factories);
         }
-        if (commonType == LONG || commonType == DATETIME || commonType == UNSIGNED_LONG) {
+        if (commonType == LONG || commonType == DATETIME || commonType == DATE_NANOS || commonType == UNSIGNED_LONG) {
             return new InLongEvaluator.Factory(source(), lhs, factories);
         }
         if (commonType == KEYWORD
@@ -297,6 +385,39 @@ static boolean process(BitSet nulls, BitSet mvs, long lhs, long[] rhs) {
         return false;
     }
 
+    /**
+     * Processor for mixed millisecond and nanosecond dates, where the "value" (aka lhs) is in nanoseconds
+     * and the "list" (aka rhs) is in milliseconds
+     */
+    static boolean processNanosMillis(BitSet nulls, BitSet mvs, long lhs, long[] rhs) {
+        for (int i = 0; i < rhs.length; i++) {
+            if ((nulls != null && nulls.get(i)) || (mvs != null && mvs.get(i))) {
+                continue;
+            }
+            if (DateUtils.compareNanosToMillis(lhs, rhs[i]) == 0) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     *  Processor for mixed millisecond and nanosecond dates, where the "value" (aka lhs) is in milliseoncds
+     *  and the "list" (aka rhs) is in nanoseconds
+     */
+    static boolean processMillisNanos(BitSet nulls, BitSet mvs, long lhs, long[] rhs) {
+        for (int i = 0; i < rhs.length; i++) {
+            if ((nulls != null && nulls.get(i)) || (mvs != null && mvs.get(i))) {
+                continue;
+            }
+            Boolean compResult = DateUtils.compareNanosToMillis(rhs[i], lhs) == 0;
+            if (compResult == Boolean.TRUE) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     static boolean process(BitSet nulls, BitSet mvs, double lhs, double[] rhs) {
         for (int i = 0; i < rhs.length; i++) {
             if ((nulls != null && nulls.get(i)) || (mvs != null && mvs.get(i))) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java
index c731e44197f2e..01564644bf5c7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.compute.ann.Evaluator;
 import org.elasticsearch.compute.ann.Fixed;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -77,9 +78,9 @@ public static Automaton automaton(BytesRef val) {
     }
 
     @Override
-    public Boolean fold() {
-        BytesRef leftVal = BytesRefs.toBytesRef(left().fold());
-        BytesRef rightVal = BytesRefs.toBytesRef(right().fold());
+    public Boolean fold(FoldContext ctx) {
+        BytesRef leftVal = BytesRefs.toBytesRef(left().fold(ctx));
+        BytesRef rightVal = BytesRefs.toBytesRef(right().fold(ctx));
         if (leftVal == null || rightVal == null) {
             return null;
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java
index f5704239993f9..7ea95c764f36c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.evaluator.mapper.ExpressionMapper;
@@ -28,15 +29,15 @@ public class InsensitiveEqualsMapper extends ExpressionMapper<InsensitiveEquals>
         InsensitiveEqualsEvaluator.Factory::new;
 
     @Override
-    public final ExpressionEvaluator.Factory map(InsensitiveEquals bc, Layout layout) {
+    public final ExpressionEvaluator.Factory map(FoldContext foldCtx, InsensitiveEquals bc, Layout layout) {
         DataType leftType = bc.left().dataType();
         DataType rightType = bc.right().dataType();
 
-        var leftEval = toEvaluator(bc.left(), layout);
-        var rightEval = toEvaluator(bc.right(), layout);
+        var leftEval = toEvaluator(foldCtx, bc.left(), layout);
+        var rightEval = toEvaluator(foldCtx, bc.right(), layout);
         if (DataType.isString(leftType)) {
             if (bc.right().foldable() && DataType.isString(rightType)) {
-                BytesRef rightVal = BytesRefs.toBytesRef(bc.right().fold());
+                BytesRef rightVal = BytesRefs.toBytesRef(bc.right().fold(FoldContext.small() /* TODO remove me */));
                 Automaton automaton = InsensitiveEquals.automaton(rightVal);
                 return dvrCtx -> new InsensitiveEqualsConstantEvaluator(
                     bc.source(),
@@ -51,13 +52,14 @@ public final ExpressionEvaluator.Factory map(InsensitiveEquals bc, Layout layout
     }
 
     public static ExpressionEvaluator.Factory castToEvaluator(
+        FoldContext foldCtx,
         InsensitiveEquals op,
         Layout layout,
         DataType required,
         TriFunction<Source, ExpressionEvaluator.Factory, ExpressionEvaluator.Factory, ExpressionEvaluator.Factory> factory
     ) {
-        var lhs = Cast.cast(op.source(), op.left().dataType(), required, toEvaluator(op.left(), layout));
-        var rhs = Cast.cast(op.source(), op.right().dataType(), required, toEvaluator(op.right(), layout));
+        var lhs = Cast.cast(op.source(), op.left().dataType(), required, toEvaluator(foldCtx, op.left(), layout));
+        var rhs = Cast.cast(op.source(), op.right().dataType(), required, toEvaluator(foldCtx, op.right(), layout));
         return factory.apply(op.source(), lhs, rhs);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/X-InEvaluator.java.st b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/X-InEvaluator.java.st
index f63815f475d8f..0cdea82119cae 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/X-InEvaluator.java.st
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/X-InEvaluator.java.st
@@ -42,7 +42,7 @@ import java.util.BitSet;
  * {@link EvalOperator.ExpressionEvaluator} implementation for {@link In}.
  * This class is generated. Edit {@code X-InEvaluator.java.st} instead.
  */
-public class In$Type$Evaluator implements EvalOperator.ExpressionEvaluator {
+public class In$Name$Evaluator implements EvalOperator.ExpressionEvaluator {
     private final Source source;
 
     private final EvalOperator.ExpressionEvaluator lhs;
@@ -53,7 +53,7 @@ public class In$Type$Evaluator implements EvalOperator.ExpressionEvaluator {
 
     private Warnings warnings;
 
-    public In$Type$Evaluator(
+    public In$Name$Evaluator(
         Source source,
         EvalOperator.ExpressionEvaluator lhs,
         EvalOperator.ExpressionEvaluator[] rhs,
@@ -166,6 +166,9 @@ $if(boolean)$
                 foundMatch = lhsBlock.getBoolean(lhsBlock.getFirstValueIndex(p)) ? hasTrue : hasFalse;
 $elseif(BytesRef)$
                 foundMatch = In.process(nulls, mvs, lhsBlock.getBytesRef(lhsBlock.getFirstValueIndex(p), lhsScratch), rhsValues);
+$elseif(NanosMillis)$
+                // Even though these are longs, they're in different units and need special handling
+                foundMatch = In.processNanosMilils(nulls, mvs, lhsBlock.get$Type$(lhsBlock.getFirstValueIndex(p)), rhsValues);
 $else$
                 foundMatch = In.process(nulls, mvs, lhsBlock.get$Type$(lhsBlock.getFirstValueIndex(p)), rhsValues);
 $endif$
@@ -230,6 +233,10 @@ $if(BytesRef)$
                 result.appendBoolean(In.process(null, null, lhsVector.getBytesRef(p, lhsScratch), rhsValues));
 $elseif(boolean)$
                 result.appendBoolean(lhsVector.getBoolean(p) ? hasTrue : hasFalse);
+$elseif(nanosMillis)$
+                result.appendBoolean(In.processNanosMillis(null, null, lhsVector.get$Type$(p), rhsValues));
+$elseif(millisNanos)$
+                result.appendBoolean(In.processMillisNanos(null, null, lhsVector.get$Type$(p), rhsValues));
 $else$
                 result.appendBoolean(In.process(null, null, lhsVector.get$Type$(p), rhsValues));
 $endif$
@@ -240,7 +247,7 @@ $endif$
 
     @Override
     public String toString() {
-        return "In$Type$Evaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]";
+        return "In$Name$Evaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]";
     }
 
     @Override
@@ -272,16 +279,16 @@ $endif$
         }
 
         @Override
-        public In$Type$Evaluator get(DriverContext context) {
+        public In$Name$Evaluator get(DriverContext context) {
             EvalOperator.ExpressionEvaluator[] rhs = Arrays.stream(this.rhs)
                 .map(a -> a.get(context))
                 .toArray(EvalOperator.ExpressionEvaluator[]::new);
-            return new In$Type$Evaluator(source, lhs.get(context), rhs, context);
+            return new In$Name$Evaluator(source, lhs.get(context), rhs, context);
         }
 
         @Override
         public String toString() {
-            return "In$Type$Evaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]";
+            return "In$Name$Evaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]";
         }
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java
index d3fc9e15e2e04..1edab8ce0e4a6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java
@@ -15,15 +15,16 @@
 
 import java.io.IOException;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
-import java.util.stream.Collectors;
 
-public class EsIndex implements Writeable {
+import static java.util.stream.Collectors.toMap;
 
-    private final String name;
-    private final Map<String, EsField> mapping;
-    private final Map<String, IndexMode> indexNameWithModes;
+public record EsIndex(String name, Map<String, EsField> mapping, Map<String, IndexMode> indexNameWithModes) implements Writeable {
+
+    public EsIndex {
+        assert name != null;
+        assert mapping != null;
+    }
 
     /**
      * Intended for tests. Returns an index with an empty index mode map.
@@ -32,56 +33,32 @@ public EsIndex(String name, Map<String, EsField> mapping) {
         this(name, mapping, Map.of());
     }
 
-    public EsIndex(String name, Map<String, EsField> mapping, Map<String, IndexMode> indexNameWithModes) {
-        assert name != null;
-        assert mapping != null;
-        this.name = name;
-        this.mapping = mapping;
-        this.indexNameWithModes = indexNameWithModes;
-    }
-
-    public EsIndex(StreamInput in) throws IOException {
-        this(in.readString(), in.readImmutableMap(StreamInput::readString, EsField::readFrom), readIndexNameWithModes(in));
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(name());
-        out.writeMap(mapping(), (o, x) -> x.writeTo(out));
-        writeIndexNameWithModes(indexNameWithModes, out);
-    }
-
-    @SuppressWarnings("unchecked")
-    private static Map<String, IndexMode> readIndexNameWithModes(StreamInput in) throws IOException {
+    public static EsIndex readFrom(StreamInput in) throws IOException {
+        String name = in.readString();
+        Map<String, EsField> mapping = in.readImmutableMap(StreamInput::readString, EsField::readFrom);
+        Map<String, IndexMode> indexNameWithModes;
         if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) {
-            return in.readMap(IndexMode::readFrom);
+            indexNameWithModes = in.readMap(IndexMode::readFrom);
         } else {
+            @SuppressWarnings("unchecked")
             Set<String> indices = (Set<String>) in.readGenericValue();
             assert indices != null;
-            return indices.stream().collect(Collectors.toMap(e -> e, e -> IndexMode.STANDARD));
+            indexNameWithModes = indices.stream().collect(toMap(e -> e, e -> IndexMode.STANDARD));
         }
+        return new EsIndex(name, mapping, indexNameWithModes);
     }
 
-    private static void writeIndexNameWithModes(Map<String, IndexMode> concreteIndices, StreamOutput out) throws IOException {
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(name());
+        out.writeMap(mapping(), (o, x) -> x.writeTo(out));
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) {
-            out.writeMap(concreteIndices, (o, v) -> IndexMode.writeTo(v, out));
+            out.writeMap(indexNameWithModes, (o, v) -> IndexMode.writeTo(v, out));
         } else {
-            out.writeGenericValue(concreteIndices.keySet());
+            out.writeGenericValue(indexNameWithModes.keySet());
         }
     }
 
-    public String name() {
-        return name;
-    }
-
-    public Map<String, EsField> mapping() {
-        return mapping;
-    }
-
-    public Map<String, IndexMode> indexNameWithModes() {
-        return indexNameWithModes;
-    }
-
     public Set<String> concreteIndices() {
         return indexNameWithModes.keySet();
     }
@@ -90,25 +67,4 @@ public Set<String> concreteIndices() {
     public String toString() {
         return name;
     }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(name, mapping, indexNameWithModes);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) {
-            return true;
-        }
-
-        if (obj == null || getClass() != obj.getClass()) {
-            return false;
-        }
-
-        EsIndex other = (EsIndex) obj;
-        return Objects.equals(name, other.name)
-            && Objects.equals(mapping, other.mapping)
-            && Objects.equals(indexNameWithModes, other.indexNameWithModes);
-    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalOptimizerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalOptimizerContext.java
index ef5cf50c76541..183008f900c5d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalOptimizerContext.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalOptimizerContext.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.optimizer;
 
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.session.Configuration;
 import org.elasticsearch.xpack.esql.stats.SearchStats;
 
@@ -15,8 +16,8 @@
 public final class LocalLogicalOptimizerContext extends LogicalOptimizerContext {
     private final SearchStats searchStats;
 
-    public LocalLogicalOptimizerContext(Configuration configuration, SearchStats searchStats) {
-        super(configuration);
+    public LocalLogicalOptimizerContext(Configuration configuration, FoldContext foldCtx, SearchStats searchStats) {
+        super(configuration, foldCtx);
         this.searchStats = searchStats;
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java
index c11e1a4ec49e4..22e07b45310fb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java
@@ -7,7 +7,8 @@
 
 package org.elasticsearch.xpack.esql.optimizer;
 
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.session.Configuration;
 import org.elasticsearch.xpack.esql.stats.SearchStats;
 
-public record LocalPhysicalOptimizerContext(Configuration configuration, SearchStats searchStats) {}
+public record LocalPhysicalOptimizerContext(Configuration configuration, FoldContext foldCtx, SearchStats searchStats) {}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java
index 1eaade043658b..a865f784137ad 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.PushTopNToSource;
 import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.ReplaceSourceAttributes;
 import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.SpatialDocValuesExtraction;
+import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.SpatialShapeBoundsExtraction;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.esql.rule.ParameterizedRuleExecutor;
 import org.elasticsearch.xpack.esql.rule.Rule;
@@ -73,7 +74,13 @@ protected List<Batch<PhysicalPlan>> rules(boolean optimizeForEsSource) {
         var pushdown = new Batch<PhysicalPlan>("Push to ES", esSourceRules.toArray(Rule[]::new));
         // add the field extraction in just one pass
         // add it at the end after all the other rules have ran
-        var fieldExtraction = new Batch<>("Field extraction", Limiter.ONCE, new InsertFieldExtraction(), new SpatialDocValuesExtraction());
+        var fieldExtraction = new Batch<>(
+            "Field extraction",
+            Limiter.ONCE,
+            new InsertFieldExtraction(),
+            new SpatialDocValuesExtraction(),
+            new SpatialShapeBoundsExtraction()
+        );
         return asList(pushdown, fieldExtraction);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalOptimizerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalOptimizerContext.java
index 67148e67cbc19..da2d583674a90 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalOptimizerContext.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalOptimizerContext.java
@@ -7,37 +7,44 @@
 
 package org.elasticsearch.xpack.esql.optimizer;
 
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.util.Objects;
 
 public class LogicalOptimizerContext {
     private final Configuration configuration;
+    private final FoldContext foldCtx;
 
-    public LogicalOptimizerContext(Configuration configuration) {
+    public LogicalOptimizerContext(Configuration configuration, FoldContext foldCtx) {
         this.configuration = configuration;
+        this.foldCtx = foldCtx;
     }
 
     public Configuration configuration() {
         return configuration;
     }
 
+    public FoldContext foldCtx() {
+        return foldCtx;
+    }
+
     @Override
     public boolean equals(Object obj) {
         if (obj == this) return true;
         if (obj == null || obj.getClass() != this.getClass()) return false;
         var that = (LogicalOptimizerContext) obj;
-        return Objects.equals(this.configuration, that.configuration);
+        return this.configuration.equals(that.configuration) && this.foldCtx.equals(that.foldCtx);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(configuration);
+        return Objects.hash(configuration, foldCtx);
     }
 
     @Override
     public String toString() {
-        return "LogicalOptimizerContext[" + "configuration=" + configuration + ']';
+        return "LogicalOptimizerContext[configuration=" + configuration + ", foldCtx=" + foldCtx + ']';
     }
 
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
index a5f97cf961378..36150083daec0 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
@@ -49,6 +49,7 @@
 import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceRegexMatch;
 import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceRowAsLocalRelation;
 import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceStatsFilteredAggWithEval;
+import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceStringCasingWithInsensitiveEquals;
 import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceTrivialTypeConversions;
 import org.elasticsearch.xpack.esql.optimizer.rules.logical.SetAsOptimized;
 import org.elasticsearch.xpack.esql.optimizer.rules.logical.SimplifyComparisonsArithmetics;
@@ -175,6 +176,7 @@ protected static Batch<LogicalPlan> operators() {
             new CombineDisjunctions(),
             // TODO: bifunction can now (since we now have just one data types set) be pushed into the rule
             new SimplifyComparisonsArithmetics(DataType::areCompatible),
+            new ReplaceStringCasingWithInsensitiveEquals(),
             new ReplaceStatsFilteredAggWithEval(),
             new ExtractAggregateCommonFilter(),
             // prune/elimination
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java
index dce828dbf192d..7e6eddd3ef041 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.esql.optimizer;
 
-import org.elasticsearch.xpack.esql.capabilities.Validatable;
+import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware;
 import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.optimizer.rules.PlanConsistencyChecker;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
@@ -28,8 +28,8 @@ public Failures verify(LogicalPlan plan) {
 
             if (failures.hasFailures() == false) {
                 p.forEachExpression(ex -> {
-                    if (ex instanceof Validatable v) {
-                        v.validate(failures);
+                    if (ex instanceof PostOptimizationVerificationAware va) {
+                        va.postLogicalOptimizationVerification(failures);
                     }
                 });
             }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsElimination.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsElimination.java
index 3152f9b574767..5f463f2aa4c78 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsElimination.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsElimination.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 
 import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE;
@@ -28,7 +29,7 @@ public BooleanFunctionEqualsElimination() {
     }
 
     @Override
-    public Expression rule(BinaryComparison bc) {
+    public Expression rule(BinaryComparison bc, LogicalOptimizerContext ctx) {
         if ((bc instanceof Equals || bc instanceof NotEquals) && bc.left() instanceof Function) {
             // for expression "==" or "!=" TRUE/FALSE, return the expression itself or its negated variant
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplification.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplification.java
index 73d1ea1fb6e8f..e1803872fd606 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplification.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplification.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 
 import java.util.List;
 
@@ -35,7 +36,7 @@ public BooleanSimplification() {
     }
 
     @Override
-    public Expression rule(ScalarFunction e) {
+    public Expression rule(ScalarFunction e, LogicalOptimizerContext ctx) {
         if (e instanceof And || e instanceof Or) {
             return simplifyAndOr((BinaryPredicate<?, ?, ?, ?>) e);
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisons.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisons.java
index 3f47c74aaf814..1c290a7c4c4fd 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisons.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisons.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.esql.optimizer.rules.logical;
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic;
@@ -20,6 +21,7 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -31,17 +33,17 @@ public CombineBinaryComparisons() {
     }
 
     @Override
-    public Expression rule(BinaryLogic e) {
+    public Expression rule(BinaryLogic e, LogicalOptimizerContext ctx) {
         if (e instanceof And and) {
-            return combine(and);
+            return combine(ctx.foldCtx(), and);
         } else if (e instanceof Or or) {
-            return combine(or);
+            return combine(ctx.foldCtx(), or);
         }
         return e;
     }
 
     // combine conjunction
-    private static Expression combine(And and) {
+    private static Expression combine(FoldContext ctx, And and) {
         List<BinaryComparison> bcs = new ArrayList<>();
         List<Expression> exps = new ArrayList<>();
         boolean changed = false;
@@ -58,13 +60,13 @@ private static Expression combine(And and) {
         });
         for (Expression ex : andExps) {
             if (ex instanceof BinaryComparison bc && (ex instanceof Equals || ex instanceof NotEquals) == false) {
-                if (bc.right().foldable() && (findExistingComparison(bc, bcs, true))) {
+                if (bc.right().foldable() && (findExistingComparison(ctx, bc, bcs, true))) {
                     changed = true;
                 } else {
                     bcs.add(bc);
                 }
             } else if (ex instanceof NotEquals neq) {
-                if (neq.right().foldable() && notEqualsIsRemovableFromConjunction(neq, bcs)) {
+                if (neq.right().foldable() && notEqualsIsRemovableFromConjunction(ctx, neq, bcs)) {
                     // the non-equality can simply be dropped: either superfluous or has been merged with an updated range/inequality
                     changed = true;
                 } else { // not foldable OR not overlapping
@@ -78,13 +80,13 @@ private static Expression combine(And and) {
     }
 
     // combine disjunction
-    private static Expression combine(Or or) {
+    private static Expression combine(FoldContext ctx, Or or) {
         List<BinaryComparison> bcs = new ArrayList<>();
         List<Expression> exps = new ArrayList<>();
         boolean changed = false;
         for (Expression ex : Predicates.splitOr(or)) {
             if (ex instanceof BinaryComparison bc) {
-                if (bc.right().foldable() && findExistingComparison(bc, bcs, false)) {
+                if (bc.right().foldable() && findExistingComparison(ctx, bc, bcs, false)) {
                     changed = true;
                 } else {
                     bcs.add(bc);
@@ -100,8 +102,8 @@ private static Expression combine(Or or) {
      * Find commonalities between the given comparison in the given list.
      * The method can be applied both for conjunctive (AND) or disjunctive purposes (OR).
      */
-    private static boolean findExistingComparison(BinaryComparison main, List<BinaryComparison> bcs, boolean conjunctive) {
-        Object value = main.right().fold();
+    private static boolean findExistingComparison(FoldContext ctx, BinaryComparison main, List<BinaryComparison> bcs, boolean conjunctive) {
+        Object value = main.right().fold(ctx);
         // NB: the loop modifies the list (hence why the int is used)
         for (int i = 0; i < bcs.size(); i++) {
             BinaryComparison other = bcs.get(i);
@@ -113,7 +115,7 @@ private static boolean findExistingComparison(BinaryComparison main, List<Binary
             if ((other instanceof GreaterThan || other instanceof GreaterThanOrEqual)
                 && (main instanceof GreaterThan || main instanceof GreaterThanOrEqual)) {
                 if (main.left().semanticEquals(other.left())) {
-                    Integer compare = BinaryComparison.compare(value, other.right().fold());
+                    Integer compare = BinaryComparison.compare(value, other.right().fold(ctx));
                     if (compare != null) {
                         // AND
                         if ((conjunctive &&
@@ -140,7 +142,7 @@ private static boolean findExistingComparison(BinaryComparison main, List<Binary
             else if ((other instanceof LessThan || other instanceof LessThanOrEqual)
                 && (main instanceof LessThan || main instanceof LessThanOrEqual)) {
                     if (main.left().semanticEquals(other.left())) {
-                        Integer compare = BinaryComparison.compare(value, other.right().fold());
+                        Integer compare = BinaryComparison.compare(value, other.right().fold(ctx));
                         if (compare != null) {
                             // AND
                             if ((conjunctive &&
@@ -167,8 +169,8 @@ else if ((other instanceof LessThan || other instanceof LessThanOrEqual)
         return false;
     }
 
-    private static boolean notEqualsIsRemovableFromConjunction(NotEquals notEquals, List<BinaryComparison> bcs) {
-        Object neqVal = notEquals.right().fold();
+    private static boolean notEqualsIsRemovableFromConjunction(FoldContext ctx, NotEquals notEquals, List<BinaryComparison> bcs) {
+        Object neqVal = notEquals.right().fold(ctx);
         Integer comp;
 
         // check on "condition-overlapping" inequalities:
@@ -183,7 +185,7 @@ private static boolean notEqualsIsRemovableFromConjunction(NotEquals notEquals,
             BinaryComparison bc = bcs.get(i);
             if (notEquals.left().semanticEquals(bc.left())) {
                 if (bc instanceof LessThan || bc instanceof LessThanOrEqual) {
-                    comp = bc.right().foldable() ? BinaryComparison.compare(neqVal, bc.right().fold()) : null;
+                    comp = bc.right().foldable() ? BinaryComparison.compare(neqVal, bc.right().fold(ctx)) : null;
                     if (comp != null) {
                         if (comp >= 0) {
                             if (comp == 0 && bc instanceof LessThanOrEqual) { // a != 2 AND a <= 2 -> a < 2
@@ -193,7 +195,7 @@ private static boolean notEqualsIsRemovableFromConjunction(NotEquals notEquals,
                         } // else: comp < 0 : a != 2 AND a </<= 3 -> nop
                     } // else: non-comparable, nop
                 } else if (bc instanceof GreaterThan || bc instanceof GreaterThanOrEqual) {
-                    comp = bc.right().foldable() ? BinaryComparison.compare(neqVal, bc.right().fold()) : null;
+                    comp = bc.right().foldable() ? BinaryComparison.compare(neqVal, bc.right().fold(ctx)) : null;
                     if (comp != null) {
                         if (comp <= 0) {
                             if (comp == 0 && bc instanceof GreaterThanOrEqual) { // a != 2 AND a >= 2 -> a > 2
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctions.java
index 5cb377de47efc..e1cda9cb149d4 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctions.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctions.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 
 import java.time.ZoneId;
 import java.util.ArrayList;
@@ -61,7 +62,7 @@ protected static CIDRMatch createCIDRMatch(Expression k, List<Expression> v) {
     }
 
     @Override
-    public Expression rule(Or or) {
+    public Expression rule(Or or, LogicalOptimizerContext ctx) {
         Expression e = or;
         // look only at equals, In and CIDRMatch
         List<Expression> exps = splitOr(e);
@@ -78,7 +79,7 @@ public Expression rule(Or or) {
                 if (eq.right().foldable()) {
                     ins.computeIfAbsent(eq.left(), k -> new LinkedHashSet<>()).add(eq.right());
                     if (eq.left().dataType() == DataType.IP) {
-                        Object value = eq.right().fold();
+                        Object value = eq.right().fold(ctx.foldCtx());
                         // ImplicitCasting and ConstantFolding(includes explicit casting) are applied before CombineDisjunctions.
                         // They fold the input IP string to an internal IP format. These happen to Equals and IN, but not for CIDRMatch,
                         // as CIDRMatch takes strings as input, ImplicitCasting does not apply to it, and the first input to CIDRMatch is a
@@ -101,7 +102,7 @@ public Expression rule(Or or) {
                 if (in.value().dataType() == DataType.IP) {
                     List<Expression> values = new ArrayList<>(in.list().size());
                     for (Expression i : in.list()) {
-                        Object value = i.fold();
+                        Object value = i.fold(ctx.foldCtx());
                         // Same as Equals.
                         if (value instanceof BytesRef bytesRef) {
                             value = ipToString(bytesRef);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFolding.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFolding.java
index 82fe2c6bddf50..27eec8de59020 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFolding.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFolding.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 
 public final class ConstantFolding extends OptimizerRules.OptimizerExpressionRule<Expression> {
 
@@ -17,7 +18,7 @@ public ConstantFolding() {
     }
 
     @Override
-    public Expression rule(Expression e) {
-        return e.foldable() ? Literal.of(e) : e;
+    public Expression rule(Expression e, LogicalOptimizerContext ctx) {
+        return e.foldable() ? Literal.of(ctx.foldCtx(), e) : e;
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConvertStringToByteRef.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConvertStringToByteRef.java
index 0604750883f14..b716d8f012d21 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConvertStringToByteRef.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConvertStringToByteRef.java
@@ -10,6 +10,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -21,7 +22,8 @@ public ConvertStringToByteRef() {
     }
 
     @Override
-    protected Expression rule(Literal lit) {
+    protected Expression rule(Literal lit, LogicalOptimizerContext ctx) {
+        // TODO we shouldn't be emitting String into Literals at all
         Object value = lit.value();
 
         if (value == null) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java
index 747864625e65c..cf4c7f19baafe 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 
 public class FoldNull extends OptimizerRules.OptimizerExpressionRule<Expression> {
 
@@ -23,7 +24,7 @@ public FoldNull() {
     }
 
     @Override
-    public Expression rule(Expression e) {
+    public Expression rule(Expression e, LogicalOptimizerContext ctx) {
         Expression result = tryReplaceIsNullIsNotNull(e);
 
         // convert an aggregate null filter into a false
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRight.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRight.java
index d96c73d5ee4f1..6504e6042c33a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRight.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRight.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 
 public final class LiteralsOnTheRight extends OptimizerRules.OptimizerExpressionRule<BinaryOperator<?, ?, ?, ?>> {
 
@@ -17,7 +18,7 @@ public LiteralsOnTheRight() {
     }
 
     @Override
-    public BinaryOperator<?, ?, ?, ?> rule(BinaryOperator<?, ?, ?, ?> be) {
+    public BinaryOperator<?, ?, ?, ?> rule(BinaryOperator<?, ?, ?, ?> be, LogicalOptimizerContext ctx) {
         return be.left() instanceof Literal && (be.right() instanceof Literal) == false ? be.swapLeftAndRight() : be;
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/OptimizerRules.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/OptimizerRules.java
index 2a0b2a6af36aa..169ac2ac8c0fe 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/OptimizerRules.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/OptimizerRules.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.util.ReflectionUtils;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.rule.ParameterizedRule;
 import org.elasticsearch.xpack.esql.rule.Rule;
@@ -36,7 +37,10 @@ public final LogicalPlan apply(LogicalPlan plan) {
         protected abstract LogicalPlan rule(SubPlan plan);
     }
 
-    public abstract static class OptimizerExpressionRule<E extends Expression> extends Rule<LogicalPlan, LogicalPlan> {
+    public abstract static class OptimizerExpressionRule<E extends Expression> extends ParameterizedRule<
+        LogicalPlan,
+        LogicalPlan,
+        LogicalOptimizerContext> {
 
         private final TransformDirection direction;
         // overriding type token which returns the correct class but does an uncheck cast to LogicalPlan due to its generic bound
@@ -49,17 +53,13 @@ public OptimizerExpressionRule(TransformDirection direction) {
         }
 
         @Override
-        public final LogicalPlan apply(LogicalPlan plan) {
+        public final LogicalPlan apply(LogicalPlan plan, LogicalOptimizerContext ctx) {
             return direction == TransformDirection.DOWN
-                ? plan.transformExpressionsDown(expressionTypeToken, this::rule)
-                : plan.transformExpressionsUp(expressionTypeToken, this::rule);
+                ? plan.transformExpressionsDown(expressionTypeToken, e -> rule(e, ctx))
+                : plan.transformExpressionsUp(expressionTypeToken, e -> rule(e, ctx));
         }
 
-        protected LogicalPlan rule(LogicalPlan plan) {
-            return plan;
-        }
-
-        protected abstract Expression rule(E e);
+        protected abstract Expression rule(E e, LogicalOptimizerContext ctx);
 
         public Class<E> expressionToken() {
             return expressionTypeToken;
@@ -82,6 +82,7 @@ protected ParameterizedOptimizerRule(TransformDirection direction) {
             this.direction = direction;
         }
 
+        @Override
         public final LogicalPlan apply(LogicalPlan plan, P context) {
             return direction == TransformDirection.DOWN
                 ? plan.transformDown(typeToken(), t -> rule(t, context))
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PartiallyFoldCase.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PartiallyFoldCase.java
index 118e4fc170520..0111c7cdd806a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PartiallyFoldCase.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PartiallyFoldCase.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 
 import static org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules.TransformDirection.DOWN;
 
@@ -28,7 +29,7 @@ public PartiallyFoldCase() {
     }
 
     @Override
-    protected Expression rule(Case c) {
-        return c.partiallyFold();
+    protected Expression rule(Case c, LogicalOptimizerContext ctx) {
+        return c.partiallyFold(ctx.foldCtx());
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEmptyRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEmptyRelation.java
index 8437b79454884..b6f185c856693 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEmptyRelation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEmptyRelation.java
@@ -12,9 +12,11 @@
 import org.elasticsearch.compute.data.BlockUtils;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Count;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan;
@@ -26,15 +28,18 @@
 import java.util.List;
 
 @SuppressWarnings("removal")
-public class PropagateEmptyRelation extends OptimizerRules.OptimizerRule<UnaryPlan> {
+public class PropagateEmptyRelation extends OptimizerRules.ParameterizedOptimizerRule<UnaryPlan, LogicalOptimizerContext> {
+    public PropagateEmptyRelation() {
+        super(OptimizerRules.TransformDirection.DOWN);
+    }
 
     @Override
-    protected LogicalPlan rule(UnaryPlan plan) {
+    protected LogicalPlan rule(UnaryPlan plan, LogicalOptimizerContext ctx) {
         LogicalPlan p = plan;
         if (plan.child() instanceof LocalRelation local && local.supplier() == LocalSupplier.EMPTY) {
             // only care about non-grouped aggs might return something (count)
             if (plan instanceof Aggregate agg && agg.groupings().isEmpty()) {
-                List<Block> emptyBlocks = aggsFromEmpty(agg.aggregates());
+                List<Block> emptyBlocks = aggsFromEmpty(ctx.foldCtx(), agg.aggregates());
                 p = replacePlanByRelation(plan, LocalSupplier.of(emptyBlocks.toArray(Block[]::new)));
             } else {
                 p = PruneEmptyPlans.skipPlan(plan);
@@ -43,14 +48,14 @@ protected LogicalPlan rule(UnaryPlan plan) {
         return p;
     }
 
-    private List<Block> aggsFromEmpty(List<? extends NamedExpression> aggs) {
+    private List<Block> aggsFromEmpty(FoldContext foldCtx, List<? extends NamedExpression> aggs) {
         List<Block> blocks = new ArrayList<>();
         var blockFactory = PlannerUtils.NON_BREAKING_BLOCK_FACTORY;
         int i = 0;
         for (var agg : aggs) {
             // there needs to be an alias
             if (Alias.unwrap(agg) instanceof AggregateFunction aggFunc) {
-                aggOutput(agg, aggFunc, blockFactory, blocks);
+                aggOutput(foldCtx, agg, aggFunc, blockFactory, blocks);
             } else {
                 throw new EsqlIllegalArgumentException("Did not expect a non-aliased aggregation {}", agg);
             }
@@ -61,9 +66,15 @@ private List<Block> aggsFromEmpty(List<? extends NamedExpression> aggs) {
     /**
      * The folded aggregation output - this variant is for the coordinator/final.
      */
-    protected void aggOutput(NamedExpression agg, AggregateFunction aggFunc, BlockFactory blockFactory, List<Block> blocks) {
+    protected void aggOutput(
+        FoldContext foldCtx,
+        NamedExpression agg,
+        AggregateFunction aggFunc,
+        BlockFactory blockFactory,
+        List<Block> blocks
+    ) {
         // look for count(literal) with literal != null
-        Object value = aggFunc instanceof Count count && (count.foldable() == false || count.fold() != null) ? 0L : null;
+        Object value = aggFunc instanceof Count count && (count.foldable() == false || count.fold(foldCtx) != null) ? 0L : null;
         var wrapper = BlockUtils.wrapperFor(blockFactory, PlannerUtils.toElementType(aggFunc.dataType()), 1);
         wrapper.accept(value);
         blocks.add(wrapper.builder().build());
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEquals.java
index 0bd98db1e1d7a..5a1677f2759e3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEquals.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEquals.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -41,17 +42,18 @@ public PropagateEquals() {
         super(OptimizerRules.TransformDirection.DOWN);
     }
 
-    public Expression rule(BinaryLogic e) {
+    @Override
+    public Expression rule(BinaryLogic e, LogicalOptimizerContext ctx) {
         if (e instanceof And) {
-            return propagate((And) e);
+            return propagate((And) e, ctx);
         } else if (e instanceof Or) {
-            return propagate((Or) e);
+            return propagate((Or) e, ctx);
         }
         return e;
     }
 
     // combine conjunction
-    private static Expression propagate(And and) {
+    private static Expression propagate(And and, LogicalOptimizerContext ctx) {
         List<Range> ranges = new ArrayList<>();
         // Only equalities, not-equalities and inequalities with a foldable .right are extracted separately;
         // the others go into the general 'exps'.
@@ -72,7 +74,7 @@ private static Expression propagate(And and) {
                 if (otherEq.right().foldable() && DataType.isDateTime(otherEq.left().dataType()) == false) {
                     for (BinaryComparison eq : equals) {
                         if (otherEq.left().semanticEquals(eq.left())) {
-                            Integer comp = BinaryComparison.compare(eq.right().fold(), otherEq.right().fold());
+                            Integer comp = BinaryComparison.compare(eq.right().fold(ctx.foldCtx()), otherEq.right().fold(ctx.foldCtx()));
                             if (comp != null) {
                                 // var cannot be equal to two different values at the same time
                                 if (comp != 0) {
@@ -108,7 +110,7 @@ private static Expression propagate(And and) {
 
         // check
         for (BinaryComparison eq : equals) {
-            Object eqValue = eq.right().fold();
+            Object eqValue = eq.right().fold(ctx.foldCtx());
 
             for (Iterator<Range> iterator = ranges.iterator(); iterator.hasNext();) {
                 Range range = iterator.next();
@@ -116,7 +118,7 @@ private static Expression propagate(And and) {
                 if (range.value().semanticEquals(eq.left())) {
                     // if equals is outside the interval, evaluate the whole expression to FALSE
                     if (range.lower().foldable()) {
-                        Integer compare = BinaryComparison.compare(range.lower().fold(), eqValue);
+                        Integer compare = BinaryComparison.compare(range.lower().fold(ctx.foldCtx()), eqValue);
                         if (compare != null && (
                         // eq outside the lower boundary
                         compare > 0 ||
@@ -126,7 +128,7 @@ private static Expression propagate(And and) {
                         }
                     }
                     if (range.upper().foldable()) {
-                        Integer compare = BinaryComparison.compare(range.upper().fold(), eqValue);
+                        Integer compare = BinaryComparison.compare(range.upper().fold(ctx.foldCtx()), eqValue);
                         if (compare != null && (
                         // eq outside the upper boundary
                         compare < 0 ||
@@ -146,7 +148,7 @@ private static Expression propagate(And and) {
             for (Iterator<NotEquals> iter = notEquals.iterator(); iter.hasNext();) {
                 NotEquals neq = iter.next();
                 if (eq.left().semanticEquals(neq.left())) {
-                    Integer comp = BinaryComparison.compare(eqValue, neq.right().fold());
+                    Integer comp = BinaryComparison.compare(eqValue, neq.right().fold(ctx.foldCtx()));
                     if (comp != null) {
                         if (comp == 0) { // clashing and conflicting: a = 1 AND a != 1
                             return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
@@ -162,7 +164,7 @@ private static Expression propagate(And and) {
             for (Iterator<BinaryComparison> iter = inequalities.iterator(); iter.hasNext();) {
                 BinaryComparison bc = iter.next();
                 if (eq.left().semanticEquals(bc.left())) {
-                    Integer compare = BinaryComparison.compare(eqValue, bc.right().fold());
+                    Integer compare = BinaryComparison.compare(eqValue, bc.right().fold(ctx.foldCtx()));
                     if (compare != null) {
                         if (bc instanceof LessThan || bc instanceof LessThanOrEqual) { // a = 2 AND a </<= ?
                             if ((compare == 0 && bc instanceof LessThan) || // a = 2 AND a < 2
@@ -191,7 +193,7 @@ private static Expression propagate(And and) {
     // a = 2 OR a < 3 -> a < 3; a = 2 OR a < 1 -> nop
     // a = 2 OR 3 < a < 5 -> nop; a = 2 OR 1 < a < 3 -> 1 < a < 3; a = 2 OR 0 < a < 1 -> nop
     // a = 2 OR a != 2 -> TRUE; a = 2 OR a = 5 -> nop; a = 2 OR a != 5 -> a != 5
-    private static Expression propagate(Or or) {
+    private static Expression propagate(Or or, LogicalOptimizerContext ctx) {
         List<Expression> exps = new ArrayList<>();
         List<Equals> equals = new ArrayList<>(); // foldable right term Equals
         List<NotEquals> notEquals = new ArrayList<>(); // foldable right term NotEquals
@@ -230,13 +232,13 @@ private static Expression propagate(Or or) {
         // evaluate the impact of each Equal over the different types of Expressions
         for (Iterator<Equals> iterEq = equals.iterator(); iterEq.hasNext();) {
             Equals eq = iterEq.next();
-            Object eqValue = eq.right().fold();
+            Object eqValue = eq.right().fold(ctx.foldCtx());
             boolean removeEquals = false;
 
             // Equals OR NotEquals
             for (NotEquals neq : notEquals) {
                 if (eq.left().semanticEquals(neq.left())) { // a = 2 OR a != ? -> ...
-                    Integer comp = BinaryComparison.compare(eqValue, neq.right().fold());
+                    Integer comp = BinaryComparison.compare(eqValue, neq.right().fold(ctx.foldCtx()));
                     if (comp != null) {
                         if (comp == 0) { // a = 2 OR a != 2 -> TRUE
                             return TRUE;
@@ -257,8 +259,12 @@ private static Expression propagate(Or or) {
             for (int i = 0; i < ranges.size(); i++) { // might modify list, so use index loop
                 Range range = ranges.get(i);
                 if (eq.left().semanticEquals(range.value())) {
-                    Integer lowerComp = range.lower().foldable() ? BinaryComparison.compare(eqValue, range.lower().fold()) : null;
-                    Integer upperComp = range.upper().foldable() ? BinaryComparison.compare(eqValue, range.upper().fold()) : null;
+                    Integer lowerComp = range.lower().foldable()
+                        ? BinaryComparison.compare(eqValue, range.lower().fold(ctx.foldCtx()))
+                        : null;
+                    Integer upperComp = range.upper().foldable()
+                        ? BinaryComparison.compare(eqValue, range.upper().fold(ctx.foldCtx()))
+                        : null;
 
                     if (lowerComp != null && lowerComp == 0) {
                         if (range.includeLower() == false) { // a = 2 OR 2 < a < ? -> 2 <= a < ?
@@ -312,7 +318,7 @@ private static Expression propagate(Or or) {
             for (int i = 0; i < inequalities.size(); i++) {
                 BinaryComparison bc = inequalities.get(i);
                 if (eq.left().semanticEquals(bc.left())) {
-                    Integer comp = BinaryComparison.compare(eqValue, bc.right().fold());
+                    Integer comp = BinaryComparison.compare(eqValue, bc.right().fold(ctx.foldCtx()));
                     if (comp != null) {
                         if (bc instanceof GreaterThan || bc instanceof GreaterThanOrEqual) {
                             if (comp < 0) { // a = 1 OR a > 2 -> nop
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEvalFoldables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEvalFoldables.java
index 73eaa9220fd84..66cdc992a91cb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEvalFoldables.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEvalFoldables.java
@@ -12,19 +12,20 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.Filter;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.rule.Rule;
+import org.elasticsearch.xpack.esql.rule.ParameterizedRule;
 
 /**
  * Replace any reference attribute with its source, if it does not affect the result.
  * This avoids ulterior look-ups between attributes and its source across nodes.
  */
-public final class PropagateEvalFoldables extends Rule<LogicalPlan, LogicalPlan> {
+public final class PropagateEvalFoldables extends ParameterizedRule<LogicalPlan, LogicalPlan, LogicalOptimizerContext> {
 
     @Override
-    public LogicalPlan apply(LogicalPlan plan) {
+    public LogicalPlan apply(LogicalPlan plan, LogicalOptimizerContext ctx) {
         var collectRefs = new AttributeMap<Expression>();
 
         java.util.function.Function<ReferenceAttribute, Expression> replaceReference = r -> collectRefs.resolve(r, r);
@@ -39,7 +40,7 @@ public LogicalPlan apply(LogicalPlan plan) {
                 shouldCollect = c.foldable();
             }
             if (shouldCollect) {
-                collectRefs.put(a.toAttribute(), Literal.of(c));
+                collectRefs.put(a.toAttribute(), Literal.of(ctx.foldCtx(), c));
             }
         });
         if (collectRefs.isEmpty()) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullable.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullable.java
index 738ca83b47e42..e3165180e331c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullable.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullable.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull;
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull;
 import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 
 import java.util.ArrayList;
 import java.util.LinkedHashSet;
@@ -33,7 +34,7 @@ public PropagateNullable() {
     }
 
     @Override
-    public Expression rule(And and) {
+    public Expression rule(And and, LogicalOptimizerContext ctx) {
         List<Expression> splits = Predicates.splitAnd(and);
 
         Set<Expression> nullExpressions = new LinkedHashSet<>();
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneColumns.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneColumns.java
index e01608c546090..1c7e765c6bd59 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneColumns.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneColumns.java
@@ -9,16 +9,16 @@
 
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockUtils;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
-import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
+import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.Limit;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.plan.logical.Project;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier;
 import org.elasticsearch.xpack.esql.planner.PlannerUtils;
@@ -34,13 +34,17 @@ public final class PruneColumns extends Rule<LogicalPlan, LogicalPlan> {
 
     @Override
     public LogicalPlan apply(LogicalPlan plan) {
-        var used = new AttributeSet();
-        // don't remove Evals without any Project/Aggregate (which might not occur as the last node in the plan)
-        var seenProjection = new Holder<>(Boolean.FALSE);
-
-        // start top-to-bottom
-        // and track used references
+        // track used references
+        var used = plan.outputSet();
+        // while going top-to-bottom (upstream)
         var pl = plan.transformDown(p -> {
+            // Note: It is NOT required to do anything special for binary plans like JOINs. It is perfectly fine that transformDown descends
+            // first into the left side, adding all kinds of attributes to the `used` set, and then descends into the right side - even
+            // though the `used` set will contain stuff only used in the left hand side. That's because any attribute that is used in the
+            // left hand side must have been created in the left side as well. Even field attributes belonging to the same index fields will
+            // have different name ids in the left and right hand sides - as in the extreme example
+            // `FROM lookup_idx | LOOKUP JOIN lookup_idx ON key_field`.
+
             // skip nodes that simply pass the input through
             if (p instanceof Limit) {
                 return p;
@@ -53,7 +57,7 @@ public LogicalPlan apply(LogicalPlan plan) {
             do {
                 recheck = false;
                 if (p instanceof Aggregate aggregate) {
-                    var remaining = seenProjection.get() ? removeUnused(aggregate.aggregates(), used) : null;
+                    var remaining = removeUnused(aggregate.aggregates(), used);
 
                     if (remaining != null) {
                         if (remaining.isEmpty()) {
@@ -87,10 +91,8 @@ public LogicalPlan apply(LogicalPlan plan) {
                             );
                         }
                     }
-
-                    seenProjection.set(Boolean.TRUE);
                 } else if (p instanceof Eval eval) {
-                    var remaining = seenProjection.get() ? removeUnused(eval.fields(), used) : null;
+                    var remaining = removeUnused(eval.fields(), used);
                     // no fields, no eval
                     if (remaining != null) {
                         if (remaining.isEmpty()) {
@@ -100,8 +102,16 @@ public LogicalPlan apply(LogicalPlan plan) {
                             p = new Eval(eval.source(), eval.child(), remaining);
                         }
                     }
-                } else if (p instanceof Project) {
-                    seenProjection.set(Boolean.TRUE);
+                } else if (p instanceof EsRelation esRelation && esRelation.indexMode() == IndexMode.LOOKUP) {
+                    // Normally, pruning EsRelation has no effect because InsertFieldExtraction only extracts the required fields, anyway.
+                    // The field extraction for LOOKUP JOIN works differently, however - we extract all fields (other than the join key)
+                    // that the EsRelation has.
+                    var remaining = removeUnused(esRelation.output(), used);
+                    // TODO: LookupFromIndexOperator cannot handle 0 lookup fields, yet. That means 1 field in total (key field + lookup).
+                    // https://github.com/elastic/elasticsearch/issues/118778
+                    if (remaining != null && remaining.size() > 1) {
+                        p = new EsRelation(esRelation.source(), esRelation.index(), remaining, esRelation.indexMode(), esRelation.frozen());
+                    }
                 }
             } while (recheck);
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimits.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimits.java
index 1cacebdf27cd2..969a6bb713eca 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimits.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimits.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.esql.optimizer.rules.logical;
 
 import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
@@ -20,14 +21,18 @@
 import org.elasticsearch.xpack.esql.plan.logical.join.Join;
 import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes;
 
-public final class PushDownAndCombineLimits extends OptimizerRules.OptimizerRule<Limit> {
+public final class PushDownAndCombineLimits extends OptimizerRules.ParameterizedOptimizerRule<Limit, LogicalOptimizerContext> {
+
+    public PushDownAndCombineLimits() {
+        super(OptimizerRules.TransformDirection.DOWN);
+    }
 
     @Override
-    public LogicalPlan rule(Limit limit) {
+    public LogicalPlan rule(Limit limit, LogicalOptimizerContext ctx) {
         if (limit.child() instanceof Limit childLimit) {
             var limitSource = limit.limit();
-            var l1 = (int) limitSource.fold();
-            var l2 = (int) childLimit.limit().fold();
+            var l1 = (int) limitSource.fold(ctx.foldCtx());
+            var l2 = (int) childLimit.limit().fold(ctx.foldCtx());
             return new Limit(limit.source(), Literal.of(limitSource, Math.min(l1, l2)), childLimit.child());
         } else if (limit.child() instanceof UnaryPlan unary) {
             if (unary instanceof Eval || unary instanceof Project || unary instanceof RegexExtract || unary instanceof Enrich) {
@@ -41,7 +46,7 @@ public LogicalPlan rule(Limit limit) {
                 // we add an inner limit to MvExpand and just push down the existing limit, ie.
                 // | MV_EXPAND | LIMIT N -> | LIMIT N | MV_EXPAND (with limit N)
                 var limitSource = limit.limit();
-                var limitVal = (int) limitSource.fold();
+                var limitVal = (int) limitSource.fold(ctx.foldCtx());
                 Integer mvxLimit = mvx.limit();
                 if (mvxLimit == null || mvxLimit > limitVal) {
                     mvx = new MvExpand(mvx.source(), mvx.child(), mvx.target(), mvx.expanded(), limitVal);
@@ -54,8 +59,8 @@ public LogicalPlan rule(Limit limit) {
             else {
                 Limit descendantLimit = descendantLimit(unary);
                 if (descendantLimit != null) {
-                    var l1 = (int) limit.limit().fold();
-                    var l2 = (int) descendantLimit.limit().fold();
+                    var l1 = (int) limit.limit().fold(ctx.foldCtx());
+                    var l2 = (int) descendantLimit.limit().fold(ctx.foldCtx());
                     if (l2 <= l1) {
                         return new Limit(limit.source(), Literal.of(limit.limit(), l2), limit.child());
                     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatch.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatch.java
index 1a8f8a164cc1b..7953b2b28eaaa 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatch.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatch.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.StringPattern;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.parser.ParsingException;
 
 public final class ReplaceRegexMatch extends OptimizerRules.OptimizerExpressionRule<RegexMatch<?>> {
@@ -23,7 +24,7 @@ public ReplaceRegexMatch() {
     }
 
     @Override
-    public Expression rule(RegexMatch<?> regexMatch) {
+    public Expression rule(RegexMatch<?> regexMatch, LogicalOptimizerContext ctx) {
         Expression e = regexMatch;
         StringPattern pattern = regexMatch.pattern();
         boolean matchesAll;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRowAsLocalRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRowAsLocalRelation.java
index eebeb1dc14f48..9e7b6ce80422d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRowAsLocalRelation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRowAsLocalRelation.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.esql.optimizer.rules.logical;
 
 import org.elasticsearch.compute.data.BlockUtils;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.plan.logical.Row;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
@@ -17,13 +18,16 @@
 import java.util.ArrayList;
 import java.util.List;
 
-public final class ReplaceRowAsLocalRelation extends OptimizerRules.OptimizerRule<Row> {
+public final class ReplaceRowAsLocalRelation extends OptimizerRules.ParameterizedOptimizerRule<Row, LogicalOptimizerContext> {
+    public ReplaceRowAsLocalRelation() {
+        super(OptimizerRules.TransformDirection.DOWN);
+    }
 
     @Override
-    protected LogicalPlan rule(Row row) {
+    protected LogicalPlan rule(Row row, LogicalOptimizerContext context) {
         var fields = row.fields();
         List<Object> values = new ArrayList<>(fields.size());
-        fields.forEach(f -> values.add(f.child().fold()));
+        fields.forEach(f -> values.add(f.child().fold(context.foldCtx())));
         var blocks = BlockUtils.fromListRow(PlannerUtils.NON_BREAKING_BLOCK_FACTORY, values);
         return new LocalRelation(row.source(), row.output(), LocalSupplier.of(blocks));
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsFilteredAggWithEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsFilteredAggWithEval.java
index 2cafcc2e07052..a7e56a5f25fc8 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsFilteredAggWithEval.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsFilteredAggWithEval.java
@@ -49,7 +49,7 @@ protected LogicalPlan rule(Aggregate aggregate) {
                 && alias.child() instanceof AggregateFunction aggFunction
                 && aggFunction.hasFilter()
                 && aggFunction.filter() instanceof Literal literal
-                && Boolean.FALSE.equals(literal.fold())) {
+                && Boolean.FALSE.equals(literal.value())) {
 
                 Object value = aggFunction instanceof Count || aggFunction instanceof CountDistinct ? 0L : null;
                 Alias newAlias = alias.replaceChild(Literal.of(aggFunction, value));
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveEquals.java
new file mode 100644
index 0000000000000..053441bce5e1f
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveEquals.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.optimizer.rules.logical;
+
+import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
+import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull;
+import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.ChangeCase;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.InsensitiveEquals;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
+
+public class ReplaceStringCasingWithInsensitiveEquals extends OptimizerRules.OptimizerExpressionRule<ScalarFunction> {
+
+    public ReplaceStringCasingWithInsensitiveEquals() {
+        super(OptimizerRules.TransformDirection.DOWN);
+    }
+
+    @Override
+    protected Expression rule(ScalarFunction sf, LogicalOptimizerContext ctx) {
+        Expression e = sf;
+        if (sf instanceof BinaryComparison bc) {
+            e = rewriteBinaryComparison(ctx, sf, bc, false);
+        } else if (sf instanceof Not not && not.field() instanceof BinaryComparison bc) {
+            e = rewriteBinaryComparison(ctx, sf, bc, true);
+        }
+        return e;
+    }
+
+    private static Expression rewriteBinaryComparison(
+        LogicalOptimizerContext ctx,
+        ScalarFunction sf,
+        BinaryComparison bc,
+        boolean negated
+    ) {
+        Expression e = sf;
+        if (bc.left() instanceof ChangeCase changeCase && bc.right().foldable()) {
+            if (bc instanceof Equals) {
+                e = replaceChangeCase(ctx, bc, changeCase, negated);
+            } else if (bc instanceof NotEquals) { // not actually used currently, `!=` is built as `NOT(==)` already
+                e = replaceChangeCase(ctx, bc, changeCase, negated == false);
+            }
+        }
+        return e;
+    }
+
+    private static Expression replaceChangeCase(LogicalOptimizerContext ctx, BinaryComparison bc, ChangeCase changeCase, boolean negated) {
+        var foldedRight = BytesRefs.toString(bc.right().fold(ctx.foldCtx()));
+        var field = unwrapCase(changeCase.field());
+        var e = changeCase.caseType().matchesCase(foldedRight)
+            ? new InsensitiveEquals(bc.source(), field, bc.right())
+            : Literal.of(bc, Boolean.FALSE);
+        if (negated) {
+            e = e instanceof Literal ? new IsNotNull(e.source(), field) : new Not(e.source(), e);
+        }
+        return e;
+    }
+
+    private static Expression unwrapCase(Expression e) {
+        for (; e instanceof ChangeCase cc; e = cc.field()) {
+        }
+        return e;
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SimplifyComparisonsArithmetics.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SimplifyComparisonsArithmetics.java
index d3a9970896c16..60ff161651f2d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SimplifyComparisonsArithmetics.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SimplifyComparisonsArithmetics.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.esql.optimizer.rules.logical;
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -15,6 +16,7 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.BinaryComparisonInversible;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Sub;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 
 import java.time.DateTimeException;
 import java.util.List;
@@ -41,20 +43,20 @@ public SimplifyComparisonsArithmetics(BiFunction<DataType, DataType, Boolean> ty
     }
 
     @Override
-    protected Expression rule(BinaryComparison bc) {
+    protected Expression rule(BinaryComparison bc, LogicalOptimizerContext ctx) {
         // optimize only once the expression has a literal on the right side of the binary comparison
         if (bc.right() instanceof Literal) {
             if (bc.left() instanceof ArithmeticOperation) {
-                return simplifyBinaryComparison(bc);
+                return simplifyBinaryComparison(ctx.foldCtx(), bc);
             }
             if (bc.left() instanceof Neg) {
-                return foldNegation(bc);
+                return foldNegation(ctx.foldCtx(), bc);
             }
         }
         return bc;
     }
 
-    private Expression simplifyBinaryComparison(BinaryComparison comparison) {
+    private Expression simplifyBinaryComparison(FoldContext foldContext, BinaryComparison comparison) {
         ArithmeticOperation operation = (ArithmeticOperation) comparison.left();
         // Use symbol comp: SQL operations aren't available in this package (as dependencies)
         String opSymbol = operation.symbol();
@@ -64,9 +66,9 @@ private Expression simplifyBinaryComparison(BinaryComparison comparison) {
         }
         OperationSimplifier simplification = null;
         if (isMulOrDiv(opSymbol)) {
-            simplification = new MulDivSimplifier(comparison);
+            simplification = new MulDivSimplifier(foldContext, comparison);
         } else if (opSymbol.equals(ADD.symbol()) || opSymbol.equals(SUB.symbol())) {
-            simplification = new AddSubSimplifier(comparison);
+            simplification = new AddSubSimplifier(foldContext, comparison);
         }
 
         return (simplification == null || simplification.isUnsafe(typesCompatible)) ? comparison : simplification.apply();
@@ -76,16 +78,16 @@ private static boolean isMulOrDiv(String opSymbol) {
         return opSymbol.equals(MUL.symbol()) || opSymbol.equals(DIV.symbol());
     }
 
-    private static Expression foldNegation(BinaryComparison bc) {
+    private static Expression foldNegation(FoldContext ctx, BinaryComparison bc) {
         Literal bcLiteral = (Literal) bc.right();
-        Expression literalNeg = tryFolding(new Neg(bcLiteral.source(), bcLiteral));
+        Expression literalNeg = tryFolding(ctx, new Neg(bcLiteral.source(), bcLiteral));
         return literalNeg == null ? bc : bc.reverse().replaceChildren(asList(((Neg) bc.left()).field(), literalNeg));
     }
 
-    private static Expression tryFolding(Expression expression) {
+    private static Expression tryFolding(FoldContext ctx, Expression expression) {
         if (expression.foldable()) {
             try {
-                expression = new Literal(expression.source(), expression.fold(), expression.dataType());
+                expression = new Literal(expression.source(), expression.fold(ctx), expression.dataType());
             } catch (ArithmeticException | DateTimeException e) {
                 // null signals that folding would result in an over-/underflow (such as Long.MAX_VALUE+1); the optimisation is skipped.
                 expression = null;
@@ -95,6 +97,7 @@ private static Expression tryFolding(Expression expression) {
     }
 
     private abstract static class OperationSimplifier {
+        final FoldContext foldContext;
         final BinaryComparison comparison;
         final Literal bcLiteral;
         final ArithmeticOperation operation;
@@ -102,7 +105,8 @@ private abstract static class OperationSimplifier {
         final Expression opRight;
         final Literal opLiteral;
 
-        OperationSimplifier(BinaryComparison comparison) {
+        OperationSimplifier(FoldContext foldContext, BinaryComparison comparison) {
+            this.foldContext = foldContext;
             this.comparison = comparison;
             operation = (ArithmeticOperation) comparison.left();
             bcLiteral = (Literal) comparison.right();
@@ -151,7 +155,7 @@ final Expression apply() {
 
             Expression bcRightExpression = ((BinaryComparisonInversible) operation).binaryComparisonInverse()
                 .create(bcl.source(), bcl, opRight);
-            bcRightExpression = tryFolding(bcRightExpression);
+            bcRightExpression = tryFolding(foldContext, bcRightExpression);
             return bcRightExpression != null
                 ? postProcess((BinaryComparison) comparison.replaceChildren(List.of(opLeft, bcRightExpression)))
                 : comparison;
@@ -169,8 +173,8 @@ Expression postProcess(BinaryComparison binaryComparison) {
 
     private static class AddSubSimplifier extends OperationSimplifier {
 
-        AddSubSimplifier(BinaryComparison comparison) {
-            super(comparison);
+        AddSubSimplifier(FoldContext foldContext, BinaryComparison comparison) {
+            super(foldContext, comparison);
         }
 
         @Override
@@ -182,7 +186,7 @@ boolean isOpUnsafe() {
 
             if (operation.symbol().equals(SUB.symbol()) && opRight instanceof Literal == false) { // such as: 1 - x > -MAX
                 // if next simplification step would fail on overflow anyways, skip the optimisation already
-                return tryFolding(new Sub(EMPTY, opLeft, bcLiteral)) == null;
+                return tryFolding(foldContext, new Sub(EMPTY, opLeft, bcLiteral)) == null;
             }
 
             return false;
@@ -194,8 +198,8 @@ private static class MulDivSimplifier extends OperationSimplifier {
         private final boolean isDiv; // and not MUL.
         private final int opRightSign; // sign of the right operand in: (left) (op) (right) (comp) (literal)
 
-        MulDivSimplifier(BinaryComparison comparison) {
-            super(comparison);
+        MulDivSimplifier(FoldContext foldContext, BinaryComparison comparison) {
+            super(foldContext, comparison);
             isDiv = operation.symbol().equals(DIV.symbol());
             opRightSign = sign(opRight);
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SkipQueryOnLimitZero.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SkipQueryOnLimitZero.java
index 5d98d941bb207..c6d62dee0ba42 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SkipQueryOnLimitZero.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SkipQueryOnLimitZero.java
@@ -7,14 +7,19 @@
 
 package org.elasticsearch.xpack.esql.optimizer.rules.logical;
 
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.plan.logical.Limit;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 
-public final class SkipQueryOnLimitZero extends OptimizerRules.OptimizerRule<Limit> {
+public final class SkipQueryOnLimitZero extends OptimizerRules.ParameterizedOptimizerRule<Limit, LogicalOptimizerContext> {
+    public SkipQueryOnLimitZero() {
+        super(OptimizerRules.TransformDirection.DOWN);
+    }
+
     @Override
-    protected LogicalPlan rule(Limit limit) {
+    protected LogicalPlan rule(Limit limit, LogicalOptimizerContext ctx) {
         if (limit.limit().foldable()) {
-            if (Integer.valueOf(0).equals((limit.limit().fold()))) {
+            if (Integer.valueOf(0).equals((limit.limit().fold(ctx.foldCtx())))) {
                 return PruneEmptyPlans.skipPlan(limit);
             }
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SplitInWithFoldableValue.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SplitInWithFoldableValue.java
index 9e9ae6a9a559d..870464feb4867 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SplitInWithFoldableValue.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SplitInWithFoldableValue.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -25,7 +26,7 @@ public SplitInWithFoldableValue() {
     }
 
     @Override
-    public Expression rule(In in) {
+    public Expression rule(In in, LogicalOptimizerContext ctx) {
         if (in.value().foldable()) {
             List<Expression> foldables = new ArrayList<>(in.list().size());
             List<Expression> nonFoldables = new ArrayList<>(in.list().size());
@@ -36,7 +37,7 @@ public Expression rule(In in) {
                     nonFoldables.add(e);
                 }
             });
-            if (foldables.size() > 0 && nonFoldables.size() > 0) {
+            if (foldables.isEmpty() == false && nonFoldables.isEmpty() == false) {
                 In withFoldables = new In(in.source(), in.value(), foldables);
                 In withoutFoldables = new In(in.source(), in.value(), nonFoldables);
                 return new Or(in.source(), withFoldables, withoutFoldables);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java
index c8369d2b08a34..62a00b79d7333 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules.OptimizerExpressionRule;
 import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules.TransformDirection;
 
@@ -21,7 +22,7 @@ public SubstituteFilteredExpression() {
     }
 
     @Override
-    protected Expression rule(FilteredExpression filteredExpression) {
+    protected Expression rule(FilteredExpression filteredExpression, LogicalOptimizerContext ctx) {
         return filteredExpression.surrogate();
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSpatialSurrogates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSpatialSurrogates.java
index 93512d80e1708..4b68ee941bc92 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSpatialSurrogates.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSpatialSurrogates.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.esql.optimizer.rules.logical;
 
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 
 /**
  * Currently this works similarly to SurrogateExpression, leaving the logic inside the expressions,
@@ -23,7 +24,7 @@ public SubstituteSpatialSurrogates() {
     }
 
     @Override
-    protected SpatialRelatesFunction rule(SpatialRelatesFunction function) {
+    protected SpatialRelatesFunction rule(SpatialRelatesFunction function, LogicalOptimizerContext ctx) {
         return function.surrogate();
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/LocalPropagateEmptyRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/LocalPropagateEmptyRelation.java
index d29da1354ef3c..9259a50d5ff9e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/LocalPropagateEmptyRelation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/LocalPropagateEmptyRelation.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BlockUtils;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
@@ -25,19 +26,24 @@
  * Local aggregation can only produce intermediate state that get wired into the global agg.
  */
 public class LocalPropagateEmptyRelation extends PropagateEmptyRelation {
-
     /**
      * Local variant of the aggregation that returns the intermediate value.
      */
     @Override
-    protected void aggOutput(NamedExpression agg, AggregateFunction aggFunc, BlockFactory blockFactory, List<Block> blocks) {
+    protected void aggOutput(
+        FoldContext foldCtx,
+        NamedExpression agg,
+        AggregateFunction aggFunc,
+        BlockFactory blockFactory,
+        List<Block> blocks
+    ) {
         List<Attribute> output = AbstractPhysicalOperationProviders.intermediateAttributes(List.of(agg), List.of());
         for (Attribute o : output) {
             DataType dataType = o.dataType();
             // boolean right now is used for the internal #seen so always return true
             var value = dataType == DataType.BOOLEAN ? true
                 // look for count(literal) with literal != null
-                : aggFunc instanceof Count count && (count.foldable() == false || count.fold() != null) ? 0L
+                : aggFunc instanceof Count count && (count.foldable() == false || count.fold(foldCtx) != null) ? 0L
                 // otherwise nullify
                 : null;
             var wrapper = BlockUtils.wrapperFor(blockFactory, PlannerUtils.toElementType(dataType), 1);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceMissingFieldWithNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceMissingFieldWithNull.java
index 096f72f7694e1..e41e500aad110 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceMissingFieldWithNull.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceMissingFieldWithNull.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.esql.optimizer.rules.logical.local;
 
 import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
@@ -41,10 +42,17 @@ public class ReplaceMissingFieldWithNull extends ParameterizedRule<LogicalPlan,
 
     @Override
     public LogicalPlan apply(LogicalPlan plan, LocalLogicalOptimizerContext localLogicalOptimizerContext) {
-        return plan.transformUp(p -> missingToNull(p, localLogicalOptimizerContext.searchStats()));
+        AttributeSet lookupFields = new AttributeSet();
+        plan.forEachUp(EsRelation.class, esRelation -> {
+            if (esRelation.indexMode() == IndexMode.LOOKUP) {
+                lookupFields.addAll(esRelation.output());
+            }
+        });
+
+        return plan.transformUp(p -> missingToNull(p, localLogicalOptimizerContext.searchStats(), lookupFields));
     }
 
-    private LogicalPlan missingToNull(LogicalPlan plan, SearchStats stats) {
+    private LogicalPlan missingToNull(LogicalPlan plan, SearchStats stats, AttributeSet lookupFields) {
         if (plan instanceof EsRelation || plan instanceof LocalRelation) {
             return plan;
         }
@@ -95,7 +103,9 @@ else if (plan instanceof Project project) {
                 plan = plan.transformExpressionsOnlyUp(
                     FieldAttribute.class,
                     // Do not use the attribute name, this can deviate from the field name for union types.
-                    f -> stats.exists(f.fieldName()) ? f : Literal.of(f, null)
+                    // Also skip fields from lookup indices because we do not have stats for these.
+                    // TODO: We do have stats for lookup indices in case they are being used in the FROM clause; this can be refined.
+                    f -> stats.exists(f.fieldName()) || lookupFields.contains(f) ? f : Literal.of(f, null)
                 );
             }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java
index dfb1dbc8bc8f3..afeab28745c65 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.AttributeMap;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
@@ -76,22 +77,33 @@ public class EnableSpatialDistancePushdown extends PhysicalOptimizerRules.Parame
     protected PhysicalPlan rule(FilterExec filterExec, LocalPhysicalOptimizerContext ctx) {
         PhysicalPlan plan = filterExec;
         if (filterExec.child() instanceof EsQueryExec esQueryExec) {
-            plan = rewrite(filterExec, esQueryExec, LucenePushdownPredicates.from(ctx.searchStats()));
+            plan = rewrite(ctx.foldCtx(), filterExec, esQueryExec, LucenePushdownPredicates.from(ctx.searchStats()));
         } else if (filterExec.child() instanceof EvalExec evalExec && evalExec.child() instanceof EsQueryExec esQueryExec) {
-            plan = rewriteBySplittingFilter(filterExec, evalExec, esQueryExec, LucenePushdownPredicates.from(ctx.searchStats()));
+            plan = rewriteBySplittingFilter(
+                ctx.foldCtx(),
+                filterExec,
+                evalExec,
+                esQueryExec,
+                LucenePushdownPredicates.from(ctx.searchStats())
+            );
         }
 
         return plan;
     }
 
-    private FilterExec rewrite(FilterExec filterExec, EsQueryExec esQueryExec, LucenePushdownPredicates lucenePushdownPredicates) {
+    private FilterExec rewrite(
+        FoldContext ctx,
+        FilterExec filterExec,
+        EsQueryExec esQueryExec,
+        LucenePushdownPredicates lucenePushdownPredicates
+    ) {
         // Find and rewrite any binary comparisons that involve a distance function and a literal
         var rewritten = filterExec.condition().transformDown(EsqlBinaryComparison.class, comparison -> {
             ComparisonType comparisonType = ComparisonType.from(comparison.getFunctionType());
             if (comparison.left() instanceof StDistance dist && comparison.right().foldable()) {
-                return rewriteComparison(comparison, dist, comparison.right(), comparisonType);
+                return rewriteComparison(ctx, comparison, dist, comparison.right(), comparisonType);
             } else if (comparison.right() instanceof StDistance dist && comparison.left().foldable()) {
-                return rewriteComparison(comparison, dist, comparison.left(), ComparisonType.invert(comparisonType));
+                return rewriteComparison(ctx, comparison, dist, comparison.left(), ComparisonType.invert(comparisonType));
             }
             return comparison;
         });
@@ -120,6 +132,7 @@ private FilterExec rewrite(FilterExec filterExec, EsQueryExec esQueryExec, Lucen
      * </pre>
      */
     private PhysicalPlan rewriteBySplittingFilter(
+        FoldContext ctx,
         FilterExec filterExec,
         EvalExec evalExec,
         EsQueryExec esQueryExec,
@@ -142,7 +155,7 @@ private PhysicalPlan rewriteBySplittingFilter(
         for (Expression exp : splitAnd(filterExec.condition())) {
             Expression resExp = exp.transformUp(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r));
             // Find and rewrite any binary comparisons that involve a distance function and a literal
-            var rewritten = rewriteDistanceFilters(resExp, distances);
+            var rewritten = rewriteDistanceFilters(ctx, resExp, distances);
             // If all pushable StDistance functions were found and re-written, we need to re-write the FILTER/EVAL combination
             if (rewritten.equals(resExp) == false && canPushToSource(rewritten, lucenePushdownPredicates)) {
                 pushable.add(rewritten);
@@ -181,40 +194,42 @@ private Map<NameId, StDistance> getPushableDistances(List<Alias> aliases, Lucene
         return distances;
     }
 
-    private Expression rewriteDistanceFilters(Expression expr, Map<NameId, StDistance> distances) {
+    private Expression rewriteDistanceFilters(FoldContext ctx, Expression expr, Map<NameId, StDistance> distances) {
         return expr.transformDown(EsqlBinaryComparison.class, comparison -> {
             ComparisonType comparisonType = ComparisonType.from(comparison.getFunctionType());
             if (comparison.left() instanceof ReferenceAttribute r && distances.containsKey(r.id()) && comparison.right().foldable()) {
                 StDistance dist = distances.get(r.id());
-                return rewriteComparison(comparison, dist, comparison.right(), comparisonType);
+                return rewriteComparison(ctx, comparison, dist, comparison.right(), comparisonType);
             } else if (comparison.right() instanceof ReferenceAttribute r
                 && distances.containsKey(r.id())
                 && comparison.left().foldable()) {
                     StDistance dist = distances.get(r.id());
-                    return rewriteComparison(comparison, dist, comparison.left(), ComparisonType.invert(comparisonType));
+                    return rewriteComparison(ctx, comparison, dist, comparison.left(), ComparisonType.invert(comparisonType));
                 }
             return comparison;
         });
     }
 
     private Expression rewriteComparison(
+        FoldContext ctx,
         EsqlBinaryComparison comparison,
         StDistance dist,
         Expression literal,
         ComparisonType comparisonType
     ) {
-        Object value = literal.fold();
+        Object value = literal.fold(ctx);
         if (value instanceof Number number) {
             if (dist.right().foldable()) {
-                return rewriteDistanceFilter(comparison, dist.left(), dist.right(), number, comparisonType);
+                return rewriteDistanceFilter(ctx, comparison, dist.left(), dist.right(), number, comparisonType);
             } else if (dist.left().foldable()) {
-                return rewriteDistanceFilter(comparison, dist.right(), dist.left(), number, comparisonType);
+                return rewriteDistanceFilter(ctx, comparison, dist.right(), dist.left(), number, comparisonType);
             }
         }
         return comparison;
     }
 
     private Expression rewriteDistanceFilter(
+        FoldContext ctx,
         EsqlBinaryComparison comparison,
         Expression spatialExp,
         Expression literalExp,
@@ -222,7 +237,7 @@ private Expression rewriteDistanceFilter(
         ComparisonType comparisonType
     ) {
         DataType shapeDataType = getShapeDataType(spatialExp);
-        Geometry geometry = SpatialRelatesUtils.makeGeometryFromLiteral(literalExp);
+        Geometry geometry = SpatialRelatesUtils.makeGeometryFromLiteral(ctx, literalExp);
         if (geometry instanceof Point point) {
             double distance = number.doubleValue();
             Source source = comparison.source();
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java
index 6fcdd538fdfc8..dae84bb6b6494 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 import org.elasticsearch.xpack.esql.core.util.Queries;
 import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction;
-import org.elasticsearch.xpack.esql.expression.function.fulltext.Term;
 import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch;
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.BinarySpatialFunction;
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction;
@@ -252,8 +251,6 @@ static boolean canPushToSource(Expression exp, LucenePushdownPredicates lucenePu
                 && Expressions.foldable(cidrMatch.matches());
         } else if (exp instanceof SpatialRelatesFunction spatial) {
             return canPushSpatialFunctionToSource(spatial, lucenePushdownPredicates);
-        } else if (exp instanceof Term term) {
-            return term.field() instanceof FieldAttribute && DataType.isString(term.field().dataType());
         } else if (exp instanceof FullTextFunction) {
             return true;
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java
index 2b531257e594a..24df3c1db234e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.xpack.esql.core.expression.AttributeMap;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
@@ -61,7 +62,7 @@ public class PushTopNToSource extends PhysicalOptimizerRules.ParameterizedOptimi
 
     @Override
     protected PhysicalPlan rule(TopNExec topNExec, LocalPhysicalOptimizerContext ctx) {
-        Pushable pushable = evaluatePushable(topNExec, LucenePushdownPredicates.from(ctx.searchStats()));
+        Pushable pushable = evaluatePushable(ctx.foldCtx(), topNExec, LucenePushdownPredicates.from(ctx.searchStats()));
         return pushable.rewrite(topNExec);
     }
 
@@ -95,18 +96,18 @@ private EsQueryExec.Sort sort() {
             return new EsQueryExec.GeoDistanceSort(fieldAttribute.exactAttribute(), order.direction(), point.getLat(), point.getLon());
         }
 
-        private static PushableGeoDistance from(StDistance distance, Order order) {
+        private static PushableGeoDistance from(FoldContext ctx, StDistance distance, Order order) {
             if (distance.left() instanceof Attribute attr && distance.right().foldable()) {
-                return from(attr, distance.right(), order);
+                return from(ctx, attr, distance.right(), order);
             } else if (distance.right() instanceof Attribute attr && distance.left().foldable()) {
-                return from(attr, distance.left(), order);
+                return from(ctx, attr, distance.left(), order);
             }
             return null;
         }
 
-        private static PushableGeoDistance from(Attribute attr, Expression foldable, Order order) {
+        private static PushableGeoDistance from(FoldContext ctx, Attribute attr, Expression foldable, Order order) {
             if (attr instanceof FieldAttribute fieldAttribute) {
-                Geometry geometry = SpatialRelatesUtils.makeGeometryFromLiteral(foldable);
+                Geometry geometry = SpatialRelatesUtils.makeGeometryFromLiteral(ctx, foldable);
                 if (geometry instanceof Point point) {
                     return new PushableGeoDistance(fieldAttribute, order, point);
                 }
@@ -122,7 +123,7 @@ public PhysicalPlan rewrite(TopNExec topNExec) {
         }
     }
 
-    private static Pushable evaluatePushable(TopNExec topNExec, LucenePushdownPredicates lucenePushdownPredicates) {
+    private static Pushable evaluatePushable(FoldContext ctx, TopNExec topNExec, LucenePushdownPredicates lucenePushdownPredicates) {
         PhysicalPlan child = topNExec.child();
         if (child instanceof EsQueryExec queryExec
             && queryExec.canPushSorts()
@@ -164,7 +165,7 @@ && canPushDownOrders(topNExec.order(), lucenePushdownPredicates)) {
                     if (distances.containsKey(resolvedAttribute.id())) {
                         StDistance distance = distances.get(resolvedAttribute.id());
                         StDistance d = (StDistance) distance.transformDown(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r));
-                        PushableGeoDistance pushableGeoDistance = PushableGeoDistance.from(d, order);
+                        PushableGeoDistance pushableGeoDistance = PushableGeoDistance.from(ctx, d, order);
                         if (pushableGeoDistance != null) {
                             pushableSorts.add(pushableGeoDistance.sort());
                         } else {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialDocValuesExtraction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialDocValuesExtraction.java
index 0f1c32e94f867..f66ed5c8e4ec1 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialDocValuesExtraction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialDocValuesExtraction.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialAggregateFunction;
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.BinarySpatialFunction;
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction;
@@ -156,6 +157,9 @@ private boolean foundField(Expression expression, Set<FieldAttribute> foundAttri
     /**
      * This function disallows the use of more than one field for doc-values extraction in the same spatial relation function.
      * This is because comparing two doc-values fields is not supported in the current implementation.
+     * This also rejects fields that do not have doc-values in the field mapping, as well as rejecting geo_shape and cartesian_shape
+     * because we do not yet support full doc-values extraction for non-point geometries. We do have aggregations that support
+     * shapes, and to prevent them triggering this rule on non-point geometries we have to explicitly disallow them here.
      */
     private boolean allowedForDocValues(
         FieldAttribute fieldAttribute,
@@ -166,6 +170,9 @@ private boolean allowedForDocValues(
         if (stats.hasDocValues(fieldAttribute.fieldName()) == false) {
             return false;
         }
+        if (fieldAttribute.dataType() == DataType.GEO_SHAPE || fieldAttribute.dataType() == DataType.CARTESIAN_SHAPE) {
+            return false;
+        }
         var candidateDocValuesAttributes = new HashSet<>(foundAttributes);
         candidateDocValuesAttributes.add(fieldAttribute);
         var spatialRelatesAttributes = new HashSet<FieldAttribute>();
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java
new file mode 100644
index 0000000000000..f6f087064a02f
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.optimizer.rules.physical.local;
+
+import org.elasticsearch.lucene.spatial.GeometryDocValueWriter;
+import org.elasticsearch.xpack.esql.core.expression.Alias;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialExtent;
+import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalOptimizerContext;
+import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerRules.ParameterizedOptimizerRule;
+import org.elasticsearch.xpack.esql.plan.physical.AggregateExec;
+import org.elasticsearch.xpack.esql.plan.physical.EvalExec;
+import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec;
+import org.elasticsearch.xpack.esql.plan.physical.FilterExec;
+import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
+import org.elasticsearch.xpack.esql.plan.physical.UnaryExec;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * This rule is responsible for marking spatial shape fields whose extent can be extracted from the binary representation encoded by
+ * {@link GeometryDocValueWriter}.
+ * This is a very specific optimization that is only used in the context of <code>ST_EXTENT_AGG</code> aggregations.
+ * Normally spatial fields are extracted from source values because this maintains original precision, but is very slow.
+ * Simply extracting the spatial bounds from the binary encoding loses both precision and geometry topological information for shapes.
+ * For this reason we only consider extract the extent under very specific conditions:
+ * <ul>
+ *     <li>The spatial data is of type GEO_SHAPE or CARTESIAN_SHAPE.</li>
+ *     <li>The spatial data is consumed directly by an <code>ST_EXTENT_AGG</code>.</li>
+ *     <li>The spatial data is not consumed by any other operation. While is this is stricter than necessary,
+ *     it is a good enough approximation for now. For example, an aggregation like {@code count} shouldn't stop this optimization,
+ *     not a check like {@code isNotNull}.</li>
+ * </ul>
+ */
+public class SpatialShapeBoundsExtraction extends ParameterizedOptimizerRule<AggregateExec, LocalPhysicalOptimizerContext> {
+    @Override
+    protected PhysicalPlan rule(AggregateExec aggregate, LocalPhysicalOptimizerContext ctx) {
+        var foundAttributes = new HashSet<Attribute>();
+
+        return aggregate.transformDown(UnaryExec.class, exec -> {
+            switch (exec) {
+                case AggregateExec agg -> {
+                    List<AggregateFunction> aggregateFunctions = agg.aggregates()
+                        .stream()
+                        .flatMap(e -> SpatialShapeBoundsExtraction.extractAggregateFunction(e).stream())
+                        .toList();
+                    List<SpatialExtent> spatialExtents = aggregateFunctions.stream()
+                        .filter(SpatialExtent.class::isInstance)
+                        .map(SpatialExtent.class::cast)
+                        .toList();
+                    List<AggregateFunction> nonSpatialExtents = aggregateFunctions.stream()
+                        .filter(a -> a instanceof SpatialExtent == false)
+                        .toList();
+                    // While we currently do not have any non-extent aggregations which apply to shapes, we might have them in the future.
+                    Set<EsField> fieldsAppearingInNonSpatialExtents = nonSpatialExtents.stream()
+                        .flatMap(af -> af.references().stream())
+                        .filter(FieldAttribute.class::isInstance)
+                        .map(f -> ((FieldAttribute) f).field())
+                        .collect(Collectors.toSet());
+                    spatialExtents.stream()
+                        .map(SpatialExtent::field)
+                        .filter(FieldAttribute.class::isInstance)
+                        .map(FieldAttribute.class::cast)
+                        .filter(
+                            f -> isShape(f.field().getDataType())
+                                && fieldsAppearingInNonSpatialExtents.contains(f.field()) == false
+                                && ctx.searchStats().hasDocValues(f.fieldName())
+                        )
+                        .forEach(foundAttributes::add);
+                }
+                case EvalExec evalExec -> foundAttributes.removeAll(evalExec.references());
+                case FilterExec filterExec -> foundAttributes.removeAll(filterExec.condition().references());
+                case FieldExtractExec fieldExtractExec -> {
+                    var boundsAttributes = new HashSet<>(foundAttributes);
+                    boundsAttributes.retainAll(fieldExtractExec.attributesToExtract());
+                    if (boundsAttributes.isEmpty() == false) {
+                        exec = fieldExtractExec.withBoundsAttributes(boundsAttributes);
+                    }
+                }
+                default -> { // Do nothing
+                }
+            }
+            return exec;
+        });
+    }
+
+    private static boolean isShape(DataType dataType) {
+        return dataType == DataType.GEO_SHAPE || dataType == DataType.CARTESIAN_SHAPE;
+    }
+
+    private static Optional<AggregateFunction> extractAggregateFunction(NamedExpression expr) {
+        return expr instanceof Alias as && as.child() instanceof AggregateFunction af ? Optional.of(af) : Optional.empty();
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp
index c5b37fa411f65..492df7fbc1608 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp
@@ -283,8 +283,6 @@ indexPattern
 clusterString
 indexString
 metadata
-metadataOption
-deprecated_metadata
 metricsCommand
 evalCommand
 statsCommand
@@ -330,4 +328,4 @@ joinPredicate
 
 
 atn:
-[4, 1, 128, 639, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 142, 8, 1, 10, 1, 12, 1, 145, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 153, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 173, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 185, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 192, 8, 5, 10, 5, 12, 5, 195, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 202, 8, 5, 1, 5, 1, 5, 1, 5, 3, 5, 207, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 215, 8, 5, 10, 5, 12, 5, 218, 9, 5, 1, 6, 1, 6, 3, 6, 222, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 229, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 234, 8, 6, 1, 7, 1, 7, 1, 7, 3, 7, 239, 8, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 249, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 255, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 263, 8, 9, 10, 9, 12, 9, 266, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 276, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 281, 8, 10, 10, 10, 12, 10, 284, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 292, 8, 11, 10, 11, 12, 11, 295, 9, 11, 3, 11, 297, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 5, 15, 311, 8, 15, 10, 15, 12, 15, 314, 9, 15, 1, 16, 1, 16, 1, 16, 3, 16, 319, 8, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 5, 17, 327, 8, 17, 10, 17, 12, 17, 330, 9, 17, 1, 17, 3, 17, 333, 8, 17, 1, 18, 1, 18, 1, 18, 3, 18, 338, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21, 3, 21, 348, 8, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 354, 8, 22, 10, 22, 12, 22, 357, 9, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 367, 8, 24, 10, 24, 12, 24, 370, 9, 24, 1, 24, 3, 24, 373, 8, 24, 1, 24, 1, 24, 3, 24, 377, 8, 24, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 3, 26, 384, 8, 26, 1, 26, 1, 26, 3, 26, 388, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 393, 8, 27, 10, 27, 12, 27, 396, 9, 27, 1, 28, 1, 28, 1, 28, 3, 28, 401, 8, 28, 1, 29, 1, 29, 1, 29, 5, 29, 406, 8, 29, 10, 29, 12, 29, 409, 9, 29, 1, 30, 1, 30, 1, 30, 5, 30, 414, 8, 30, 10, 30, 12, 30, 417, 9, 30, 1, 31, 1, 31, 1, 31, 5, 31, 422, 8, 31, 10, 31, 12, 31, 425, 9, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 3, 33, 432, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 447, 8, 34, 10, 34, 12, 34, 450, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 458, 8, 34, 10, 34, 12, 34, 461, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 469, 8, 34, 10, 34, 12, 34, 472, 9, 34, 1, 34, 1, 34, 3, 34, 476, 8, 34, 1, 35, 1, 35, 3, 35, 480, 8, 35, 1, 36, 1, 36, 1, 36, 3, 36, 485, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 494, 8, 38, 10, 38, 12, 38, 497, 9, 38, 1, 39, 1, 39, 3, 39, 501, 8, 39, 1, 39, 1, 39, 3, 39, 505, 8, 39, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 517, 8, 42, 10, 42, 12, 42, 520, 9, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 530, 8, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 5, 47, 542, 8, 47, 10, 47, 12, 47, 545, 9, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 3, 50, 555, 8, 50, 1, 51, 3, 51, 558, 8, 51, 1, 51, 1, 51, 1, 52, 3, 52, 563, 8, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 585, 8, 58, 1, 58, 1, 58, 1, 58, 1, 58, 5, 58, 591, 8, 58, 10, 58, 12, 58, 594, 9, 58, 3, 58, 596, 8, 58, 1, 59, 1, 59, 1, 59, 3, 59, 601, 8, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 3, 61, 614, 8, 61, 1, 62, 3, 62, 617, 8, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 3, 63, 626, 8, 63, 1, 64, 1, 64, 1, 64, 1, 64, 5, 64, 632, 8, 64, 10, 64, 12, 64, 635, 9, 64, 1, 65, 1, 65, 1, 65, 0, 4, 2, 10, 18, 20, 66, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 0, 9, 1, 0, 64, 65, 1, 0, 66, 68, 2, 0, 30, 30, 81, 81, 1, 0, 72, 73, 2, 0, 35, 35, 40, 40, 2, 0, 43, 43, 46, 46, 2, 0, 42, 42, 56, 56, 2, 0, 57, 57, 59, 63, 1, 0, 22, 24, 665, 0, 132, 1, 0, 0, 0, 2, 135, 1, 0, 0, 0, 4, 152, 1, 0, 0, 0, 6, 172, 1, 0, 0, 0, 8, 174, 1, 0, 0, 0, 10, 206, 1, 0, 0, 0, 12, 233, 1, 0, 0, 0, 14, 235, 1, 0, 0, 0, 16, 248, 1, 0, 0, 0, 18, 254, 1, 0, 0, 0, 20, 275, 1, 0, 0, 0, 22, 285, 1, 0, 0, 0, 24, 300, 1, 0, 0, 0, 26, 302, 1, 0, 0, 0, 28, 304, 1, 0, 0, 0, 30, 307, 1, 0, 0, 0, 32, 318, 1, 0, 0, 0, 34, 322, 1, 0, 0, 0, 36, 337, 1, 0, 0, 0, 38, 341, 1, 0, 0, 0, 40, 343, 1, 0, 0, 0, 42, 347, 1, 0, 0, 0, 44, 349, 1, 0, 0, 0, 46, 358, 1, 0, 0, 0, 48, 362, 1, 0, 0, 0, 50, 378, 1, 0, 0, 0, 52, 381, 1, 0, 0, 0, 54, 389, 1, 0, 0, 0, 56, 397, 1, 0, 0, 0, 58, 402, 1, 0, 0, 0, 60, 410, 1, 0, 0, 0, 62, 418, 1, 0, 0, 0, 64, 426, 1, 0, 0, 0, 66, 431, 1, 0, 0, 0, 68, 475, 1, 0, 0, 0, 70, 479, 1, 0, 0, 0, 72, 484, 1, 0, 0, 0, 74, 486, 1, 0, 0, 0, 76, 489, 1, 0, 0, 0, 78, 498, 1, 0, 0, 0, 80, 506, 1, 0, 0, 0, 82, 509, 1, 0, 0, 0, 84, 512, 1, 0, 0, 0, 86, 521, 1, 0, 0, 0, 88, 525, 1, 0, 0, 0, 90, 531, 1, 0, 0, 0, 92, 535, 1, 0, 0, 0, 94, 538, 1, 0, 0, 0, 96, 546, 1, 0, 0, 0, 98, 550, 1, 0, 0, 0, 100, 554, 1, 0, 0, 0, 102, 557, 1, 0, 0, 0, 104, 562, 1, 0, 0, 0, 106, 566, 1, 0, 0, 0, 108, 568, 1, 0, 0, 0, 110, 570, 1, 0, 0, 0, 112, 573, 1, 0, 0, 0, 114, 577, 1, 0, 0, 0, 116, 580, 1, 0, 0, 0, 118, 600, 1, 0, 0, 0, 120, 604, 1, 0, 0, 0, 122, 609, 1, 0, 0, 0, 124, 616, 1, 0, 0, 0, 126, 622, 1, 0, 0, 0, 128, 627, 1, 0, 0, 0, 130, 636, 1, 0, 0, 0, 132, 133, 3, 2, 1, 0, 133, 134, 5, 0, 0, 1, 134, 1, 1, 0, 0, 0, 135, 136, 6, 1, -1, 0, 136, 137, 3, 4, 2, 0, 137, 143, 1, 0, 0, 0, 138, 139, 10, 1, 0, 0, 139, 140, 5, 29, 0, 0, 140, 142, 3, 6, 3, 0, 141, 138, 1, 0, 0, 0, 142, 145, 1, 0, 0, 0, 143, 141, 1, 0, 0, 0, 143, 144, 1, 0, 0, 0, 144, 3, 1, 0, 0, 0, 145, 143, 1, 0, 0, 0, 146, 153, 3, 110, 55, 0, 147, 153, 3, 34, 17, 0, 148, 153, 3, 28, 14, 0, 149, 153, 3, 114, 57, 0, 150, 151, 4, 2, 1, 0, 151, 153, 3, 48, 24, 0, 152, 146, 1, 0, 0, 0, 152, 147, 1, 0, 0, 0, 152, 148, 1, 0, 0, 0, 152, 149, 1, 0, 0, 0, 152, 150, 1, 0, 0, 0, 153, 5, 1, 0, 0, 0, 154, 173, 3, 50, 25, 0, 155, 173, 3, 8, 4, 0, 156, 173, 3, 80, 40, 0, 157, 173, 3, 74, 37, 0, 158, 173, 3, 52, 26, 0, 159, 173, 3, 76, 38, 0, 160, 173, 3, 82, 41, 0, 161, 173, 3, 84, 42, 0, 162, 173, 3, 88, 44, 0, 163, 173, 3, 90, 45, 0, 164, 173, 3, 116, 58, 0, 165, 173, 3, 92, 46, 0, 166, 167, 4, 3, 2, 0, 167, 173, 3, 122, 61, 0, 168, 169, 4, 3, 3, 0, 169, 173, 3, 120, 60, 0, 170, 171, 4, 3, 4, 0, 171, 173, 3, 124, 62, 0, 172, 154, 1, 0, 0, 0, 172, 155, 1, 0, 0, 0, 172, 156, 1, 0, 0, 0, 172, 157, 1, 0, 0, 0, 172, 158, 1, 0, 0, 0, 172, 159, 1, 0, 0, 0, 172, 160, 1, 0, 0, 0, 172, 161, 1, 0, 0, 0, 172, 162, 1, 0, 0, 0, 172, 163, 1, 0, 0, 0, 172, 164, 1, 0, 0, 0, 172, 165, 1, 0, 0, 0, 172, 166, 1, 0, 0, 0, 172, 168, 1, 0, 0, 0, 172, 170, 1, 0, 0, 0, 173, 7, 1, 0, 0, 0, 174, 175, 5, 16, 0, 0, 175, 176, 3, 10, 5, 0, 176, 9, 1, 0, 0, 0, 177, 178, 6, 5, -1, 0, 178, 179, 5, 49, 0, 0, 179, 207, 3, 10, 5, 8, 180, 207, 3, 16, 8, 0, 181, 207, 3, 12, 6, 0, 182, 184, 3, 16, 8, 0, 183, 185, 5, 49, 0, 0, 184, 183, 1, 0, 0, 0, 184, 185, 1, 0, 0, 0, 185, 186, 1, 0, 0, 0, 186, 187, 5, 44, 0, 0, 187, 188, 5, 48, 0, 0, 188, 193, 3, 16, 8, 0, 189, 190, 5, 39, 0, 0, 190, 192, 3, 16, 8, 0, 191, 189, 1, 0, 0, 0, 192, 195, 1, 0, 0, 0, 193, 191, 1, 0, 0, 0, 193, 194, 1, 0, 0, 0, 194, 196, 1, 0, 0, 0, 195, 193, 1, 0, 0, 0, 196, 197, 5, 55, 0, 0, 197, 207, 1, 0, 0, 0, 198, 199, 3, 16, 8, 0, 199, 201, 5, 45, 0, 0, 200, 202, 5, 49, 0, 0, 201, 200, 1, 0, 0, 0, 201, 202, 1, 0, 0, 0, 202, 203, 1, 0, 0, 0, 203, 204, 5, 50, 0, 0, 204, 207, 1, 0, 0, 0, 205, 207, 3, 14, 7, 0, 206, 177, 1, 0, 0, 0, 206, 180, 1, 0, 0, 0, 206, 181, 1, 0, 0, 0, 206, 182, 1, 0, 0, 0, 206, 198, 1, 0, 0, 0, 206, 205, 1, 0, 0, 0, 207, 216, 1, 0, 0, 0, 208, 209, 10, 5, 0, 0, 209, 210, 5, 34, 0, 0, 210, 215, 3, 10, 5, 6, 211, 212, 10, 4, 0, 0, 212, 213, 5, 52, 0, 0, 213, 215, 3, 10, 5, 5, 214, 208, 1, 0, 0, 0, 214, 211, 1, 0, 0, 0, 215, 218, 1, 0, 0, 0, 216, 214, 1, 0, 0, 0, 216, 217, 1, 0, 0, 0, 217, 11, 1, 0, 0, 0, 218, 216, 1, 0, 0, 0, 219, 221, 3, 16, 8, 0, 220, 222, 5, 49, 0, 0, 221, 220, 1, 0, 0, 0, 221, 222, 1, 0, 0, 0, 222, 223, 1, 0, 0, 0, 223, 224, 5, 47, 0, 0, 224, 225, 3, 106, 53, 0, 225, 234, 1, 0, 0, 0, 226, 228, 3, 16, 8, 0, 227, 229, 5, 49, 0, 0, 228, 227, 1, 0, 0, 0, 228, 229, 1, 0, 0, 0, 229, 230, 1, 0, 0, 0, 230, 231, 5, 54, 0, 0, 231, 232, 3, 106, 53, 0, 232, 234, 1, 0, 0, 0, 233, 219, 1, 0, 0, 0, 233, 226, 1, 0, 0, 0, 234, 13, 1, 0, 0, 0, 235, 238, 3, 58, 29, 0, 236, 237, 5, 37, 0, 0, 237, 239, 3, 26, 13, 0, 238, 236, 1, 0, 0, 0, 238, 239, 1, 0, 0, 0, 239, 240, 1, 0, 0, 0, 240, 241, 5, 38, 0, 0, 241, 242, 3, 68, 34, 0, 242, 15, 1, 0, 0, 0, 243, 249, 3, 18, 9, 0, 244, 245, 3, 18, 9, 0, 245, 246, 3, 108, 54, 0, 246, 247, 3, 18, 9, 0, 247, 249, 1, 0, 0, 0, 248, 243, 1, 0, 0, 0, 248, 244, 1, 0, 0, 0, 249, 17, 1, 0, 0, 0, 250, 251, 6, 9, -1, 0, 251, 255, 3, 20, 10, 0, 252, 253, 7, 0, 0, 0, 253, 255, 3, 18, 9, 3, 254, 250, 1, 0, 0, 0, 254, 252, 1, 0, 0, 0, 255, 264, 1, 0, 0, 0, 256, 257, 10, 2, 0, 0, 257, 258, 7, 1, 0, 0, 258, 263, 3, 18, 9, 3, 259, 260, 10, 1, 0, 0, 260, 261, 7, 0, 0, 0, 261, 263, 3, 18, 9, 2, 262, 256, 1, 0, 0, 0, 262, 259, 1, 0, 0, 0, 263, 266, 1, 0, 0, 0, 264, 262, 1, 0, 0, 0, 264, 265, 1, 0, 0, 0, 265, 19, 1, 0, 0, 0, 266, 264, 1, 0, 0, 0, 267, 268, 6, 10, -1, 0, 268, 276, 3, 68, 34, 0, 269, 276, 3, 58, 29, 0, 270, 276, 3, 22, 11, 0, 271, 272, 5, 48, 0, 0, 272, 273, 3, 10, 5, 0, 273, 274, 5, 55, 0, 0, 274, 276, 1, 0, 0, 0, 275, 267, 1, 0, 0, 0, 275, 269, 1, 0, 0, 0, 275, 270, 1, 0, 0, 0, 275, 271, 1, 0, 0, 0, 276, 282, 1, 0, 0, 0, 277, 278, 10, 1, 0, 0, 278, 279, 5, 37, 0, 0, 279, 281, 3, 26, 13, 0, 280, 277, 1, 0, 0, 0, 281, 284, 1, 0, 0, 0, 282, 280, 1, 0, 0, 0, 282, 283, 1, 0, 0, 0, 283, 21, 1, 0, 0, 0, 284, 282, 1, 0, 0, 0, 285, 286, 3, 24, 12, 0, 286, 296, 5, 48, 0, 0, 287, 297, 5, 66, 0, 0, 288, 293, 3, 10, 5, 0, 289, 290, 5, 39, 0, 0, 290, 292, 3, 10, 5, 0, 291, 289, 1, 0, 0, 0, 292, 295, 1, 0, 0, 0, 293, 291, 1, 0, 0, 0, 293, 294, 1, 0, 0, 0, 294, 297, 1, 0, 0, 0, 295, 293, 1, 0, 0, 0, 296, 287, 1, 0, 0, 0, 296, 288, 1, 0, 0, 0, 296, 297, 1, 0, 0, 0, 297, 298, 1, 0, 0, 0, 298, 299, 5, 55, 0, 0, 299, 23, 1, 0, 0, 0, 300, 301, 3, 72, 36, 0, 301, 25, 1, 0, 0, 0, 302, 303, 3, 64, 32, 0, 303, 27, 1, 0, 0, 0, 304, 305, 5, 12, 0, 0, 305, 306, 3, 30, 15, 0, 306, 29, 1, 0, 0, 0, 307, 312, 3, 32, 16, 0, 308, 309, 5, 39, 0, 0, 309, 311, 3, 32, 16, 0, 310, 308, 1, 0, 0, 0, 311, 314, 1, 0, 0, 0, 312, 310, 1, 0, 0, 0, 312, 313, 1, 0, 0, 0, 313, 31, 1, 0, 0, 0, 314, 312, 1, 0, 0, 0, 315, 316, 3, 58, 29, 0, 316, 317, 5, 36, 0, 0, 317, 319, 1, 0, 0, 0, 318, 315, 1, 0, 0, 0, 318, 319, 1, 0, 0, 0, 319, 320, 1, 0, 0, 0, 320, 321, 3, 10, 5, 0, 321, 33, 1, 0, 0, 0, 322, 323, 5, 6, 0, 0, 323, 328, 3, 36, 18, 0, 324, 325, 5, 39, 0, 0, 325, 327, 3, 36, 18, 0, 326, 324, 1, 0, 0, 0, 327, 330, 1, 0, 0, 0, 328, 326, 1, 0, 0, 0, 328, 329, 1, 0, 0, 0, 329, 332, 1, 0, 0, 0, 330, 328, 1, 0, 0, 0, 331, 333, 3, 42, 21, 0, 332, 331, 1, 0, 0, 0, 332, 333, 1, 0, 0, 0, 333, 35, 1, 0, 0, 0, 334, 335, 3, 38, 19, 0, 335, 336, 5, 38, 0, 0, 336, 338, 1, 0, 0, 0, 337, 334, 1, 0, 0, 0, 337, 338, 1, 0, 0, 0, 338, 339, 1, 0, 0, 0, 339, 340, 3, 40, 20, 0, 340, 37, 1, 0, 0, 0, 341, 342, 5, 81, 0, 0, 342, 39, 1, 0, 0, 0, 343, 344, 7, 2, 0, 0, 344, 41, 1, 0, 0, 0, 345, 348, 3, 44, 22, 0, 346, 348, 3, 46, 23, 0, 347, 345, 1, 0, 0, 0, 347, 346, 1, 0, 0, 0, 348, 43, 1, 0, 0, 0, 349, 350, 5, 80, 0, 0, 350, 355, 5, 81, 0, 0, 351, 352, 5, 39, 0, 0, 352, 354, 5, 81, 0, 0, 353, 351, 1, 0, 0, 0, 354, 357, 1, 0, 0, 0, 355, 353, 1, 0, 0, 0, 355, 356, 1, 0, 0, 0, 356, 45, 1, 0, 0, 0, 357, 355, 1, 0, 0, 0, 358, 359, 5, 70, 0, 0, 359, 360, 3, 44, 22, 0, 360, 361, 5, 71, 0, 0, 361, 47, 1, 0, 0, 0, 362, 363, 5, 19, 0, 0, 363, 368, 3, 36, 18, 0, 364, 365, 5, 39, 0, 0, 365, 367, 3, 36, 18, 0, 366, 364, 1, 0, 0, 0, 367, 370, 1, 0, 0, 0, 368, 366, 1, 0, 0, 0, 368, 369, 1, 0, 0, 0, 369, 372, 1, 0, 0, 0, 370, 368, 1, 0, 0, 0, 371, 373, 3, 54, 27, 0, 372, 371, 1, 0, 0, 0, 372, 373, 1, 0, 0, 0, 373, 376, 1, 0, 0, 0, 374, 375, 5, 33, 0, 0, 375, 377, 3, 30, 15, 0, 376, 374, 1, 0, 0, 0, 376, 377, 1, 0, 0, 0, 377, 49, 1, 0, 0, 0, 378, 379, 5, 4, 0, 0, 379, 380, 3, 30, 15, 0, 380, 51, 1, 0, 0, 0, 381, 383, 5, 15, 0, 0, 382, 384, 3, 54, 27, 0, 383, 382, 1, 0, 0, 0, 383, 384, 1, 0, 0, 0, 384, 387, 1, 0, 0, 0, 385, 386, 5, 33, 0, 0, 386, 388, 3, 30, 15, 0, 387, 385, 1, 0, 0, 0, 387, 388, 1, 0, 0, 0, 388, 53, 1, 0, 0, 0, 389, 394, 3, 56, 28, 0, 390, 391, 5, 39, 0, 0, 391, 393, 3, 56, 28, 0, 392, 390, 1, 0, 0, 0, 393, 396, 1, 0, 0, 0, 394, 392, 1, 0, 0, 0, 394, 395, 1, 0, 0, 0, 395, 55, 1, 0, 0, 0, 396, 394, 1, 0, 0, 0, 397, 400, 3, 32, 16, 0, 398, 399, 5, 16, 0, 0, 399, 401, 3, 10, 5, 0, 400, 398, 1, 0, 0, 0, 400, 401, 1, 0, 0, 0, 401, 57, 1, 0, 0, 0, 402, 407, 3, 72, 36, 0, 403, 404, 5, 41, 0, 0, 404, 406, 3, 72, 36, 0, 405, 403, 1, 0, 0, 0, 406, 409, 1, 0, 0, 0, 407, 405, 1, 0, 0, 0, 407, 408, 1, 0, 0, 0, 408, 59, 1, 0, 0, 0, 409, 407, 1, 0, 0, 0, 410, 415, 3, 66, 33, 0, 411, 412, 5, 41, 0, 0, 412, 414, 3, 66, 33, 0, 413, 411, 1, 0, 0, 0, 414, 417, 1, 0, 0, 0, 415, 413, 1, 0, 0, 0, 415, 416, 1, 0, 0, 0, 416, 61, 1, 0, 0, 0, 417, 415, 1, 0, 0, 0, 418, 423, 3, 60, 30, 0, 419, 420, 5, 39, 0, 0, 420, 422, 3, 60, 30, 0, 421, 419, 1, 0, 0, 0, 422, 425, 1, 0, 0, 0, 423, 421, 1, 0, 0, 0, 423, 424, 1, 0, 0, 0, 424, 63, 1, 0, 0, 0, 425, 423, 1, 0, 0, 0, 426, 427, 7, 3, 0, 0, 427, 65, 1, 0, 0, 0, 428, 432, 5, 85, 0, 0, 429, 430, 4, 33, 10, 0, 430, 432, 3, 70, 35, 0, 431, 428, 1, 0, 0, 0, 431, 429, 1, 0, 0, 0, 432, 67, 1, 0, 0, 0, 433, 476, 5, 50, 0, 0, 434, 435, 3, 104, 52, 0, 435, 436, 5, 72, 0, 0, 436, 476, 1, 0, 0, 0, 437, 476, 3, 102, 51, 0, 438, 476, 3, 104, 52, 0, 439, 476, 3, 98, 49, 0, 440, 476, 3, 70, 35, 0, 441, 476, 3, 106, 53, 0, 442, 443, 5, 70, 0, 0, 443, 448, 3, 100, 50, 0, 444, 445, 5, 39, 0, 0, 445, 447, 3, 100, 50, 0, 446, 444, 1, 0, 0, 0, 447, 450, 1, 0, 0, 0, 448, 446, 1, 0, 0, 0, 448, 449, 1, 0, 0, 0, 449, 451, 1, 0, 0, 0, 450, 448, 1, 0, 0, 0, 451, 452, 5, 71, 0, 0, 452, 476, 1, 0, 0, 0, 453, 454, 5, 70, 0, 0, 454, 459, 3, 98, 49, 0, 455, 456, 5, 39, 0, 0, 456, 458, 3, 98, 49, 0, 457, 455, 1, 0, 0, 0, 458, 461, 1, 0, 0, 0, 459, 457, 1, 0, 0, 0, 459, 460, 1, 0, 0, 0, 460, 462, 1, 0, 0, 0, 461, 459, 1, 0, 0, 0, 462, 463, 5, 71, 0, 0, 463, 476, 1, 0, 0, 0, 464, 465, 5, 70, 0, 0, 465, 470, 3, 106, 53, 0, 466, 467, 5, 39, 0, 0, 467, 469, 3, 106, 53, 0, 468, 466, 1, 0, 0, 0, 469, 472, 1, 0, 0, 0, 470, 468, 1, 0, 0, 0, 470, 471, 1, 0, 0, 0, 471, 473, 1, 0, 0, 0, 472, 470, 1, 0, 0, 0, 473, 474, 5, 71, 0, 0, 474, 476, 1, 0, 0, 0, 475, 433, 1, 0, 0, 0, 475, 434, 1, 0, 0, 0, 475, 437, 1, 0, 0, 0, 475, 438, 1, 0, 0, 0, 475, 439, 1, 0, 0, 0, 475, 440, 1, 0, 0, 0, 475, 441, 1, 0, 0, 0, 475, 442, 1, 0, 0, 0, 475, 453, 1, 0, 0, 0, 475, 464, 1, 0, 0, 0, 476, 69, 1, 0, 0, 0, 477, 480, 5, 53, 0, 0, 478, 480, 5, 69, 0, 0, 479, 477, 1, 0, 0, 0, 479, 478, 1, 0, 0, 0, 480, 71, 1, 0, 0, 0, 481, 485, 3, 64, 32, 0, 482, 483, 4, 36, 11, 0, 483, 485, 3, 70, 35, 0, 484, 481, 1, 0, 0, 0, 484, 482, 1, 0, 0, 0, 485, 73, 1, 0, 0, 0, 486, 487, 5, 9, 0, 0, 487, 488, 5, 31, 0, 0, 488, 75, 1, 0, 0, 0, 489, 490, 5, 14, 0, 0, 490, 495, 3, 78, 39, 0, 491, 492, 5, 39, 0, 0, 492, 494, 3, 78, 39, 0, 493, 491, 1, 0, 0, 0, 494, 497, 1, 0, 0, 0, 495, 493, 1, 0, 0, 0, 495, 496, 1, 0, 0, 0, 496, 77, 1, 0, 0, 0, 497, 495, 1, 0, 0, 0, 498, 500, 3, 10, 5, 0, 499, 501, 7, 4, 0, 0, 500, 499, 1, 0, 0, 0, 500, 501, 1, 0, 0, 0, 501, 504, 1, 0, 0, 0, 502, 503, 5, 51, 0, 0, 503, 505, 7, 5, 0, 0, 504, 502, 1, 0, 0, 0, 504, 505, 1, 0, 0, 0, 505, 79, 1, 0, 0, 0, 506, 507, 5, 8, 0, 0, 507, 508, 3, 62, 31, 0, 508, 81, 1, 0, 0, 0, 509, 510, 5, 2, 0, 0, 510, 511, 3, 62, 31, 0, 511, 83, 1, 0, 0, 0, 512, 513, 5, 11, 0, 0, 513, 518, 3, 86, 43, 0, 514, 515, 5, 39, 0, 0, 515, 517, 3, 86, 43, 0, 516, 514, 1, 0, 0, 0, 517, 520, 1, 0, 0, 0, 518, 516, 1, 0, 0, 0, 518, 519, 1, 0, 0, 0, 519, 85, 1, 0, 0, 0, 520, 518, 1, 0, 0, 0, 521, 522, 3, 60, 30, 0, 522, 523, 5, 89, 0, 0, 523, 524, 3, 60, 30, 0, 524, 87, 1, 0, 0, 0, 525, 526, 5, 1, 0, 0, 526, 527, 3, 20, 10, 0, 527, 529, 3, 106, 53, 0, 528, 530, 3, 94, 47, 0, 529, 528, 1, 0, 0, 0, 529, 530, 1, 0, 0, 0, 530, 89, 1, 0, 0, 0, 531, 532, 5, 7, 0, 0, 532, 533, 3, 20, 10, 0, 533, 534, 3, 106, 53, 0, 534, 91, 1, 0, 0, 0, 535, 536, 5, 10, 0, 0, 536, 537, 3, 58, 29, 0, 537, 93, 1, 0, 0, 0, 538, 543, 3, 96, 48, 0, 539, 540, 5, 39, 0, 0, 540, 542, 3, 96, 48, 0, 541, 539, 1, 0, 0, 0, 542, 545, 1, 0, 0, 0, 543, 541, 1, 0, 0, 0, 543, 544, 1, 0, 0, 0, 544, 95, 1, 0, 0, 0, 545, 543, 1, 0, 0, 0, 546, 547, 3, 64, 32, 0, 547, 548, 5, 36, 0, 0, 548, 549, 3, 68, 34, 0, 549, 97, 1, 0, 0, 0, 550, 551, 7, 6, 0, 0, 551, 99, 1, 0, 0, 0, 552, 555, 3, 102, 51, 0, 553, 555, 3, 104, 52, 0, 554, 552, 1, 0, 0, 0, 554, 553, 1, 0, 0, 0, 555, 101, 1, 0, 0, 0, 556, 558, 7, 0, 0, 0, 557, 556, 1, 0, 0, 0, 557, 558, 1, 0, 0, 0, 558, 559, 1, 0, 0, 0, 559, 560, 5, 32, 0, 0, 560, 103, 1, 0, 0, 0, 561, 563, 7, 0, 0, 0, 562, 561, 1, 0, 0, 0, 562, 563, 1, 0, 0, 0, 563, 564, 1, 0, 0, 0, 564, 565, 5, 31, 0, 0, 565, 105, 1, 0, 0, 0, 566, 567, 5, 30, 0, 0, 567, 107, 1, 0, 0, 0, 568, 569, 7, 7, 0, 0, 569, 109, 1, 0, 0, 0, 570, 571, 5, 5, 0, 0, 571, 572, 3, 112, 56, 0, 572, 111, 1, 0, 0, 0, 573, 574, 5, 70, 0, 0, 574, 575, 3, 2, 1, 0, 575, 576, 5, 71, 0, 0, 576, 113, 1, 0, 0, 0, 577, 578, 5, 13, 0, 0, 578, 579, 5, 105, 0, 0, 579, 115, 1, 0, 0, 0, 580, 581, 5, 3, 0, 0, 581, 584, 5, 95, 0, 0, 582, 583, 5, 93, 0, 0, 583, 585, 3, 60, 30, 0, 584, 582, 1, 0, 0, 0, 584, 585, 1, 0, 0, 0, 585, 595, 1, 0, 0, 0, 586, 587, 5, 94, 0, 0, 587, 592, 3, 118, 59, 0, 588, 589, 5, 39, 0, 0, 589, 591, 3, 118, 59, 0, 590, 588, 1, 0, 0, 0, 591, 594, 1, 0, 0, 0, 592, 590, 1, 0, 0, 0, 592, 593, 1, 0, 0, 0, 593, 596, 1, 0, 0, 0, 594, 592, 1, 0, 0, 0, 595, 586, 1, 0, 0, 0, 595, 596, 1, 0, 0, 0, 596, 117, 1, 0, 0, 0, 597, 598, 3, 60, 30, 0, 598, 599, 5, 36, 0, 0, 599, 601, 1, 0, 0, 0, 600, 597, 1, 0, 0, 0, 600, 601, 1, 0, 0, 0, 601, 602, 1, 0, 0, 0, 602, 603, 3, 60, 30, 0, 603, 119, 1, 0, 0, 0, 604, 605, 5, 18, 0, 0, 605, 606, 3, 36, 18, 0, 606, 607, 5, 93, 0, 0, 607, 608, 3, 62, 31, 0, 608, 121, 1, 0, 0, 0, 609, 610, 5, 17, 0, 0, 610, 613, 3, 54, 27, 0, 611, 612, 5, 33, 0, 0, 612, 614, 3, 30, 15, 0, 613, 611, 1, 0, 0, 0, 613, 614, 1, 0, 0, 0, 614, 123, 1, 0, 0, 0, 615, 617, 7, 8, 0, 0, 616, 615, 1, 0, 0, 0, 616, 617, 1, 0, 0, 0, 617, 618, 1, 0, 0, 0, 618, 619, 5, 20, 0, 0, 619, 620, 3, 126, 63, 0, 620, 621, 3, 128, 64, 0, 621, 125, 1, 0, 0, 0, 622, 625, 3, 64, 32, 0, 623, 624, 5, 89, 0, 0, 624, 626, 3, 64, 32, 0, 625, 623, 1, 0, 0, 0, 625, 626, 1, 0, 0, 0, 626, 127, 1, 0, 0, 0, 627, 628, 5, 93, 0, 0, 628, 633, 3, 130, 65, 0, 629, 630, 5, 39, 0, 0, 630, 632, 3, 130, 65, 0, 631, 629, 1, 0, 0, 0, 632, 635, 1, 0, 0, 0, 633, 631, 1, 0, 0, 0, 633, 634, 1, 0, 0, 0, 634, 129, 1, 0, 0, 0, 635, 633, 1, 0, 0, 0, 636, 637, 3, 16, 8, 0, 637, 131, 1, 0, 0, 0, 62, 143, 152, 172, 184, 193, 201, 206, 214, 216, 221, 228, 233, 238, 248, 254, 262, 264, 275, 282, 293, 296, 312, 318, 328, 332, 337, 347, 355, 368, 372, 376, 383, 387, 394, 400, 407, 415, 423, 431, 448, 459, 470, 475, 479, 484, 495, 500, 504, 518, 529, 543, 554, 557, 562, 584, 592, 595, 600, 613, 616, 625, 633]
\ No newline at end of file
+[4, 1, 128, 627, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 138, 8, 1, 10, 1, 12, 1, 141, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 149, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 169, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 181, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 188, 8, 5, 10, 5, 12, 5, 191, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 198, 8, 5, 1, 5, 1, 5, 1, 5, 3, 5, 203, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 211, 8, 5, 10, 5, 12, 5, 214, 9, 5, 1, 6, 1, 6, 3, 6, 218, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 225, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 230, 8, 6, 1, 7, 1, 7, 1, 7, 3, 7, 235, 8, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 245, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 251, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 259, 8, 9, 10, 9, 12, 9, 262, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 272, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 277, 8, 10, 10, 10, 12, 10, 280, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 288, 8, 11, 10, 11, 12, 11, 291, 9, 11, 3, 11, 293, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 5, 15, 307, 8, 15, 10, 15, 12, 15, 310, 9, 15, 1, 16, 1, 16, 1, 16, 3, 16, 315, 8, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 5, 17, 323, 8, 17, 10, 17, 12, 17, 326, 9, 17, 1, 17, 3, 17, 329, 8, 17, 1, 18, 1, 18, 1, 18, 3, 18, 334, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 346, 8, 21, 10, 21, 12, 21, 349, 9, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 355, 8, 22, 10, 22, 12, 22, 358, 9, 22, 1, 22, 3, 22, 361, 8, 22, 1, 22, 1, 22, 3, 22, 365, 8, 22, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 3, 24, 372, 8, 24, 1, 24, 1, 24, 3, 24, 376, 8, 24, 1, 25, 1, 25, 1, 25, 5, 25, 381, 8, 25, 10, 25, 12, 25, 384, 9, 25, 1, 26, 1, 26, 1, 26, 3, 26, 389, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 394, 8, 27, 10, 27, 12, 27, 397, 9, 27, 1, 28, 1, 28, 1, 28, 5, 28, 402, 8, 28, 10, 28, 12, 28, 405, 9, 28, 1, 29, 1, 29, 1, 29, 5, 29, 410, 8, 29, 10, 29, 12, 29, 413, 9, 29, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 3, 31, 420, 8, 31, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 435, 8, 32, 10, 32, 12, 32, 438, 9, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 446, 8, 32, 10, 32, 12, 32, 449, 9, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 457, 8, 32, 10, 32, 12, 32, 460, 9, 32, 1, 32, 1, 32, 3, 32, 464, 8, 32, 1, 33, 1, 33, 3, 33, 468, 8, 33, 1, 34, 1, 34, 1, 34, 3, 34, 473, 8, 34, 1, 35, 1, 35, 1, 35, 1, 36, 1, 36, 1, 36, 1, 36, 5, 36, 482, 8, 36, 10, 36, 12, 36, 485, 9, 36, 1, 37, 1, 37, 3, 37, 489, 8, 37, 1, 37, 1, 37, 3, 37, 493, 8, 37, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 1, 40, 5, 40, 505, 8, 40, 10, 40, 12, 40, 508, 9, 40, 1, 41, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 3, 42, 518, 8, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 5, 45, 530, 8, 45, 10, 45, 12, 45, 533, 9, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 48, 1, 48, 3, 48, 543, 8, 48, 1, 49, 3, 49, 546, 8, 49, 1, 49, 1, 49, 1, 50, 3, 50, 551, 8, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 3, 56, 573, 8, 56, 1, 56, 1, 56, 1, 56, 1, 56, 5, 56, 579, 8, 56, 10, 56, 12, 56, 582, 9, 56, 3, 56, 584, 8, 56, 1, 57, 1, 57, 1, 57, 3, 57, 589, 8, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 3, 59, 602, 8, 59, 1, 60, 3, 60, 605, 8, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 3, 61, 614, 8, 61, 1, 62, 1, 62, 1, 62, 1, 62, 5, 62, 620, 8, 62, 10, 62, 12, 62, 623, 9, 62, 1, 63, 1, 63, 1, 63, 0, 4, 2, 10, 18, 20, 64, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 0, 9, 1, 0, 64, 65, 1, 0, 66, 68, 2, 0, 30, 30, 81, 81, 1, 0, 72, 73, 2, 0, 35, 35, 40, 40, 2, 0, 43, 43, 46, 46, 2, 0, 42, 42, 56, 56, 2, 0, 57, 57, 59, 63, 1, 0, 22, 24, 654, 0, 128, 1, 0, 0, 0, 2, 131, 1, 0, 0, 0, 4, 148, 1, 0, 0, 0, 6, 168, 1, 0, 0, 0, 8, 170, 1, 0, 0, 0, 10, 202, 1, 0, 0, 0, 12, 229, 1, 0, 0, 0, 14, 231, 1, 0, 0, 0, 16, 244, 1, 0, 0, 0, 18, 250, 1, 0, 0, 0, 20, 271, 1, 0, 0, 0, 22, 281, 1, 0, 0, 0, 24, 296, 1, 0, 0, 0, 26, 298, 1, 0, 0, 0, 28, 300, 1, 0, 0, 0, 30, 303, 1, 0, 0, 0, 32, 314, 1, 0, 0, 0, 34, 318, 1, 0, 0, 0, 36, 333, 1, 0, 0, 0, 38, 337, 1, 0, 0, 0, 40, 339, 1, 0, 0, 0, 42, 341, 1, 0, 0, 0, 44, 350, 1, 0, 0, 0, 46, 366, 1, 0, 0, 0, 48, 369, 1, 0, 0, 0, 50, 377, 1, 0, 0, 0, 52, 385, 1, 0, 0, 0, 54, 390, 1, 0, 0, 0, 56, 398, 1, 0, 0, 0, 58, 406, 1, 0, 0, 0, 60, 414, 1, 0, 0, 0, 62, 419, 1, 0, 0, 0, 64, 463, 1, 0, 0, 0, 66, 467, 1, 0, 0, 0, 68, 472, 1, 0, 0, 0, 70, 474, 1, 0, 0, 0, 72, 477, 1, 0, 0, 0, 74, 486, 1, 0, 0, 0, 76, 494, 1, 0, 0, 0, 78, 497, 1, 0, 0, 0, 80, 500, 1, 0, 0, 0, 82, 509, 1, 0, 0, 0, 84, 513, 1, 0, 0, 0, 86, 519, 1, 0, 0, 0, 88, 523, 1, 0, 0, 0, 90, 526, 1, 0, 0, 0, 92, 534, 1, 0, 0, 0, 94, 538, 1, 0, 0, 0, 96, 542, 1, 0, 0, 0, 98, 545, 1, 0, 0, 0, 100, 550, 1, 0, 0, 0, 102, 554, 1, 0, 0, 0, 104, 556, 1, 0, 0, 0, 106, 558, 1, 0, 0, 0, 108, 561, 1, 0, 0, 0, 110, 565, 1, 0, 0, 0, 112, 568, 1, 0, 0, 0, 114, 588, 1, 0, 0, 0, 116, 592, 1, 0, 0, 0, 118, 597, 1, 0, 0, 0, 120, 604, 1, 0, 0, 0, 122, 610, 1, 0, 0, 0, 124, 615, 1, 0, 0, 0, 126, 624, 1, 0, 0, 0, 128, 129, 3, 2, 1, 0, 129, 130, 5, 0, 0, 1, 130, 1, 1, 0, 0, 0, 131, 132, 6, 1, -1, 0, 132, 133, 3, 4, 2, 0, 133, 139, 1, 0, 0, 0, 134, 135, 10, 1, 0, 0, 135, 136, 5, 29, 0, 0, 136, 138, 3, 6, 3, 0, 137, 134, 1, 0, 0, 0, 138, 141, 1, 0, 0, 0, 139, 137, 1, 0, 0, 0, 139, 140, 1, 0, 0, 0, 140, 3, 1, 0, 0, 0, 141, 139, 1, 0, 0, 0, 142, 149, 3, 106, 53, 0, 143, 149, 3, 34, 17, 0, 144, 149, 3, 28, 14, 0, 145, 149, 3, 110, 55, 0, 146, 147, 4, 2, 1, 0, 147, 149, 3, 44, 22, 0, 148, 142, 1, 0, 0, 0, 148, 143, 1, 0, 0, 0, 148, 144, 1, 0, 0, 0, 148, 145, 1, 0, 0, 0, 148, 146, 1, 0, 0, 0, 149, 5, 1, 0, 0, 0, 150, 169, 3, 46, 23, 0, 151, 169, 3, 8, 4, 0, 152, 169, 3, 76, 38, 0, 153, 169, 3, 70, 35, 0, 154, 169, 3, 48, 24, 0, 155, 169, 3, 72, 36, 0, 156, 169, 3, 78, 39, 0, 157, 169, 3, 80, 40, 0, 158, 169, 3, 84, 42, 0, 159, 169, 3, 86, 43, 0, 160, 169, 3, 112, 56, 0, 161, 169, 3, 88, 44, 0, 162, 163, 4, 3, 2, 0, 163, 169, 3, 118, 59, 0, 164, 165, 4, 3, 3, 0, 165, 169, 3, 116, 58, 0, 166, 167, 4, 3, 4, 0, 167, 169, 3, 120, 60, 0, 168, 150, 1, 0, 0, 0, 168, 151, 1, 0, 0, 0, 168, 152, 1, 0, 0, 0, 168, 153, 1, 0, 0, 0, 168, 154, 1, 0, 0, 0, 168, 155, 1, 0, 0, 0, 168, 156, 1, 0, 0, 0, 168, 157, 1, 0, 0, 0, 168, 158, 1, 0, 0, 0, 168, 159, 1, 0, 0, 0, 168, 160, 1, 0, 0, 0, 168, 161, 1, 0, 0, 0, 168, 162, 1, 0, 0, 0, 168, 164, 1, 0, 0, 0, 168, 166, 1, 0, 0, 0, 169, 7, 1, 0, 0, 0, 170, 171, 5, 16, 0, 0, 171, 172, 3, 10, 5, 0, 172, 9, 1, 0, 0, 0, 173, 174, 6, 5, -1, 0, 174, 175, 5, 49, 0, 0, 175, 203, 3, 10, 5, 8, 176, 203, 3, 16, 8, 0, 177, 203, 3, 12, 6, 0, 178, 180, 3, 16, 8, 0, 179, 181, 5, 49, 0, 0, 180, 179, 1, 0, 0, 0, 180, 181, 1, 0, 0, 0, 181, 182, 1, 0, 0, 0, 182, 183, 5, 44, 0, 0, 183, 184, 5, 48, 0, 0, 184, 189, 3, 16, 8, 0, 185, 186, 5, 39, 0, 0, 186, 188, 3, 16, 8, 0, 187, 185, 1, 0, 0, 0, 188, 191, 1, 0, 0, 0, 189, 187, 1, 0, 0, 0, 189, 190, 1, 0, 0, 0, 190, 192, 1, 0, 0, 0, 191, 189, 1, 0, 0, 0, 192, 193, 5, 55, 0, 0, 193, 203, 1, 0, 0, 0, 194, 195, 3, 16, 8, 0, 195, 197, 5, 45, 0, 0, 196, 198, 5, 49, 0, 0, 197, 196, 1, 0, 0, 0, 197, 198, 1, 0, 0, 0, 198, 199, 1, 0, 0, 0, 199, 200, 5, 50, 0, 0, 200, 203, 1, 0, 0, 0, 201, 203, 3, 14, 7, 0, 202, 173, 1, 0, 0, 0, 202, 176, 1, 0, 0, 0, 202, 177, 1, 0, 0, 0, 202, 178, 1, 0, 0, 0, 202, 194, 1, 0, 0, 0, 202, 201, 1, 0, 0, 0, 203, 212, 1, 0, 0, 0, 204, 205, 10, 5, 0, 0, 205, 206, 5, 34, 0, 0, 206, 211, 3, 10, 5, 6, 207, 208, 10, 4, 0, 0, 208, 209, 5, 52, 0, 0, 209, 211, 3, 10, 5, 5, 210, 204, 1, 0, 0, 0, 210, 207, 1, 0, 0, 0, 211, 214, 1, 0, 0, 0, 212, 210, 1, 0, 0, 0, 212, 213, 1, 0, 0, 0, 213, 11, 1, 0, 0, 0, 214, 212, 1, 0, 0, 0, 215, 217, 3, 16, 8, 0, 216, 218, 5, 49, 0, 0, 217, 216, 1, 0, 0, 0, 217, 218, 1, 0, 0, 0, 218, 219, 1, 0, 0, 0, 219, 220, 5, 47, 0, 0, 220, 221, 3, 102, 51, 0, 221, 230, 1, 0, 0, 0, 222, 224, 3, 16, 8, 0, 223, 225, 5, 49, 0, 0, 224, 223, 1, 0, 0, 0, 224, 225, 1, 0, 0, 0, 225, 226, 1, 0, 0, 0, 226, 227, 5, 54, 0, 0, 227, 228, 3, 102, 51, 0, 228, 230, 1, 0, 0, 0, 229, 215, 1, 0, 0, 0, 229, 222, 1, 0, 0, 0, 230, 13, 1, 0, 0, 0, 231, 234, 3, 54, 27, 0, 232, 233, 5, 37, 0, 0, 233, 235, 3, 26, 13, 0, 234, 232, 1, 0, 0, 0, 234, 235, 1, 0, 0, 0, 235, 236, 1, 0, 0, 0, 236, 237, 5, 38, 0, 0, 237, 238, 3, 64, 32, 0, 238, 15, 1, 0, 0, 0, 239, 245, 3, 18, 9, 0, 240, 241, 3, 18, 9, 0, 241, 242, 3, 104, 52, 0, 242, 243, 3, 18, 9, 0, 243, 245, 1, 0, 0, 0, 244, 239, 1, 0, 0, 0, 244, 240, 1, 0, 0, 0, 245, 17, 1, 0, 0, 0, 246, 247, 6, 9, -1, 0, 247, 251, 3, 20, 10, 0, 248, 249, 7, 0, 0, 0, 249, 251, 3, 18, 9, 3, 250, 246, 1, 0, 0, 0, 250, 248, 1, 0, 0, 0, 251, 260, 1, 0, 0, 0, 252, 253, 10, 2, 0, 0, 253, 254, 7, 1, 0, 0, 254, 259, 3, 18, 9, 3, 255, 256, 10, 1, 0, 0, 256, 257, 7, 0, 0, 0, 257, 259, 3, 18, 9, 2, 258, 252, 1, 0, 0, 0, 258, 255, 1, 0, 0, 0, 259, 262, 1, 0, 0, 0, 260, 258, 1, 0, 0, 0, 260, 261, 1, 0, 0, 0, 261, 19, 1, 0, 0, 0, 262, 260, 1, 0, 0, 0, 263, 264, 6, 10, -1, 0, 264, 272, 3, 64, 32, 0, 265, 272, 3, 54, 27, 0, 266, 272, 3, 22, 11, 0, 267, 268, 5, 48, 0, 0, 268, 269, 3, 10, 5, 0, 269, 270, 5, 55, 0, 0, 270, 272, 1, 0, 0, 0, 271, 263, 1, 0, 0, 0, 271, 265, 1, 0, 0, 0, 271, 266, 1, 0, 0, 0, 271, 267, 1, 0, 0, 0, 272, 278, 1, 0, 0, 0, 273, 274, 10, 1, 0, 0, 274, 275, 5, 37, 0, 0, 275, 277, 3, 26, 13, 0, 276, 273, 1, 0, 0, 0, 277, 280, 1, 0, 0, 0, 278, 276, 1, 0, 0, 0, 278, 279, 1, 0, 0, 0, 279, 21, 1, 0, 0, 0, 280, 278, 1, 0, 0, 0, 281, 282, 3, 24, 12, 0, 282, 292, 5, 48, 0, 0, 283, 293, 5, 66, 0, 0, 284, 289, 3, 10, 5, 0, 285, 286, 5, 39, 0, 0, 286, 288, 3, 10, 5, 0, 287, 285, 1, 0, 0, 0, 288, 291, 1, 0, 0, 0, 289, 287, 1, 0, 0, 0, 289, 290, 1, 0, 0, 0, 290, 293, 1, 0, 0, 0, 291, 289, 1, 0, 0, 0, 292, 283, 1, 0, 0, 0, 292, 284, 1, 0, 0, 0, 292, 293, 1, 0, 0, 0, 293, 294, 1, 0, 0, 0, 294, 295, 5, 55, 0, 0, 295, 23, 1, 0, 0, 0, 296, 297, 3, 68, 34, 0, 297, 25, 1, 0, 0, 0, 298, 299, 3, 60, 30, 0, 299, 27, 1, 0, 0, 0, 300, 301, 5, 12, 0, 0, 301, 302, 3, 30, 15, 0, 302, 29, 1, 0, 0, 0, 303, 308, 3, 32, 16, 0, 304, 305, 5, 39, 0, 0, 305, 307, 3, 32, 16, 0, 306, 304, 1, 0, 0, 0, 307, 310, 1, 0, 0, 0, 308, 306, 1, 0, 0, 0, 308, 309, 1, 0, 0, 0, 309, 31, 1, 0, 0, 0, 310, 308, 1, 0, 0, 0, 311, 312, 3, 54, 27, 0, 312, 313, 5, 36, 0, 0, 313, 315, 1, 0, 0, 0, 314, 311, 1, 0, 0, 0, 314, 315, 1, 0, 0, 0, 315, 316, 1, 0, 0, 0, 316, 317, 3, 10, 5, 0, 317, 33, 1, 0, 0, 0, 318, 319, 5, 6, 0, 0, 319, 324, 3, 36, 18, 0, 320, 321, 5, 39, 0, 0, 321, 323, 3, 36, 18, 0, 322, 320, 1, 0, 0, 0, 323, 326, 1, 0, 0, 0, 324, 322, 1, 0, 0, 0, 324, 325, 1, 0, 0, 0, 325, 328, 1, 0, 0, 0, 326, 324, 1, 0, 0, 0, 327, 329, 3, 42, 21, 0, 328, 327, 1, 0, 0, 0, 328, 329, 1, 0, 0, 0, 329, 35, 1, 0, 0, 0, 330, 331, 3, 38, 19, 0, 331, 332, 5, 38, 0, 0, 332, 334, 1, 0, 0, 0, 333, 330, 1, 0, 0, 0, 333, 334, 1, 0, 0, 0, 334, 335, 1, 0, 0, 0, 335, 336, 3, 40, 20, 0, 336, 37, 1, 0, 0, 0, 337, 338, 5, 81, 0, 0, 338, 39, 1, 0, 0, 0, 339, 340, 7, 2, 0, 0, 340, 41, 1, 0, 0, 0, 341, 342, 5, 80, 0, 0, 342, 347, 5, 81, 0, 0, 343, 344, 5, 39, 0, 0, 344, 346, 5, 81, 0, 0, 345, 343, 1, 0, 0, 0, 346, 349, 1, 0, 0, 0, 347, 345, 1, 0, 0, 0, 347, 348, 1, 0, 0, 0, 348, 43, 1, 0, 0, 0, 349, 347, 1, 0, 0, 0, 350, 351, 5, 19, 0, 0, 351, 356, 3, 36, 18, 0, 352, 353, 5, 39, 0, 0, 353, 355, 3, 36, 18, 0, 354, 352, 1, 0, 0, 0, 355, 358, 1, 0, 0, 0, 356, 354, 1, 0, 0, 0, 356, 357, 1, 0, 0, 0, 357, 360, 1, 0, 0, 0, 358, 356, 1, 0, 0, 0, 359, 361, 3, 50, 25, 0, 360, 359, 1, 0, 0, 0, 360, 361, 1, 0, 0, 0, 361, 364, 1, 0, 0, 0, 362, 363, 5, 33, 0, 0, 363, 365, 3, 30, 15, 0, 364, 362, 1, 0, 0, 0, 364, 365, 1, 0, 0, 0, 365, 45, 1, 0, 0, 0, 366, 367, 5, 4, 0, 0, 367, 368, 3, 30, 15, 0, 368, 47, 1, 0, 0, 0, 369, 371, 5, 15, 0, 0, 370, 372, 3, 50, 25, 0, 371, 370, 1, 0, 0, 0, 371, 372, 1, 0, 0, 0, 372, 375, 1, 0, 0, 0, 373, 374, 5, 33, 0, 0, 374, 376, 3, 30, 15, 0, 375, 373, 1, 0, 0, 0, 375, 376, 1, 0, 0, 0, 376, 49, 1, 0, 0, 0, 377, 382, 3, 52, 26, 0, 378, 379, 5, 39, 0, 0, 379, 381, 3, 52, 26, 0, 380, 378, 1, 0, 0, 0, 381, 384, 1, 0, 0, 0, 382, 380, 1, 0, 0, 0, 382, 383, 1, 0, 0, 0, 383, 51, 1, 0, 0, 0, 384, 382, 1, 0, 0, 0, 385, 388, 3, 32, 16, 0, 386, 387, 5, 16, 0, 0, 387, 389, 3, 10, 5, 0, 388, 386, 1, 0, 0, 0, 388, 389, 1, 0, 0, 0, 389, 53, 1, 0, 0, 0, 390, 395, 3, 68, 34, 0, 391, 392, 5, 41, 0, 0, 392, 394, 3, 68, 34, 0, 393, 391, 1, 0, 0, 0, 394, 397, 1, 0, 0, 0, 395, 393, 1, 0, 0, 0, 395, 396, 1, 0, 0, 0, 396, 55, 1, 0, 0, 0, 397, 395, 1, 0, 0, 0, 398, 403, 3, 62, 31, 0, 399, 400, 5, 41, 0, 0, 400, 402, 3, 62, 31, 0, 401, 399, 1, 0, 0, 0, 402, 405, 1, 0, 0, 0, 403, 401, 1, 0, 0, 0, 403, 404, 1, 0, 0, 0, 404, 57, 1, 0, 0, 0, 405, 403, 1, 0, 0, 0, 406, 411, 3, 56, 28, 0, 407, 408, 5, 39, 0, 0, 408, 410, 3, 56, 28, 0, 409, 407, 1, 0, 0, 0, 410, 413, 1, 0, 0, 0, 411, 409, 1, 0, 0, 0, 411, 412, 1, 0, 0, 0, 412, 59, 1, 0, 0, 0, 413, 411, 1, 0, 0, 0, 414, 415, 7, 3, 0, 0, 415, 61, 1, 0, 0, 0, 416, 420, 5, 85, 0, 0, 417, 418, 4, 31, 10, 0, 418, 420, 3, 66, 33, 0, 419, 416, 1, 0, 0, 0, 419, 417, 1, 0, 0, 0, 420, 63, 1, 0, 0, 0, 421, 464, 5, 50, 0, 0, 422, 423, 3, 100, 50, 0, 423, 424, 5, 72, 0, 0, 424, 464, 1, 0, 0, 0, 425, 464, 3, 98, 49, 0, 426, 464, 3, 100, 50, 0, 427, 464, 3, 94, 47, 0, 428, 464, 3, 66, 33, 0, 429, 464, 3, 102, 51, 0, 430, 431, 5, 70, 0, 0, 431, 436, 3, 96, 48, 0, 432, 433, 5, 39, 0, 0, 433, 435, 3, 96, 48, 0, 434, 432, 1, 0, 0, 0, 435, 438, 1, 0, 0, 0, 436, 434, 1, 0, 0, 0, 436, 437, 1, 0, 0, 0, 437, 439, 1, 0, 0, 0, 438, 436, 1, 0, 0, 0, 439, 440, 5, 71, 0, 0, 440, 464, 1, 0, 0, 0, 441, 442, 5, 70, 0, 0, 442, 447, 3, 94, 47, 0, 443, 444, 5, 39, 0, 0, 444, 446, 3, 94, 47, 0, 445, 443, 1, 0, 0, 0, 446, 449, 1, 0, 0, 0, 447, 445, 1, 0, 0, 0, 447, 448, 1, 0, 0, 0, 448, 450, 1, 0, 0, 0, 449, 447, 1, 0, 0, 0, 450, 451, 5, 71, 0, 0, 451, 464, 1, 0, 0, 0, 452, 453, 5, 70, 0, 0, 453, 458, 3, 102, 51, 0, 454, 455, 5, 39, 0, 0, 455, 457, 3, 102, 51, 0, 456, 454, 1, 0, 0, 0, 457, 460, 1, 0, 0, 0, 458, 456, 1, 0, 0, 0, 458, 459, 1, 0, 0, 0, 459, 461, 1, 0, 0, 0, 460, 458, 1, 0, 0, 0, 461, 462, 5, 71, 0, 0, 462, 464, 1, 0, 0, 0, 463, 421, 1, 0, 0, 0, 463, 422, 1, 0, 0, 0, 463, 425, 1, 0, 0, 0, 463, 426, 1, 0, 0, 0, 463, 427, 1, 0, 0, 0, 463, 428, 1, 0, 0, 0, 463, 429, 1, 0, 0, 0, 463, 430, 1, 0, 0, 0, 463, 441, 1, 0, 0, 0, 463, 452, 1, 0, 0, 0, 464, 65, 1, 0, 0, 0, 465, 468, 5, 53, 0, 0, 466, 468, 5, 69, 0, 0, 467, 465, 1, 0, 0, 0, 467, 466, 1, 0, 0, 0, 468, 67, 1, 0, 0, 0, 469, 473, 3, 60, 30, 0, 470, 471, 4, 34, 11, 0, 471, 473, 3, 66, 33, 0, 472, 469, 1, 0, 0, 0, 472, 470, 1, 0, 0, 0, 473, 69, 1, 0, 0, 0, 474, 475, 5, 9, 0, 0, 475, 476, 5, 31, 0, 0, 476, 71, 1, 0, 0, 0, 477, 478, 5, 14, 0, 0, 478, 483, 3, 74, 37, 0, 479, 480, 5, 39, 0, 0, 480, 482, 3, 74, 37, 0, 481, 479, 1, 0, 0, 0, 482, 485, 1, 0, 0, 0, 483, 481, 1, 0, 0, 0, 483, 484, 1, 0, 0, 0, 484, 73, 1, 0, 0, 0, 485, 483, 1, 0, 0, 0, 486, 488, 3, 10, 5, 0, 487, 489, 7, 4, 0, 0, 488, 487, 1, 0, 0, 0, 488, 489, 1, 0, 0, 0, 489, 492, 1, 0, 0, 0, 490, 491, 5, 51, 0, 0, 491, 493, 7, 5, 0, 0, 492, 490, 1, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 75, 1, 0, 0, 0, 494, 495, 5, 8, 0, 0, 495, 496, 3, 58, 29, 0, 496, 77, 1, 0, 0, 0, 497, 498, 5, 2, 0, 0, 498, 499, 3, 58, 29, 0, 499, 79, 1, 0, 0, 0, 500, 501, 5, 11, 0, 0, 501, 506, 3, 82, 41, 0, 502, 503, 5, 39, 0, 0, 503, 505, 3, 82, 41, 0, 504, 502, 1, 0, 0, 0, 505, 508, 1, 0, 0, 0, 506, 504, 1, 0, 0, 0, 506, 507, 1, 0, 0, 0, 507, 81, 1, 0, 0, 0, 508, 506, 1, 0, 0, 0, 509, 510, 3, 56, 28, 0, 510, 511, 5, 89, 0, 0, 511, 512, 3, 56, 28, 0, 512, 83, 1, 0, 0, 0, 513, 514, 5, 1, 0, 0, 514, 515, 3, 20, 10, 0, 515, 517, 3, 102, 51, 0, 516, 518, 3, 90, 45, 0, 517, 516, 1, 0, 0, 0, 517, 518, 1, 0, 0, 0, 518, 85, 1, 0, 0, 0, 519, 520, 5, 7, 0, 0, 520, 521, 3, 20, 10, 0, 521, 522, 3, 102, 51, 0, 522, 87, 1, 0, 0, 0, 523, 524, 5, 10, 0, 0, 524, 525, 3, 54, 27, 0, 525, 89, 1, 0, 0, 0, 526, 531, 3, 92, 46, 0, 527, 528, 5, 39, 0, 0, 528, 530, 3, 92, 46, 0, 529, 527, 1, 0, 0, 0, 530, 533, 1, 0, 0, 0, 531, 529, 1, 0, 0, 0, 531, 532, 1, 0, 0, 0, 532, 91, 1, 0, 0, 0, 533, 531, 1, 0, 0, 0, 534, 535, 3, 60, 30, 0, 535, 536, 5, 36, 0, 0, 536, 537, 3, 64, 32, 0, 537, 93, 1, 0, 0, 0, 538, 539, 7, 6, 0, 0, 539, 95, 1, 0, 0, 0, 540, 543, 3, 98, 49, 0, 541, 543, 3, 100, 50, 0, 542, 540, 1, 0, 0, 0, 542, 541, 1, 0, 0, 0, 543, 97, 1, 0, 0, 0, 544, 546, 7, 0, 0, 0, 545, 544, 1, 0, 0, 0, 545, 546, 1, 0, 0, 0, 546, 547, 1, 0, 0, 0, 547, 548, 5, 32, 0, 0, 548, 99, 1, 0, 0, 0, 549, 551, 7, 0, 0, 0, 550, 549, 1, 0, 0, 0, 550, 551, 1, 0, 0, 0, 551, 552, 1, 0, 0, 0, 552, 553, 5, 31, 0, 0, 553, 101, 1, 0, 0, 0, 554, 555, 5, 30, 0, 0, 555, 103, 1, 0, 0, 0, 556, 557, 7, 7, 0, 0, 557, 105, 1, 0, 0, 0, 558, 559, 5, 5, 0, 0, 559, 560, 3, 108, 54, 0, 560, 107, 1, 0, 0, 0, 561, 562, 5, 70, 0, 0, 562, 563, 3, 2, 1, 0, 563, 564, 5, 71, 0, 0, 564, 109, 1, 0, 0, 0, 565, 566, 5, 13, 0, 0, 566, 567, 5, 105, 0, 0, 567, 111, 1, 0, 0, 0, 568, 569, 5, 3, 0, 0, 569, 572, 5, 95, 0, 0, 570, 571, 5, 93, 0, 0, 571, 573, 3, 56, 28, 0, 572, 570, 1, 0, 0, 0, 572, 573, 1, 0, 0, 0, 573, 583, 1, 0, 0, 0, 574, 575, 5, 94, 0, 0, 575, 580, 3, 114, 57, 0, 576, 577, 5, 39, 0, 0, 577, 579, 3, 114, 57, 0, 578, 576, 1, 0, 0, 0, 579, 582, 1, 0, 0, 0, 580, 578, 1, 0, 0, 0, 580, 581, 1, 0, 0, 0, 581, 584, 1, 0, 0, 0, 582, 580, 1, 0, 0, 0, 583, 574, 1, 0, 0, 0, 583, 584, 1, 0, 0, 0, 584, 113, 1, 0, 0, 0, 585, 586, 3, 56, 28, 0, 586, 587, 5, 36, 0, 0, 587, 589, 1, 0, 0, 0, 588, 585, 1, 0, 0, 0, 588, 589, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 591, 3, 56, 28, 0, 591, 115, 1, 0, 0, 0, 592, 593, 5, 18, 0, 0, 593, 594, 3, 36, 18, 0, 594, 595, 5, 93, 0, 0, 595, 596, 3, 58, 29, 0, 596, 117, 1, 0, 0, 0, 597, 598, 5, 17, 0, 0, 598, 601, 3, 50, 25, 0, 599, 600, 5, 33, 0, 0, 600, 602, 3, 30, 15, 0, 601, 599, 1, 0, 0, 0, 601, 602, 1, 0, 0, 0, 602, 119, 1, 0, 0, 0, 603, 605, 7, 8, 0, 0, 604, 603, 1, 0, 0, 0, 604, 605, 1, 0, 0, 0, 605, 606, 1, 0, 0, 0, 606, 607, 5, 20, 0, 0, 607, 608, 3, 122, 61, 0, 608, 609, 3, 124, 62, 0, 609, 121, 1, 0, 0, 0, 610, 613, 3, 60, 30, 0, 611, 612, 5, 89, 0, 0, 612, 614, 3, 60, 30, 0, 613, 611, 1, 0, 0, 0, 613, 614, 1, 0, 0, 0, 614, 123, 1, 0, 0, 0, 615, 616, 5, 93, 0, 0, 616, 621, 3, 126, 63, 0, 617, 618, 5, 39, 0, 0, 618, 620, 3, 126, 63, 0, 619, 617, 1, 0, 0, 0, 620, 623, 1, 0, 0, 0, 621, 619, 1, 0, 0, 0, 621, 622, 1, 0, 0, 0, 622, 125, 1, 0, 0, 0, 623, 621, 1, 0, 0, 0, 624, 625, 3, 16, 8, 0, 625, 127, 1, 0, 0, 0, 61, 139, 148, 168, 180, 189, 197, 202, 210, 212, 217, 224, 229, 234, 244, 250, 258, 260, 271, 278, 289, 292, 308, 314, 324, 328, 333, 347, 356, 360, 364, 371, 375, 382, 388, 395, 403, 411, 419, 436, 447, 458, 463, 467, 472, 483, 488, 492, 506, 517, 531, 542, 545, 550, 572, 580, 583, 588, 601, 604, 613, 621]
\ No newline at end of file
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java
index a56035364641d..e272dc7f477a4 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java
@@ -59,35 +59,34 @@ public class EsqlBaseParser extends ParserConfig {
     RULE_primaryExpression = 10, RULE_functionExpression = 11, RULE_functionName = 12, 
     RULE_dataType = 13, RULE_rowCommand = 14, RULE_fields = 15, RULE_field = 16, 
     RULE_fromCommand = 17, RULE_indexPattern = 18, RULE_clusterString = 19, 
-    RULE_indexString = 20, RULE_metadata = 21, RULE_metadataOption = 22, RULE_deprecated_metadata = 23, 
-    RULE_metricsCommand = 24, RULE_evalCommand = 25, RULE_statsCommand = 26, 
-    RULE_aggFields = 27, RULE_aggField = 28, RULE_qualifiedName = 29, RULE_qualifiedNamePattern = 30, 
-    RULE_qualifiedNamePatterns = 31, RULE_identifier = 32, RULE_identifierPattern = 33, 
-    RULE_constant = 34, RULE_parameter = 35, RULE_identifierOrParameter = 36, 
-    RULE_limitCommand = 37, RULE_sortCommand = 38, RULE_orderExpression = 39, 
-    RULE_keepCommand = 40, RULE_dropCommand = 41, RULE_renameCommand = 42, 
-    RULE_renameClause = 43, RULE_dissectCommand = 44, RULE_grokCommand = 45, 
-    RULE_mvExpandCommand = 46, RULE_commandOptions = 47, RULE_commandOption = 48, 
-    RULE_booleanValue = 49, RULE_numericValue = 50, RULE_decimalValue = 51, 
-    RULE_integerValue = 52, RULE_string = 53, RULE_comparisonOperator = 54, 
-    RULE_explainCommand = 55, RULE_subqueryExpression = 56, RULE_showCommand = 57, 
-    RULE_enrichCommand = 58, RULE_enrichWithClause = 59, RULE_lookupCommand = 60, 
-    RULE_inlinestatsCommand = 61, RULE_joinCommand = 62, RULE_joinTarget = 63, 
-    RULE_joinCondition = 64, RULE_joinPredicate = 65;
+    RULE_indexString = 20, RULE_metadata = 21, RULE_metricsCommand = 22, RULE_evalCommand = 23, 
+    RULE_statsCommand = 24, RULE_aggFields = 25, RULE_aggField = 26, RULE_qualifiedName = 27, 
+    RULE_qualifiedNamePattern = 28, RULE_qualifiedNamePatterns = 29, RULE_identifier = 30, 
+    RULE_identifierPattern = 31, RULE_constant = 32, RULE_parameter = 33, 
+    RULE_identifierOrParameter = 34, RULE_limitCommand = 35, RULE_sortCommand = 36, 
+    RULE_orderExpression = 37, RULE_keepCommand = 38, RULE_dropCommand = 39, 
+    RULE_renameCommand = 40, RULE_renameClause = 41, RULE_dissectCommand = 42, 
+    RULE_grokCommand = 43, RULE_mvExpandCommand = 44, RULE_commandOptions = 45, 
+    RULE_commandOption = 46, RULE_booleanValue = 47, RULE_numericValue = 48, 
+    RULE_decimalValue = 49, RULE_integerValue = 50, RULE_string = 51, RULE_comparisonOperator = 52, 
+    RULE_explainCommand = 53, RULE_subqueryExpression = 54, RULE_showCommand = 55, 
+    RULE_enrichCommand = 56, RULE_enrichWithClause = 57, RULE_lookupCommand = 58, 
+    RULE_inlinestatsCommand = 59, RULE_joinCommand = 60, RULE_joinTarget = 61, 
+    RULE_joinCondition = 62, RULE_joinPredicate = 63;
   private static String[] makeRuleNames() {
     return new String[] {
       "singleStatement", "query", "sourceCommand", "processingCommand", "whereCommand", 
       "booleanExpression", "regexBooleanExpression", "matchBooleanExpression", 
       "valueExpression", "operatorExpression", "primaryExpression", "functionExpression", 
       "functionName", "dataType", "rowCommand", "fields", "field", "fromCommand", 
-      "indexPattern", "clusterString", "indexString", "metadata", "metadataOption", 
-      "deprecated_metadata", "metricsCommand", "evalCommand", "statsCommand", 
-      "aggFields", "aggField", "qualifiedName", "qualifiedNamePattern", "qualifiedNamePatterns", 
-      "identifier", "identifierPattern", "constant", "parameter", "identifierOrParameter", 
-      "limitCommand", "sortCommand", "orderExpression", "keepCommand", "dropCommand", 
-      "renameCommand", "renameClause", "dissectCommand", "grokCommand", "mvExpandCommand", 
-      "commandOptions", "commandOption", "booleanValue", "numericValue", "decimalValue", 
-      "integerValue", "string", "comparisonOperator", "explainCommand", "subqueryExpression", 
+      "indexPattern", "clusterString", "indexString", "metadata", "metricsCommand", 
+      "evalCommand", "statsCommand", "aggFields", "aggField", "qualifiedName", 
+      "qualifiedNamePattern", "qualifiedNamePatterns", "identifier", "identifierPattern", 
+      "constant", "parameter", "identifierOrParameter", "limitCommand", "sortCommand", 
+      "orderExpression", "keepCommand", "dropCommand", "renameCommand", "renameClause", 
+      "dissectCommand", "grokCommand", "mvExpandCommand", "commandOptions", 
+      "commandOption", "booleanValue", "numericValue", "decimalValue", "integerValue", 
+      "string", "comparisonOperator", "explainCommand", "subqueryExpression", 
       "showCommand", "enrichCommand", "enrichWithClause", "lookupCommand", 
       "inlinestatsCommand", "joinCommand", "joinTarget", "joinCondition", "joinPredicate"
     };
@@ -226,9 +225,9 @@ public final SingleStatementContext singleStatement() throws RecognitionExceptio
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(132);
+      setState(128);
       query(0);
-      setState(133);
+      setState(129);
       match(EOF);
       }
     }
@@ -324,11 +323,11 @@ private QueryContext query(int _p) throws RecognitionException {
       _ctx = _localctx;
       _prevctx = _localctx;
 
-      setState(136);
+      setState(132);
       sourceCommand();
       }
       _ctx.stop = _input.LT(-1);
-      setState(143);
+      setState(139);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -339,16 +338,16 @@ private QueryContext query(int _p) throws RecognitionException {
           {
           _localctx = new CompositeQueryContext(new QueryContext(_parentctx, _parentState));
           pushNewRecursionContext(_localctx, _startState, RULE_query);
-          setState(138);
+          setState(134);
           if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
-          setState(139);
+          setState(135);
           match(PIPE);
-          setState(140);
+          setState(136);
           processingCommand();
           }
           } 
         }
-        setState(145);
+        setState(141);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
       }
@@ -406,43 +405,43 @@ public final SourceCommandContext sourceCommand() throws RecognitionException {
     SourceCommandContext _localctx = new SourceCommandContext(_ctx, getState());
     enterRule(_localctx, 4, RULE_sourceCommand);
     try {
-      setState(152);
+      setState(148);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,1,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(146);
+        setState(142);
         explainCommand();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(147);
+        setState(143);
         fromCommand();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(148);
+        setState(144);
         rowCommand();
         }
         break;
       case 4:
         enterOuterAlt(_localctx, 4);
         {
-        setState(149);
+        setState(145);
         showCommand();
         }
         break;
       case 5:
         enterOuterAlt(_localctx, 5);
         {
-        setState(150);
+        setState(146);
         if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()");
-        setState(151);
+        setState(147);
         metricsCommand();
         }
         break;
@@ -530,117 +529,117 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce
     ProcessingCommandContext _localctx = new ProcessingCommandContext(_ctx, getState());
     enterRule(_localctx, 6, RULE_processingCommand);
     try {
-      setState(172);
+      setState(168);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(154);
+        setState(150);
         evalCommand();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(155);
+        setState(151);
         whereCommand();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(156);
+        setState(152);
         keepCommand();
         }
         break;
       case 4:
         enterOuterAlt(_localctx, 4);
         {
-        setState(157);
+        setState(153);
         limitCommand();
         }
         break;
       case 5:
         enterOuterAlt(_localctx, 5);
         {
-        setState(158);
+        setState(154);
         statsCommand();
         }
         break;
       case 6:
         enterOuterAlt(_localctx, 6);
         {
-        setState(159);
+        setState(155);
         sortCommand();
         }
         break;
       case 7:
         enterOuterAlt(_localctx, 7);
         {
-        setState(160);
+        setState(156);
         dropCommand();
         }
         break;
       case 8:
         enterOuterAlt(_localctx, 8);
         {
-        setState(161);
+        setState(157);
         renameCommand();
         }
         break;
       case 9:
         enterOuterAlt(_localctx, 9);
         {
-        setState(162);
+        setState(158);
         dissectCommand();
         }
         break;
       case 10:
         enterOuterAlt(_localctx, 10);
         {
-        setState(163);
+        setState(159);
         grokCommand();
         }
         break;
       case 11:
         enterOuterAlt(_localctx, 11);
         {
-        setState(164);
+        setState(160);
         enrichCommand();
         }
         break;
       case 12:
         enterOuterAlt(_localctx, 12);
         {
-        setState(165);
+        setState(161);
         mvExpandCommand();
         }
         break;
       case 13:
         enterOuterAlt(_localctx, 13);
         {
-        setState(166);
+        setState(162);
         if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()");
-        setState(167);
+        setState(163);
         inlinestatsCommand();
         }
         break;
       case 14:
         enterOuterAlt(_localctx, 14);
         {
-        setState(168);
+        setState(164);
         if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()");
-        setState(169);
+        setState(165);
         lookupCommand();
         }
         break;
       case 15:
         enterOuterAlt(_localctx, 15);
         {
-        setState(170);
+        setState(166);
         if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()");
-        setState(171);
+        setState(167);
         joinCommand();
         }
         break;
@@ -689,9 +688,9 @@ public final WhereCommandContext whereCommand() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(174);
+      setState(170);
       match(WHERE);
-      setState(175);
+      setState(171);
       booleanExpression(0);
       }
     }
@@ -907,7 +906,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(206);
+      setState(202);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) {
       case 1:
@@ -916,9 +915,9 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(178);
+        setState(174);
         match(NOT);
-        setState(179);
+        setState(175);
         booleanExpression(8);
         }
         break;
@@ -927,7 +926,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new BooleanDefaultContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(180);
+        setState(176);
         valueExpression();
         }
         break;
@@ -936,7 +935,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new RegexExpressionContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(181);
+        setState(177);
         regexBooleanExpression();
         }
         break;
@@ -945,41 +944,41 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new LogicalInContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(182);
+        setState(178);
         valueExpression();
-        setState(184);
+        setState(180);
         _errHandler.sync(this);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(183);
+          setState(179);
           match(NOT);
           }
         }
 
-        setState(186);
+        setState(182);
         match(IN);
-        setState(187);
+        setState(183);
         match(LP);
-        setState(188);
+        setState(184);
         valueExpression();
-        setState(193);
+        setState(189);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(189);
+          setState(185);
           match(COMMA);
-          setState(190);
+          setState(186);
           valueExpression();
           }
           }
-          setState(195);
+          setState(191);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(196);
+        setState(192);
         match(RP);
         }
         break;
@@ -988,21 +987,21 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new IsNullContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(198);
+        setState(194);
         valueExpression();
-        setState(199);
+        setState(195);
         match(IS);
-        setState(201);
+        setState(197);
         _errHandler.sync(this);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(200);
+          setState(196);
           match(NOT);
           }
         }
 
-        setState(203);
+        setState(199);
         match(NULL);
         }
         break;
@@ -1011,13 +1010,13 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new MatchExpressionContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(205);
+        setState(201);
         matchBooleanExpression();
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(216);
+      setState(212);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,8,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1025,7 +1024,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(214);
+          setState(210);
           _errHandler.sync(this);
           switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) {
           case 1:
@@ -1033,11 +1032,11 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
             _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState));
             ((LogicalBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression);
-            setState(208);
+            setState(204);
             if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
-            setState(209);
+            setState(205);
             ((LogicalBinaryContext)_localctx).operator = match(AND);
-            setState(210);
+            setState(206);
             ((LogicalBinaryContext)_localctx).right = booleanExpression(6);
             }
             break;
@@ -1046,18 +1045,18 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
             _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState));
             ((LogicalBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression);
-            setState(211);
+            setState(207);
             if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
-            setState(212);
+            setState(208);
             ((LogicalBinaryContext)_localctx).operator = match(OR);
-            setState(213);
+            setState(209);
             ((LogicalBinaryContext)_localctx).right = booleanExpression(5);
             }
             break;
           }
           } 
         }
-        setState(218);
+        setState(214);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,8,_ctx);
       }
@@ -1112,48 +1111,48 @@ public final RegexBooleanExpressionContext regexBooleanExpression() throws Recog
     enterRule(_localctx, 12, RULE_regexBooleanExpression);
     int _la;
     try {
-      setState(233);
+      setState(229);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(219);
+        setState(215);
         valueExpression();
-        setState(221);
+        setState(217);
         _errHandler.sync(this);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(220);
+          setState(216);
           match(NOT);
           }
         }
 
-        setState(223);
+        setState(219);
         ((RegexBooleanExpressionContext)_localctx).kind = match(LIKE);
-        setState(224);
+        setState(220);
         ((RegexBooleanExpressionContext)_localctx).pattern = string();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(226);
+        setState(222);
         valueExpression();
-        setState(228);
+        setState(224);
         _errHandler.sync(this);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(227);
+          setState(223);
           match(NOT);
           }
         }
 
-        setState(230);
+        setState(226);
         ((RegexBooleanExpressionContext)_localctx).kind = match(RLIKE);
-        setState(231);
+        setState(227);
         ((RegexBooleanExpressionContext)_localctx).pattern = string();
         }
         break;
@@ -1213,23 +1212,23 @@ public final MatchBooleanExpressionContext matchBooleanExpression() throws Recog
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(235);
+      setState(231);
       ((MatchBooleanExpressionContext)_localctx).fieldExp = qualifiedName();
-      setState(238);
+      setState(234);
       _errHandler.sync(this);
       _la = _input.LA(1);
       if (_la==CAST_OP) {
         {
-        setState(236);
+        setState(232);
         match(CAST_OP);
-        setState(237);
+        setState(233);
         ((MatchBooleanExpressionContext)_localctx).fieldType = dataType();
         }
       }
 
-      setState(240);
+      setState(236);
       match(COLON);
-      setState(241);
+      setState(237);
       ((MatchBooleanExpressionContext)_localctx).matchQuery = constant();
       }
     }
@@ -1313,14 +1312,14 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio
     ValueExpressionContext _localctx = new ValueExpressionContext(_ctx, getState());
     enterRule(_localctx, 16, RULE_valueExpression);
     try {
-      setState(248);
+      setState(244);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) {
       case 1:
         _localctx = new ValueExpressionDefaultContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(243);
+        setState(239);
         operatorExpression(0);
         }
         break;
@@ -1328,11 +1327,11 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio
         _localctx = new ComparisonContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(244);
+        setState(240);
         ((ComparisonContext)_localctx).left = operatorExpression(0);
-        setState(245);
+        setState(241);
         comparisonOperator();
-        setState(246);
+        setState(242);
         ((ComparisonContext)_localctx).right = operatorExpression(0);
         }
         break;
@@ -1457,7 +1456,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(254);
+      setState(250);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) {
       case 1:
@@ -1466,7 +1465,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(251);
+        setState(247);
         primaryExpression(0);
         }
         break;
@@ -1475,7 +1474,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
         _localctx = new ArithmeticUnaryContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(252);
+        setState(248);
         ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1);
         _la = _input.LA(1);
         if ( !(_la==PLUS || _la==MINUS) ) {
@@ -1486,13 +1485,13 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
           _errHandler.reportMatch(this);
           consume();
         }
-        setState(253);
+        setState(249);
         operatorExpression(3);
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(264);
+      setState(260);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,16,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1500,7 +1499,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(262);
+          setState(258);
           _errHandler.sync(this);
           switch ( getInterpreter().adaptivePredict(_input,15,_ctx) ) {
           case 1:
@@ -1508,9 +1507,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
             _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState));
             ((ArithmeticBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression);
-            setState(256);
+            setState(252);
             if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(257);
+            setState(253);
             ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1);
             _la = _input.LA(1);
             if ( !(((((_la - 66)) & ~0x3f) == 0 && ((1L << (_la - 66)) & 7L) != 0)) ) {
@@ -1521,7 +1520,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
               _errHandler.reportMatch(this);
               consume();
             }
-            setState(258);
+            setState(254);
             ((ArithmeticBinaryContext)_localctx).right = operatorExpression(3);
             }
             break;
@@ -1530,9 +1529,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
             _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState));
             ((ArithmeticBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression);
-            setState(259);
+            setState(255);
             if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
-            setState(260);
+            setState(256);
             ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1);
             _la = _input.LA(1);
             if ( !(_la==PLUS || _la==MINUS) ) {
@@ -1543,14 +1542,14 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
               _errHandler.reportMatch(this);
               consume();
             }
-            setState(261);
+            setState(257);
             ((ArithmeticBinaryContext)_localctx).right = operatorExpression(2);
             }
             break;
           }
           } 
         }
-        setState(266);
+        setState(262);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,16,_ctx);
       }
@@ -1708,7 +1707,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(275);
+      setState(271);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,17,_ctx) ) {
       case 1:
@@ -1717,7 +1716,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(268);
+        setState(264);
         constant();
         }
         break;
@@ -1726,7 +1725,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
         _localctx = new DereferenceContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(269);
+        setState(265);
         qualifiedName();
         }
         break;
@@ -1735,7 +1734,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
         _localctx = new FunctionContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(270);
+        setState(266);
         functionExpression();
         }
         break;
@@ -1744,17 +1743,17 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
         _localctx = new ParenthesizedExpressionContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(271);
+        setState(267);
         match(LP);
-        setState(272);
+        setState(268);
         booleanExpression(0);
-        setState(273);
+        setState(269);
         match(RP);
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(282);
+      setState(278);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,18,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1765,16 +1764,16 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
           {
           _localctx = new InlineCastContext(new PrimaryExpressionContext(_parentctx, _parentState));
           pushNewRecursionContext(_localctx, _startState, RULE_primaryExpression);
-          setState(277);
+          setState(273);
           if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
-          setState(278);
+          setState(274);
           match(CAST_OP);
-          setState(279);
+          setState(275);
           dataType();
           }
           } 
         }
-        setState(284);
+        setState(280);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,18,_ctx);
       }
@@ -1836,37 +1835,37 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(285);
+      setState(281);
       functionName();
-      setState(286);
+      setState(282);
       match(LP);
-      setState(296);
+      setState(292);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) {
       case 1:
         {
-        setState(287);
+        setState(283);
         match(ASTERISK);
         }
         break;
       case 2:
         {
         {
-        setState(288);
+        setState(284);
         booleanExpression(0);
-        setState(293);
+        setState(289);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(289);
+          setState(285);
           match(COMMA);
-          setState(290);
+          setState(286);
           booleanExpression(0);
           }
           }
-          setState(295);
+          setState(291);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
@@ -1874,7 +1873,7 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx
         }
         break;
       }
-      setState(298);
+      setState(294);
       match(RP);
       }
     }
@@ -1920,7 +1919,7 @@ public final FunctionNameContext functionName() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(300);
+      setState(296);
       identifierOrParameter();
       }
     }
@@ -1978,7 +1977,7 @@ public final DataTypeContext dataType() throws RecognitionException {
       _localctx = new ToDataTypeContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(302);
+      setState(298);
       identifier();
       }
     }
@@ -2025,9 +2024,9 @@ public final RowCommandContext rowCommand() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(304);
+      setState(300);
       match(ROW);
-      setState(305);
+      setState(301);
       fields();
       }
     }
@@ -2081,23 +2080,23 @@ public final FieldsContext fields() throws RecognitionException {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(307);
+      setState(303);
       field();
-      setState(312);
+      setState(308);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(308);
+          setState(304);
           match(COMMA);
-          setState(309);
+          setState(305);
           field();
           }
           } 
         }
-        setState(314);
+        setState(310);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
       }
@@ -2149,19 +2148,19 @@ public final FieldContext field() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(318);
+      setState(314);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) {
       case 1:
         {
-        setState(315);
+        setState(311);
         qualifiedName();
-        setState(316);
+        setState(312);
         match(ASSIGN);
         }
         break;
       }
-      setState(320);
+      setState(316);
       booleanExpression(0);
       }
     }
@@ -2219,34 +2218,34 @@ public final FromCommandContext fromCommand() throws RecognitionException {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(322);
+      setState(318);
       match(FROM);
-      setState(323);
+      setState(319);
       indexPattern();
-      setState(328);
+      setState(324);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(324);
+          setState(320);
           match(COMMA);
-          setState(325);
+          setState(321);
           indexPattern();
           }
           } 
         }
-        setState(330);
+        setState(326);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
       }
-      setState(332);
+      setState(328);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
       case 1:
         {
-        setState(331);
+        setState(327);
         metadata();
         }
         break;
@@ -2299,19 +2298,19 @@ public final IndexPatternContext indexPattern() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(337);
+      setState(333);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) {
       case 1:
         {
-        setState(334);
+        setState(330);
         clusterString();
-        setState(335);
+        setState(331);
         match(COLON);
         }
         break;
       }
-      setState(339);
+      setState(335);
       indexString();
       }
     }
@@ -2355,7 +2354,7 @@ public final ClusterStringContext clusterString() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(341);
+      setState(337);
       match(UNQUOTED_SOURCE);
       }
     }
@@ -2401,7 +2400,7 @@ public final IndexStringContext indexString() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(343);
+      setState(339);
       _la = _input.LA(1);
       if ( !(_la==QUOTED_STRING || _la==UNQUOTED_SOURCE) ) {
       _errHandler.recoverInline(this);
@@ -2426,11 +2425,14 @@ public final IndexStringContext indexString() throws RecognitionException {
 
   @SuppressWarnings("CheckReturnValue")
   public static class MetadataContext extends ParserRuleContext {
-    public MetadataOptionContext metadataOption() {
-      return getRuleContext(MetadataOptionContext.class,0);
+    public TerminalNode METADATA() { return getToken(EsqlBaseParser.METADATA, 0); }
+    public List<TerminalNode> UNQUOTED_SOURCE() { return getTokens(EsqlBaseParser.UNQUOTED_SOURCE); }
+    public TerminalNode UNQUOTED_SOURCE(int i) {
+      return getToken(EsqlBaseParser.UNQUOTED_SOURCE, i);
     }
-    public Deprecated_metadataContext deprecated_metadata() {
-      return getRuleContext(Deprecated_metadataContext.class,0);
+    public List<TerminalNode> COMMA() { return getTokens(EsqlBaseParser.COMMA); }
+    public TerminalNode COMMA(int i) {
+      return getToken(EsqlBaseParser.COMMA, i);
     }
     @SuppressWarnings("this-escape")
     public MetadataContext(ParserRuleContext parent, int invokingState) {
@@ -2455,98 +2457,31 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
   public final MetadataContext metadata() throws RecognitionException {
     MetadataContext _localctx = new MetadataContext(_ctx, getState());
     enterRule(_localctx, 42, RULE_metadata);
-    try {
-      setState(347);
-      _errHandler.sync(this);
-      switch (_input.LA(1)) {
-      case METADATA:
-        enterOuterAlt(_localctx, 1);
-        {
-        setState(345);
-        metadataOption();
-        }
-        break;
-      case OPENING_BRACKET:
-        enterOuterAlt(_localctx, 2);
-        {
-        setState(346);
-        deprecated_metadata();
-        }
-        break;
-      default:
-        throw new NoViableAltException(this);
-      }
-    }
-    catch (RecognitionException re) {
-      _localctx.exception = re;
-      _errHandler.reportError(this, re);
-      _errHandler.recover(this, re);
-    }
-    finally {
-      exitRule();
-    }
-    return _localctx;
-  }
-
-  @SuppressWarnings("CheckReturnValue")
-  public static class MetadataOptionContext extends ParserRuleContext {
-    public TerminalNode METADATA() { return getToken(EsqlBaseParser.METADATA, 0); }
-    public List<TerminalNode> UNQUOTED_SOURCE() { return getTokens(EsqlBaseParser.UNQUOTED_SOURCE); }
-    public TerminalNode UNQUOTED_SOURCE(int i) {
-      return getToken(EsqlBaseParser.UNQUOTED_SOURCE, i);
-    }
-    public List<TerminalNode> COMMA() { return getTokens(EsqlBaseParser.COMMA); }
-    public TerminalNode COMMA(int i) {
-      return getToken(EsqlBaseParser.COMMA, i);
-    }
-    @SuppressWarnings("this-escape")
-    public MetadataOptionContext(ParserRuleContext parent, int invokingState) {
-      super(parent, invokingState);
-    }
-    @Override public int getRuleIndex() { return RULE_metadataOption; }
-    @Override
-    public void enterRule(ParseTreeListener listener) {
-      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterMetadataOption(this);
-    }
-    @Override
-    public void exitRule(ParseTreeListener listener) {
-      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitMetadataOption(this);
-    }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor<? extends T>)visitor).visitMetadataOption(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-
-  public final MetadataOptionContext metadataOption() throws RecognitionException {
-    MetadataOptionContext _localctx = new MetadataOptionContext(_ctx, getState());
-    enterRule(_localctx, 44, RULE_metadataOption);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(349);
+      setState(341);
       match(METADATA);
-      setState(350);
+      setState(342);
       match(UNQUOTED_SOURCE);
-      setState(355);
+      setState(347);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,27,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,26,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(351);
+          setState(343);
           match(COMMA);
-          setState(352);
+          setState(344);
           match(UNQUOTED_SOURCE);
           }
           } 
         }
-        setState(357);
+        setState(349);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,27,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,26,_ctx);
       }
       }
     }
@@ -2561,57 +2496,6 @@ public final MetadataOptionContext metadataOption() throws RecognitionException
     return _localctx;
   }
 
-  @SuppressWarnings("CheckReturnValue")
-  public static class Deprecated_metadataContext extends ParserRuleContext {
-    public TerminalNode OPENING_BRACKET() { return getToken(EsqlBaseParser.OPENING_BRACKET, 0); }
-    public MetadataOptionContext metadataOption() {
-      return getRuleContext(MetadataOptionContext.class,0);
-    }
-    public TerminalNode CLOSING_BRACKET() { return getToken(EsqlBaseParser.CLOSING_BRACKET, 0); }
-    public Deprecated_metadataContext(ParserRuleContext parent, int invokingState) {
-      super(parent, invokingState);
-    }
-    @Override public int getRuleIndex() { return RULE_deprecated_metadata; }
-    @Override
-    public void enterRule(ParseTreeListener listener) {
-      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterDeprecated_metadata(this);
-    }
-    @Override
-    public void exitRule(ParseTreeListener listener) {
-      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitDeprecated_metadata(this);
-    }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor<? extends T>)visitor).visitDeprecated_metadata(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-
-  public final Deprecated_metadataContext deprecated_metadata() throws RecognitionException {
-    Deprecated_metadataContext _localctx = new Deprecated_metadataContext(_ctx, getState());
-    enterRule(_localctx, 46, RULE_deprecated_metadata);
-    try {
-      enterOuterAlt(_localctx, 1);
-      {
-      setState(358);
-      match(OPENING_BRACKET);
-      setState(359);
-      metadataOption();
-      setState(360);
-      match(CLOSING_BRACKET);
-      }
-    }
-    catch (RecognitionException re) {
-      _localctx.exception = re;
-      _errHandler.reportError(this, re);
-      _errHandler.recover(this, re);
-    }
-    finally {
-      exitRule();
-    }
-    return _localctx;
-  }
-
   @SuppressWarnings("CheckReturnValue")
   public static class MetricsCommandContext extends ParserRuleContext {
     public AggFieldsContext aggregates;
@@ -2656,51 +2540,51 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final MetricsCommandContext metricsCommand() throws RecognitionException {
     MetricsCommandContext _localctx = new MetricsCommandContext(_ctx, getState());
-    enterRule(_localctx, 48, RULE_metricsCommand);
+    enterRule(_localctx, 44, RULE_metricsCommand);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(362);
+      setState(350);
       match(DEV_METRICS);
-      setState(363);
+      setState(351);
       indexPattern();
-      setState(368);
+      setState(356);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,28,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,27,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(364);
+          setState(352);
           match(COMMA);
-          setState(365);
+          setState(353);
           indexPattern();
           }
           } 
         }
-        setState(370);
+        setState(358);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,28,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,27,_ctx);
       }
-      setState(372);
+      setState(360);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) {
       case 1:
         {
-        setState(371);
+        setState(359);
         ((MetricsCommandContext)_localctx).aggregates = aggFields();
         }
         break;
       }
-      setState(376);
+      setState(364);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         {
-        setState(374);
+        setState(362);
         match(BY);
-        setState(375);
+        setState(363);
         ((MetricsCommandContext)_localctx).grouping = fields();
         }
         break;
@@ -2746,13 +2630,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final EvalCommandContext evalCommand() throws RecognitionException {
     EvalCommandContext _localctx = new EvalCommandContext(_ctx, getState());
-    enterRule(_localctx, 50, RULE_evalCommand);
+    enterRule(_localctx, 46, RULE_evalCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(378);
+      setState(366);
       match(EVAL);
-      setState(379);
+      setState(367);
       fields();
       }
     }
@@ -2801,30 +2685,30 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final StatsCommandContext statsCommand() throws RecognitionException {
     StatsCommandContext _localctx = new StatsCommandContext(_ctx, getState());
-    enterRule(_localctx, 52, RULE_statsCommand);
+    enterRule(_localctx, 48, RULE_statsCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(381);
+      setState(369);
       match(STATS);
-      setState(383);
+      setState(371);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         {
-        setState(382);
+        setState(370);
         ((StatsCommandContext)_localctx).stats = aggFields();
         }
         break;
       }
-      setState(387);
+      setState(375);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) {
       case 1:
         {
-        setState(385);
+        setState(373);
         match(BY);
-        setState(386);
+        setState(374);
         ((StatsCommandContext)_localctx).grouping = fields();
         }
         break;
@@ -2876,30 +2760,30 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final AggFieldsContext aggFields() throws RecognitionException {
     AggFieldsContext _localctx = new AggFieldsContext(_ctx, getState());
-    enterRule(_localctx, 54, RULE_aggFields);
+    enterRule(_localctx, 50, RULE_aggFields);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(389);
+      setState(377);
       aggField();
-      setState(394);
+      setState(382);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,33,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,32,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(390);
+          setState(378);
           match(COMMA);
-          setState(391);
+          setState(379);
           aggField();
           }
           } 
         }
-        setState(396);
+        setState(384);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,33,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,32,_ctx);
       }
       }
     }
@@ -2945,20 +2829,20 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final AggFieldContext aggField() throws RecognitionException {
     AggFieldContext _localctx = new AggFieldContext(_ctx, getState());
-    enterRule(_localctx, 56, RULE_aggField);
+    enterRule(_localctx, 52, RULE_aggField);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(397);
+      setState(385);
       field();
-      setState(400);
+      setState(388);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) {
       case 1:
         {
-        setState(398);
+        setState(386);
         match(WHERE);
-        setState(399);
+        setState(387);
         booleanExpression(0);
         }
         break;
@@ -3010,30 +2894,30 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final QualifiedNameContext qualifiedName() throws RecognitionException {
     QualifiedNameContext _localctx = new QualifiedNameContext(_ctx, getState());
-    enterRule(_localctx, 58, RULE_qualifiedName);
+    enterRule(_localctx, 54, RULE_qualifiedName);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(402);
+      setState(390);
       identifierOrParameter();
-      setState(407);
+      setState(395);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,35,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,34,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(403);
+          setState(391);
           match(DOT);
-          setState(404);
+          setState(392);
           identifierOrParameter();
           }
           } 
         }
-        setState(409);
+        setState(397);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,35,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,34,_ctx);
       }
       }
     }
@@ -3082,30 +2966,30 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final QualifiedNamePatternContext qualifiedNamePattern() throws RecognitionException {
     QualifiedNamePatternContext _localctx = new QualifiedNamePatternContext(_ctx, getState());
-    enterRule(_localctx, 60, RULE_qualifiedNamePattern);
+    enterRule(_localctx, 56, RULE_qualifiedNamePattern);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(410);
+      setState(398);
       identifierPattern();
-      setState(415);
+      setState(403);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,36,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,35,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(411);
+          setState(399);
           match(DOT);
-          setState(412);
+          setState(400);
           identifierPattern();
           }
           } 
         }
-        setState(417);
+        setState(405);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,36,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,35,_ctx);
       }
       }
     }
@@ -3154,30 +3038,30 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final QualifiedNamePatternsContext qualifiedNamePatterns() throws RecognitionException {
     QualifiedNamePatternsContext _localctx = new QualifiedNamePatternsContext(_ctx, getState());
-    enterRule(_localctx, 62, RULE_qualifiedNamePatterns);
+    enterRule(_localctx, 58, RULE_qualifiedNamePatterns);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(418);
+      setState(406);
       qualifiedNamePattern();
-      setState(423);
+      setState(411);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,37,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,36,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(419);
+          setState(407);
           match(COMMA);
-          setState(420);
+          setState(408);
           qualifiedNamePattern();
           }
           } 
         }
-        setState(425);
+        setState(413);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,37,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,36,_ctx);
       }
       }
     }
@@ -3218,12 +3102,12 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final IdentifierContext identifier() throws RecognitionException {
     IdentifierContext _localctx = new IdentifierContext(_ctx, getState());
-    enterRule(_localctx, 64, RULE_identifier);
+    enterRule(_localctx, 60, RULE_identifier);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(426);
+      setState(414);
       _la = _input.LA(1);
       if ( !(_la==UNQUOTED_IDENTIFIER || _la==QUOTED_IDENTIFIER) ) {
       _errHandler.recoverInline(this);
@@ -3274,24 +3158,24 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final IdentifierPatternContext identifierPattern() throws RecognitionException {
     IdentifierPatternContext _localctx = new IdentifierPatternContext(_ctx, getState());
-    enterRule(_localctx, 66, RULE_identifierPattern);
+    enterRule(_localctx, 62, RULE_identifierPattern);
     try {
-      setState(431);
+      setState(419);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,37,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(428);
+        setState(416);
         match(ID_PATTERN);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(429);
+        setState(417);
         if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()");
-        setState(430);
+        setState(418);
         parameter();
         }
         break;
@@ -3561,17 +3445,17 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final ConstantContext constant() throws RecognitionException {
     ConstantContext _localctx = new ConstantContext(_ctx, getState());
-    enterRule(_localctx, 68, RULE_constant);
+    enterRule(_localctx, 64, RULE_constant);
     int _la;
     try {
-      setState(475);
+      setState(463);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,42,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) {
       case 1:
         _localctx = new NullLiteralContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(433);
+        setState(421);
         match(NULL);
         }
         break;
@@ -3579,9 +3463,9 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new QualifiedIntegerLiteralContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(434);
+        setState(422);
         integerValue();
-        setState(435);
+        setState(423);
         match(UNQUOTED_IDENTIFIER);
         }
         break;
@@ -3589,7 +3473,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new DecimalLiteralContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(437);
+        setState(425);
         decimalValue();
         }
         break;
@@ -3597,7 +3481,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new IntegerLiteralContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(438);
+        setState(426);
         integerValue();
         }
         break;
@@ -3605,7 +3489,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new BooleanLiteralContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(439);
+        setState(427);
         booleanValue();
         }
         break;
@@ -3613,7 +3497,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new InputParameterContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(440);
+        setState(428);
         parameter();
         }
         break;
@@ -3621,7 +3505,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new StringLiteralContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(441);
+        setState(429);
         string();
         }
         break;
@@ -3629,27 +3513,27 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new NumericArrayLiteralContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(442);
+        setState(430);
         match(OPENING_BRACKET);
-        setState(443);
+        setState(431);
         numericValue();
-        setState(448);
+        setState(436);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(444);
+          setState(432);
           match(COMMA);
-          setState(445);
+          setState(433);
           numericValue();
           }
           }
-          setState(450);
+          setState(438);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(451);
+        setState(439);
         match(CLOSING_BRACKET);
         }
         break;
@@ -3657,27 +3541,27 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new BooleanArrayLiteralContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(453);
+        setState(441);
         match(OPENING_BRACKET);
-        setState(454);
+        setState(442);
         booleanValue();
-        setState(459);
+        setState(447);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(455);
+          setState(443);
           match(COMMA);
-          setState(456);
+          setState(444);
           booleanValue();
           }
           }
-          setState(461);
+          setState(449);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(462);
+        setState(450);
         match(CLOSING_BRACKET);
         }
         break;
@@ -3685,27 +3569,27 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new StringArrayLiteralContext(_localctx);
         enterOuterAlt(_localctx, 10);
         {
-        setState(464);
+        setState(452);
         match(OPENING_BRACKET);
-        setState(465);
+        setState(453);
         string();
-        setState(470);
+        setState(458);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(466);
+          setState(454);
           match(COMMA);
-          setState(467);
+          setState(455);
           string();
           }
           }
-          setState(472);
+          setState(460);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(473);
+        setState(461);
         match(CLOSING_BRACKET);
         }
         break;
@@ -3777,16 +3661,16 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final ParameterContext parameter() throws RecognitionException {
     ParameterContext _localctx = new ParameterContext(_ctx, getState());
-    enterRule(_localctx, 70, RULE_parameter);
+    enterRule(_localctx, 66, RULE_parameter);
     try {
-      setState(479);
+      setState(467);
       _errHandler.sync(this);
       switch (_input.LA(1)) {
       case PARAM:
         _localctx = new InputParamContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(477);
+        setState(465);
         match(PARAM);
         }
         break;
@@ -3794,7 +3678,7 @@ public final ParameterContext parameter() throws RecognitionException {
         _localctx = new InputNamedOrPositionalParamContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(478);
+        setState(466);
         match(NAMED_OR_POSITIONAL_PARAM);
         }
         break;
@@ -3843,24 +3727,24 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final IdentifierOrParameterContext identifierOrParameter() throws RecognitionException {
     IdentifierOrParameterContext _localctx = new IdentifierOrParameterContext(_ctx, getState());
-    enterRule(_localctx, 72, RULE_identifierOrParameter);
+    enterRule(_localctx, 68, RULE_identifierOrParameter);
     try {
-      setState(484);
+      setState(472);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,44,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,43,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(481);
+        setState(469);
         identifier();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(482);
+        setState(470);
         if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()");
-        setState(483);
+        setState(471);
         parameter();
         }
         break;
@@ -3903,13 +3787,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final LimitCommandContext limitCommand() throws RecognitionException {
     LimitCommandContext _localctx = new LimitCommandContext(_ctx, getState());
-    enterRule(_localctx, 74, RULE_limitCommand);
+    enterRule(_localctx, 70, RULE_limitCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(486);
+      setState(474);
       match(LIMIT);
-      setState(487);
+      setState(475);
       match(INTEGER_LITERAL);
       }
     }
@@ -3959,32 +3843,32 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final SortCommandContext sortCommand() throws RecognitionException {
     SortCommandContext _localctx = new SortCommandContext(_ctx, getState());
-    enterRule(_localctx, 76, RULE_sortCommand);
+    enterRule(_localctx, 72, RULE_sortCommand);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(489);
+      setState(477);
       match(SORT);
-      setState(490);
+      setState(478);
       orderExpression();
-      setState(495);
+      setState(483);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,45,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,44,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(491);
+          setState(479);
           match(COMMA);
-          setState(492);
+          setState(480);
           orderExpression();
           }
           } 
         }
-        setState(497);
+        setState(485);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,45,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,44,_ctx);
       }
       }
     }
@@ -4033,19 +3917,19 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final OrderExpressionContext orderExpression() throws RecognitionException {
     OrderExpressionContext _localctx = new OrderExpressionContext(_ctx, getState());
-    enterRule(_localctx, 78, RULE_orderExpression);
+    enterRule(_localctx, 74, RULE_orderExpression);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(498);
+      setState(486);
       booleanExpression(0);
-      setState(500);
+      setState(488);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,45,_ctx) ) {
       case 1:
         {
-        setState(499);
+        setState(487);
         ((OrderExpressionContext)_localctx).ordering = _input.LT(1);
         _la = _input.LA(1);
         if ( !(_la==ASC || _la==DESC) ) {
@@ -4059,14 +3943,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio
         }
         break;
       }
-      setState(504);
+      setState(492);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) {
       case 1:
         {
-        setState(502);
+        setState(490);
         match(NULLS);
-        setState(503);
+        setState(491);
         ((OrderExpressionContext)_localctx).nullOrdering = _input.LT(1);
         _la = _input.LA(1);
         if ( !(_la==FIRST || _la==LAST) ) {
@@ -4121,13 +4005,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final KeepCommandContext keepCommand() throws RecognitionException {
     KeepCommandContext _localctx = new KeepCommandContext(_ctx, getState());
-    enterRule(_localctx, 80, RULE_keepCommand);
+    enterRule(_localctx, 76, RULE_keepCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(506);
+      setState(494);
       match(KEEP);
-      setState(507);
+      setState(495);
       qualifiedNamePatterns();
       }
     }
@@ -4170,13 +4054,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final DropCommandContext dropCommand() throws RecognitionException {
     DropCommandContext _localctx = new DropCommandContext(_ctx, getState());
-    enterRule(_localctx, 82, RULE_dropCommand);
+    enterRule(_localctx, 78, RULE_dropCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(509);
+      setState(497);
       match(DROP);
-      setState(510);
+      setState(498);
       qualifiedNamePatterns();
       }
     }
@@ -4226,32 +4110,32 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final RenameCommandContext renameCommand() throws RecognitionException {
     RenameCommandContext _localctx = new RenameCommandContext(_ctx, getState());
-    enterRule(_localctx, 84, RULE_renameCommand);
+    enterRule(_localctx, 80, RULE_renameCommand);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(512);
+      setState(500);
       match(RENAME);
-      setState(513);
+      setState(501);
       renameClause();
-      setState(518);
+      setState(506);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,48,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,47,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(514);
+          setState(502);
           match(COMMA);
-          setState(515);
+          setState(503);
           renameClause();
           }
           } 
         }
-        setState(520);
+        setState(508);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,48,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,47,_ctx);
       }
       }
     }
@@ -4299,15 +4183,15 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final RenameClauseContext renameClause() throws RecognitionException {
     RenameClauseContext _localctx = new RenameClauseContext(_ctx, getState());
-    enterRule(_localctx, 86, RULE_renameClause);
+    enterRule(_localctx, 82, RULE_renameClause);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(521);
+      setState(509);
       ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern();
-      setState(522);
+      setState(510);
       match(AS);
-      setState(523);
+      setState(511);
       ((RenameClauseContext)_localctx).newName = qualifiedNamePattern();
       }
     }
@@ -4356,22 +4240,22 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final DissectCommandContext dissectCommand() throws RecognitionException {
     DissectCommandContext _localctx = new DissectCommandContext(_ctx, getState());
-    enterRule(_localctx, 88, RULE_dissectCommand);
+    enterRule(_localctx, 84, RULE_dissectCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(525);
+      setState(513);
       match(DISSECT);
-      setState(526);
+      setState(514);
       primaryExpression(0);
-      setState(527);
+      setState(515);
       string();
-      setState(529);
+      setState(517);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,49,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,48,_ctx) ) {
       case 1:
         {
-        setState(528);
+        setState(516);
         commandOptions();
         }
         break;
@@ -4420,15 +4304,15 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final GrokCommandContext grokCommand() throws RecognitionException {
     GrokCommandContext _localctx = new GrokCommandContext(_ctx, getState());
-    enterRule(_localctx, 90, RULE_grokCommand);
+    enterRule(_localctx, 86, RULE_grokCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(531);
+      setState(519);
       match(GROK);
-      setState(532);
+      setState(520);
       primaryExpression(0);
-      setState(533);
+      setState(521);
       string();
       }
     }
@@ -4471,13 +4355,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final MvExpandCommandContext mvExpandCommand() throws RecognitionException {
     MvExpandCommandContext _localctx = new MvExpandCommandContext(_ctx, getState());
-    enterRule(_localctx, 92, RULE_mvExpandCommand);
+    enterRule(_localctx, 88, RULE_mvExpandCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(535);
+      setState(523);
       match(MV_EXPAND);
-      setState(536);
+      setState(524);
       qualifiedName();
       }
     }
@@ -4526,30 +4410,30 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final CommandOptionsContext commandOptions() throws RecognitionException {
     CommandOptionsContext _localctx = new CommandOptionsContext(_ctx, getState());
-    enterRule(_localctx, 94, RULE_commandOptions);
+    enterRule(_localctx, 90, RULE_commandOptions);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(538);
+      setState(526);
       commandOption();
-      setState(543);
+      setState(531);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,50,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,49,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(539);
+          setState(527);
           match(COMMA);
-          setState(540);
+          setState(528);
           commandOption();
           }
           } 
         }
-        setState(545);
+        setState(533);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,50,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,49,_ctx);
       }
       }
     }
@@ -4595,15 +4479,15 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final CommandOptionContext commandOption() throws RecognitionException {
     CommandOptionContext _localctx = new CommandOptionContext(_ctx, getState());
-    enterRule(_localctx, 96, RULE_commandOption);
+    enterRule(_localctx, 92, RULE_commandOption);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(546);
+      setState(534);
       identifier();
-      setState(547);
+      setState(535);
       match(ASSIGN);
-      setState(548);
+      setState(536);
       constant();
       }
     }
@@ -4644,12 +4528,12 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final BooleanValueContext booleanValue() throws RecognitionException {
     BooleanValueContext _localctx = new BooleanValueContext(_ctx, getState());
-    enterRule(_localctx, 98, RULE_booleanValue);
+    enterRule(_localctx, 94, RULE_booleanValue);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(550);
+      setState(538);
       _la = _input.LA(1);
       if ( !(_la==FALSE || _la==TRUE) ) {
       _errHandler.recoverInline(this);
@@ -4702,22 +4586,22 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final NumericValueContext numericValue() throws RecognitionException {
     NumericValueContext _localctx = new NumericValueContext(_ctx, getState());
-    enterRule(_localctx, 100, RULE_numericValue);
+    enterRule(_localctx, 96, RULE_numericValue);
     try {
-      setState(554);
+      setState(542);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,50,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(552);
+        setState(540);
         decimalValue();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(553);
+        setState(541);
         integerValue();
         }
         break;
@@ -4761,17 +4645,17 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final DecimalValueContext decimalValue() throws RecognitionException {
     DecimalValueContext _localctx = new DecimalValueContext(_ctx, getState());
-    enterRule(_localctx, 102, RULE_decimalValue);
+    enterRule(_localctx, 98, RULE_decimalValue);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(557);
+      setState(545);
       _errHandler.sync(this);
       _la = _input.LA(1);
       if (_la==PLUS || _la==MINUS) {
         {
-        setState(556);
+        setState(544);
         _la = _input.LA(1);
         if ( !(_la==PLUS || _la==MINUS) ) {
         _errHandler.recoverInline(this);
@@ -4784,7 +4668,7 @@ public final DecimalValueContext decimalValue() throws RecognitionException {
         }
       }
 
-      setState(559);
+      setState(547);
       match(DECIMAL_LITERAL);
       }
     }
@@ -4826,17 +4710,17 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final IntegerValueContext integerValue() throws RecognitionException {
     IntegerValueContext _localctx = new IntegerValueContext(_ctx, getState());
-    enterRule(_localctx, 104, RULE_integerValue);
+    enterRule(_localctx, 100, RULE_integerValue);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(562);
+      setState(550);
       _errHandler.sync(this);
       _la = _input.LA(1);
       if (_la==PLUS || _la==MINUS) {
         {
-        setState(561);
+        setState(549);
         _la = _input.LA(1);
         if ( !(_la==PLUS || _la==MINUS) ) {
         _errHandler.recoverInline(this);
@@ -4849,7 +4733,7 @@ public final IntegerValueContext integerValue() throws RecognitionException {
         }
       }
 
-      setState(564);
+      setState(552);
       match(INTEGER_LITERAL);
       }
     }
@@ -4889,11 +4773,11 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final StringContext string() throws RecognitionException {
     StringContext _localctx = new StringContext(_ctx, getState());
-    enterRule(_localctx, 106, RULE_string);
+    enterRule(_localctx, 102, RULE_string);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(566);
+      setState(554);
       match(QUOTED_STRING);
       }
     }
@@ -4938,12 +4822,12 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final ComparisonOperatorContext comparisonOperator() throws RecognitionException {
     ComparisonOperatorContext _localctx = new ComparisonOperatorContext(_ctx, getState());
-    enterRule(_localctx, 108, RULE_comparisonOperator);
+    enterRule(_localctx, 104, RULE_comparisonOperator);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(568);
+      setState(556);
       _la = _input.LA(1);
       if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & -432345564227567616L) != 0)) ) {
       _errHandler.recoverInline(this);
@@ -4994,13 +4878,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final ExplainCommandContext explainCommand() throws RecognitionException {
     ExplainCommandContext _localctx = new ExplainCommandContext(_ctx, getState());
-    enterRule(_localctx, 110, RULE_explainCommand);
+    enterRule(_localctx, 106, RULE_explainCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(570);
+      setState(558);
       match(EXPLAIN);
-      setState(571);
+      setState(559);
       subqueryExpression();
       }
     }
@@ -5044,15 +4928,15 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final SubqueryExpressionContext subqueryExpression() throws RecognitionException {
     SubqueryExpressionContext _localctx = new SubqueryExpressionContext(_ctx, getState());
-    enterRule(_localctx, 112, RULE_subqueryExpression);
+    enterRule(_localctx, 108, RULE_subqueryExpression);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(573);
+      setState(561);
       match(OPENING_BRACKET);
-      setState(574);
+      setState(562);
       query(0);
-      setState(575);
+      setState(563);
       match(CLOSING_BRACKET);
       }
     }
@@ -5104,14 +4988,14 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final ShowCommandContext showCommand() throws RecognitionException {
     ShowCommandContext _localctx = new ShowCommandContext(_ctx, getState());
-    enterRule(_localctx, 114, RULE_showCommand);
+    enterRule(_localctx, 110, RULE_showCommand);
     try {
       _localctx = new ShowInfoContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(577);
+      setState(565);
       match(SHOW);
-      setState(578);
+      setState(566);
       match(INFO);
       }
     }
@@ -5169,53 +5053,53 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final EnrichCommandContext enrichCommand() throws RecognitionException {
     EnrichCommandContext _localctx = new EnrichCommandContext(_ctx, getState());
-    enterRule(_localctx, 116, RULE_enrichCommand);
+    enterRule(_localctx, 112, RULE_enrichCommand);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(580);
+      setState(568);
       match(ENRICH);
-      setState(581);
+      setState(569);
       ((EnrichCommandContext)_localctx).policyName = match(ENRICH_POLICY_NAME);
-      setState(584);
+      setState(572);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,54,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,53,_ctx) ) {
       case 1:
         {
-        setState(582);
+        setState(570);
         match(ON);
-        setState(583);
+        setState(571);
         ((EnrichCommandContext)_localctx).matchField = qualifiedNamePattern();
         }
         break;
       }
-      setState(595);
+      setState(583);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,56,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,55,_ctx) ) {
       case 1:
         {
-        setState(586);
+        setState(574);
         match(WITH);
-        setState(587);
+        setState(575);
         enrichWithClause();
-        setState(592);
+        setState(580);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,55,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,54,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(588);
+            setState(576);
             match(COMMA);
-            setState(589);
+            setState(577);
             enrichWithClause();
             }
             } 
           }
-          setState(594);
+          setState(582);
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,55,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,54,_ctx);
         }
         }
         break;
@@ -5266,23 +5150,23 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final EnrichWithClauseContext enrichWithClause() throws RecognitionException {
     EnrichWithClauseContext _localctx = new EnrichWithClauseContext(_ctx, getState());
-    enterRule(_localctx, 118, RULE_enrichWithClause);
+    enterRule(_localctx, 114, RULE_enrichWithClause);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(600);
+      setState(588);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,57,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,56,_ctx) ) {
       case 1:
         {
-        setState(597);
+        setState(585);
         ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern();
-        setState(598);
+        setState(586);
         match(ASSIGN);
         }
         break;
       }
-      setState(602);
+      setState(590);
       ((EnrichWithClauseContext)_localctx).enrichField = qualifiedNamePattern();
       }
     }
@@ -5331,17 +5215,17 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final LookupCommandContext lookupCommand() throws RecognitionException {
     LookupCommandContext _localctx = new LookupCommandContext(_ctx, getState());
-    enterRule(_localctx, 120, RULE_lookupCommand);
+    enterRule(_localctx, 116, RULE_lookupCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(604);
+      setState(592);
       match(DEV_LOOKUP);
-      setState(605);
+      setState(593);
       ((LookupCommandContext)_localctx).tableName = indexPattern();
-      setState(606);
+      setState(594);
       match(ON);
-      setState(607);
+      setState(595);
       ((LookupCommandContext)_localctx).matchFields = qualifiedNamePatterns();
       }
     }
@@ -5390,22 +5274,22 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionException {
     InlinestatsCommandContext _localctx = new InlinestatsCommandContext(_ctx, getState());
-    enterRule(_localctx, 122, RULE_inlinestatsCommand);
+    enterRule(_localctx, 118, RULE_inlinestatsCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(609);
+      setState(597);
       match(DEV_INLINESTATS);
-      setState(610);
+      setState(598);
       ((InlinestatsCommandContext)_localctx).stats = aggFields();
-      setState(613);
+      setState(601);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,58,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,57,_ctx) ) {
       case 1:
         {
-        setState(611);
+        setState(599);
         match(BY);
-        setState(612);
+        setState(600);
         ((InlinestatsCommandContext)_localctx).grouping = fields();
         }
         break;
@@ -5458,17 +5342,17 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final JoinCommandContext joinCommand() throws RecognitionException {
     JoinCommandContext _localctx = new JoinCommandContext(_ctx, getState());
-    enterRule(_localctx, 124, RULE_joinCommand);
+    enterRule(_localctx, 120, RULE_joinCommand);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(616);
+      setState(604);
       _errHandler.sync(this);
       _la = _input.LA(1);
       if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 29360128L) != 0)) {
         {
-        setState(615);
+        setState(603);
         ((JoinCommandContext)_localctx).type = _input.LT(1);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 29360128L) != 0)) ) {
@@ -5482,11 +5366,11 @@ public final JoinCommandContext joinCommand() throws RecognitionException {
         }
       }
 
-      setState(618);
+      setState(606);
       match(DEV_JOIN);
-      setState(619);
+      setState(607);
       joinTarget();
-      setState(620);
+      setState(608);
       joinCondition();
       }
     }
@@ -5534,21 +5418,21 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final JoinTargetContext joinTarget() throws RecognitionException {
     JoinTargetContext _localctx = new JoinTargetContext(_ctx, getState());
-    enterRule(_localctx, 126, RULE_joinTarget);
+    enterRule(_localctx, 122, RULE_joinTarget);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(622);
+      setState(610);
       ((JoinTargetContext)_localctx).index = identifier();
-      setState(625);
+      setState(613);
       _errHandler.sync(this);
       _la = _input.LA(1);
       if (_la==AS) {
         {
-        setState(623);
+        setState(611);
         match(AS);
-        setState(624);
+        setState(612);
         ((JoinTargetContext)_localctx).alias = identifier();
         }
       }
@@ -5601,32 +5485,32 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final JoinConditionContext joinCondition() throws RecognitionException {
     JoinConditionContext _localctx = new JoinConditionContext(_ctx, getState());
-    enterRule(_localctx, 128, RULE_joinCondition);
+    enterRule(_localctx, 124, RULE_joinCondition);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(627);
+      setState(615);
       match(ON);
-      setState(628);
+      setState(616);
       joinPredicate();
-      setState(633);
+      setState(621);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,61,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,60,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(629);
+          setState(617);
           match(COMMA);
-          setState(630);
+          setState(618);
           joinPredicate();
           }
           } 
         }
-        setState(635);
+        setState(623);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,61,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,60,_ctx);
       }
       }
     }
@@ -5668,11 +5552,11 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final JoinPredicateContext joinPredicate() throws RecognitionException {
     JoinPredicateContext _localctx = new JoinPredicateContext(_ctx, getState());
-    enterRule(_localctx, 130, RULE_joinPredicate);
+    enterRule(_localctx, 126, RULE_joinPredicate);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(636);
+      setState(624);
       valueExpression();
       }
     }
@@ -5701,9 +5585,9 @@ public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
       return operatorExpression_sempred((OperatorExpressionContext)_localctx, predIndex);
     case 10:
       return primaryExpression_sempred((PrimaryExpressionContext)_localctx, predIndex);
-    case 33:
+    case 31:
       return identifierPattern_sempred((IdentifierPatternContext)_localctx, predIndex);
-    case 36:
+    case 34:
       return identifierOrParameter_sempred((IdentifierOrParameterContext)_localctx, predIndex);
     }
     return true;
@@ -5774,7 +5658,7 @@ private boolean identifierOrParameter_sempred(IdentifierOrParameterContext _loca
   }
 
   public static final String _serializedATN =
-    "\u0004\u0001\u0080\u027f\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+
+    "\u0004\u0001\u0080\u0273\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+
     "\u0002\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004"+
     "\u0002\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007"+
     "\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b"+
@@ -5790,392 +5674,385 @@ private boolean identifierOrParameter_sempred(IdentifierOrParameterContext _loca
     ",\u0002-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u0007"+
     "1\u00022\u00072\u00023\u00073\u00024\u00074\u00025\u00075\u00026\u0007"+
     "6\u00027\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0002;\u0007"+
-    ";\u0002<\u0007<\u0002=\u0007=\u0002>\u0007>\u0002?\u0007?\u0002@\u0007"+
-    "@\u0002A\u0007A\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001"+
-    "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0005\u0001\u008e"+
-    "\b\u0001\n\u0001\f\u0001\u0091\t\u0001\u0001\u0002\u0001\u0002\u0001\u0002"+
-    "\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u0099\b\u0002\u0001\u0003"+
-    "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+
-    "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+
-    "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0003\u0003"+
-    "\u00ad\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005"+
-    "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005"+
-    "\u00b9\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+
-    "\u0005\u0005\u00c0\b\u0005\n\u0005\f\u0005\u00c3\t\u0005\u0001\u0005\u0001"+
-    "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00ca\b\u0005\u0001"+
-    "\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00cf\b\u0005\u0001\u0005\u0001"+
-    "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00d7"+
-    "\b\u0005\n\u0005\f\u0005\u00da\t\u0005\u0001\u0006\u0001\u0006\u0003\u0006"+
-    "\u00de\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006"+
-    "\u0003\u0006\u00e5\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006"+
-    "\u00ea\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0003\u0007\u00ef\b"+
-    "\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001"+
-    "\b\u0001\b\u0003\b\u00f9\b\b\u0001\t\u0001\t\u0001\t\u0001\t\u0003\t\u00ff"+
-    "\b\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0005\t\u0107\b\t"+
-    "\n\t\f\t\u010a\t\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+
-    "\n\u0001\n\u0003\n\u0114\b\n\u0001\n\u0001\n\u0001\n\u0005\n\u0119\b\n"+
-    "\n\n\f\n\u011c\t\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001"+
-    "\u000b\u0001\u000b\u0005\u000b\u0124\b\u000b\n\u000b\f\u000b\u0127\t\u000b"+
-    "\u0003\u000b\u0129\b\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001"+
-    "\r\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f"+
-    "\u0001\u000f\u0005\u000f\u0137\b\u000f\n\u000f\f\u000f\u013a\t\u000f\u0001"+
-    "\u0010\u0001\u0010\u0001\u0010\u0003\u0010\u013f\b\u0010\u0001\u0010\u0001"+
-    "\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0005\u0011\u0147"+
-    "\b\u0011\n\u0011\f\u0011\u014a\t\u0011\u0001\u0011\u0003\u0011\u014d\b"+
-    "\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0003\u0012\u0152\b\u0012\u0001"+
-    "\u0012\u0001\u0012\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001"+
-    "\u0015\u0001\u0015\u0003\u0015\u015c\b\u0015\u0001\u0016\u0001\u0016\u0001"+
-    "\u0016\u0001\u0016\u0005\u0016\u0162\b\u0016\n\u0016\f\u0016\u0165\t\u0016"+
-    "\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018"+
-    "\u0001\u0018\u0001\u0018\u0005\u0018\u016f\b\u0018\n\u0018\f\u0018\u0172"+
-    "\t\u0018\u0001\u0018\u0003\u0018\u0175\b\u0018\u0001\u0018\u0001\u0018"+
-    "\u0003\u0018\u0179\b\u0018\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u001a"+
-    "\u0001\u001a\u0003\u001a\u0180\b\u001a\u0001\u001a\u0001\u001a\u0003\u001a"+
-    "\u0184\b\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0005\u001b\u0189\b"+
-    "\u001b\n\u001b\f\u001b\u018c\t\u001b\u0001\u001c\u0001\u001c\u0001\u001c"+
-    "\u0003\u001c\u0191\b\u001c\u0001\u001d\u0001\u001d\u0001\u001d\u0005\u001d"+
-    "\u0196\b\u001d\n\u001d\f\u001d\u0199\t\u001d\u0001\u001e\u0001\u001e\u0001"+
-    "\u001e\u0005\u001e\u019e\b\u001e\n\u001e\f\u001e\u01a1\t\u001e\u0001\u001f"+
-    "\u0001\u001f\u0001\u001f\u0005\u001f\u01a6\b\u001f\n\u001f\f\u001f\u01a9"+
-    "\t\u001f\u0001 \u0001 \u0001!\u0001!\u0001!\u0003!\u01b0\b!\u0001\"\u0001"+
-    "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001"+
-    "\"\u0001\"\u0001\"\u0005\"\u01bf\b\"\n\"\f\"\u01c2\t\"\u0001\"\u0001\""+
-    "\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01ca\b\"\n\"\f\"\u01cd\t\"\u0001"+
-    "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01d5\b\"\n\"\f\"\u01d8"+
-    "\t\"\u0001\"\u0001\"\u0003\"\u01dc\b\"\u0001#\u0001#\u0003#\u01e0\b#\u0001"+
-    "$\u0001$\u0001$\u0003$\u01e5\b$\u0001%\u0001%\u0001%\u0001&\u0001&\u0001"+
-    "&\u0001&\u0005&\u01ee\b&\n&\f&\u01f1\t&\u0001\'\u0001\'\u0003\'\u01f5"+
-    "\b\'\u0001\'\u0001\'\u0003\'\u01f9\b\'\u0001(\u0001(\u0001(\u0001)\u0001"+
-    ")\u0001)\u0001*\u0001*\u0001*\u0001*\u0005*\u0205\b*\n*\f*\u0208\t*\u0001"+
-    "+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001,\u0001,\u0003,\u0212\b,\u0001"+
-    "-\u0001-\u0001-\u0001-\u0001.\u0001.\u0001.\u0001/\u0001/\u0001/\u0005"+
-    "/\u021e\b/\n/\f/\u0221\t/\u00010\u00010\u00010\u00010\u00011\u00011\u0001"+
-    "2\u00012\u00032\u022b\b2\u00013\u00033\u022e\b3\u00013\u00013\u00014\u0003"+
-    "4\u0233\b4\u00014\u00014\u00015\u00015\u00016\u00016\u00017\u00017\u0001"+
-    "7\u00018\u00018\u00018\u00018\u00019\u00019\u00019\u0001:\u0001:\u0001"+
-    ":\u0001:\u0003:\u0249\b:\u0001:\u0001:\u0001:\u0001:\u0005:\u024f\b:\n"+
-    ":\f:\u0252\t:\u0003:\u0254\b:\u0001;\u0001;\u0001;\u0003;\u0259\b;\u0001"+
-    ";\u0001;\u0001<\u0001<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001=\u0001"+
-    "=\u0003=\u0266\b=\u0001>\u0003>\u0269\b>\u0001>\u0001>\u0001>\u0001>\u0001"+
-    "?\u0001?\u0001?\u0003?\u0272\b?\u0001@\u0001@\u0001@\u0001@\u0005@\u0278"+
-    "\b@\n@\f@\u027b\t@\u0001A\u0001A\u0001A\u0000\u0004\u0002\n\u0012\u0014"+
-    "B\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012\u0014\u0016\u0018\u001a"+
-    "\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\^`bdfhjlnprtvxz|~\u0080\u0082"+
-    "\u0000\t\u0001\u0000@A\u0001\u0000BD\u0002\u0000\u001e\u001eQQ\u0001\u0000"+
-    "HI\u0002\u0000##((\u0002\u0000++..\u0002\u0000**88\u0002\u000099;?\u0001"+
-    "\u0000\u0016\u0018\u0299\u0000\u0084\u0001\u0000\u0000\u0000\u0002\u0087"+
-    "\u0001\u0000\u0000\u0000\u0004\u0098\u0001\u0000\u0000\u0000\u0006\u00ac"+
-    "\u0001\u0000\u0000\u0000\b\u00ae\u0001\u0000\u0000\u0000\n\u00ce\u0001"+
-    "\u0000\u0000\u0000\f\u00e9\u0001\u0000\u0000\u0000\u000e\u00eb\u0001\u0000"+
-    "\u0000\u0000\u0010\u00f8\u0001\u0000\u0000\u0000\u0012\u00fe\u0001\u0000"+
-    "\u0000\u0000\u0014\u0113\u0001\u0000\u0000\u0000\u0016\u011d\u0001\u0000"+
-    "\u0000\u0000\u0018\u012c\u0001\u0000\u0000\u0000\u001a\u012e\u0001\u0000"+
-    "\u0000\u0000\u001c\u0130\u0001\u0000\u0000\u0000\u001e\u0133\u0001\u0000"+
-    "\u0000\u0000 \u013e\u0001\u0000\u0000\u0000\"\u0142\u0001\u0000\u0000"+
-    "\u0000$\u0151\u0001\u0000\u0000\u0000&\u0155\u0001\u0000\u0000\u0000("+
-    "\u0157\u0001\u0000\u0000\u0000*\u015b\u0001\u0000\u0000\u0000,\u015d\u0001"+
-    "\u0000\u0000\u0000.\u0166\u0001\u0000\u0000\u00000\u016a\u0001\u0000\u0000"+
-    "\u00002\u017a\u0001\u0000\u0000\u00004\u017d\u0001\u0000\u0000\u00006"+
-    "\u0185\u0001\u0000\u0000\u00008\u018d\u0001\u0000\u0000\u0000:\u0192\u0001"+
-    "\u0000\u0000\u0000<\u019a\u0001\u0000\u0000\u0000>\u01a2\u0001\u0000\u0000"+
-    "\u0000@\u01aa\u0001\u0000\u0000\u0000B\u01af\u0001\u0000\u0000\u0000D"+
-    "\u01db\u0001\u0000\u0000\u0000F\u01df\u0001\u0000\u0000\u0000H\u01e4\u0001"+
-    "\u0000\u0000\u0000J\u01e6\u0001\u0000\u0000\u0000L\u01e9\u0001\u0000\u0000"+
-    "\u0000N\u01f2\u0001\u0000\u0000\u0000P\u01fa\u0001\u0000\u0000\u0000R"+
-    "\u01fd\u0001\u0000\u0000\u0000T\u0200\u0001\u0000\u0000\u0000V\u0209\u0001"+
-    "\u0000\u0000\u0000X\u020d\u0001\u0000\u0000\u0000Z\u0213\u0001\u0000\u0000"+
-    "\u0000\\\u0217\u0001\u0000\u0000\u0000^\u021a\u0001\u0000\u0000\u0000"+
-    "`\u0222\u0001\u0000\u0000\u0000b\u0226\u0001\u0000\u0000\u0000d\u022a"+
-    "\u0001\u0000\u0000\u0000f\u022d\u0001\u0000\u0000\u0000h\u0232\u0001\u0000"+
-    "\u0000\u0000j\u0236\u0001\u0000\u0000\u0000l\u0238\u0001\u0000\u0000\u0000"+
-    "n\u023a\u0001\u0000\u0000\u0000p\u023d\u0001\u0000\u0000\u0000r\u0241"+
-    "\u0001\u0000\u0000\u0000t\u0244\u0001\u0000\u0000\u0000v\u0258\u0001\u0000"+
-    "\u0000\u0000x\u025c\u0001\u0000\u0000\u0000z\u0261\u0001\u0000\u0000\u0000"+
-    "|\u0268\u0001\u0000\u0000\u0000~\u026e\u0001\u0000\u0000\u0000\u0080\u0273"+
-    "\u0001\u0000\u0000\u0000\u0082\u027c\u0001\u0000\u0000\u0000\u0084\u0085"+
-    "\u0003\u0002\u0001\u0000\u0085\u0086\u0005\u0000\u0000\u0001\u0086\u0001"+
-    "\u0001\u0000\u0000\u0000\u0087\u0088\u0006\u0001\uffff\uffff\u0000\u0088"+
-    "\u0089\u0003\u0004\u0002\u0000\u0089\u008f\u0001\u0000\u0000\u0000\u008a"+
-    "\u008b\n\u0001\u0000\u0000\u008b\u008c\u0005\u001d\u0000\u0000\u008c\u008e"+
-    "\u0003\u0006\u0003\u0000\u008d\u008a\u0001\u0000\u0000\u0000\u008e\u0091"+
-    "\u0001\u0000\u0000\u0000\u008f\u008d\u0001\u0000\u0000\u0000\u008f\u0090"+
-    "\u0001\u0000\u0000\u0000\u0090\u0003\u0001\u0000\u0000\u0000\u0091\u008f"+
-    "\u0001\u0000\u0000\u0000\u0092\u0099\u0003n7\u0000\u0093\u0099\u0003\""+
-    "\u0011\u0000\u0094\u0099\u0003\u001c\u000e\u0000\u0095\u0099\u0003r9\u0000"+
-    "\u0096\u0097\u0004\u0002\u0001\u0000\u0097\u0099\u00030\u0018\u0000\u0098"+
-    "\u0092\u0001\u0000\u0000\u0000\u0098\u0093\u0001\u0000\u0000\u0000\u0098"+
-    "\u0094\u0001\u0000\u0000\u0000\u0098\u0095\u0001\u0000\u0000\u0000\u0098"+
-    "\u0096\u0001\u0000\u0000\u0000\u0099\u0005\u0001\u0000\u0000\u0000\u009a"+
-    "\u00ad\u00032\u0019\u0000\u009b\u00ad\u0003\b\u0004\u0000\u009c\u00ad"+
-    "\u0003P(\u0000\u009d\u00ad\u0003J%\u0000\u009e\u00ad\u00034\u001a\u0000"+
-    "\u009f\u00ad\u0003L&\u0000\u00a0\u00ad\u0003R)\u0000\u00a1\u00ad\u0003"+
-    "T*\u0000\u00a2\u00ad\u0003X,\u0000\u00a3\u00ad\u0003Z-\u0000\u00a4\u00ad"+
-    "\u0003t:\u0000\u00a5\u00ad\u0003\\.\u0000\u00a6\u00a7\u0004\u0003\u0002"+
-    "\u0000\u00a7\u00ad\u0003z=\u0000\u00a8\u00a9\u0004\u0003\u0003\u0000\u00a9"+
-    "\u00ad\u0003x<\u0000\u00aa\u00ab\u0004\u0003\u0004\u0000\u00ab\u00ad\u0003"+
-    "|>\u0000\u00ac\u009a\u0001\u0000\u0000\u0000\u00ac\u009b\u0001\u0000\u0000"+
-    "\u0000\u00ac\u009c\u0001\u0000\u0000\u0000\u00ac\u009d\u0001\u0000\u0000"+
-    "\u0000\u00ac\u009e\u0001\u0000\u0000\u0000\u00ac\u009f\u0001\u0000\u0000"+
-    "\u0000\u00ac\u00a0\u0001\u0000\u0000\u0000\u00ac\u00a1\u0001\u0000\u0000"+
-    "\u0000\u00ac\u00a2\u0001\u0000\u0000\u0000\u00ac\u00a3\u0001\u0000\u0000"+
-    "\u0000\u00ac\u00a4\u0001\u0000\u0000\u0000\u00ac\u00a5\u0001\u0000\u0000"+
-    "\u0000\u00ac\u00a6\u0001\u0000\u0000\u0000\u00ac\u00a8\u0001\u0000\u0000"+
-    "\u0000\u00ac\u00aa\u0001\u0000\u0000\u0000\u00ad\u0007\u0001\u0000\u0000"+
-    "\u0000\u00ae\u00af\u0005\u0010\u0000\u0000\u00af\u00b0\u0003\n\u0005\u0000"+
-    "\u00b0\t\u0001\u0000\u0000\u0000\u00b1\u00b2\u0006\u0005\uffff\uffff\u0000"+
-    "\u00b2\u00b3\u00051\u0000\u0000\u00b3\u00cf\u0003\n\u0005\b\u00b4\u00cf"+
-    "\u0003\u0010\b\u0000\u00b5\u00cf\u0003\f\u0006\u0000\u00b6\u00b8\u0003"+
-    "\u0010\b\u0000\u00b7\u00b9\u00051\u0000\u0000\u00b8\u00b7\u0001\u0000"+
-    "\u0000\u0000\u00b8\u00b9\u0001\u0000\u0000\u0000\u00b9\u00ba\u0001\u0000"+
-    "\u0000\u0000\u00ba\u00bb\u0005,\u0000\u0000\u00bb\u00bc\u00050\u0000\u0000"+
-    "\u00bc\u00c1\u0003\u0010\b\u0000\u00bd\u00be\u0005\'\u0000\u0000\u00be"+
-    "\u00c0\u0003\u0010\b\u0000\u00bf\u00bd\u0001\u0000\u0000\u0000\u00c0\u00c3"+
-    "\u0001\u0000\u0000\u0000\u00c1\u00bf\u0001\u0000\u0000\u0000\u00c1\u00c2"+
-    "\u0001\u0000\u0000\u0000\u00c2\u00c4\u0001\u0000\u0000\u0000\u00c3\u00c1"+
-    "\u0001\u0000\u0000\u0000\u00c4\u00c5\u00057\u0000\u0000\u00c5\u00cf\u0001"+
-    "\u0000\u0000\u0000\u00c6\u00c7\u0003\u0010\b\u0000\u00c7\u00c9\u0005-"+
-    "\u0000\u0000\u00c8\u00ca\u00051\u0000\u0000\u00c9\u00c8\u0001\u0000\u0000"+
-    "\u0000\u00c9\u00ca\u0001\u0000\u0000\u0000\u00ca\u00cb\u0001\u0000\u0000"+
-    "\u0000\u00cb\u00cc\u00052\u0000\u0000\u00cc\u00cf\u0001\u0000\u0000\u0000"+
-    "\u00cd\u00cf\u0003\u000e\u0007\u0000\u00ce\u00b1\u0001\u0000\u0000\u0000"+
-    "\u00ce\u00b4\u0001\u0000\u0000\u0000\u00ce\u00b5\u0001\u0000\u0000\u0000"+
-    "\u00ce\u00b6\u0001\u0000\u0000\u0000\u00ce\u00c6\u0001\u0000\u0000\u0000"+
-    "\u00ce\u00cd\u0001\u0000\u0000\u0000\u00cf\u00d8\u0001\u0000\u0000\u0000"+
-    "\u00d0\u00d1\n\u0005\u0000\u0000\u00d1\u00d2\u0005\"\u0000\u0000\u00d2"+
-    "\u00d7\u0003\n\u0005\u0006\u00d3\u00d4\n\u0004\u0000\u0000\u00d4\u00d5"+
-    "\u00054\u0000\u0000\u00d5\u00d7\u0003\n\u0005\u0005\u00d6\u00d0\u0001"+
-    "\u0000\u0000\u0000\u00d6\u00d3\u0001\u0000\u0000\u0000\u00d7\u00da\u0001"+
-    "\u0000\u0000\u0000\u00d8\u00d6\u0001\u0000\u0000\u0000\u00d8\u00d9\u0001"+
-    "\u0000\u0000\u0000\u00d9\u000b\u0001\u0000\u0000\u0000\u00da\u00d8\u0001"+
-    "\u0000\u0000\u0000\u00db\u00dd\u0003\u0010\b\u0000\u00dc\u00de\u00051"+
-    "\u0000\u0000\u00dd\u00dc\u0001\u0000\u0000\u0000\u00dd\u00de\u0001\u0000"+
-    "\u0000\u0000\u00de\u00df\u0001\u0000\u0000\u0000\u00df\u00e0\u0005/\u0000"+
-    "\u0000\u00e0\u00e1\u0003j5\u0000\u00e1\u00ea\u0001\u0000\u0000\u0000\u00e2"+
-    "\u00e4\u0003\u0010\b\u0000\u00e3\u00e5\u00051\u0000\u0000\u00e4\u00e3"+
-    "\u0001\u0000\u0000\u0000\u00e4\u00e5\u0001\u0000\u0000\u0000\u00e5\u00e6"+
-    "\u0001\u0000\u0000\u0000\u00e6\u00e7\u00056\u0000\u0000\u00e7\u00e8\u0003"+
-    "j5\u0000\u00e8\u00ea\u0001\u0000\u0000\u0000\u00e9\u00db\u0001\u0000\u0000"+
-    "\u0000\u00e9\u00e2\u0001\u0000\u0000\u0000\u00ea\r\u0001\u0000\u0000\u0000"+
-    "\u00eb\u00ee\u0003:\u001d\u0000\u00ec\u00ed\u0005%\u0000\u0000\u00ed\u00ef"+
-    "\u0003\u001a\r\u0000\u00ee\u00ec\u0001\u0000\u0000\u0000\u00ee\u00ef\u0001"+
-    "\u0000\u0000\u0000\u00ef\u00f0\u0001\u0000\u0000\u0000\u00f0\u00f1\u0005"+
-    "&\u0000\u0000\u00f1\u00f2\u0003D\"\u0000\u00f2\u000f\u0001\u0000\u0000"+
-    "\u0000\u00f3\u00f9\u0003\u0012\t\u0000\u00f4\u00f5\u0003\u0012\t\u0000"+
-    "\u00f5\u00f6\u0003l6\u0000\u00f6\u00f7\u0003\u0012\t\u0000\u00f7\u00f9"+
-    "\u0001\u0000\u0000\u0000\u00f8\u00f3\u0001\u0000\u0000\u0000\u00f8\u00f4"+
-    "\u0001\u0000\u0000\u0000\u00f9\u0011\u0001\u0000\u0000\u0000\u00fa\u00fb"+
-    "\u0006\t\uffff\uffff\u0000\u00fb\u00ff\u0003\u0014\n\u0000\u00fc\u00fd"+
-    "\u0007\u0000\u0000\u0000\u00fd\u00ff\u0003\u0012\t\u0003\u00fe\u00fa\u0001"+
-    "\u0000\u0000\u0000\u00fe\u00fc\u0001\u0000\u0000\u0000\u00ff\u0108\u0001"+
-    "\u0000\u0000\u0000\u0100\u0101\n\u0002\u0000\u0000\u0101\u0102\u0007\u0001"+
-    "\u0000\u0000\u0102\u0107\u0003\u0012\t\u0003\u0103\u0104\n\u0001\u0000"+
-    "\u0000\u0104\u0105\u0007\u0000\u0000\u0000\u0105\u0107\u0003\u0012\t\u0002"+
-    "\u0106\u0100\u0001\u0000\u0000\u0000\u0106\u0103\u0001\u0000\u0000\u0000"+
-    "\u0107\u010a\u0001\u0000\u0000\u0000\u0108\u0106\u0001\u0000\u0000\u0000"+
-    "\u0108\u0109\u0001\u0000\u0000\u0000\u0109\u0013\u0001\u0000\u0000\u0000"+
-    "\u010a\u0108\u0001\u0000\u0000\u0000\u010b\u010c\u0006\n\uffff\uffff\u0000"+
-    "\u010c\u0114\u0003D\"\u0000\u010d\u0114\u0003:\u001d\u0000\u010e\u0114"+
-    "\u0003\u0016\u000b\u0000\u010f\u0110\u00050\u0000\u0000\u0110\u0111\u0003"+
-    "\n\u0005\u0000\u0111\u0112\u00057\u0000\u0000\u0112\u0114\u0001\u0000"+
-    "\u0000\u0000\u0113\u010b\u0001\u0000\u0000\u0000\u0113\u010d\u0001\u0000"+
-    "\u0000\u0000\u0113\u010e\u0001\u0000\u0000\u0000\u0113\u010f\u0001\u0000"+
-    "\u0000\u0000\u0114\u011a\u0001\u0000\u0000\u0000\u0115\u0116\n\u0001\u0000"+
-    "\u0000\u0116\u0117\u0005%\u0000\u0000\u0117\u0119\u0003\u001a\r\u0000"+
-    "\u0118\u0115\u0001\u0000\u0000\u0000\u0119\u011c\u0001\u0000\u0000\u0000"+
-    "\u011a\u0118\u0001\u0000\u0000\u0000\u011a\u011b\u0001\u0000\u0000\u0000"+
-    "\u011b\u0015\u0001\u0000\u0000\u0000\u011c\u011a\u0001\u0000\u0000\u0000"+
-    "\u011d\u011e\u0003\u0018\f\u0000\u011e\u0128\u00050\u0000\u0000\u011f"+
-    "\u0129\u0005B\u0000\u0000\u0120\u0125\u0003\n\u0005\u0000\u0121\u0122"+
-    "\u0005\'\u0000\u0000\u0122\u0124\u0003\n\u0005\u0000\u0123\u0121\u0001"+
-    "\u0000\u0000\u0000\u0124\u0127\u0001\u0000\u0000\u0000\u0125\u0123\u0001"+
-    "\u0000\u0000\u0000\u0125\u0126\u0001\u0000\u0000\u0000\u0126\u0129\u0001"+
-    "\u0000\u0000\u0000\u0127\u0125\u0001\u0000\u0000\u0000\u0128\u011f\u0001"+
-    "\u0000\u0000\u0000\u0128\u0120\u0001\u0000\u0000\u0000\u0128\u0129\u0001"+
-    "\u0000\u0000\u0000\u0129\u012a\u0001\u0000\u0000\u0000\u012a\u012b\u0005"+
-    "7\u0000\u0000\u012b\u0017\u0001\u0000\u0000\u0000\u012c\u012d\u0003H$"+
-    "\u0000\u012d\u0019\u0001\u0000\u0000\u0000\u012e\u012f\u0003@ \u0000\u012f"+
-    "\u001b\u0001\u0000\u0000\u0000\u0130\u0131\u0005\f\u0000\u0000\u0131\u0132"+
-    "\u0003\u001e\u000f\u0000\u0132\u001d\u0001\u0000\u0000\u0000\u0133\u0138"+
-    "\u0003 \u0010\u0000\u0134\u0135\u0005\'\u0000\u0000\u0135\u0137\u0003"+
-    " \u0010\u0000\u0136\u0134\u0001\u0000\u0000\u0000\u0137\u013a\u0001\u0000"+
-    "\u0000\u0000\u0138\u0136\u0001\u0000\u0000\u0000\u0138\u0139\u0001\u0000"+
-    "\u0000\u0000\u0139\u001f\u0001\u0000\u0000\u0000\u013a\u0138\u0001\u0000"+
-    "\u0000\u0000\u013b\u013c\u0003:\u001d\u0000\u013c\u013d\u0005$\u0000\u0000"+
-    "\u013d\u013f\u0001\u0000\u0000\u0000\u013e\u013b\u0001\u0000\u0000\u0000"+
-    "\u013e\u013f\u0001\u0000\u0000\u0000\u013f\u0140\u0001\u0000\u0000\u0000"+
-    "\u0140\u0141\u0003\n\u0005\u0000\u0141!\u0001\u0000\u0000\u0000\u0142"+
-    "\u0143\u0005\u0006\u0000\u0000\u0143\u0148\u0003$\u0012\u0000\u0144\u0145"+
-    "\u0005\'\u0000\u0000\u0145\u0147\u0003$\u0012\u0000\u0146\u0144\u0001"+
-    "\u0000\u0000\u0000\u0147\u014a\u0001\u0000\u0000\u0000\u0148\u0146\u0001"+
-    "\u0000\u0000\u0000\u0148\u0149\u0001\u0000\u0000\u0000\u0149\u014c\u0001"+
-    "\u0000\u0000\u0000\u014a\u0148\u0001\u0000\u0000\u0000\u014b\u014d\u0003"+
-    "*\u0015\u0000\u014c\u014b\u0001\u0000\u0000\u0000\u014c\u014d\u0001\u0000"+
-    "\u0000\u0000\u014d#\u0001\u0000\u0000\u0000\u014e\u014f\u0003&\u0013\u0000"+
-    "\u014f\u0150\u0005&\u0000\u0000\u0150\u0152\u0001\u0000\u0000\u0000\u0151"+
-    "\u014e\u0001\u0000\u0000\u0000\u0151\u0152\u0001\u0000\u0000\u0000\u0152"+
-    "\u0153\u0001\u0000\u0000\u0000\u0153\u0154\u0003(\u0014\u0000\u0154%\u0001"+
-    "\u0000\u0000\u0000\u0155\u0156\u0005Q\u0000\u0000\u0156\'\u0001\u0000"+
-    "\u0000\u0000\u0157\u0158\u0007\u0002\u0000\u0000\u0158)\u0001\u0000\u0000"+
-    "\u0000\u0159\u015c\u0003,\u0016\u0000\u015a\u015c\u0003.\u0017\u0000\u015b"+
-    "\u0159\u0001\u0000\u0000\u0000\u015b\u015a\u0001\u0000\u0000\u0000\u015c"+
-    "+\u0001\u0000\u0000\u0000\u015d\u015e\u0005P\u0000\u0000\u015e\u0163\u0005"+
-    "Q\u0000\u0000\u015f\u0160\u0005\'\u0000\u0000\u0160\u0162\u0005Q\u0000"+
-    "\u0000\u0161\u015f\u0001\u0000\u0000\u0000\u0162\u0165\u0001\u0000\u0000"+
-    "\u0000\u0163\u0161\u0001\u0000\u0000\u0000\u0163\u0164\u0001\u0000\u0000"+
-    "\u0000\u0164-\u0001\u0000\u0000\u0000\u0165\u0163\u0001\u0000\u0000\u0000"+
-    "\u0166\u0167\u0005F\u0000\u0000\u0167\u0168\u0003,\u0016\u0000\u0168\u0169"+
-    "\u0005G\u0000\u0000\u0169/\u0001\u0000\u0000\u0000\u016a\u016b\u0005\u0013"+
-    "\u0000\u0000\u016b\u0170\u0003$\u0012\u0000\u016c\u016d\u0005\'\u0000"+
-    "\u0000\u016d\u016f\u0003$\u0012\u0000\u016e\u016c\u0001\u0000\u0000\u0000"+
-    "\u016f\u0172\u0001\u0000\u0000\u0000\u0170\u016e\u0001\u0000\u0000\u0000"+
-    "\u0170\u0171\u0001\u0000\u0000\u0000\u0171\u0174\u0001\u0000\u0000\u0000"+
-    "\u0172\u0170\u0001\u0000\u0000\u0000\u0173\u0175\u00036\u001b\u0000\u0174"+
-    "\u0173\u0001\u0000\u0000\u0000\u0174\u0175\u0001\u0000\u0000\u0000\u0175"+
-    "\u0178\u0001\u0000\u0000\u0000\u0176\u0177\u0005!\u0000\u0000\u0177\u0179"+
-    "\u0003\u001e\u000f\u0000\u0178\u0176\u0001\u0000\u0000\u0000\u0178\u0179"+
-    "\u0001\u0000\u0000\u0000\u01791\u0001\u0000\u0000\u0000\u017a\u017b\u0005"+
-    "\u0004\u0000\u0000\u017b\u017c\u0003\u001e\u000f\u0000\u017c3\u0001\u0000"+
-    "\u0000\u0000\u017d\u017f\u0005\u000f\u0000\u0000\u017e\u0180\u00036\u001b"+
-    "\u0000\u017f\u017e\u0001\u0000\u0000\u0000\u017f\u0180\u0001\u0000\u0000"+
-    "\u0000\u0180\u0183\u0001\u0000\u0000\u0000\u0181\u0182\u0005!\u0000\u0000"+
-    "\u0182\u0184\u0003\u001e\u000f\u0000\u0183\u0181\u0001\u0000\u0000\u0000"+
-    "\u0183\u0184\u0001\u0000\u0000\u0000\u01845\u0001\u0000\u0000\u0000\u0185"+
-    "\u018a\u00038\u001c\u0000\u0186\u0187\u0005\'\u0000\u0000\u0187\u0189"+
-    "\u00038\u001c\u0000\u0188\u0186\u0001\u0000\u0000\u0000\u0189\u018c\u0001"+
-    "\u0000\u0000\u0000\u018a\u0188\u0001\u0000\u0000\u0000\u018a\u018b\u0001"+
-    "\u0000\u0000\u0000\u018b7\u0001\u0000\u0000\u0000\u018c\u018a\u0001\u0000"+
-    "\u0000\u0000\u018d\u0190\u0003 \u0010\u0000\u018e\u018f\u0005\u0010\u0000"+
-    "\u0000\u018f\u0191\u0003\n\u0005\u0000\u0190\u018e\u0001\u0000\u0000\u0000"+
-    "\u0190\u0191\u0001\u0000\u0000\u0000\u01919\u0001\u0000\u0000\u0000\u0192"+
-    "\u0197\u0003H$\u0000\u0193\u0194\u0005)\u0000\u0000\u0194\u0196\u0003"+
-    "H$\u0000\u0195\u0193\u0001\u0000\u0000\u0000\u0196\u0199\u0001\u0000\u0000"+
-    "\u0000\u0197\u0195\u0001\u0000\u0000\u0000\u0197\u0198\u0001\u0000\u0000"+
-    "\u0000\u0198;\u0001\u0000\u0000\u0000\u0199\u0197\u0001\u0000\u0000\u0000"+
-    "\u019a\u019f\u0003B!\u0000\u019b\u019c\u0005)\u0000\u0000\u019c\u019e"+
-    "\u0003B!\u0000\u019d\u019b\u0001\u0000\u0000\u0000\u019e\u01a1\u0001\u0000"+
-    "\u0000\u0000\u019f\u019d\u0001\u0000\u0000\u0000\u019f\u01a0\u0001\u0000"+
-    "\u0000\u0000\u01a0=\u0001\u0000\u0000\u0000\u01a1\u019f\u0001\u0000\u0000"+
-    "\u0000\u01a2\u01a7\u0003<\u001e\u0000\u01a3\u01a4\u0005\'\u0000\u0000"+
-    "\u01a4\u01a6\u0003<\u001e\u0000\u01a5\u01a3\u0001\u0000\u0000\u0000\u01a6"+
-    "\u01a9\u0001\u0000\u0000\u0000\u01a7\u01a5\u0001\u0000\u0000\u0000\u01a7"+
-    "\u01a8\u0001\u0000\u0000\u0000\u01a8?\u0001\u0000\u0000\u0000\u01a9\u01a7"+
-    "\u0001\u0000\u0000\u0000\u01aa\u01ab\u0007\u0003\u0000\u0000\u01abA\u0001"+
-    "\u0000\u0000\u0000\u01ac\u01b0\u0005U\u0000\u0000\u01ad\u01ae\u0004!\n"+
-    "\u0000\u01ae\u01b0\u0003F#\u0000\u01af\u01ac\u0001\u0000\u0000\u0000\u01af"+
-    "\u01ad\u0001\u0000\u0000\u0000\u01b0C\u0001\u0000\u0000\u0000\u01b1\u01dc"+
-    "\u00052\u0000\u0000\u01b2\u01b3\u0003h4\u0000\u01b3\u01b4\u0005H\u0000"+
-    "\u0000\u01b4\u01dc\u0001\u0000\u0000\u0000\u01b5\u01dc\u0003f3\u0000\u01b6"+
-    "\u01dc\u0003h4\u0000\u01b7\u01dc\u0003b1\u0000\u01b8\u01dc\u0003F#\u0000"+
-    "\u01b9\u01dc\u0003j5\u0000\u01ba\u01bb\u0005F\u0000\u0000\u01bb\u01c0"+
-    "\u0003d2\u0000\u01bc\u01bd\u0005\'\u0000\u0000\u01bd\u01bf\u0003d2\u0000"+
-    "\u01be\u01bc\u0001\u0000\u0000\u0000\u01bf\u01c2\u0001\u0000\u0000\u0000"+
-    "\u01c0\u01be\u0001\u0000\u0000\u0000\u01c0\u01c1\u0001\u0000\u0000\u0000"+
-    "\u01c1\u01c3\u0001\u0000\u0000\u0000\u01c2\u01c0\u0001\u0000\u0000\u0000"+
-    "\u01c3\u01c4\u0005G\u0000\u0000\u01c4\u01dc\u0001\u0000\u0000\u0000\u01c5"+
-    "\u01c6\u0005F\u0000\u0000\u01c6\u01cb\u0003b1\u0000\u01c7\u01c8\u0005"+
-    "\'\u0000\u0000\u01c8\u01ca\u0003b1\u0000\u01c9\u01c7\u0001\u0000\u0000"+
-    "\u0000\u01ca\u01cd\u0001\u0000\u0000\u0000\u01cb\u01c9\u0001\u0000\u0000"+
-    "\u0000\u01cb\u01cc\u0001\u0000\u0000\u0000\u01cc\u01ce\u0001\u0000\u0000"+
-    "\u0000\u01cd\u01cb\u0001\u0000\u0000\u0000\u01ce\u01cf\u0005G\u0000\u0000"+
-    "\u01cf\u01dc\u0001\u0000\u0000\u0000\u01d0\u01d1\u0005F\u0000\u0000\u01d1"+
-    "\u01d6\u0003j5\u0000\u01d2\u01d3\u0005\'\u0000\u0000\u01d3\u01d5\u0003"+
-    "j5\u0000\u01d4\u01d2\u0001\u0000\u0000\u0000\u01d5\u01d8\u0001\u0000\u0000"+
-    "\u0000\u01d6\u01d4\u0001\u0000\u0000\u0000\u01d6\u01d7\u0001\u0000\u0000"+
-    "\u0000\u01d7\u01d9\u0001\u0000\u0000\u0000\u01d8\u01d6\u0001\u0000\u0000"+
-    "\u0000\u01d9\u01da\u0005G\u0000\u0000\u01da\u01dc\u0001\u0000\u0000\u0000"+
-    "\u01db\u01b1\u0001\u0000\u0000\u0000\u01db\u01b2\u0001\u0000\u0000\u0000"+
-    "\u01db\u01b5\u0001\u0000\u0000\u0000\u01db\u01b6\u0001\u0000\u0000\u0000"+
-    "\u01db\u01b7\u0001\u0000\u0000\u0000\u01db\u01b8\u0001\u0000\u0000\u0000"+
-    "\u01db\u01b9\u0001\u0000\u0000\u0000\u01db\u01ba\u0001\u0000\u0000\u0000"+
-    "\u01db\u01c5\u0001\u0000\u0000\u0000\u01db\u01d0\u0001\u0000\u0000\u0000"+
-    "\u01dcE\u0001\u0000\u0000\u0000\u01dd\u01e0\u00055\u0000\u0000\u01de\u01e0"+
-    "\u0005E\u0000\u0000\u01df\u01dd\u0001\u0000\u0000\u0000\u01df\u01de\u0001"+
-    "\u0000\u0000\u0000\u01e0G\u0001\u0000\u0000\u0000\u01e1\u01e5\u0003@ "+
-    "\u0000\u01e2\u01e3\u0004$\u000b\u0000\u01e3\u01e5\u0003F#\u0000\u01e4"+
-    "\u01e1\u0001\u0000\u0000\u0000\u01e4\u01e2\u0001\u0000\u0000\u0000\u01e5"+
-    "I\u0001\u0000\u0000\u0000\u01e6\u01e7\u0005\t\u0000\u0000\u01e7\u01e8"+
-    "\u0005\u001f\u0000\u0000\u01e8K\u0001\u0000\u0000\u0000\u01e9\u01ea\u0005"+
-    "\u000e\u0000\u0000\u01ea\u01ef\u0003N\'\u0000\u01eb\u01ec\u0005\'\u0000"+
-    "\u0000\u01ec\u01ee\u0003N\'\u0000\u01ed\u01eb\u0001\u0000\u0000\u0000"+
-    "\u01ee\u01f1\u0001\u0000\u0000\u0000\u01ef\u01ed\u0001\u0000\u0000\u0000"+
-    "\u01ef\u01f0\u0001\u0000\u0000\u0000\u01f0M\u0001\u0000\u0000\u0000\u01f1"+
-    "\u01ef\u0001\u0000\u0000\u0000\u01f2\u01f4\u0003\n\u0005\u0000\u01f3\u01f5"+
-    "\u0007\u0004\u0000\u0000\u01f4\u01f3\u0001\u0000\u0000\u0000\u01f4\u01f5"+
-    "\u0001\u0000\u0000\u0000\u01f5\u01f8\u0001\u0000\u0000\u0000\u01f6\u01f7"+
-    "\u00053\u0000\u0000\u01f7\u01f9\u0007\u0005\u0000\u0000\u01f8\u01f6\u0001"+
-    "\u0000\u0000\u0000\u01f8\u01f9\u0001\u0000\u0000\u0000\u01f9O\u0001\u0000"+
-    "\u0000\u0000\u01fa\u01fb\u0005\b\u0000\u0000\u01fb\u01fc\u0003>\u001f"+
-    "\u0000\u01fcQ\u0001\u0000\u0000\u0000\u01fd\u01fe\u0005\u0002\u0000\u0000"+
-    "\u01fe\u01ff\u0003>\u001f\u0000\u01ffS\u0001\u0000\u0000\u0000\u0200\u0201"+
-    "\u0005\u000b\u0000\u0000\u0201\u0206\u0003V+\u0000\u0202\u0203\u0005\'"+
-    "\u0000\u0000\u0203\u0205\u0003V+\u0000\u0204\u0202\u0001\u0000\u0000\u0000"+
-    "\u0205\u0208\u0001\u0000\u0000\u0000\u0206\u0204\u0001\u0000\u0000\u0000"+
-    "\u0206\u0207\u0001\u0000\u0000\u0000\u0207U\u0001\u0000\u0000\u0000\u0208"+
-    "\u0206\u0001\u0000\u0000\u0000\u0209\u020a\u0003<\u001e\u0000\u020a\u020b"+
-    "\u0005Y\u0000\u0000\u020b\u020c\u0003<\u001e\u0000\u020cW\u0001\u0000"+
-    "\u0000\u0000\u020d\u020e\u0005\u0001\u0000\u0000\u020e\u020f\u0003\u0014"+
-    "\n\u0000\u020f\u0211\u0003j5\u0000\u0210\u0212\u0003^/\u0000\u0211\u0210"+
-    "\u0001\u0000\u0000\u0000\u0211\u0212\u0001\u0000\u0000\u0000\u0212Y\u0001"+
-    "\u0000\u0000\u0000\u0213\u0214\u0005\u0007\u0000\u0000\u0214\u0215\u0003"+
-    "\u0014\n\u0000\u0215\u0216\u0003j5\u0000\u0216[\u0001\u0000\u0000\u0000"+
-    "\u0217\u0218\u0005\n\u0000\u0000\u0218\u0219\u0003:\u001d\u0000\u0219"+
-    "]\u0001\u0000\u0000\u0000\u021a\u021f\u0003`0\u0000\u021b\u021c\u0005"+
-    "\'\u0000\u0000\u021c\u021e\u0003`0\u0000\u021d\u021b\u0001\u0000\u0000"+
-    "\u0000\u021e\u0221\u0001\u0000\u0000\u0000\u021f\u021d\u0001\u0000\u0000"+
-    "\u0000\u021f\u0220\u0001\u0000\u0000\u0000\u0220_\u0001\u0000\u0000\u0000"+
-    "\u0221\u021f\u0001\u0000\u0000\u0000\u0222\u0223\u0003@ \u0000\u0223\u0224"+
-    "\u0005$\u0000\u0000\u0224\u0225\u0003D\"\u0000\u0225a\u0001\u0000\u0000"+
-    "\u0000\u0226\u0227\u0007\u0006\u0000\u0000\u0227c\u0001\u0000\u0000\u0000"+
-    "\u0228\u022b\u0003f3\u0000\u0229\u022b\u0003h4\u0000\u022a\u0228\u0001"+
-    "\u0000\u0000\u0000\u022a\u0229\u0001\u0000\u0000\u0000\u022be\u0001\u0000"+
-    "\u0000\u0000\u022c\u022e\u0007\u0000\u0000\u0000\u022d\u022c\u0001\u0000"+
-    "\u0000\u0000\u022d\u022e\u0001\u0000\u0000\u0000\u022e\u022f\u0001\u0000"+
-    "\u0000\u0000\u022f\u0230\u0005 \u0000\u0000\u0230g\u0001\u0000\u0000\u0000"+
-    "\u0231\u0233\u0007\u0000\u0000\u0000\u0232\u0231\u0001\u0000\u0000\u0000"+
-    "\u0232\u0233\u0001\u0000\u0000\u0000\u0233\u0234\u0001\u0000\u0000\u0000"+
-    "\u0234\u0235\u0005\u001f\u0000\u0000\u0235i\u0001\u0000\u0000\u0000\u0236"+
-    "\u0237\u0005\u001e\u0000\u0000\u0237k\u0001\u0000\u0000\u0000\u0238\u0239"+
-    "\u0007\u0007\u0000\u0000\u0239m\u0001\u0000\u0000\u0000\u023a\u023b\u0005"+
-    "\u0005\u0000\u0000\u023b\u023c\u0003p8\u0000\u023co\u0001\u0000\u0000"+
-    "\u0000\u023d\u023e\u0005F\u0000\u0000\u023e\u023f\u0003\u0002\u0001\u0000"+
-    "\u023f\u0240\u0005G\u0000\u0000\u0240q\u0001\u0000\u0000\u0000\u0241\u0242"+
-    "\u0005\r\u0000\u0000\u0242\u0243\u0005i\u0000\u0000\u0243s\u0001\u0000"+
-    "\u0000\u0000\u0244\u0245\u0005\u0003\u0000\u0000\u0245\u0248\u0005_\u0000"+
-    "\u0000\u0246\u0247\u0005]\u0000\u0000\u0247\u0249\u0003<\u001e\u0000\u0248"+
-    "\u0246\u0001\u0000\u0000\u0000\u0248\u0249\u0001\u0000\u0000\u0000\u0249"+
-    "\u0253\u0001\u0000\u0000\u0000\u024a\u024b\u0005^\u0000\u0000\u024b\u0250"+
-    "\u0003v;\u0000\u024c\u024d\u0005\'\u0000\u0000\u024d\u024f\u0003v;\u0000"+
-    "\u024e\u024c\u0001\u0000\u0000\u0000\u024f\u0252\u0001\u0000\u0000\u0000"+
-    "\u0250\u024e\u0001\u0000\u0000\u0000\u0250\u0251\u0001\u0000\u0000\u0000"+
-    "\u0251\u0254\u0001\u0000\u0000\u0000\u0252\u0250\u0001\u0000\u0000\u0000"+
-    "\u0253\u024a\u0001\u0000\u0000\u0000\u0253\u0254\u0001\u0000\u0000\u0000"+
-    "\u0254u\u0001\u0000\u0000\u0000\u0255\u0256\u0003<\u001e\u0000\u0256\u0257"+
-    "\u0005$\u0000\u0000\u0257\u0259\u0001\u0000\u0000\u0000\u0258\u0255\u0001"+
-    "\u0000\u0000\u0000\u0258\u0259\u0001\u0000\u0000\u0000\u0259\u025a\u0001"+
-    "\u0000\u0000\u0000\u025a\u025b\u0003<\u001e\u0000\u025bw\u0001\u0000\u0000"+
-    "\u0000\u025c\u025d\u0005\u0012\u0000\u0000\u025d\u025e\u0003$\u0012\u0000"+
-    "\u025e\u025f\u0005]\u0000\u0000\u025f\u0260\u0003>\u001f\u0000\u0260y"+
-    "\u0001\u0000\u0000\u0000\u0261\u0262\u0005\u0011\u0000\u0000\u0262\u0265"+
-    "\u00036\u001b\u0000\u0263\u0264\u0005!\u0000\u0000\u0264\u0266\u0003\u001e"+
-    "\u000f\u0000\u0265\u0263\u0001\u0000\u0000\u0000\u0265\u0266\u0001\u0000"+
-    "\u0000\u0000\u0266{\u0001\u0000\u0000\u0000\u0267\u0269\u0007\b\u0000"+
-    "\u0000\u0268\u0267\u0001\u0000\u0000\u0000\u0268\u0269\u0001\u0000\u0000"+
-    "\u0000\u0269\u026a\u0001\u0000\u0000\u0000\u026a\u026b\u0005\u0014\u0000"+
-    "\u0000\u026b\u026c\u0003~?\u0000\u026c\u026d\u0003\u0080@\u0000\u026d"+
-    "}\u0001\u0000\u0000\u0000\u026e\u0271\u0003@ \u0000\u026f\u0270\u0005"+
-    "Y\u0000\u0000\u0270\u0272\u0003@ \u0000\u0271\u026f\u0001\u0000\u0000"+
-    "\u0000\u0271\u0272\u0001\u0000\u0000\u0000\u0272\u007f\u0001\u0000\u0000"+
-    "\u0000\u0273\u0274\u0005]\u0000\u0000\u0274\u0279\u0003\u0082A\u0000\u0275"+
-    "\u0276\u0005\'\u0000\u0000\u0276\u0278\u0003\u0082A\u0000\u0277\u0275"+
-    "\u0001\u0000\u0000\u0000\u0278\u027b\u0001\u0000\u0000\u0000\u0279\u0277"+
-    "\u0001\u0000\u0000\u0000\u0279\u027a\u0001\u0000\u0000\u0000\u027a\u0081"+
-    "\u0001\u0000\u0000\u0000\u027b\u0279\u0001\u0000\u0000\u0000\u027c\u027d"+
-    "\u0003\u0010\b\u0000\u027d\u0083\u0001\u0000\u0000\u0000>\u008f\u0098"+
-    "\u00ac\u00b8\u00c1\u00c9\u00ce\u00d6\u00d8\u00dd\u00e4\u00e9\u00ee\u00f8"+
-    "\u00fe\u0106\u0108\u0113\u011a\u0125\u0128\u0138\u013e\u0148\u014c\u0151"+
-    "\u015b\u0163\u0170\u0174\u0178\u017f\u0183\u018a\u0190\u0197\u019f\u01a7"+
-    "\u01af\u01c0\u01cb\u01d6\u01db\u01df\u01e4\u01ef\u01f4\u01f8\u0206\u0211"+
-    "\u021f\u022a\u022d\u0232\u0248\u0250\u0253\u0258\u0265\u0268\u0271\u0279";
+    ";\u0002<\u0007<\u0002=\u0007=\u0002>\u0007>\u0002?\u0007?\u0001\u0000"+
+    "\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+
+    "\u0001\u0001\u0001\u0001\u0005\u0001\u008a\b\u0001\n\u0001\f\u0001\u008d"+
+    "\t\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+
+    "\u0002\u0003\u0002\u0095\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+
+    "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+
+    "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+
+    "\u0003\u0001\u0003\u0001\u0003\u0003\u0003\u00a9\b\u0003\u0001\u0004\u0001"+
+    "\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+
+    "\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00b5\b\u0005\u0001\u0005\u0001"+
+    "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00bc\b\u0005\n"+
+    "\u0005\f\u0005\u00bf\t\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+
+    "\u0005\u0001\u0005\u0003\u0005\u00c6\b\u0005\u0001\u0005\u0001\u0005\u0001"+
+    "\u0005\u0003\u0005\u00cb\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+
+    "\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00d3\b\u0005\n\u0005\f\u0005"+
+    "\u00d6\t\u0005\u0001\u0006\u0001\u0006\u0003\u0006\u00da\b\u0006\u0001"+
+    "\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00e1"+
+    "\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00e6\b\u0006"+
+    "\u0001\u0007\u0001\u0007\u0001\u0007\u0003\u0007\u00eb\b\u0007\u0001\u0007"+
+    "\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0003"+
+    "\b\u00f5\b\b\u0001\t\u0001\t\u0001\t\u0001\t\u0003\t\u00fb\b\t\u0001\t"+
+    "\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0005\t\u0103\b\t\n\t\f\t\u0106"+
+    "\t\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0003"+
+    "\n\u0110\b\n\u0001\n\u0001\n\u0001\n\u0005\n\u0115\b\n\n\n\f\n\u0118\t"+
+    "\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"+
+    "\u0005\u000b\u0120\b\u000b\n\u000b\f\u000b\u0123\t\u000b\u0003\u000b\u0125"+
+    "\b\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\r\u0001\r\u0001"+
+    "\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0005"+
+    "\u000f\u0133\b\u000f\n\u000f\f\u000f\u0136\t\u000f\u0001\u0010\u0001\u0010"+
+    "\u0001\u0010\u0003\u0010\u013b\b\u0010\u0001\u0010\u0001\u0010\u0001\u0011"+
+    "\u0001\u0011\u0001\u0011\u0001\u0011\u0005\u0011\u0143\b\u0011\n\u0011"+
+    "\f\u0011\u0146\t\u0011\u0001\u0011\u0003\u0011\u0149\b\u0011\u0001\u0012"+
+    "\u0001\u0012\u0001\u0012\u0003\u0012\u014e\b\u0012\u0001\u0012\u0001\u0012"+
+    "\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001\u0015\u0001\u0015"+
+    "\u0001\u0015\u0001\u0015\u0005\u0015\u015a\b\u0015\n\u0015\f\u0015\u015d"+
+    "\t\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0005\u0016\u0163"+
+    "\b\u0016\n\u0016\f\u0016\u0166\t\u0016\u0001\u0016\u0003\u0016\u0169\b"+
+    "\u0016\u0001\u0016\u0001\u0016\u0003\u0016\u016d\b\u0016\u0001\u0017\u0001"+
+    "\u0017\u0001\u0017\u0001\u0018\u0001\u0018\u0003\u0018\u0174\b\u0018\u0001"+
+    "\u0018\u0001\u0018\u0003\u0018\u0178\b\u0018\u0001\u0019\u0001\u0019\u0001"+
+    "\u0019\u0005\u0019\u017d\b\u0019\n\u0019\f\u0019\u0180\t\u0019\u0001\u001a"+
+    "\u0001\u001a\u0001\u001a\u0003\u001a\u0185\b\u001a\u0001\u001b\u0001\u001b"+
+    "\u0001\u001b\u0005\u001b\u018a\b\u001b\n\u001b\f\u001b\u018d\t\u001b\u0001"+
+    "\u001c\u0001\u001c\u0001\u001c\u0005\u001c\u0192\b\u001c\n\u001c\f\u001c"+
+    "\u0195\t\u001c\u0001\u001d\u0001\u001d\u0001\u001d\u0005\u001d\u019a\b"+
+    "\u001d\n\u001d\f\u001d\u019d\t\u001d\u0001\u001e\u0001\u001e\u0001\u001f"+
+    "\u0001\u001f\u0001\u001f\u0003\u001f\u01a4\b\u001f\u0001 \u0001 \u0001"+
+    " \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001"+
+    " \u0005 \u01b3\b \n \f \u01b6\t \u0001 \u0001 \u0001 \u0001 \u0001 \u0001"+
+    " \u0005 \u01be\b \n \f \u01c1\t \u0001 \u0001 \u0001 \u0001 \u0001 \u0001"+
+    " \u0005 \u01c9\b \n \f \u01cc\t \u0001 \u0001 \u0003 \u01d0\b \u0001!"+
+    "\u0001!\u0003!\u01d4\b!\u0001\"\u0001\"\u0001\"\u0003\"\u01d9\b\"\u0001"+
+    "#\u0001#\u0001#\u0001$\u0001$\u0001$\u0001$\u0005$\u01e2\b$\n$\f$\u01e5"+
+    "\t$\u0001%\u0001%\u0003%\u01e9\b%\u0001%\u0001%\u0003%\u01ed\b%\u0001"+
+    "&\u0001&\u0001&\u0001\'\u0001\'\u0001\'\u0001(\u0001(\u0001(\u0001(\u0005"+
+    "(\u01f9\b(\n(\f(\u01fc\t(\u0001)\u0001)\u0001)\u0001)\u0001*\u0001*\u0001"+
+    "*\u0001*\u0003*\u0206\b*\u0001+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001"+
+    ",\u0001-\u0001-\u0001-\u0005-\u0212\b-\n-\f-\u0215\t-\u0001.\u0001.\u0001"+
+    ".\u0001.\u0001/\u0001/\u00010\u00010\u00030\u021f\b0\u00011\u00031\u0222"+
+    "\b1\u00011\u00011\u00012\u00032\u0227\b2\u00012\u00012\u00013\u00013\u0001"+
+    "4\u00014\u00015\u00015\u00015\u00016\u00016\u00016\u00016\u00017\u0001"+
+    "7\u00017\u00018\u00018\u00018\u00018\u00038\u023d\b8\u00018\u00018\u0001"+
+    "8\u00018\u00058\u0243\b8\n8\f8\u0246\t8\u00038\u0248\b8\u00019\u00019"+
+    "\u00019\u00039\u024d\b9\u00019\u00019\u0001:\u0001:\u0001:\u0001:\u0001"+
+    ":\u0001;\u0001;\u0001;\u0001;\u0003;\u025a\b;\u0001<\u0003<\u025d\b<\u0001"+
+    "<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001=\u0003=\u0266\b=\u0001>\u0001"+
+    ">\u0001>\u0001>\u0005>\u026c\b>\n>\f>\u026f\t>\u0001?\u0001?\u0001?\u0000"+
+    "\u0004\u0002\n\u0012\u0014@\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010"+
+    "\u0012\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPR"+
+    "TVXZ\\^`bdfhjlnprtvxz|~\u0000\t\u0001\u0000@A\u0001\u0000BD\u0002\u0000"+
+    "\u001e\u001eQQ\u0001\u0000HI\u0002\u0000##((\u0002\u0000++..\u0002\u0000"+
+    "**88\u0002\u000099;?\u0001\u0000\u0016\u0018\u028e\u0000\u0080\u0001\u0000"+
+    "\u0000\u0000\u0002\u0083\u0001\u0000\u0000\u0000\u0004\u0094\u0001\u0000"+
+    "\u0000\u0000\u0006\u00a8\u0001\u0000\u0000\u0000\b\u00aa\u0001\u0000\u0000"+
+    "\u0000\n\u00ca\u0001\u0000\u0000\u0000\f\u00e5\u0001\u0000\u0000\u0000"+
+    "\u000e\u00e7\u0001\u0000\u0000\u0000\u0010\u00f4\u0001\u0000\u0000\u0000"+
+    "\u0012\u00fa\u0001\u0000\u0000\u0000\u0014\u010f\u0001\u0000\u0000\u0000"+
+    "\u0016\u0119\u0001\u0000\u0000\u0000\u0018\u0128\u0001\u0000\u0000\u0000"+
+    "\u001a\u012a\u0001\u0000\u0000\u0000\u001c\u012c\u0001\u0000\u0000\u0000"+
+    "\u001e\u012f\u0001\u0000\u0000\u0000 \u013a\u0001\u0000\u0000\u0000\""+
+    "\u013e\u0001\u0000\u0000\u0000$\u014d\u0001\u0000\u0000\u0000&\u0151\u0001"+
+    "\u0000\u0000\u0000(\u0153\u0001\u0000\u0000\u0000*\u0155\u0001\u0000\u0000"+
+    "\u0000,\u015e\u0001\u0000\u0000\u0000.\u016e\u0001\u0000\u0000\u00000"+
+    "\u0171\u0001\u0000\u0000\u00002\u0179\u0001\u0000\u0000\u00004\u0181\u0001"+
+    "\u0000\u0000\u00006\u0186\u0001\u0000\u0000\u00008\u018e\u0001\u0000\u0000"+
+    "\u0000:\u0196\u0001\u0000\u0000\u0000<\u019e\u0001\u0000\u0000\u0000>"+
+    "\u01a3\u0001\u0000\u0000\u0000@\u01cf\u0001\u0000\u0000\u0000B\u01d3\u0001"+
+    "\u0000\u0000\u0000D\u01d8\u0001\u0000\u0000\u0000F\u01da\u0001\u0000\u0000"+
+    "\u0000H\u01dd\u0001\u0000\u0000\u0000J\u01e6\u0001\u0000\u0000\u0000L"+
+    "\u01ee\u0001\u0000\u0000\u0000N\u01f1\u0001\u0000\u0000\u0000P\u01f4\u0001"+
+    "\u0000\u0000\u0000R\u01fd\u0001\u0000\u0000\u0000T\u0201\u0001\u0000\u0000"+
+    "\u0000V\u0207\u0001\u0000\u0000\u0000X\u020b\u0001\u0000\u0000\u0000Z"+
+    "\u020e\u0001\u0000\u0000\u0000\\\u0216\u0001\u0000\u0000\u0000^\u021a"+
+    "\u0001\u0000\u0000\u0000`\u021e\u0001\u0000\u0000\u0000b\u0221\u0001\u0000"+
+    "\u0000\u0000d\u0226\u0001\u0000\u0000\u0000f\u022a\u0001\u0000\u0000\u0000"+
+    "h\u022c\u0001\u0000\u0000\u0000j\u022e\u0001\u0000\u0000\u0000l\u0231"+
+    "\u0001\u0000\u0000\u0000n\u0235\u0001\u0000\u0000\u0000p\u0238\u0001\u0000"+
+    "\u0000\u0000r\u024c\u0001\u0000\u0000\u0000t\u0250\u0001\u0000\u0000\u0000"+
+    "v\u0255\u0001\u0000\u0000\u0000x\u025c\u0001\u0000\u0000\u0000z\u0262"+
+    "\u0001\u0000\u0000\u0000|\u0267\u0001\u0000\u0000\u0000~\u0270\u0001\u0000"+
+    "\u0000\u0000\u0080\u0081\u0003\u0002\u0001\u0000\u0081\u0082\u0005\u0000"+
+    "\u0000\u0001\u0082\u0001\u0001\u0000\u0000\u0000\u0083\u0084\u0006\u0001"+
+    "\uffff\uffff\u0000\u0084\u0085\u0003\u0004\u0002\u0000\u0085\u008b\u0001"+
+    "\u0000\u0000\u0000\u0086\u0087\n\u0001\u0000\u0000\u0087\u0088\u0005\u001d"+
+    "\u0000\u0000\u0088\u008a\u0003\u0006\u0003\u0000\u0089\u0086\u0001\u0000"+
+    "\u0000\u0000\u008a\u008d\u0001\u0000\u0000\u0000\u008b\u0089\u0001\u0000"+
+    "\u0000\u0000\u008b\u008c\u0001\u0000\u0000\u0000\u008c\u0003\u0001\u0000"+
+    "\u0000\u0000\u008d\u008b\u0001\u0000\u0000\u0000\u008e\u0095\u0003j5\u0000"+
+    "\u008f\u0095\u0003\"\u0011\u0000\u0090\u0095\u0003\u001c\u000e\u0000\u0091"+
+    "\u0095\u0003n7\u0000\u0092\u0093\u0004\u0002\u0001\u0000\u0093\u0095\u0003"+
+    ",\u0016\u0000\u0094\u008e\u0001\u0000\u0000\u0000\u0094\u008f\u0001\u0000"+
+    "\u0000\u0000\u0094\u0090\u0001\u0000\u0000\u0000\u0094\u0091\u0001\u0000"+
+    "\u0000\u0000\u0094\u0092\u0001\u0000\u0000\u0000\u0095\u0005\u0001\u0000"+
+    "\u0000\u0000\u0096\u00a9\u0003.\u0017\u0000\u0097\u00a9\u0003\b\u0004"+
+    "\u0000\u0098\u00a9\u0003L&\u0000\u0099\u00a9\u0003F#\u0000\u009a\u00a9"+
+    "\u00030\u0018\u0000\u009b\u00a9\u0003H$\u0000\u009c\u00a9\u0003N\'\u0000"+
+    "\u009d\u00a9\u0003P(\u0000\u009e\u00a9\u0003T*\u0000\u009f\u00a9\u0003"+
+    "V+\u0000\u00a0\u00a9\u0003p8\u0000\u00a1\u00a9\u0003X,\u0000\u00a2\u00a3"+
+    "\u0004\u0003\u0002\u0000\u00a3\u00a9\u0003v;\u0000\u00a4\u00a5\u0004\u0003"+
+    "\u0003\u0000\u00a5\u00a9\u0003t:\u0000\u00a6\u00a7\u0004\u0003\u0004\u0000"+
+    "\u00a7\u00a9\u0003x<\u0000\u00a8\u0096\u0001\u0000\u0000\u0000\u00a8\u0097"+
+    "\u0001\u0000\u0000\u0000\u00a8\u0098\u0001\u0000\u0000\u0000\u00a8\u0099"+
+    "\u0001\u0000\u0000\u0000\u00a8\u009a\u0001\u0000\u0000\u0000\u00a8\u009b"+
+    "\u0001\u0000\u0000\u0000\u00a8\u009c\u0001\u0000\u0000\u0000\u00a8\u009d"+
+    "\u0001\u0000\u0000\u0000\u00a8\u009e\u0001\u0000\u0000\u0000\u00a8\u009f"+
+    "\u0001\u0000\u0000\u0000\u00a8\u00a0\u0001\u0000\u0000\u0000\u00a8\u00a1"+
+    "\u0001\u0000\u0000\u0000\u00a8\u00a2\u0001\u0000\u0000\u0000\u00a8\u00a4"+
+    "\u0001\u0000\u0000\u0000\u00a8\u00a6\u0001\u0000\u0000\u0000\u00a9\u0007"+
+    "\u0001\u0000\u0000\u0000\u00aa\u00ab\u0005\u0010\u0000\u0000\u00ab\u00ac"+
+    "\u0003\n\u0005\u0000\u00ac\t\u0001\u0000\u0000\u0000\u00ad\u00ae\u0006"+
+    "\u0005\uffff\uffff\u0000\u00ae\u00af\u00051\u0000\u0000\u00af\u00cb\u0003"+
+    "\n\u0005\b\u00b0\u00cb\u0003\u0010\b\u0000\u00b1\u00cb\u0003\f\u0006\u0000"+
+    "\u00b2\u00b4\u0003\u0010\b\u0000\u00b3\u00b5\u00051\u0000\u0000\u00b4"+
+    "\u00b3\u0001\u0000\u0000\u0000\u00b4\u00b5\u0001\u0000\u0000\u0000\u00b5"+
+    "\u00b6\u0001\u0000\u0000\u0000\u00b6\u00b7\u0005,\u0000\u0000\u00b7\u00b8"+
+    "\u00050\u0000\u0000\u00b8\u00bd\u0003\u0010\b\u0000\u00b9\u00ba\u0005"+
+    "\'\u0000\u0000\u00ba\u00bc\u0003\u0010\b\u0000\u00bb\u00b9\u0001\u0000"+
+    "\u0000\u0000\u00bc\u00bf\u0001\u0000\u0000\u0000\u00bd\u00bb\u0001\u0000"+
+    "\u0000\u0000\u00bd\u00be\u0001\u0000\u0000\u0000\u00be\u00c0\u0001\u0000"+
+    "\u0000\u0000\u00bf\u00bd\u0001\u0000\u0000\u0000\u00c0\u00c1\u00057\u0000"+
+    "\u0000\u00c1\u00cb\u0001\u0000\u0000\u0000\u00c2\u00c3\u0003\u0010\b\u0000"+
+    "\u00c3\u00c5\u0005-\u0000\u0000\u00c4\u00c6\u00051\u0000\u0000\u00c5\u00c4"+
+    "\u0001\u0000\u0000\u0000\u00c5\u00c6\u0001\u0000\u0000\u0000\u00c6\u00c7"+
+    "\u0001\u0000\u0000\u0000\u00c7\u00c8\u00052\u0000\u0000\u00c8\u00cb\u0001"+
+    "\u0000\u0000\u0000\u00c9\u00cb\u0003\u000e\u0007\u0000\u00ca\u00ad\u0001"+
+    "\u0000\u0000\u0000\u00ca\u00b0\u0001\u0000\u0000\u0000\u00ca\u00b1\u0001"+
+    "\u0000\u0000\u0000\u00ca\u00b2\u0001\u0000\u0000\u0000\u00ca\u00c2\u0001"+
+    "\u0000\u0000\u0000\u00ca\u00c9\u0001\u0000\u0000\u0000\u00cb\u00d4\u0001"+
+    "\u0000\u0000\u0000\u00cc\u00cd\n\u0005\u0000\u0000\u00cd\u00ce\u0005\""+
+    "\u0000\u0000\u00ce\u00d3\u0003\n\u0005\u0006\u00cf\u00d0\n\u0004\u0000"+
+    "\u0000\u00d0\u00d1\u00054\u0000\u0000\u00d1\u00d3\u0003\n\u0005\u0005"+
+    "\u00d2\u00cc\u0001\u0000\u0000\u0000\u00d2\u00cf\u0001\u0000\u0000\u0000"+
+    "\u00d3\u00d6\u0001\u0000\u0000\u0000\u00d4\u00d2\u0001\u0000\u0000\u0000"+
+    "\u00d4\u00d5\u0001\u0000\u0000\u0000\u00d5\u000b\u0001\u0000\u0000\u0000"+
+    "\u00d6\u00d4\u0001\u0000\u0000\u0000\u00d7\u00d9\u0003\u0010\b\u0000\u00d8"+
+    "\u00da\u00051\u0000\u0000\u00d9\u00d8\u0001\u0000\u0000\u0000\u00d9\u00da"+
+    "\u0001\u0000\u0000\u0000\u00da\u00db\u0001\u0000\u0000\u0000\u00db\u00dc"+
+    "\u0005/\u0000\u0000\u00dc\u00dd\u0003f3\u0000\u00dd\u00e6\u0001\u0000"+
+    "\u0000\u0000\u00de\u00e0\u0003\u0010\b\u0000\u00df\u00e1\u00051\u0000"+
+    "\u0000\u00e0\u00df\u0001\u0000\u0000\u0000\u00e0\u00e1\u0001\u0000\u0000"+
+    "\u0000\u00e1\u00e2\u0001\u0000\u0000\u0000\u00e2\u00e3\u00056\u0000\u0000"+
+    "\u00e3\u00e4\u0003f3\u0000\u00e4\u00e6\u0001\u0000\u0000\u0000\u00e5\u00d7"+
+    "\u0001\u0000\u0000\u0000\u00e5\u00de\u0001\u0000\u0000\u0000\u00e6\r\u0001"+
+    "\u0000\u0000\u0000\u00e7\u00ea\u00036\u001b\u0000\u00e8\u00e9\u0005%\u0000"+
+    "\u0000\u00e9\u00eb\u0003\u001a\r\u0000\u00ea\u00e8\u0001\u0000\u0000\u0000"+
+    "\u00ea\u00eb\u0001\u0000\u0000\u0000\u00eb\u00ec\u0001\u0000\u0000\u0000"+
+    "\u00ec\u00ed\u0005&\u0000\u0000\u00ed\u00ee\u0003@ \u0000\u00ee\u000f"+
+    "\u0001\u0000\u0000\u0000\u00ef\u00f5\u0003\u0012\t\u0000\u00f0\u00f1\u0003"+
+    "\u0012\t\u0000\u00f1\u00f2\u0003h4\u0000\u00f2\u00f3\u0003\u0012\t\u0000"+
+    "\u00f3\u00f5\u0001\u0000\u0000\u0000\u00f4\u00ef\u0001\u0000\u0000\u0000"+
+    "\u00f4\u00f0\u0001\u0000\u0000\u0000\u00f5\u0011\u0001\u0000\u0000\u0000"+
+    "\u00f6\u00f7\u0006\t\uffff\uffff\u0000\u00f7\u00fb\u0003\u0014\n\u0000"+
+    "\u00f8\u00f9\u0007\u0000\u0000\u0000\u00f9\u00fb\u0003\u0012\t\u0003\u00fa"+
+    "\u00f6\u0001\u0000\u0000\u0000\u00fa\u00f8\u0001\u0000\u0000\u0000\u00fb"+
+    "\u0104\u0001\u0000\u0000\u0000\u00fc\u00fd\n\u0002\u0000\u0000\u00fd\u00fe"+
+    "\u0007\u0001\u0000\u0000\u00fe\u0103\u0003\u0012\t\u0003\u00ff\u0100\n"+
+    "\u0001\u0000\u0000\u0100\u0101\u0007\u0000\u0000\u0000\u0101\u0103\u0003"+
+    "\u0012\t\u0002\u0102\u00fc\u0001\u0000\u0000\u0000\u0102\u00ff\u0001\u0000"+
+    "\u0000\u0000\u0103\u0106\u0001\u0000\u0000\u0000\u0104\u0102\u0001\u0000"+
+    "\u0000\u0000\u0104\u0105\u0001\u0000\u0000\u0000\u0105\u0013\u0001\u0000"+
+    "\u0000\u0000\u0106\u0104\u0001\u0000\u0000\u0000\u0107\u0108\u0006\n\uffff"+
+    "\uffff\u0000\u0108\u0110\u0003@ \u0000\u0109\u0110\u00036\u001b\u0000"+
+    "\u010a\u0110\u0003\u0016\u000b\u0000\u010b\u010c\u00050\u0000\u0000\u010c"+
+    "\u010d\u0003\n\u0005\u0000\u010d\u010e\u00057\u0000\u0000\u010e\u0110"+
+    "\u0001\u0000\u0000\u0000\u010f\u0107\u0001\u0000\u0000\u0000\u010f\u0109"+
+    "\u0001\u0000\u0000\u0000\u010f\u010a\u0001\u0000\u0000\u0000\u010f\u010b"+
+    "\u0001\u0000\u0000\u0000\u0110\u0116\u0001\u0000\u0000\u0000\u0111\u0112"+
+    "\n\u0001\u0000\u0000\u0112\u0113\u0005%\u0000\u0000\u0113\u0115\u0003"+
+    "\u001a\r\u0000\u0114\u0111\u0001\u0000\u0000\u0000\u0115\u0118\u0001\u0000"+
+    "\u0000\u0000\u0116\u0114\u0001\u0000\u0000\u0000\u0116\u0117\u0001\u0000"+
+    "\u0000\u0000\u0117\u0015\u0001\u0000\u0000\u0000\u0118\u0116\u0001\u0000"+
+    "\u0000\u0000\u0119\u011a\u0003\u0018\f\u0000\u011a\u0124\u00050\u0000"+
+    "\u0000\u011b\u0125\u0005B\u0000\u0000\u011c\u0121\u0003\n\u0005\u0000"+
+    "\u011d\u011e\u0005\'\u0000\u0000\u011e\u0120\u0003\n\u0005\u0000\u011f"+
+    "\u011d\u0001\u0000\u0000\u0000\u0120\u0123\u0001\u0000\u0000\u0000\u0121"+
+    "\u011f\u0001\u0000\u0000\u0000\u0121\u0122\u0001\u0000\u0000\u0000\u0122"+
+    "\u0125\u0001\u0000\u0000\u0000\u0123\u0121\u0001\u0000\u0000\u0000\u0124"+
+    "\u011b\u0001\u0000\u0000\u0000\u0124\u011c\u0001\u0000\u0000\u0000\u0124"+
+    "\u0125\u0001\u0000\u0000\u0000\u0125\u0126\u0001\u0000\u0000\u0000\u0126"+
+    "\u0127\u00057\u0000\u0000\u0127\u0017\u0001\u0000\u0000\u0000\u0128\u0129"+
+    "\u0003D\"\u0000\u0129\u0019\u0001\u0000\u0000\u0000\u012a\u012b\u0003"+
+    "<\u001e\u0000\u012b\u001b\u0001\u0000\u0000\u0000\u012c\u012d\u0005\f"+
+    "\u0000\u0000\u012d\u012e\u0003\u001e\u000f\u0000\u012e\u001d\u0001\u0000"+
+    "\u0000\u0000\u012f\u0134\u0003 \u0010\u0000\u0130\u0131\u0005\'\u0000"+
+    "\u0000\u0131\u0133\u0003 \u0010\u0000\u0132\u0130\u0001\u0000\u0000\u0000"+
+    "\u0133\u0136\u0001\u0000\u0000\u0000\u0134\u0132\u0001\u0000\u0000\u0000"+
+    "\u0134\u0135\u0001\u0000\u0000\u0000\u0135\u001f\u0001\u0000\u0000\u0000"+
+    "\u0136\u0134\u0001\u0000\u0000\u0000\u0137\u0138\u00036\u001b\u0000\u0138"+
+    "\u0139\u0005$\u0000\u0000\u0139\u013b\u0001\u0000\u0000\u0000\u013a\u0137"+
+    "\u0001\u0000\u0000\u0000\u013a\u013b\u0001\u0000\u0000\u0000\u013b\u013c"+
+    "\u0001\u0000\u0000\u0000\u013c\u013d\u0003\n\u0005\u0000\u013d!\u0001"+
+    "\u0000\u0000\u0000\u013e\u013f\u0005\u0006\u0000\u0000\u013f\u0144\u0003"+
+    "$\u0012\u0000\u0140\u0141\u0005\'\u0000\u0000\u0141\u0143\u0003$\u0012"+
+    "\u0000\u0142\u0140\u0001\u0000\u0000\u0000\u0143\u0146\u0001\u0000\u0000"+
+    "\u0000\u0144\u0142\u0001\u0000\u0000\u0000\u0144\u0145\u0001\u0000\u0000"+
+    "\u0000\u0145\u0148\u0001\u0000\u0000\u0000\u0146\u0144\u0001\u0000\u0000"+
+    "\u0000\u0147\u0149\u0003*\u0015\u0000\u0148\u0147\u0001\u0000\u0000\u0000"+
+    "\u0148\u0149\u0001\u0000\u0000\u0000\u0149#\u0001\u0000\u0000\u0000\u014a"+
+    "\u014b\u0003&\u0013\u0000\u014b\u014c\u0005&\u0000\u0000\u014c\u014e\u0001"+
+    "\u0000\u0000\u0000\u014d\u014a\u0001\u0000\u0000\u0000\u014d\u014e\u0001"+
+    "\u0000\u0000\u0000\u014e\u014f\u0001\u0000\u0000\u0000\u014f\u0150\u0003"+
+    "(\u0014\u0000\u0150%\u0001\u0000\u0000\u0000\u0151\u0152\u0005Q\u0000"+
+    "\u0000\u0152\'\u0001\u0000\u0000\u0000\u0153\u0154\u0007\u0002\u0000\u0000"+
+    "\u0154)\u0001\u0000\u0000\u0000\u0155\u0156\u0005P\u0000\u0000\u0156\u015b"+
+    "\u0005Q\u0000\u0000\u0157\u0158\u0005\'\u0000\u0000\u0158\u015a\u0005"+
+    "Q\u0000\u0000\u0159\u0157\u0001\u0000\u0000\u0000\u015a\u015d\u0001\u0000"+
+    "\u0000\u0000\u015b\u0159\u0001\u0000\u0000\u0000\u015b\u015c\u0001\u0000"+
+    "\u0000\u0000\u015c+\u0001\u0000\u0000\u0000\u015d\u015b\u0001\u0000\u0000"+
+    "\u0000\u015e\u015f\u0005\u0013\u0000\u0000\u015f\u0164\u0003$\u0012\u0000"+
+    "\u0160\u0161\u0005\'\u0000\u0000\u0161\u0163\u0003$\u0012\u0000\u0162"+
+    "\u0160\u0001\u0000\u0000\u0000\u0163\u0166\u0001\u0000\u0000\u0000\u0164"+
+    "\u0162\u0001\u0000\u0000\u0000\u0164\u0165\u0001\u0000\u0000\u0000\u0165"+
+    "\u0168\u0001\u0000\u0000\u0000\u0166\u0164\u0001\u0000\u0000\u0000\u0167"+
+    "\u0169\u00032\u0019\u0000\u0168\u0167\u0001\u0000\u0000\u0000\u0168\u0169"+
+    "\u0001\u0000\u0000\u0000\u0169\u016c\u0001\u0000\u0000\u0000\u016a\u016b"+
+    "\u0005!\u0000\u0000\u016b\u016d\u0003\u001e\u000f\u0000\u016c\u016a\u0001"+
+    "\u0000\u0000\u0000\u016c\u016d\u0001\u0000\u0000\u0000\u016d-\u0001\u0000"+
+    "\u0000\u0000\u016e\u016f\u0005\u0004\u0000\u0000\u016f\u0170\u0003\u001e"+
+    "\u000f\u0000\u0170/\u0001\u0000\u0000\u0000\u0171\u0173\u0005\u000f\u0000"+
+    "\u0000\u0172\u0174\u00032\u0019\u0000\u0173\u0172\u0001\u0000\u0000\u0000"+
+    "\u0173\u0174\u0001\u0000\u0000\u0000\u0174\u0177\u0001\u0000\u0000\u0000"+
+    "\u0175\u0176\u0005!\u0000\u0000\u0176\u0178\u0003\u001e\u000f\u0000\u0177"+
+    "\u0175\u0001\u0000\u0000\u0000\u0177\u0178\u0001\u0000\u0000\u0000\u0178"+
+    "1\u0001\u0000\u0000\u0000\u0179\u017e\u00034\u001a\u0000\u017a\u017b\u0005"+
+    "\'\u0000\u0000\u017b\u017d\u00034\u001a\u0000\u017c\u017a\u0001\u0000"+
+    "\u0000\u0000\u017d\u0180\u0001\u0000\u0000\u0000\u017e\u017c\u0001\u0000"+
+    "\u0000\u0000\u017e\u017f\u0001\u0000\u0000\u0000\u017f3\u0001\u0000\u0000"+
+    "\u0000\u0180\u017e\u0001\u0000\u0000\u0000\u0181\u0184\u0003 \u0010\u0000"+
+    "\u0182\u0183\u0005\u0010\u0000\u0000\u0183\u0185\u0003\n\u0005\u0000\u0184"+
+    "\u0182\u0001\u0000\u0000\u0000\u0184\u0185\u0001\u0000\u0000\u0000\u0185"+
+    "5\u0001\u0000\u0000\u0000\u0186\u018b\u0003D\"\u0000\u0187\u0188\u0005"+
+    ")\u0000\u0000\u0188\u018a\u0003D\"\u0000\u0189\u0187\u0001\u0000\u0000"+
+    "\u0000\u018a\u018d\u0001\u0000\u0000\u0000\u018b\u0189\u0001\u0000\u0000"+
+    "\u0000\u018b\u018c\u0001\u0000\u0000\u0000\u018c7\u0001\u0000\u0000\u0000"+
+    "\u018d\u018b\u0001\u0000\u0000\u0000\u018e\u0193\u0003>\u001f\u0000\u018f"+
+    "\u0190\u0005)\u0000\u0000\u0190\u0192\u0003>\u001f\u0000\u0191\u018f\u0001"+
+    "\u0000\u0000\u0000\u0192\u0195\u0001\u0000\u0000\u0000\u0193\u0191\u0001"+
+    "\u0000\u0000\u0000\u0193\u0194\u0001\u0000\u0000\u0000\u01949\u0001\u0000"+
+    "\u0000\u0000\u0195\u0193\u0001\u0000\u0000\u0000\u0196\u019b\u00038\u001c"+
+    "\u0000\u0197\u0198\u0005\'\u0000\u0000\u0198\u019a\u00038\u001c\u0000"+
+    "\u0199\u0197\u0001\u0000\u0000\u0000\u019a\u019d\u0001\u0000\u0000\u0000"+
+    "\u019b\u0199\u0001\u0000\u0000\u0000\u019b\u019c\u0001\u0000\u0000\u0000"+
+    "\u019c;\u0001\u0000\u0000\u0000\u019d\u019b\u0001\u0000\u0000\u0000\u019e"+
+    "\u019f\u0007\u0003\u0000\u0000\u019f=\u0001\u0000\u0000\u0000\u01a0\u01a4"+
+    "\u0005U\u0000\u0000\u01a1\u01a2\u0004\u001f\n\u0000\u01a2\u01a4\u0003"+
+    "B!\u0000\u01a3\u01a0\u0001\u0000\u0000\u0000\u01a3\u01a1\u0001\u0000\u0000"+
+    "\u0000\u01a4?\u0001\u0000\u0000\u0000\u01a5\u01d0\u00052\u0000\u0000\u01a6"+
+    "\u01a7\u0003d2\u0000\u01a7\u01a8\u0005H\u0000\u0000\u01a8\u01d0\u0001"+
+    "\u0000\u0000\u0000\u01a9\u01d0\u0003b1\u0000\u01aa\u01d0\u0003d2\u0000"+
+    "\u01ab\u01d0\u0003^/\u0000\u01ac\u01d0\u0003B!\u0000\u01ad\u01d0\u0003"+
+    "f3\u0000\u01ae\u01af\u0005F\u0000\u0000\u01af\u01b4\u0003`0\u0000\u01b0"+
+    "\u01b1\u0005\'\u0000\u0000\u01b1\u01b3\u0003`0\u0000\u01b2\u01b0\u0001"+
+    "\u0000\u0000\u0000\u01b3\u01b6\u0001\u0000\u0000\u0000\u01b4\u01b2\u0001"+
+    "\u0000\u0000\u0000\u01b4\u01b5\u0001\u0000\u0000\u0000\u01b5\u01b7\u0001"+
+    "\u0000\u0000\u0000\u01b6\u01b4\u0001\u0000\u0000\u0000\u01b7\u01b8\u0005"+
+    "G\u0000\u0000\u01b8\u01d0\u0001\u0000\u0000\u0000\u01b9\u01ba\u0005F\u0000"+
+    "\u0000\u01ba\u01bf\u0003^/\u0000\u01bb\u01bc\u0005\'\u0000\u0000\u01bc"+
+    "\u01be\u0003^/\u0000\u01bd\u01bb\u0001\u0000\u0000\u0000\u01be\u01c1\u0001"+
+    "\u0000\u0000\u0000\u01bf\u01bd\u0001\u0000\u0000\u0000\u01bf\u01c0\u0001"+
+    "\u0000\u0000\u0000\u01c0\u01c2\u0001\u0000\u0000\u0000\u01c1\u01bf\u0001"+
+    "\u0000\u0000\u0000\u01c2\u01c3\u0005G\u0000\u0000\u01c3\u01d0\u0001\u0000"+
+    "\u0000\u0000\u01c4\u01c5\u0005F\u0000\u0000\u01c5\u01ca\u0003f3\u0000"+
+    "\u01c6\u01c7\u0005\'\u0000\u0000\u01c7\u01c9\u0003f3\u0000\u01c8\u01c6"+
+    "\u0001\u0000\u0000\u0000\u01c9\u01cc\u0001\u0000\u0000\u0000\u01ca\u01c8"+
+    "\u0001\u0000\u0000\u0000\u01ca\u01cb\u0001\u0000\u0000\u0000\u01cb\u01cd"+
+    "\u0001\u0000\u0000\u0000\u01cc\u01ca\u0001\u0000\u0000\u0000\u01cd\u01ce"+
+    "\u0005G\u0000\u0000\u01ce\u01d0\u0001\u0000\u0000\u0000\u01cf\u01a5\u0001"+
+    "\u0000\u0000\u0000\u01cf\u01a6\u0001\u0000\u0000\u0000\u01cf\u01a9\u0001"+
+    "\u0000\u0000\u0000\u01cf\u01aa\u0001\u0000\u0000\u0000\u01cf\u01ab\u0001"+
+    "\u0000\u0000\u0000\u01cf\u01ac\u0001\u0000\u0000\u0000\u01cf\u01ad\u0001"+
+    "\u0000\u0000\u0000\u01cf\u01ae\u0001\u0000\u0000\u0000\u01cf\u01b9\u0001"+
+    "\u0000\u0000\u0000\u01cf\u01c4\u0001\u0000\u0000\u0000\u01d0A\u0001\u0000"+
+    "\u0000\u0000\u01d1\u01d4\u00055\u0000\u0000\u01d2\u01d4\u0005E\u0000\u0000"+
+    "\u01d3\u01d1\u0001\u0000\u0000\u0000\u01d3\u01d2\u0001\u0000\u0000\u0000"+
+    "\u01d4C\u0001\u0000\u0000\u0000\u01d5\u01d9\u0003<\u001e\u0000\u01d6\u01d7"+
+    "\u0004\"\u000b\u0000\u01d7\u01d9\u0003B!\u0000\u01d8\u01d5\u0001\u0000"+
+    "\u0000\u0000\u01d8\u01d6\u0001\u0000\u0000\u0000\u01d9E\u0001\u0000\u0000"+
+    "\u0000\u01da\u01db\u0005\t\u0000\u0000\u01db\u01dc\u0005\u001f\u0000\u0000"+
+    "\u01dcG\u0001\u0000\u0000\u0000\u01dd\u01de\u0005\u000e\u0000\u0000\u01de"+
+    "\u01e3\u0003J%\u0000\u01df\u01e0\u0005\'\u0000\u0000\u01e0\u01e2\u0003"+
+    "J%\u0000\u01e1\u01df\u0001\u0000\u0000\u0000\u01e2\u01e5\u0001\u0000\u0000"+
+    "\u0000\u01e3\u01e1\u0001\u0000\u0000\u0000\u01e3\u01e4\u0001\u0000\u0000"+
+    "\u0000\u01e4I\u0001\u0000\u0000\u0000\u01e5\u01e3\u0001\u0000\u0000\u0000"+
+    "\u01e6\u01e8\u0003\n\u0005\u0000\u01e7\u01e9\u0007\u0004\u0000\u0000\u01e8"+
+    "\u01e7\u0001\u0000\u0000\u0000\u01e8\u01e9\u0001\u0000\u0000\u0000\u01e9"+
+    "\u01ec\u0001\u0000\u0000\u0000\u01ea\u01eb\u00053\u0000\u0000\u01eb\u01ed"+
+    "\u0007\u0005\u0000\u0000\u01ec\u01ea\u0001\u0000\u0000\u0000\u01ec\u01ed"+
+    "\u0001\u0000\u0000\u0000\u01edK\u0001\u0000\u0000\u0000\u01ee\u01ef\u0005"+
+    "\b\u0000\u0000\u01ef\u01f0\u0003:\u001d\u0000\u01f0M\u0001\u0000\u0000"+
+    "\u0000\u01f1\u01f2\u0005\u0002\u0000\u0000\u01f2\u01f3\u0003:\u001d\u0000"+
+    "\u01f3O\u0001\u0000\u0000\u0000\u01f4\u01f5\u0005\u000b\u0000\u0000\u01f5"+
+    "\u01fa\u0003R)\u0000\u01f6\u01f7\u0005\'\u0000\u0000\u01f7\u01f9\u0003"+
+    "R)\u0000\u01f8\u01f6\u0001\u0000\u0000\u0000\u01f9\u01fc\u0001\u0000\u0000"+
+    "\u0000\u01fa\u01f8\u0001\u0000\u0000\u0000\u01fa\u01fb\u0001\u0000\u0000"+
+    "\u0000\u01fbQ\u0001\u0000\u0000\u0000\u01fc\u01fa\u0001\u0000\u0000\u0000"+
+    "\u01fd\u01fe\u00038\u001c\u0000\u01fe\u01ff\u0005Y\u0000\u0000\u01ff\u0200"+
+    "\u00038\u001c\u0000\u0200S\u0001\u0000\u0000\u0000\u0201\u0202\u0005\u0001"+
+    "\u0000\u0000\u0202\u0203\u0003\u0014\n\u0000\u0203\u0205\u0003f3\u0000"+
+    "\u0204\u0206\u0003Z-\u0000\u0205\u0204\u0001\u0000\u0000\u0000\u0205\u0206"+
+    "\u0001\u0000\u0000\u0000\u0206U\u0001\u0000\u0000\u0000\u0207\u0208\u0005"+
+    "\u0007\u0000\u0000\u0208\u0209\u0003\u0014\n\u0000\u0209\u020a\u0003f"+
+    "3\u0000\u020aW\u0001\u0000\u0000\u0000\u020b\u020c\u0005\n\u0000\u0000"+
+    "\u020c\u020d\u00036\u001b\u0000\u020dY\u0001\u0000\u0000\u0000\u020e\u0213"+
+    "\u0003\\.\u0000\u020f\u0210\u0005\'\u0000\u0000\u0210\u0212\u0003\\.\u0000"+
+    "\u0211\u020f\u0001\u0000\u0000\u0000\u0212\u0215\u0001\u0000\u0000\u0000"+
+    "\u0213\u0211\u0001\u0000\u0000\u0000\u0213\u0214\u0001\u0000\u0000\u0000"+
+    "\u0214[\u0001\u0000\u0000\u0000\u0215\u0213\u0001\u0000\u0000\u0000\u0216"+
+    "\u0217\u0003<\u001e\u0000\u0217\u0218\u0005$\u0000\u0000\u0218\u0219\u0003"+
+    "@ \u0000\u0219]\u0001\u0000\u0000\u0000\u021a\u021b\u0007\u0006\u0000"+
+    "\u0000\u021b_\u0001\u0000\u0000\u0000\u021c\u021f\u0003b1\u0000\u021d"+
+    "\u021f\u0003d2\u0000\u021e\u021c\u0001\u0000\u0000\u0000\u021e\u021d\u0001"+
+    "\u0000\u0000\u0000\u021fa\u0001\u0000\u0000\u0000\u0220\u0222\u0007\u0000"+
+    "\u0000\u0000\u0221\u0220\u0001\u0000\u0000\u0000\u0221\u0222\u0001\u0000"+
+    "\u0000\u0000\u0222\u0223\u0001\u0000\u0000\u0000\u0223\u0224\u0005 \u0000"+
+    "\u0000\u0224c\u0001\u0000\u0000\u0000\u0225\u0227\u0007\u0000\u0000\u0000"+
+    "\u0226\u0225\u0001\u0000\u0000\u0000\u0226\u0227\u0001\u0000\u0000\u0000"+
+    "\u0227\u0228\u0001\u0000\u0000\u0000\u0228\u0229\u0005\u001f\u0000\u0000"+
+    "\u0229e\u0001\u0000\u0000\u0000\u022a\u022b\u0005\u001e\u0000\u0000\u022b"+
+    "g\u0001\u0000\u0000\u0000\u022c\u022d\u0007\u0007\u0000\u0000\u022di\u0001"+
+    "\u0000\u0000\u0000\u022e\u022f\u0005\u0005\u0000\u0000\u022f\u0230\u0003"+
+    "l6\u0000\u0230k\u0001\u0000\u0000\u0000\u0231\u0232\u0005F\u0000\u0000"+
+    "\u0232\u0233\u0003\u0002\u0001\u0000\u0233\u0234\u0005G\u0000\u0000\u0234"+
+    "m\u0001\u0000\u0000\u0000\u0235\u0236\u0005\r\u0000\u0000\u0236\u0237"+
+    "\u0005i\u0000\u0000\u0237o\u0001\u0000\u0000\u0000\u0238\u0239\u0005\u0003"+
+    "\u0000\u0000\u0239\u023c\u0005_\u0000\u0000\u023a\u023b\u0005]\u0000\u0000"+
+    "\u023b\u023d\u00038\u001c\u0000\u023c\u023a\u0001\u0000\u0000\u0000\u023c"+
+    "\u023d\u0001\u0000\u0000\u0000\u023d\u0247\u0001\u0000\u0000\u0000\u023e"+
+    "\u023f\u0005^\u0000\u0000\u023f\u0244\u0003r9\u0000\u0240\u0241\u0005"+
+    "\'\u0000\u0000\u0241\u0243\u0003r9\u0000\u0242\u0240\u0001\u0000\u0000"+
+    "\u0000\u0243\u0246\u0001\u0000\u0000\u0000\u0244\u0242\u0001\u0000\u0000"+
+    "\u0000\u0244\u0245\u0001\u0000\u0000\u0000\u0245\u0248\u0001\u0000\u0000"+
+    "\u0000\u0246\u0244\u0001\u0000\u0000\u0000\u0247\u023e\u0001\u0000\u0000"+
+    "\u0000\u0247\u0248\u0001\u0000\u0000\u0000\u0248q\u0001\u0000\u0000\u0000"+
+    "\u0249\u024a\u00038\u001c\u0000\u024a\u024b\u0005$\u0000\u0000\u024b\u024d"+
+    "\u0001\u0000\u0000\u0000\u024c\u0249\u0001\u0000\u0000\u0000\u024c\u024d"+
+    "\u0001\u0000\u0000\u0000\u024d\u024e\u0001\u0000\u0000\u0000\u024e\u024f"+
+    "\u00038\u001c\u0000\u024fs\u0001\u0000\u0000\u0000\u0250\u0251\u0005\u0012"+
+    "\u0000\u0000\u0251\u0252\u0003$\u0012\u0000\u0252\u0253\u0005]\u0000\u0000"+
+    "\u0253\u0254\u0003:\u001d\u0000\u0254u\u0001\u0000\u0000\u0000\u0255\u0256"+
+    "\u0005\u0011\u0000\u0000\u0256\u0259\u00032\u0019\u0000\u0257\u0258\u0005"+
+    "!\u0000\u0000\u0258\u025a\u0003\u001e\u000f\u0000\u0259\u0257\u0001\u0000"+
+    "\u0000\u0000\u0259\u025a\u0001\u0000\u0000\u0000\u025aw\u0001\u0000\u0000"+
+    "\u0000\u025b\u025d\u0007\b\u0000\u0000\u025c\u025b\u0001\u0000\u0000\u0000"+
+    "\u025c\u025d\u0001\u0000\u0000\u0000\u025d\u025e\u0001\u0000\u0000\u0000"+
+    "\u025e\u025f\u0005\u0014\u0000\u0000\u025f\u0260\u0003z=\u0000\u0260\u0261"+
+    "\u0003|>\u0000\u0261y\u0001\u0000\u0000\u0000\u0262\u0265\u0003<\u001e"+
+    "\u0000\u0263\u0264\u0005Y\u0000\u0000\u0264\u0266\u0003<\u001e\u0000\u0265"+
+    "\u0263\u0001\u0000\u0000\u0000\u0265\u0266\u0001\u0000\u0000\u0000\u0266"+
+    "{\u0001\u0000\u0000\u0000\u0267\u0268\u0005]\u0000\u0000\u0268\u026d\u0003"+
+    "~?\u0000\u0269\u026a\u0005\'\u0000\u0000\u026a\u026c\u0003~?\u0000\u026b"+
+    "\u0269\u0001\u0000\u0000\u0000\u026c\u026f\u0001\u0000\u0000\u0000\u026d"+
+    "\u026b\u0001\u0000\u0000\u0000\u026d\u026e\u0001\u0000\u0000\u0000\u026e"+
+    "}\u0001\u0000\u0000\u0000\u026f\u026d\u0001\u0000\u0000\u0000\u0270\u0271"+
+    "\u0003\u0010\b\u0000\u0271\u007f\u0001\u0000\u0000\u0000=\u008b\u0094"+
+    "\u00a8\u00b4\u00bd\u00c5\u00ca\u00d2\u00d4\u00d9\u00e0\u00e5\u00ea\u00f4"+
+    "\u00fa\u0102\u0104\u010f\u0116\u0121\u0124\u0134\u013a\u0144\u0148\u014d"+
+    "\u015b\u0164\u0168\u016c\u0173\u0177\u017e\u0184\u018b\u0193\u019b\u01a3"+
+    "\u01b4\u01bf\u01ca\u01cf\u01d3\u01d8\u01e3\u01e8\u01ec\u01fa\u0205\u0213"+
+    "\u021e\u0221\u0226\u023c\u0244\u0247\u024c\u0259\u025c\u0265\u026d";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java
index 6071219839bab..2ee0efe52dfbe 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java
@@ -452,30 +452,6 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener {
    * <p>The default implementation does nothing.</p>
    */
   @Override public void exitMetadata(EsqlBaseParser.MetadataContext ctx) { }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation does nothing.</p>
-   */
-  @Override public void enterMetadataOption(EsqlBaseParser.MetadataOptionContext ctx) { }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation does nothing.</p>
-   */
-  @Override public void exitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx) { }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation does nothing.</p>
-   */
-  @Override public void enterDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx) { }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation does nothing.</p>
-   */
-  @Override public void exitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx) { }
   /**
    * {@inheritDoc}
    *
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java
index afe7146923791..c5c1b0dfa7d7d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java
@@ -272,20 +272,6 @@ public class EsqlBaseParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> im
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitMetadata(EsqlBaseParser.MetadataContext ctx) { return visitChildren(ctx); }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation returns the result of calling
-   * {@link #visitChildren} on {@code ctx}.</p>
-   */
-  @Override public T visitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx) { return visitChildren(ctx); }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation returns the result of calling
-   * {@link #visitChildren} on {@code ctx}.</p>
-   */
-  @Override public T visitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java
index 0faca2541c9ad..f45184e920658 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java
@@ -415,26 +415,6 @@ public interface EsqlBaseParserListener extends ParseTreeListener {
    * @param ctx the parse tree
    */
   void exitMetadata(EsqlBaseParser.MetadataContext ctx);
-  /**
-   * Enter a parse tree produced by {@link EsqlBaseParser#metadataOption}.
-   * @param ctx the parse tree
-   */
-  void enterMetadataOption(EsqlBaseParser.MetadataOptionContext ctx);
-  /**
-   * Exit a parse tree produced by {@link EsqlBaseParser#metadataOption}.
-   * @param ctx the parse tree
-   */
-  void exitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx);
-  /**
-   * Enter a parse tree produced by {@link EsqlBaseParser#deprecated_metadata}.
-   * @param ctx the parse tree
-   */
-  void enterDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx);
-  /**
-   * Exit a parse tree produced by {@link EsqlBaseParser#deprecated_metadata}.
-   * @param ctx the parse tree
-   */
-  void exitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx);
   /**
    * Enter a parse tree produced by {@link EsqlBaseParser#metricsCommand}.
    * @param ctx the parse tree
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java
index e91cd6670e971..30c5e0ce78092 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java
@@ -254,18 +254,6 @@ public interface EsqlBaseParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitMetadata(EsqlBaseParser.MetadataContext ctx);
-  /**
-   * Visit a parse tree produced by {@link EsqlBaseParser#metadataOption}.
-   * @param ctx the parse tree
-   * @return the visitor result
-   */
-  T visitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx);
-  /**
-   * Visit a parse tree produced by {@link EsqlBaseParser#deprecated_metadata}.
-   * @param ctx the parse tree
-   * @return the visitor result
-   */
-  T visitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx);
   /**
    * Visit a parse tree produced by {@link EsqlBaseParser#metricsCommand}.
    * @param ctx the parse tree
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java
index 2e55b4df1e223..9538e3ba495db 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java
@@ -70,11 +70,7 @@ private <T> T invokeParser(
         BiFunction<AstBuilder, ParserRuleContext, T> result
     ) {
         if (query.length() > MAX_LENGTH) {
-            throw new org.elasticsearch.xpack.esql.core.ParsingException(
-                "ESQL statement is too large [{} characters > {}]",
-                query.length(),
-                MAX_LENGTH
-            );
+            throw new ParsingException("ESQL statement is too large [{} characters > {}]", query.length(), MAX_LENGTH);
         }
         try {
             EsqlBaseLexer lexer = new EsqlBaseLexer(CharStreams.fromString(query));
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
index 81d43bc68b79e..eb81446f9ddea 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
@@ -686,12 +687,20 @@ public Expression visitRegexBooleanExpression(EsqlBaseParser.RegexBooleanExpress
         RegexMatch<?> result = switch (type) {
             case EsqlBaseParser.LIKE -> {
                 try {
-                    yield new WildcardLike(source, left, new WildcardPattern(pattern.fold().toString()));
+                    yield new WildcardLike(
+                        source,
+                        left,
+                        new WildcardPattern(pattern.fold(FoldContext.small() /* TODO remove me */).toString())
+                    );
                 } catch (InvalidArgumentException e) {
                     throw new ParsingException(source, "Invalid pattern for LIKE [{}]: [{}]", pattern, e.getMessage());
                 }
             }
-            case EsqlBaseParser.RLIKE -> new RLike(source, left, new RLikePattern(pattern.fold().toString()));
+            case EsqlBaseParser.RLIKE -> new RLike(
+                source,
+                left,
+                new RLikePattern(pattern.fold(FoldContext.small() /* TODO remove me */).toString())
+            );
             default -> throw new ParsingException("Invalid predicate type for [{}]", source.text());
         };
         return ctx.NOT() == null ? result : new Not(source, result);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
index 24398afa18010..4b7c0118acda3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
@@ -34,7 +35,6 @@
 import org.elasticsearch.xpack.esql.expression.Order;
 import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern;
 import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction;
-import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.MetadataOptionContext;
 import org.elasticsearch.xpack.esql.plan.TableIdentifier;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.Dissect;
@@ -71,7 +71,6 @@
 import java.util.function.Function;
 
 import static java.util.Collections.emptyList;
-import static org.elasticsearch.common.logging.HeaderWarning.addWarning;
 import static org.elasticsearch.xpack.esql.core.util.StringUtils.WILDCARD;
 import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputExpressions;
 import static org.elasticsearch.xpack.esql.parser.ParserUtils.source;
@@ -157,7 +156,7 @@ public PlanFactory visitEvalCommand(EsqlBaseParser.EvalCommandContext ctx) {
     public PlanFactory visitGrokCommand(EsqlBaseParser.GrokCommandContext ctx) {
         return p -> {
             Source source = source(ctx);
-            String pattern = visitString(ctx.string()).fold().toString();
+            String pattern = visitString(ctx.string()).fold(FoldContext.small() /* TODO remove me */).toString();
             Grok.Parser grokParser;
             try {
                 grokParser = Grok.pattern(source, pattern);
@@ -188,7 +187,7 @@ private void validateGrokPattern(Source source, Grok.Parser grokParser, String p
     @Override
     public PlanFactory visitDissectCommand(EsqlBaseParser.DissectCommandContext ctx) {
         return p -> {
-            String pattern = visitString(ctx.string()).fold().toString();
+            String pattern = visitString(ctx.string()).fold(FoldContext.small() /* TODO remove me */).toString();
             Map<String, Object> options = visitCommandOptions(ctx.commandOptions());
             String appendSeparator = "";
             for (Map.Entry<String, Object> item : options.entrySet()) {
@@ -243,7 +242,7 @@ public Map<String, Object> visitCommandOptions(EsqlBaseParser.CommandOptionsCont
         }
         Map<String, Object> result = new HashMap<>();
         for (EsqlBaseParser.CommandOptionContext option : ctx.commandOption()) {
-            result.put(visitIdentifier(option.identifier()), expression(option.constant()).fold());
+            result.put(visitIdentifier(option.identifier()), expression(option.constant()).fold(FoldContext.small() /* TODO remove me */));
         }
         return result;
     }
@@ -260,21 +259,7 @@ public LogicalPlan visitFromCommand(EsqlBaseParser.FromCommandContext ctx) {
         TableIdentifier table = new TableIdentifier(source, null, visitIndexPattern(ctx.indexPattern()));
         Map<String, Attribute> metadataMap = new LinkedHashMap<>();
         if (ctx.metadata() != null) {
-            var deprecatedContext = ctx.metadata().deprecated_metadata();
-            MetadataOptionContext metadataOptionContext = null;
-            if (deprecatedContext != null) {
-                var s = source(deprecatedContext).source();
-                addWarning(
-                    "Line {}:{}: Square brackets '[]' need to be removed in FROM METADATA declaration",
-                    s.getLineNumber(),
-                    s.getColumnNumber()
-                );
-                metadataOptionContext = deprecatedContext.metadataOption();
-            } else {
-                metadataOptionContext = ctx.metadata().metadataOption();
-
-            }
-            for (var c : metadataOptionContext.UNQUOTED_SOURCE()) {
+            for (var c : ctx.metadata().UNQUOTED_SOURCE()) {
                 String id = c.getText();
                 Source src = source(c);
                 if (MetadataAttribute.isSupported(id) == false // TODO: drop check below once METADATA_SCORE is no longer snapshot-only
@@ -564,6 +549,11 @@ public PlanFactory visitJoinCommand(EsqlBaseParser.JoinCommandContext ctx) {
             }
         }
 
+        var matchFieldsCount = joinFields.size();
+        if (matchFieldsCount > 1) {
+            throw new ParsingException(source, "JOIN ON clause only supports one field at the moment, found [{}]", matchFieldsCount);
+        }
+
         return p -> new LookupJoin(source, p, right, joinFields);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParserUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParserUtils.java
index 89b1ae4e37a68..398c6c5aafbb2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParserUtils.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParserUtils.java
@@ -14,7 +14,6 @@
 import org.antlr.v4.runtime.tree.ParseTreeVisitor;
 import org.antlr.v4.runtime.tree.TerminalNode;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.xpack.esql.core.ParsingException;
 import org.elasticsearch.xpack.esql.core.tree.Location;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.util.Check;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParsingException.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParsingException.java
index 484a655fc2988..c25ab92437bfc 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParsingException.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParsingException.java
@@ -21,7 +21,7 @@ public ParsingException(String message, Exception cause, int line, int charPosit
         this.charPositionInLine = charPositionInLine + 1;
     }
 
-    ParsingException(String message, Object... args) {
+    public ParsingException(String message, Object... args) {
         this(Source.EMPTY, message, args);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java
index e362c9646a8e0..0111d23fac281 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java
@@ -10,15 +10,24 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.esql.capabilities.PostAnalysisVerificationAware;
+import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.AttributeMap;
 import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.Rate;
+import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize;
+import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
 import java.io.IOException;
@@ -26,9 +35,11 @@
 import java.util.Objects;
 
 import static java.util.Collections.emptyList;
+import static org.elasticsearch.xpack.esql.common.Failure.fail;
 import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes;
+import static org.elasticsearch.xpack.esql.plan.logical.Filter.checkFilterConditionDataType;
 
-public class Aggregate extends UnaryPlan {
+public class Aggregate extends UnaryPlan implements PostAnalysisVerificationAware {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
         LogicalPlan.class,
         "Aggregate",
@@ -191,4 +202,260 @@ public boolean equals(Object obj) {
             && Objects.equals(aggregates, other.aggregates)
             && Objects.equals(child(), other.child());
     }
+
+    @Override
+    public void postAnalysisVerification(Failures failures) {
+        AttributeSet groupRefs = new AttributeSet();
+        // check grouping
+        // The grouping can not be an aggregate function
+        groupings.forEach(e -> {
+            e.forEachUp(g -> {
+                if (g instanceof AggregateFunction af) {
+                    failures.add(fail(g, "cannot use an aggregate [{}] for grouping", af));
+                } else if (g instanceof GroupingFunction gf) {
+                    gf.children()
+                        .forEach(
+                            c -> c.forEachDown(
+                                GroupingFunction.class,
+                                inner -> failures.add(
+                                    fail(
+                                        inner,
+                                        "cannot nest grouping functions; found [{}] inside [{}]",
+                                        inner.sourceText(),
+                                        gf.sourceText()
+                                    )
+                                )
+                            )
+                        );
+                }
+            });
+            // keep the grouping attributes (common case)
+            Attribute attr = Expressions.attribute(e);
+            if (attr != null) {
+                groupRefs.add(attr);
+            }
+            if (e instanceof FieldAttribute f && f.dataType().isCounter()) {
+                failures.add(fail(e, "cannot group by on [{}] type for grouping [{}]", f.dataType().typeName(), e.sourceText()));
+            }
+        });
+
+        // check aggregates - accept only aggregate functions or expressions over grouping
+        // don't allow the group by itself to avoid duplicates in the output
+        // and since the groups are copied, only look at the declared aggregates
+        // List<? extends NamedExpression> aggs = agg.aggregates();
+        aggregates.subList(0, aggregates.size() - groupings.size()).forEach(e -> {
+            var exp = Alias.unwrap(e);
+            if (exp.foldable()) {
+                failures.add(fail(exp, "expected an aggregate function but found [{}]", exp.sourceText()));
+            }
+            // traverse the tree to find invalid matches
+            checkInvalidNamedExpressionUsage(exp, groupings, groupRefs, failures, 0);
+        });
+        if (aggregateType() == Aggregate.AggregateType.METRICS) {
+            aggregates.forEach(a -> checkRateAggregates(a, 0, failures));
+        } else {
+            forEachExpression(
+                Rate.class,
+                r -> failures.add(fail(r, "the rate aggregate[{}] can only be used within the metrics command", r.sourceText()))
+            );
+        }
+        checkCategorizeGrouping(failures);
+
+    }
+
+    /**
+     * Check CATEGORIZE grouping function usages.
+     * <p>
+     *     Some of those checks are temporary, until the required syntax or engine changes are implemented.
+     * </p>
+     */
+    private void checkCategorizeGrouping(Failures failures) {
+        // Forbid CATEGORIZE grouping function with other groupings
+        if (groupings.size() > 1) {
+            groupings.subList(1, groupings.size()).forEach(g -> {
+                g.forEachDown(
+                    Categorize.class,
+                    categorize -> failures.add(
+                        fail(
+                            categorize,
+                            "CATEGORIZE grouping function [{}] can only be in the first grouping expression",
+                            categorize.sourceText()
+                        )
+                    )
+                );
+            });
+        }
+
+        // Forbid CATEGORIZE grouping functions not being top level groupings
+        groupings.forEach(g -> {
+            // Check all CATEGORIZE but the top level one
+            Alias.unwrap(g)
+                .children()
+                .forEach(
+                    child -> child.forEachDown(
+                        Categorize.class,
+                        c -> failures.add(
+                            fail(c, "CATEGORIZE grouping function [{}] can't be used within other expressions", c.sourceText())
+                        )
+                    )
+                );
+        });
+
+        // Forbid CATEGORIZE being used in the aggregations, unless it appears as a grouping
+        aggregates.forEach(
+            a -> a.forEachDown(
+                AggregateFunction.class,
+                aggregateFunction -> aggregateFunction.forEachDown(
+                    Categorize.class,
+                    categorize -> failures.add(
+                        fail(categorize, "cannot use CATEGORIZE grouping function [{}] within an aggregation", categorize.sourceText())
+                    )
+                )
+            )
+        );
+
+        // Forbid CATEGORIZE being referenced as a child of an aggregation function
+        AttributeMap<Categorize> categorizeByAttribute = new AttributeMap<>();
+        groupings.forEach(g -> {
+            g.forEachDown(Alias.class, alias -> {
+                if (alias.child() instanceof Categorize categorize) {
+                    categorizeByAttribute.put(alias.toAttribute(), categorize);
+                }
+            });
+        });
+        aggregates.forEach(a -> a.forEachDown(AggregateFunction.class, aggregate -> aggregate.forEachDown(Attribute.class, attribute -> {
+            var categorize = categorizeByAttribute.get(attribute);
+            if (categorize != null) {
+                failures.add(
+                    fail(attribute, "cannot reference CATEGORIZE grouping function [{}] within an aggregation", attribute.sourceText())
+                );
+            }
+        })));
+        aggregates.forEach(a -> a.forEachDown(FilteredExpression.class, fe -> fe.filter().forEachDown(Attribute.class, attribute -> {
+            var categorize = categorizeByAttribute.get(attribute);
+            if (categorize != null) {
+                failures.add(
+                    fail(
+                        attribute,
+                        "cannot reference CATEGORIZE grouping function [{}] within an aggregation filter",
+                        attribute.sourceText()
+                    )
+                );
+            }
+        })));
+    }
+
+    private static void checkRateAggregates(Expression expr, int nestedLevel, Failures failures) {
+        if (expr instanceof AggregateFunction) {
+            nestedLevel++;
+        }
+        if (expr instanceof Rate r) {
+            if (nestedLevel != 2) {
+                failures.add(
+                    fail(
+                        expr,
+                        "the rate aggregate [{}] can only be used within the metrics command and inside another aggregate",
+                        r.sourceText()
+                    )
+                );
+            }
+        }
+        for (Expression child : expr.children()) {
+            checkRateAggregates(child, nestedLevel, failures);
+        }
+    }
+
+    // traverse the expression and look either for an agg function or a grouping match
+    // stop either when no children are left, the leafs are literals or a reference attribute is given
+    private static void checkInvalidNamedExpressionUsage(
+        Expression e,
+        List<Expression> groups,
+        AttributeSet groupRefs,
+        Failures failures,
+        int level
+    ) {
+        // unwrap filtered expression
+        if (e instanceof FilteredExpression fe) {
+            e = fe.delegate();
+            // make sure they work on aggregate functions
+            if (e.anyMatch(AggregateFunction.class::isInstance) == false) {
+                Expression filter = fe.filter();
+                failures.add(fail(filter, "WHERE clause allowed only for aggregate functions, none found in [{}]", fe.sourceText()));
+            }
+            Expression f = fe.filter();
+            // check filter's data type
+            checkFilterConditionDataType(f, failures);
+            // but that the filter doesn't use grouping or aggregate functions
+            fe.filter().forEachDown(c -> {
+                if (c instanceof AggregateFunction af) {
+                    failures.add(
+                        fail(af, "cannot use aggregate function [{}] in aggregate WHERE clause [{}]", af.sourceText(), fe.sourceText())
+                    );
+                }
+                // check the grouping function against the group
+                else if (c instanceof GroupingFunction gf) {
+                    if (c instanceof Categorize
+                        || Expressions.anyMatch(groups, ex -> ex instanceof Alias a && a.child().semanticEquals(gf)) == false) {
+                        failures.add(fail(gf, "can only use grouping function [{}] as part of the BY clause", gf.sourceText()));
+                    }
+                }
+            });
+        }
+        // found an aggregate, constant or a group, bail out
+        if (e instanceof AggregateFunction af) {
+            af.field().forEachDown(AggregateFunction.class, f -> {
+                // rate aggregate is allowed to be inside another aggregate
+                if (f instanceof Rate == false) {
+                    failures.add(fail(f, "nested aggregations [{}] not allowed inside other aggregations [{}]", f, af));
+                }
+            });
+        } else if (e instanceof GroupingFunction gf) {
+            // optimizer will later unroll expressions with aggs and non-aggs with a grouping function into an EVAL, but that will no longer
+            // be verified (by check above in checkAggregate()), so do it explicitly here
+            if (Expressions.anyMatch(groups, ex -> ex instanceof Alias a && a.child().semanticEquals(gf)) == false) {
+                failures.add(fail(gf, "can only use grouping function [{}] as part of the BY clause", gf.sourceText()));
+            } else if (level == 0) {
+                addFailureOnGroupingUsedNakedInAggs(failures, gf, "function");
+            }
+        } else if (e.foldable()) {
+            // don't do anything
+        } else if (groups.contains(e) || groupRefs.contains(e)) {
+            if (level == 0) {
+                addFailureOnGroupingUsedNakedInAggs(failures, e, "key");
+            }
+        }
+        // if a reference is found, mark it as an error
+        else if (e instanceof NamedExpression ne) {
+            boolean foundInGrouping = false;
+            for (Expression g : groups) {
+                if (g.anyMatch(se -> se.semanticEquals(ne))) {
+                    foundInGrouping = true;
+                    failures.add(
+                        fail(
+                            e,
+                            "column [{}] cannot be used as an aggregate once declared in the STATS BY grouping key [{}]",
+                            ne.name(),
+                            g.sourceText()
+                        )
+                    );
+                    break;
+                }
+            }
+            if (foundInGrouping == false) {
+                failures.add(fail(e, "column [{}] must appear in the STATS BY clause or be used in an aggregate function", ne.name()));
+            }
+        }
+        // other keep on going
+        else {
+            for (Expression child : e.children()) {
+                checkInvalidNamedExpressionUsage(child, groups, groupRefs, failures, level + 1);
+            }
+        }
+    }
+
+    private static void addFailureOnGroupingUsedNakedInAggs(Failures failures, Expression e, String element) {
+        failures.add(
+            fail(e, "grouping {} [{}] cannot be used as an aggregate once declared in the STATS BY clause", element, e.sourceText())
+        );
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java
index fcb5d9dbd61cf..9b81060349815 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java
@@ -17,12 +17,15 @@
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
+import org.elasticsearch.xpack.esql.capabilities.PostAnalysisPlanVerificationAware;
+import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
@@ -39,11 +42,13 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.BiConsumer;
 
+import static org.elasticsearch.xpack.esql.common.Failure.fail;
 import static org.elasticsearch.xpack.esql.core.expression.Expressions.asAttributes;
 import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes;
 
-public class Enrich extends UnaryPlan implements GeneratingPlan<Enrich> {
+public class Enrich extends UnaryPlan implements GeneratingPlan<Enrich>, PostAnalysisPlanVerificationAware {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
         LogicalPlan.class,
         "Enrich",
@@ -116,7 +121,7 @@ private static Enrich readFrom(StreamInput in) throws IOException {
         if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
             concreteIndices = in.readMap(StreamInput::readString, StreamInput::readString);
         } else {
-            EsIndex esIndex = new EsIndex(in);
+            EsIndex esIndex = EsIndex.readFrom(in);
             if (esIndex.concreteIndices().size() > 1) {
                 throw new IllegalStateException("expected a single enrich index; got " + esIndex);
             }
@@ -145,7 +150,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeNamedWriteable(policyName());
         out.writeNamedWriteable(matchField());
         if (out.getTransportVersion().before(TransportVersions.V_8_13_0)) {
-            out.writeString(BytesRefs.toString(policyName().fold())); // old policy name
+            out.writeString(BytesRefs.toString(policyName().fold(FoldContext.small() /* TODO remove me */))); // old policy name
         }
         policy().writeTo(out);
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
@@ -274,4 +279,42 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(super.hashCode(), mode, policyName, matchField, policy, concreteIndices, enrichFields);
     }
+
+    @Override
+    public BiConsumer<LogicalPlan, Failures> postAnalysisPlanVerification() {
+        return Enrich::checkRemoteEnrich;
+    }
+
+    /**
+     * Ensure that no remote enrich is allowed after a reduction or an enrich with coordinator mode.
+     * <p>
+     * TODO:
+     * For Limit and TopN, we can insert the same node after the remote enrich (also needs to move projections around)
+     * to eliminate this limitation. Otherwise, we force users to write queries that might not perform well.
+     * For example, `FROM test | ORDER @timestamp | LIMIT 10 | ENRICH _remote:` doesn't work.
+     * In that case, users have to write it as `FROM test | ENRICH _remote: | ORDER @timestamp | LIMIT 10`,
+     * which is equivalent to bringing all data to the coordinating cluster.
+     * We might consider implementing the actual remote enrich on the coordinating cluster, however, this requires
+     * retaining the originating cluster and restructing pages for routing, which might be complicated.
+     */
+    private static void checkRemoteEnrich(LogicalPlan plan, Failures failures) {
+        boolean[] agg = { false };
+        boolean[] enrichCoord = { false };
+
+        plan.forEachUp(UnaryPlan.class, u -> {
+            if (u instanceof Aggregate) {
+                agg[0] = true;
+            } else if (u instanceof Enrich enrich && enrich.mode() == Enrich.Mode.COORDINATOR) {
+                enrichCoord[0] = true;
+            }
+            if (u instanceof Enrich enrich && enrich.mode() == Enrich.Mode.REMOTE) {
+                if (agg[0]) {
+                    failures.add(fail(enrich, "ENRICH with remote policy can't be executed after STATS"));
+                }
+                if (enrichCoord[0]) {
+                    failures.add(fail(enrich, "ENRICH with remote policy can't be executed after another ENRICH with coordinator policy"));
+                }
+            }
+        });
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java
index 794a52b8e3f89..df0b258679d4c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java
@@ -58,7 +58,7 @@ public EsRelation(Source source, EsIndex index, List<Attribute> attributes, Inde
 
     private static EsRelation readFrom(StreamInput in) throws IOException {
         Source source = Source.readFrom((PlanStreamInput) in);
-        EsIndex esIndex = new EsIndex(in);
+        EsIndex esIndex = EsIndex.readFrom(in);
         List<Attribute> attributes = in.readNamedWriteableCollectionAsList(Attribute.class);
         if (supportingEsSourceOptions(in.getTransportVersion())) {
             // We don't do anything with these strings
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java
index b24a87d365020..cbd79011032df 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java
@@ -10,6 +10,8 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.esql.capabilities.PostAnalysisVerificationAware;
+import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
@@ -20,6 +22,9 @@
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.Rate;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.plan.GeneratingPlan;
 
@@ -28,10 +33,11 @@
 import java.util.List;
 import java.util.Objects;
 
+import static org.elasticsearch.xpack.esql.common.Failure.fail;
 import static org.elasticsearch.xpack.esql.core.expression.Expressions.asAttributes;
 import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes;
 
-public class Eval extends UnaryPlan implements GeneratingPlan<Eval> {
+public class Eval extends UnaryPlan implements GeneratingPlan<Eval>, PostAnalysisVerificationAware {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Eval", Eval::new);
 
     private final List<Alias> fields;
@@ -161,4 +167,30 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(super.hashCode(), fields);
     }
+
+    @Override
+    public void postAnalysisVerification(Failures failures) {
+        fields.forEach(field -> {
+            // check supported types
+            DataType dataType = field.dataType();
+            if (DataType.isRepresentable(dataType) == false) {
+                failures.add(
+                    fail(
+                        field,
+                        "EVAL does not support type [{}] as the return data type of expression [{}]",
+                        dataType.typeName(),
+                        field.child().sourceText()
+                    )
+                );
+            }
+            // check no aggregate functions are used
+            field.forEachDown(AggregateFunction.class, af -> {
+                if (af instanceof Rate) {
+                    failures.add(fail(af, "aggregate function [{}] not allowed outside METRICS command", af.sourceText()));
+                } else {
+                    failures.add(fail(af, "aggregate function [{}] not allowed outside STATS command", af.sourceText()));
+                }
+            });
+        });
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Filter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Filter.java
index 611793b583d7a..0fae5e5831fc7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Filter.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Filter.java
@@ -9,6 +9,8 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.esql.capabilities.PostAnalysisVerificationAware;
+import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -17,12 +19,16 @@
 import java.io.IOException;
 import java.util.Objects;
 
+import static org.elasticsearch.xpack.esql.common.Failure.fail;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
+
 /**
  * A {@code Filter} is a type of Plan that performs filtering of results. In
  * {@code SELECT x FROM y WHERE z ..} the "WHERE" clause is a Filter. A
  * {@code Filter} has a "condition" Expression that does the filtering.
  */
-public class Filter extends UnaryPlan {
+public class Filter extends UnaryPlan implements PostAnalysisVerificationAware {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Filter", Filter::new);
 
     private final Expression condition;
@@ -98,4 +104,15 @@ public Filter with(Expression conditionExpr) {
     public Filter with(LogicalPlan child, Expression conditionExpr) {
         return new Filter(source(), child, conditionExpr);
     }
+
+    @Override
+    public void postAnalysisVerification(Failures failures) {
+        checkFilterConditionDataType(condition, failures);
+    }
+
+    public static void checkFilterConditionDataType(Expression expression, Failures failures) {
+        if (expression.dataType() != NULL && expression.dataType() != BOOLEAN) {
+            failures.add(fail(expression, "Condition expression needs to be boolean, found [{}]", expression.dataType()));
+        }
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/OrderBy.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/OrderBy.java
index 8756cddbc3c97..d927d78701c65 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/OrderBy.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/OrderBy.java
@@ -9,9 +9,12 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.esql.capabilities.PostAnalysisVerificationAware;
+import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.Order;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
@@ -19,7 +22,9 @@
 import java.util.List;
 import java.util.Objects;
 
-public class OrderBy extends UnaryPlan {
+import static org.elasticsearch.xpack.esql.common.Failure.fail;
+
+public class OrderBy extends UnaryPlan implements PostAnalysisVerificationAware {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "OrderBy", OrderBy::new);
 
     private final List<Order> order;
@@ -91,4 +96,16 @@ public boolean equals(Object obj) {
         OrderBy other = (OrderBy) obj;
         return Objects.equals(order, other.order) && Objects.equals(child(), other.child());
     }
+
+    @Override
+    public void postAnalysisVerification(Failures failures) {
+        /**
+         * Some datatypes are not sortable
+         */
+        order.forEach(order -> {
+            if (DataType.isSortable(order.dataType()) == false) {
+                failures.add(fail(order, "cannot sort on " + order.dataType().typeName()));
+            }
+        });
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/RegexExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/RegexExtract.java
index ab813f1a9c737..d691507b62cb3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/RegexExtract.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/RegexExtract.java
@@ -7,20 +7,24 @@
 
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.xpack.esql.capabilities.PostAnalysisVerificationAware;
+import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.plan.GeneratingPlan;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 
+import static org.elasticsearch.xpack.esql.common.Failure.fail;
 import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes;
 
-public abstract class RegexExtract extends UnaryPlan implements GeneratingPlan<RegexExtract> {
+public abstract class RegexExtract extends UnaryPlan implements GeneratingPlan<RegexExtract>, PostAnalysisVerificationAware {
     protected final Expression input;
     protected final List<Attribute> extractedFields;
 
@@ -92,4 +96,22 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(super.hashCode(), input, extractedFields);
     }
+
+    @Override
+    public void postAnalysisVerification(Failures failures) {
+        DataType type = input.dataType();
+        if (DataType.isString(type) == false) {
+            failures.add(
+                fail(
+                    input,
+                    "{} only supports KEYWORD or TEXT values, found expression [{}] type [{}]",
+                    getClass().getSimpleName(),
+                    input.sourceText(),
+                    type
+                )
+            );
+        }
+
+    }
+
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Row.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Row.java
index 57cce3c26afc7..65d1adf5e2799 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Row.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Row.java
@@ -8,17 +8,22 @@
 package org.elasticsearch.xpack.esql.plan.logical;
 
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.esql.capabilities.PostAnalysisVerificationAware;
+import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 
 import java.util.List;
 import java.util.Objects;
 
-public class Row extends LeafPlan {
+import static org.elasticsearch.xpack.esql.common.Failure.fail;
+
+public class Row extends LeafPlan implements PostAnalysisVerificationAware {
 
     private final List<Alias> fields;
 
@@ -73,4 +78,13 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(fields);
     }
+
+    @Override
+    public void postAnalysisVerification(Failures failures) {
+        fields.forEach(a -> {
+            if (DataType.isRepresentable(a.dataType()) == false) {
+                failures.add(fail(a.child(), "cannot use [{}] directly in a row assignment", a.child().sourceText()));
+            }
+        });
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java
index a2c159e506880..2b4f93602c6a7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java
@@ -10,6 +10,8 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.esql.capabilities.PostAnalysisVerificationAware;
+import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
@@ -25,10 +27,11 @@
 import java.util.List;
 import java.util.Objects;
 
+import static org.elasticsearch.xpack.esql.common.Failure.fail;
 import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes;
 import static org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes.LEFT;
 
-public class Join extends BinaryPlan {
+public class Join extends BinaryPlan implements PostAnalysisVerificationAware {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Join", Join::new);
 
     private final JoinConfig config;
@@ -207,4 +210,24 @@ public boolean equals(Object obj) {
         Join other = (Join) obj;
         return config.equals(other.config) && Objects.equals(left(), other.left()) && Objects.equals(right(), other.right());
     }
+
+    @Override
+    public void postAnalysisVerification(Failures failures) {
+        for (int i = 0; i < config.leftFields().size(); i++) {
+            Attribute leftField = config.leftFields().get(i);
+            Attribute rightField = config.rightFields().get(i);
+            if (leftField.dataType() != rightField.dataType()) {
+                failures.add(
+                    fail(
+                        leftField,
+                        "JOIN left field [{}] of type [{}] is incompatible with right field [{}] of type [{}]",
+                        leftField.name(),
+                        leftField.dataType(),
+                        rightField.name(),
+                        rightField.dataType()
+                    )
+                );
+            }
+        }
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/LookupJoin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/LookupJoin.java
index 57c8cb00baa32..4e009156072df 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/LookupJoin.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/LookupJoin.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.plan.logical.Project;
 import org.elasticsearch.xpack.esql.plan.logical.SurrogateLogicalPlan;
 import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes.UsingJoinType;
 
@@ -50,9 +49,8 @@ public LookupJoin(Source source, LogicalPlan left, LogicalPlan right, JoinConfig
      */
     @Override
     public LogicalPlan surrogate() {
-        Join normalized = new Join(source(), left(), right(), config());
         // TODO: decide whether to introduce USING or just basic ON semantics - keep the ordering out for now
-        return new Project(source(), normalized, output());
+        return new Join(source(), left(), right(), config());
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EnrichExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EnrichExec.java
index a14332ebef7c3..42cf3528f2ae6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EnrichExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EnrichExec.java
@@ -87,7 +87,7 @@ private static EnrichExec readFrom(StreamInput in) throws IOException {
             concreteIndices = in.readMap(StreamInput::readString, StreamInput::readString);
         } else {
             mode = Enrich.Mode.ANY;
-            EsIndex esIndex = new EsIndex(in);
+            EsIndex esIndex = EsIndex.readFrom(in);
             if (esIndex.concreteIndices().size() != 1) {
                 throw new IllegalStateException("expected a single concrete enrich index; got " + esIndex.concreteIndices());
             }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java
index 267b9e613abef..ab533899aaff6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java
@@ -138,7 +138,7 @@ public EsQueryExec(
      */
     public static EsQueryExec deserialize(StreamInput in) throws IOException {
         var source = Source.readFrom((PlanStreamInput) in);
-        var index = new EsIndex(in);
+        var index = EsIndex.readFrom(in);
         var indexMode = EsRelation.readIndexMode(in);
         var attrs = in.readNamedWriteableCollectionAsList(Attribute.class);
         var query = in.readOptionalNamedWriteable(QueryBuilder.class);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java
index cd167b4683493..eeeafc52f158b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java
@@ -51,7 +51,7 @@ public EsSourceExec(Source source, EsIndex index, List<Attribute> attributes, Qu
     private EsSourceExec(StreamInput in) throws IOException {
         this(
             Source.readFrom((PlanStreamInput) in),
-            new EsIndex(in),
+            EsIndex.readFrom(in),
             in.readNamedWriteableCollectionAsList(Attribute.class),
             in.readOptionalNamedWriteable(QueryBuilder.class),
             EsRelation.readIndexMode(in)
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EstimatesRowSize.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EstimatesRowSize.java
index cfb6cce2579a2..f4ab546fa7493 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EstimatesRowSize.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EstimatesRowSize.java
@@ -103,9 +103,6 @@ public String toString() {
 
     static int estimateSize(DataType dataType) {
         ElementType elementType = PlannerUtils.toElementType(dataType);
-        if (elementType == ElementType.DOC) {
-            throw new EsqlIllegalArgumentException("can't load a [doc] with field extraction");
-        }
         if (elementType == ElementType.UNKNOWN) {
             throw new EsqlIllegalArgumentException("[unknown] can't be the result of field extraction");
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/FieldExtractExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/FieldExtractExec.java
index ec996c5c84064..e9783a241f0b9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/FieldExtractExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/FieldExtractExec.java
@@ -7,9 +7,12 @@
 
 package org.elasticsearch.xpack.esql.plan.physical;
 
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
@@ -31,9 +34,10 @@ public class FieldExtractExec extends UnaryExec implements EstimatesRowSize {
     );
 
     private final List<Attribute> attributesToExtract;
-    private final Attribute sourceAttribute;
+    private final @Nullable Attribute sourceAttribute;
+
     /**
-     * Attributes that many be extracted as doc values even if that makes them
+     * Attributes that may be extracted as doc values even if that makes them
      * less accurate. This is mostly used for geo fields which lose a lot of
      * precision in their doc values, but in some cases doc values provides
      * <strong>enough</strong> precision to do the job.
@@ -43,17 +47,32 @@ public class FieldExtractExec extends UnaryExec implements EstimatesRowSize {
      */
     private final Set<Attribute> docValuesAttributes;
 
+    /**
+     * Attributes of a shape whose extent can be extracted directly from the doc-values encoded geometry.
+     * <p>
+     *     This is never serialized between nodes and only used locally.
+     * </p>
+     */
+    private final Set<Attribute> boundsAttributes;
+
     private List<Attribute> lazyOutput;
 
     public FieldExtractExec(Source source, PhysicalPlan child, List<Attribute> attributesToExtract) {
-        this(source, child, attributesToExtract, Set.of());
+        this(source, child, attributesToExtract, Set.of(), Set.of());
     }
 
-    private FieldExtractExec(Source source, PhysicalPlan child, List<Attribute> attributesToExtract, Set<Attribute> docValuesAttributes) {
+    private FieldExtractExec(
+        Source source,
+        PhysicalPlan child,
+        List<Attribute> attributesToExtract,
+        Set<Attribute> docValuesAttributes,
+        Set<Attribute> boundsAttributes
+    ) {
         super(source, child);
         this.attributesToExtract = attributesToExtract;
         this.sourceAttribute = extractSourceAttributesFrom(child);
         this.docValuesAttributes = docValuesAttributes;
+        this.boundsAttributes = boundsAttributes;
     }
 
     private FieldExtractExec(StreamInput in) throws IOException {
@@ -62,7 +81,7 @@ private FieldExtractExec(StreamInput in) throws IOException {
             in.readNamedWriteable(PhysicalPlan.class),
             in.readNamedWriteableCollectionAsList(Attribute.class)
         );
-        // docValueAttributes are only used on the data node and never serialized.
+        // docValueAttributes and boundsAttributes are only used on the data node and never serialized.
     }
 
     @Override
@@ -70,7 +89,7 @@ public void writeTo(StreamOutput out) throws IOException {
         Source.EMPTY.writeTo(out);
         out.writeNamedWriteable(child());
         out.writeNamedWriteableCollection(attributesToExtract());
-        // docValueAttributes are only used on the data node and never serialized.
+        // docValueAttributes and boundsAttributes are only used on the data node and never serialized.
     }
 
     @Override
@@ -78,7 +97,7 @@ public String getWriteableName() {
         return ENTRY.name;
     }
 
-    public static Attribute extractSourceAttributesFrom(PhysicalPlan plan) {
+    public static @Nullable Attribute extractSourceAttributesFrom(PhysicalPlan plan) {
         for (Attribute attribute : plan.outputSet()) {
             if (EsQueryExec.isSourceAttribute(attribute)) {
                 return attribute;
@@ -99,18 +118,22 @@ protected NodeInfo<FieldExtractExec> info() {
 
     @Override
     public UnaryExec replaceChild(PhysicalPlan newChild) {
-        return new FieldExtractExec(source(), newChild, attributesToExtract, docValuesAttributes);
+        return new FieldExtractExec(source(), newChild, attributesToExtract, docValuesAttributes, boundsAttributes);
     }
 
     public FieldExtractExec withDocValuesAttributes(Set<Attribute> docValuesAttributes) {
-        return new FieldExtractExec(source(), child(), attributesToExtract, docValuesAttributes);
+        return new FieldExtractExec(source(), child(), attributesToExtract, docValuesAttributes, boundsAttributes);
+    }
+
+    public FieldExtractExec withBoundsAttributes(Set<Attribute> boundsAttributes) {
+        return new FieldExtractExec(source(), child(), attributesToExtract, docValuesAttributes, boundsAttributes);
     }
 
     public List<Attribute> attributesToExtract() {
         return attributesToExtract;
     }
 
-    public Attribute sourceAttribute() {
+    public @Nullable Attribute sourceAttribute() {
         return sourceAttribute;
     }
 
@@ -118,8 +141,8 @@ public Set<Attribute> docValuesAttributes() {
         return docValuesAttributes;
     }
 
-    public boolean hasDocValuesAttribute(Attribute attr) {
-        return docValuesAttributes.contains(attr);
+    public Set<Attribute> boundsAttributes() {
+        return boundsAttributes;
     }
 
     @Override
@@ -142,7 +165,7 @@ public PhysicalPlan estimateRowSize(State state) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(attributesToExtract, docValuesAttributes, child());
+        return Objects.hash(attributesToExtract, docValuesAttributes, boundsAttributes, child());
     }
 
     @Override
@@ -158,12 +181,27 @@ public boolean equals(Object obj) {
         FieldExtractExec other = (FieldExtractExec) obj;
         return Objects.equals(attributesToExtract, other.attributesToExtract)
             && Objects.equals(docValuesAttributes, other.docValuesAttributes)
+            && Objects.equals(boundsAttributes, other.boundsAttributes)
             && Objects.equals(child(), other.child());
     }
 
     @Override
     public String nodeString() {
-        return nodeName() + NodeUtils.limitedToString(attributesToExtract) + "<" + NodeUtils.limitedToString(docValuesAttributes) + ">";
+        return Strings.format(
+            "%s<%s,%s>",
+            nodeName() + NodeUtils.limitedToString(attributesToExtract),
+            docValuesAttributes,
+            boundsAttributes
+        );
     }
 
+    public MappedFieldType.FieldExtractPreference fieldExtractPreference(Attribute attr) {
+        if (boundsAttributes.contains(attr)) {
+            return MappedFieldType.FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS;
+        }
+        if (docValuesAttributes.contains(attr)) {
+            return MappedFieldType.FieldExtractPreference.DOC_VALUES;
+        }
+        return MappedFieldType.FieldExtractPreference.NONE;
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/TopNExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/TopNExec.java
index 61e40b3fa4693..bbf0c681bec7d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/TopNExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/TopNExec.java
@@ -10,9 +10,11 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.Order;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
@@ -95,6 +97,9 @@ public Integer estimatedRowSize() {
 
     @Override
     public PhysicalPlan estimateRowSize(State state) {
+        final List<Attribute> output = output();
+        final boolean needsSortedDocIds = output.stream().anyMatch(a -> a.dataType() == DataType.DOC_DATA_TYPE);
+        state.add(needsSortedDocIds, output);
         int size = state.consumeAllFields(true);
         return Objects.equals(this.estimatedRowSize, size) ? this : new TopNExec(source(), child(), order, limit, size);
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java
index 57ba1c8016feb..072bae21da2a3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
@@ -47,9 +48,11 @@
 public abstract class AbstractPhysicalOperationProviders implements PhysicalOperationProviders {
 
     private final AggregateMapper aggregateMapper = new AggregateMapper();
+    private final FoldContext foldContext;
     private final AnalysisRegistry analysisRegistry;
 
-    AbstractPhysicalOperationProviders(AnalysisRegistry analysisRegistry) {
+    AbstractPhysicalOperationProviders(FoldContext foldContext, AnalysisRegistry analysisRegistry) {
+        this.foldContext = foldContext;
         this.analysisRegistry = analysisRegistry;
     }
 
@@ -251,6 +254,7 @@ public static List<Attribute> intermediateAttributes(List<? extends NamedExpress
     private record AggFunctionSupplierContext(AggregatorFunctionSupplier supplier, AggregatorMode mode) {}
 
     private void aggregatesToFactory(
+
         List<? extends NamedExpression> aggregates,
         AggregatorMode mode,
         Layout layout,
@@ -311,7 +315,11 @@ else if (mode == AggregatorMode.FINAL || mode == AggregatorMode.INTERMEDIATE) {
 
                     // apply the filter only in the initial phase - as the rest of the data is already filtered
                     if (aggregateFunction.hasFilter() && mode.isInputPartial() == false) {
-                        EvalOperator.ExpressionEvaluator.Factory evalFactory = EvalMapper.toEvaluator(aggregateFunction.filter(), layout);
+                        EvalOperator.ExpressionEvaluator.Factory evalFactory = EvalMapper.toEvaluator(
+                            foldContext,
+                            aggregateFunction.filter(),
+                            layout
+                        );
                         aggSupplier = new FilteredAggregatorFunctionSupplier(aggSupplier, evalFactory);
                     }
                     consumer.accept(new AggFunctionSupplierContext(aggSupplier, mode));
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java
index 17468f7afec1b..112f101ad842b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java
@@ -13,22 +13,16 @@
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.common.breaker.CircuitBreaker;
-import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.logging.HeaderWarning;
-import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.aggregation.GroupingAggregator;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.ElementType;
-import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.lucene.LuceneCountOperator;
 import org.elasticsearch.compute.lucene.LuceneOperator;
 import org.elasticsearch.compute.lucene.LuceneSourceOperator;
 import org.elasticsearch.compute.lucene.LuceneTopNSourceOperator;
 import org.elasticsearch.compute.lucene.TimeSeriesSortedSourceOperatorFactory;
 import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator;
-import org.elasticsearch.compute.operator.DriverContext;
-import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.compute.operator.Operator;
 import org.elasticsearch.compute.operator.OrdinalsGroupingOperator;
 import org.elasticsearch.compute.operator.SourceOperator;
@@ -39,7 +33,6 @@
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NestedLookup;
-import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -53,6 +46,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField;
@@ -100,8 +94,8 @@ public interface ShardContext extends org.elasticsearch.compute.lucene.ShardCont
 
     private final List<ShardContext> shardContexts;
 
-    public EsPhysicalOperationProviders(List<ShardContext> shardContexts, AnalysisRegistry analysisRegistry) {
-        super(analysisRegistry);
+    public EsPhysicalOperationProviders(FoldContext foldContext, List<ShardContext> shardContexts, AnalysisRegistry analysisRegistry) {
+        super(foldContext, analysisRegistry);
         this.shardContexts = shardContexts;
     }
 
@@ -114,12 +108,11 @@ public final PhysicalOperation fieldExtractPhysicalOperation(FieldExtractExec fi
             .toList();
         List<ValuesSourceReaderOperator.FieldInfo> fields = new ArrayList<>();
         int docChannel = source.layout.get(sourceAttr.id()).channel();
-        var docValuesAttrs = fieldExtractExec.docValuesAttributes();
         for (Attribute attr : fieldExtractExec.attributesToExtract()) {
             layout.append(attr);
             var unionTypes = findUnionTypes(attr);
             DataType dataType = attr.dataType();
-            MappedFieldType.FieldExtractPreference fieldExtractPreference = PlannerUtils.extractPreference(docValuesAttrs.contains(attr));
+            MappedFieldType.FieldExtractPreference fieldExtractPreference = fieldExtractExec.fieldExtractPreference(attr);
             ElementType elementType = PlannerUtils.toElementType(dataType, fieldExtractPreference);
             // Do not use the field attribute name, this can deviate from the field name for union types.
             String fieldName = attr instanceof FieldAttribute fa ? fa.fieldName() : attr.name();
@@ -168,7 +161,7 @@ public final PhysicalOperation sourcePhysicalOperation(EsQueryExec esQueryExec,
         List<Sort> sorts = esQueryExec.sorts();
         assert esQueryExec.estimatedRowSize() != null : "estimated row size not initialized";
         int rowEstimatedSize = esQueryExec.estimatedRowSize();
-        int limit = esQueryExec.limit() != null ? (Integer) esQueryExec.limit().fold() : NO_LIMIT;
+        int limit = esQueryExec.limit() != null ? (Integer) esQueryExec.limit().fold(context.foldCtx()) : NO_LIMIT;
         boolean scoring = esQueryExec.attrs()
             .stream()
             .anyMatch(a -> a instanceof MetadataAttribute && a.name().equals(MetadataAttribute.SCORE));
@@ -224,7 +217,7 @@ public LuceneCountOperator.Factory countSource(LocalExecutionPlannerContext cont
             querySupplier(queryBuilder),
             context.queryPragmas().dataPartitioning(),
             context.queryPragmas().taskConcurrency(),
-            limit == null ? NO_LIMIT : (Integer) limit.fold()
+            limit == null ? NO_LIMIT : (Integer) limit.fold(context.foldCtx())
         );
     }
 
@@ -345,16 +338,7 @@ public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
 
                 @Override
                 public SearchLookup lookup() {
-                    boolean syntheticSource = SourceFieldMapper.isSynthetic(indexSettings());
-                    var searchLookup = ctx.lookup();
-                    if (syntheticSource) {
-                        // in the context of scripts and when synthetic source is used the search lookup can't always be reused between
-                        // users of SearchLookup. This is only an issue when scripts fallback to _source, but since we can't always
-                        // accurately determine whether a script uses _source, we should do this for all script usages.
-                        // This lookup() method is only invoked for scripts / runtime fields, so it is ok to do here.
-                        searchLookup = searchLookup.swapSourceProvider(ctx.createSourceProvider());
-                    }
-                    return searchLookup;
+                    return ctx.lookup();
                 }
 
                 @Override
@@ -381,29 +365,13 @@ public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
         }
     }
 
-    static class TypeConvertingBlockLoader implements BlockLoader {
-        protected final BlockLoader delegate;
-        private final EvalOperator.ExpressionEvaluator convertEvaluator;
+    private static class TypeConvertingBlockLoader implements BlockLoader {
+        private final BlockLoader delegate;
+        private final TypeConverter typeConverter;
 
         protected TypeConvertingBlockLoader(BlockLoader delegate, AbstractConvertFunction convertFunction) {
             this.delegate = delegate;
-            DriverContext driverContext1 = new DriverContext(
-                BigArrays.NON_RECYCLING_INSTANCE,
-                new org.elasticsearch.compute.data.BlockFactory(
-                    new NoopCircuitBreaker(CircuitBreaker.REQUEST),
-                    BigArrays.NON_RECYCLING_INSTANCE
-                )
-            );
-            this.convertEvaluator = convertFunction.toEvaluator(e -> driverContext -> new EvalOperator.ExpressionEvaluator() {
-                @Override
-                public org.elasticsearch.compute.data.Block eval(Page page) {
-                    // This is a pass-through evaluator, since it sits directly on the source loading (no prior expressions)
-                    return page.getBlock(0);
-                }
-
-                @Override
-                public void close() {}
-            }).get(driverContext1);
+            this.typeConverter = TypeConverter.fromConvertFunction(convertFunction);
         }
 
         @Override
@@ -414,8 +382,7 @@ public Builder builder(BlockFactory factory, int expectedCount) {
 
         @Override
         public Block convert(Block block) {
-            Page page = new Page((org.elasticsearch.compute.data.Block) block);
-            return convertEvaluator.eval(page);
+            return typeConverter.convert((org.elasticsearch.compute.data.Block) block);
         }
 
         @Override
@@ -428,9 +395,7 @@ public ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws
                 @Override
                 public Block read(BlockFactory factory, Docs docs) throws IOException {
                     Block block = reader.read(factory, docs);
-                    Page page = new Page((org.elasticsearch.compute.data.Block) block);
-                    org.elasticsearch.compute.data.Block converted = convertEvaluator.eval(page);
-                    return converted;
+                    return typeConverter.convert((org.elasticsearch.compute.data.Block) block);
                 }
 
                 @Override
@@ -470,7 +435,7 @@ public SortedSetDocValues ordinals(LeafReaderContext context) {
 
         @Override
         public final String toString() {
-            return "TypeConvertingBlockLoader[delegate=" + delegate + ", convertEvaluator=" + convertEvaluator + "]";
+            return "TypeConvertingBlockLoader[delegate=" + delegate + ", typeConverter=" + typeConverter + "]";
         }
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
index 43bbf9a5f4ff1..c185bd5729879 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.TranslationAware;
 import org.elasticsearch.xpack.esql.core.expression.TypedAttribute;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction;
@@ -25,7 +26,6 @@
 import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslators;
 import org.elasticsearch.xpack.esql.core.planner.TranslatorHandler;
 import org.elasticsearch.xpack.esql.core.querydsl.query.MatchAll;
-import org.elasticsearch.xpack.esql.core.querydsl.query.MatchQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.NotQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery;
@@ -44,6 +44,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch;
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction;
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils;
+import org.elasticsearch.xpack.esql.expression.predicate.fulltext.MultiMatchQueryPredicate;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual;
@@ -53,6 +54,8 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.esql.querydsl.query.KqlQuery;
+import org.elasticsearch.xpack.esql.querydsl.query.MatchQuery;
+import org.elasticsearch.xpack.esql.querydsl.query.MultiMatchQuery;
 import org.elasticsearch.xpack.esql.querydsl.query.SpatialRelatesQuery;
 import org.elasticsearch.xpack.versionfield.Version;
 
@@ -92,7 +95,7 @@ public final class EsqlExpressionTranslators {
         new ExpressionTranslators.IsNotNulls(),
         new ExpressionTranslators.Nots(),
         new ExpressionTranslators.Likes(),
-        new ExpressionTranslators.MultiMatches(),
+        new MultiMatches(),
         new MatchFunctionTranslator(),
         new QueryStringFunctionTranslator(),
         new KqlFunctionTranslator(),
@@ -142,7 +145,7 @@ public static void checkInsensitiveComparison(InsensitiveEquals bc) {
         static Query translate(InsensitiveEquals bc) {
             TypedAttribute attribute = checkIsPushableAttribute(bc.left());
             Source source = bc.source();
-            BytesRef value = BytesRefs.toBytesRef(valueOf(bc.right()));
+            BytesRef value = BytesRefs.toBytesRef(valueOf(FoldContext.small() /* TODO remove me */, bc.right()));
             String name = pushableAttributeName(attribute);
             return new TermQuery(source, name, value.utf8ToString(), true);
         }
@@ -186,7 +189,7 @@ static Query translate(BinaryComparison bc, TranslatorHandler handler) {
             TypedAttribute attribute = checkIsPushableAttribute(bc.left());
             Source source = bc.source();
             String name = handler.nameOf(attribute);
-            Object result = bc.right().fold();
+            Object result = bc.right().fold(FoldContext.small() /* TODO remove me */);
             Object value = result;
             String format = null;
             boolean isDateLiteralComparison = false;
@@ -267,7 +270,7 @@ private static Query translateOutOfRangeComparisons(BinaryComparison bc) {
                 return null;
             }
             Source source = bc.source();
-            Object value = valueOf(bc.right());
+            Object value = valueOf(FoldContext.small() /* TODO remove me */, bc.right());
 
             // Comparisons with multi-values always return null in ESQL.
             if (value instanceof List<?>) {
@@ -367,7 +370,7 @@ public static Query doTranslate(ScalarFunction f, TranslatorHandler handler) {
             if (f instanceof CIDRMatch cm) {
                 if (cm.ipField() instanceof FieldAttribute fa && Expressions.foldable(cm.matches())) {
                     String targetFieldName = handler.nameOf(fa.exactAttribute());
-                    Set<Object> set = new LinkedHashSet<>(Expressions.fold(cm.matches()));
+                    Set<Object> set = new LinkedHashSet<>(Expressions.fold(FoldContext.small() /* TODO remove me */, cm.matches()));
 
                     Query query = new TermsQuery(f.source(), targetFieldName, set);
                     // CIDR_MATCH applies only to single values.
@@ -418,7 +421,7 @@ static Query translate(
             String name = handler.nameOf(attribute);
 
             try {
-                Geometry shape = SpatialRelatesUtils.makeGeometryFromLiteral(constantExpression);
+                Geometry shape = SpatialRelatesUtils.makeGeometryFromLiteral(FoldContext.small() /* TODO remove me */, constantExpression);
                 return new SpatialRelatesQuery(bc.source(), name, bc.queryRelation(), shape, attribute.dataType());
             } catch (IllegalArgumentException e) {
                 throw new QlIllegalArgumentException(e.getMessage(), e);
@@ -459,7 +462,7 @@ private static Query translate(In in, TranslatorHandler handler) {
                             queries.add(query);
                         }
                     } else {
-                        terms.add(valueOf(rhs));
+                        terms.add(valueOf(FoldContext.small() /* TODO remove me */, rhs));
                     }
                 }
             }
@@ -485,8 +488,8 @@ public static Query doTranslate(Range r, TranslatorHandler handler) {
         }
 
         private static RangeQuery translate(Range r, TranslatorHandler handler) {
-            Object lower = valueOf(r.lower());
-            Object upper = valueOf(r.upper());
+            Object lower = valueOf(FoldContext.small() /* TODO remove me */, r.lower());
+            Object upper = valueOf(FoldContext.small() /* TODO remove me */, r.upper());
             String format = null;
 
             DataType dataType = r.value().dataType();
@@ -537,6 +540,18 @@ private static RangeQuery translate(Range r, TranslatorHandler handler) {
         }
     }
 
+    public static class MultiMatches extends ExpressionTranslator<MultiMatchQueryPredicate> {
+
+        @Override
+        protected Query asQuery(MultiMatchQueryPredicate q, TranslatorHandler handler) {
+            return doTranslate(q, handler);
+        }
+
+        public static Query doTranslate(MultiMatchQueryPredicate q, TranslatorHandler handler) {
+            return new MultiMatchQuery(q.source(), q.query(), q.fields(), q);
+        }
+    }
+
     public static class MatchFunctionTranslator extends ExpressionTranslator<Match> {
         @Override
         protected Query asQuery(Match match, TranslatorHandler handler) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java
index b85340936497e..ecd0284c7cb57 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.compute.operator.Driver;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator.EvalOperatorFactory;
-import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.compute.operator.FilterOperator.FilterOperatorFactory;
 import org.elasticsearch.compute.operator.LocalSourceOperator;
 import org.elasticsearch.compute.operator.LocalSourceOperator.LocalSourceFactory;
@@ -55,6 +54,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
@@ -161,13 +161,14 @@ public LocalExecutionPlanner(
     /**
      * turn the given plan into a list of drivers to execute
      */
-    public LocalExecutionPlan plan(PhysicalPlan localPhysicalPlan) {
+    public LocalExecutionPlan plan(FoldContext foldCtx, PhysicalPlan localPhysicalPlan) {
         var context = new LocalExecutionPlannerContext(
             new ArrayList<>(),
             new Holder<>(DriverParallelism.SINGLE),
             configuration.pragmas(),
             bigArrays,
             blockFactory,
+            foldCtx,
             settings
         );
 
@@ -345,6 +346,8 @@ private PhysicalOperation planExchangeSource(ExchangeSourceExec exchangeSource,
     }
 
     private PhysicalOperation planTopN(TopNExec topNExec, LocalExecutionPlannerContext context) {
+        final Integer rowSize = topNExec.estimatedRowSize();
+        assert rowSize != null && rowSize > 0 : "estimated row size [" + rowSize + "] wasn't set";
         PhysicalOperation source = plan(topNExec.child(), context);
 
         ElementType[] elementTypes = new ElementType[source.layout.numberOfChannels()];
@@ -385,24 +388,8 @@ private PhysicalOperation planTopN(TopNExec topNExec, LocalExecutionPlannerConte
         } else {
             throw new EsqlIllegalArgumentException("limit only supported with literal values");
         }
-
-        // TODO Replace page size with passing estimatedRowSize down
-        /*
-         * The 2000 below is a hack to account for incoming size and to make
-         * sure the estimated row size is never 0 which'd cause a divide by 0.
-         * But we should replace this with passing the estimate into the real
-         * topn and letting it actually measure the size of rows it produces.
-         * That'll be more accurate. And we don't have a path for estimating
-         * incoming rows. And we don't need one because we can estimate.
-         */
         return source.with(
-            new TopNOperatorFactory(
-                limit,
-                asList(elementTypes),
-                asList(encoders),
-                orders,
-                context.pageSize(2000 + topNExec.estimatedRowSize())
-            ),
+            new TopNOperatorFactory(limit, asList(elementTypes), asList(encoders), orders, context.pageSize(rowSize)),
             source.layout
         );
     }
@@ -411,7 +398,7 @@ private PhysicalOperation planEval(EvalExec eval, LocalExecutionPlannerContext c
         PhysicalOperation source = plan(eval.child(), context);
 
         for (Alias field : eval.fields()) {
-            var evaluatorSupplier = EvalMapper.toEvaluator(field.child(), source.layout);
+            var evaluatorSupplier = EvalMapper.toEvaluator(context.foldCtx(), field.child(), source.layout);
             Layout.Builder layout = source.layout.builder();
             layout.append(field.toAttribute());
             source = source.with(new EvalOperatorFactory(evaluatorSupplier), layout.build());
@@ -432,7 +419,7 @@ private PhysicalOperation planDissect(DissectExec dissect, LocalExecutionPlanner
         source = source.with(
             new StringExtractOperator.StringExtractOperatorFactory(
                 patternNames,
-                EvalMapper.toEvaluator(expr, layout),
+                EvalMapper.toEvaluator(context.foldCtx(), expr, layout),
                 () -> (input) -> dissect.parser().parser().parse(input)
             ),
             layout
@@ -464,7 +451,7 @@ private PhysicalOperation planGrok(GrokExec grok, LocalExecutionPlannerContext c
         source = source.with(
             new ColumnExtractOperator.Factory(
                 types,
-                EvalMapper.toEvaluator(grok.inputExpression(), layout),
+                EvalMapper.toEvaluator(context.foldCtx(), grok.inputExpression(), layout),
                 () -> new GrokEvaluatorExtracter(grok.pattern().grok(), grok.pattern().pattern(), fieldToPos, fieldToType)
             ),
             layout
@@ -575,6 +562,15 @@ private PhysicalOperation planLookupJoin(LookupJoinExec join, LocalExecutionPlan
         if (localSourceExec.indexMode() != IndexMode.LOOKUP) {
             throw new IllegalArgumentException("can't plan [" + join + "]");
         }
+        Map<String, IndexMode> indicesWithModes = localSourceExec.index().indexNameWithModes();
+        if (indicesWithModes.size() != 1) {
+            throw new IllegalArgumentException("can't plan [" + join + "], found more than 1 index");
+        }
+        var entry = indicesWithModes.entrySet().iterator().next();
+        if (entry.getValue() != IndexMode.LOOKUP) {
+            throw new IllegalArgumentException("can't plan [" + join + "], found index with mode [" + entry.getValue() + "]");
+        }
+        String indexName = entry.getKey();
         List<Layout.ChannelAndType> matchFields = new ArrayList<>(join.leftFields().size());
         for (Attribute m : join.leftFields()) {
             Layout.ChannelAndType t = source.layout.get(m.id());
@@ -595,7 +591,7 @@ private PhysicalOperation planLookupJoin(LookupJoinExec join, LocalExecutionPlan
                 matchFields.getFirst().channel(),
                 lookupFromIndexService,
                 matchFields.getFirst().type(),
-                localSourceExec.index().name(),
+                indexName,
                 join.leftFields().getFirst().name(),
                 join.addedFields().stream().map(f -> (NamedExpression) f).toList(),
                 join.source()
@@ -604,10 +600,6 @@ private PhysicalOperation planLookupJoin(LookupJoinExec join, LocalExecutionPlan
         );
     }
 
-    private ExpressionEvaluator.Factory toEvaluator(Expression exp, Layout layout) {
-        return EvalMapper.toEvaluator(exp, layout);
-    }
-
     private PhysicalOperation planLocal(LocalSourceExec localSourceExec, LocalExecutionPlannerContext context) {
         Layout.Builder layout = new Layout.Builder();
         layout.append(localSourceExec.output());
@@ -662,12 +654,15 @@ private PhysicalOperation planProject(ProjectExec project, LocalExecutionPlanner
     private PhysicalOperation planFilter(FilterExec filter, LocalExecutionPlannerContext context) {
         PhysicalOperation source = plan(filter.child(), context);
         // TODO: should this be extracted into a separate eval block?
-        return source.with(new FilterOperatorFactory(toEvaluator(filter.condition(), source.layout)), source.layout);
+        return source.with(
+            new FilterOperatorFactory(EvalMapper.toEvaluator(context.foldCtx(), filter.condition(), source.layout)),
+            source.layout
+        );
     }
 
     private PhysicalOperation planLimit(LimitExec limit, LocalExecutionPlannerContext context) {
         PhysicalOperation source = plan(limit.child(), context);
-        return source.with(new Factory((Integer) limit.limit().fold()), source.layout);
+        return source.with(new Factory((Integer) limit.limit().fold(context.foldCtx)), source.layout);
     }
 
     private PhysicalOperation planMvExpand(MvExpandExec mvExpandExec, LocalExecutionPlannerContext context) {
@@ -749,7 +744,7 @@ public String describe() {
             return Stream.concat(
                 Stream.concat(Stream.of(sourceOperatorFactory), intermediateOperatorFactories.stream()),
                 Stream.of(sinkOperatorFactory)
-            ).map(Describable::describe).collect(joining("\n\\_", "\\_", ""));
+            ).map(describable -> describable == null ? "null" : describable.describe()).collect(joining("\n\\_", "\\_", ""));
         }
 
         @Override
@@ -788,6 +783,7 @@ public record LocalExecutionPlannerContext(
         QueryPragmas queryPragmas,
         BigArrays bigArrays,
         BlockFactory blockFactory,
+        FoldContext foldCtx,
         Settings settings
     ) {
         void addDriverFactory(DriverFactory driverFactory) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
index 37f89891860d8..2f4368155069f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
@@ -14,25 +14,26 @@
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates;
+import org.elasticsearch.xpack.esql.core.tree.Node;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.core.util.Queries;
-import org.elasticsearch.xpack.esql.index.EsIndex;
 import org.elasticsearch.xpack.esql.optimizer.LocalLogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.optimizer.LocalLogicalPlanOptimizer;
 import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalOptimizerContext;
 import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizer;
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
 import org.elasticsearch.xpack.esql.plan.logical.Filter;
+import org.elasticsearch.xpack.esql.plan.logical.join.Join;
 import org.elasticsearch.xpack.esql.plan.physical.AggregateExec;
 import org.elasticsearch.xpack.esql.plan.physical.EsSourceExec;
 import org.elasticsearch.xpack.esql.plan.physical.EstimatesRowSize;
@@ -40,6 +41,7 @@
 import org.elasticsearch.xpack.esql.plan.physical.ExchangeSinkExec;
 import org.elasticsearch.xpack.esql.plan.physical.ExchangeSourceExec;
 import org.elasticsearch.xpack.esql.plan.physical.FragmentExec;
+import org.elasticsearch.xpack.esql.plan.physical.LookupJoinExec;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.esql.planner.mapper.LocalMapper;
 import org.elasticsearch.xpack.esql.planner.mapper.Mapper;
@@ -48,9 +50,12 @@
 import org.elasticsearch.xpack.esql.stats.SearchStats;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.function.Consumer;
+import java.util.function.Function;
 
 import static java.util.Arrays.asList;
 import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.DOC_VALUES;
@@ -105,7 +110,7 @@ public static Set<String> planConcreteIndices(PhysicalPlan plan) {
             return Set.of();
         }
         var indices = new LinkedHashSet<String>();
-        plan.forEachUp(FragmentExec.class, f -> f.fragment().forEachUp(EsRelation.class, r -> indices.addAll(r.index().concreteIndices())));
+        forEachFromRelation(plan, relation -> indices.addAll(relation.index().concreteIndices()));
         return indices;
     }
 
@@ -117,26 +122,56 @@ public static String[] planOriginalIndices(PhysicalPlan plan) {
             return Strings.EMPTY_ARRAY;
         }
         var indices = new LinkedHashSet<String>();
-        plan.forEachUp(
-            FragmentExec.class,
-            f -> f.fragment().forEachUp(EsRelation.class, r -> addOriginalIndexIfNotLookup(indices, r.index()))
-        );
+        forEachFromRelation(plan, relation -> indices.addAll(asList(Strings.commaDelimitedListToStringArray(relation.index().name()))));
         return indices.toArray(String[]::new);
     }
 
-    private static void addOriginalIndexIfNotLookup(Set<String> indices, EsIndex index) {
-        if (index.indexNameWithModes().get(index.name()) != IndexMode.LOOKUP) {
-            indices.addAll(asList(Strings.commaDelimitedListToStringArray(index.name())));
+    /**
+     * Iterates over the plan and applies the action to each {@link EsRelation} node.
+     * <p>
+     *     This method ignores the right side of joins.
+     * </p>
+     */
+    private static void forEachFromRelation(PhysicalPlan plan, Consumer<EsRelation> action) {
+        // Take the non-join-side fragments
+        forEachUpWithChildren(plan, FragmentExec.class, fragment -> {
+            // Take the non-join-side relations
+            forEachUpWithChildren(
+                fragment.fragment(),
+                EsRelation.class,
+                action,
+                node -> node instanceof Join join ? List.of(join.left()) : node.children()
+            );
+        }, node -> node instanceof LookupJoinExec join ? List.of(join.left()) : node.children());
+    }
+
+    /**
+     * Similar to {@link Node#forEachUp(Consumer)}, but with a custom callback to get the node children.
+     */
+    private static <T extends Node<T>, E extends T> void forEachUpWithChildren(
+        T node,
+        Class<E> typeToken,
+        Consumer<? super E> action,
+        Function<? super T, Collection<T>> childrenGetter
+    ) {
+        childrenGetter.apply(node).forEach(c -> forEachUpWithChildren(c, typeToken, action, childrenGetter));
+        if (typeToken.isInstance(node)) {
+            action.accept(typeToken.cast(node));
         }
     }
 
-    public static PhysicalPlan localPlan(List<SearchExecutionContext> searchContexts, Configuration configuration, PhysicalPlan plan) {
-        return localPlan(configuration, plan, SearchContextStats.from(searchContexts));
+    public static PhysicalPlan localPlan(
+        List<SearchExecutionContext> searchContexts,
+        Configuration configuration,
+        FoldContext foldCtx,
+        PhysicalPlan plan
+    ) {
+        return localPlan(configuration, foldCtx, plan, SearchContextStats.from(searchContexts));
     }
 
-    public static PhysicalPlan localPlan(Configuration configuration, PhysicalPlan plan, SearchStats searchStats) {
-        final var logicalOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, searchStats));
-        var physicalOptimizer = new LocalPhysicalPlanOptimizer(new LocalPhysicalOptimizerContext(configuration, searchStats));
+    public static PhysicalPlan localPlan(Configuration configuration, FoldContext foldCtx, PhysicalPlan plan, SearchStats searchStats) {
+        final var logicalOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, foldCtx, searchStats));
+        var physicalOptimizer = new LocalPhysicalPlanOptimizer(new LocalPhysicalOptimizerContext(configuration, foldCtx, searchStats));
 
         return localPlan(plan, logicalOptimizer, physicalOptimizer);
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TypeConverter.java
new file mode 100644
index 0000000000000..4dea8a50b5c17
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TypeConverter.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.planner;
+
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
+import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction;
+
+class TypeConverter {
+    private final String evaluatorName;
+    private final ExpressionEvaluator convertEvaluator;
+
+    private TypeConverter(String evaluatorName, ExpressionEvaluator convertEvaluator) {
+        this.evaluatorName = evaluatorName;
+        this.convertEvaluator = convertEvaluator;
+    }
+
+    public static TypeConverter fromConvertFunction(AbstractConvertFunction convertFunction) {
+        DriverContext driverContext1 = new DriverContext(
+            BigArrays.NON_RECYCLING_INSTANCE,
+            new org.elasticsearch.compute.data.BlockFactory(
+                new NoopCircuitBreaker(CircuitBreaker.REQUEST),
+                BigArrays.NON_RECYCLING_INSTANCE
+            )
+        );
+        return new TypeConverter(convertFunction.functionName(), convertFunction.toEvaluator(new EvaluatorMapper.ToEvaluator() {
+            @Override
+            public ExpressionEvaluator.Factory apply(Expression expression) {
+                return driverContext -> new ExpressionEvaluator() {
+                    @Override
+                    public org.elasticsearch.compute.data.Block eval(Page page) {
+                        // This is a pass-through evaluator, since it sits directly on the source loading (no prior expressions)
+                        return page.getBlock(0);
+                    }
+
+                    @Override
+                    public void close() {}
+                };
+            }
+
+            @Override
+            public FoldContext foldCtx() {
+                throw new IllegalStateException("not folding");
+            }
+        }).get(driverContext1));
+    }
+
+    public Block convert(Block block) {
+        return convertEvaluator.eval(new Page(block));
+    }
+
+    @Override
+    public String toString() {
+        return evaluatorName;
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/MapperUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/MapperUtils.java
index e881eabb38c43..b8f539ea307c9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/MapperUtils.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/MapperUtils.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.compute.aggregation.AggregatorMode;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -90,7 +91,7 @@ static PhysicalPlan mapUnary(UnaryPlan p, PhysicalPlan child) {
                 enrich.mode(),
                 enrich.policy().getType(),
                 enrich.matchField(),
-                BytesRefs.toString(enrich.policyName().fold()),
+                BytesRefs.toString(enrich.policyName().fold(FoldContext.small() /* TODO remove me */)),
                 enrich.policy().getMatchField(),
                 enrich.concreteIndices(),
                 enrich.enrichFields()
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java
index 9b59b98a7cdc2..a38236fe60954 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java
@@ -61,14 +61,11 @@
 import org.elasticsearch.xpack.esql.action.EsqlQueryAction;
 import org.elasticsearch.xpack.esql.action.EsqlSearchShardsAction;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.enrich.EnrichLookupService;
 import org.elasticsearch.xpack.esql.enrich.LookupFromIndexService;
-import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
-import org.elasticsearch.xpack.esql.plan.logical.join.Join;
 import org.elasticsearch.xpack.esql.plan.physical.ExchangeSinkExec;
 import org.elasticsearch.xpack.esql.plan.physical.ExchangeSourceExec;
-import org.elasticsearch.xpack.esql.plan.physical.FragmentExec;
-import org.elasticsearch.xpack.esql.plan.physical.LookupJoinExec;
 import org.elasticsearch.xpack.esql.plan.physical.OutputExec;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.esql.planner.EsPhysicalOperationProviders;
@@ -81,7 +78,6 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -145,6 +141,7 @@ public void execute(
         CancellableTask rootTask,
         PhysicalPlan physicalPlan,
         Configuration configuration,
+        FoldContext foldContext,
         EsqlExecutionInfo execInfo,
         ActionListener<Result> listener
     ) {
@@ -167,11 +164,9 @@ public void execute(
         Map<String, OriginalIndices> clusterToConcreteIndices = transportService.getRemoteClusterService()
             .groupIndices(SearchRequest.DEFAULT_INDICES_OPTIONS, PlannerUtils.planConcreteIndices(physicalPlan).toArray(String[]::new));
         QueryPragmas queryPragmas = configuration.pragmas();
-        Set<String> lookupIndexNames = findLookupIndexNames(physicalPlan);
-        Set<String> concreteIndexNames = selectConcreteIndices(clusterToConcreteIndices, lookupIndexNames);
         if (dataNodePlan == null) {
-            if (concreteIndexNames.isEmpty() == false) {
-                String error = "expected no concrete indices without data node plan; got " + concreteIndexNames;
+            if (clusterToConcreteIndices.values().stream().allMatch(v -> v.indices().length == 0) == false) {
+                String error = "expected no concrete indices without data node plan; got " + clusterToConcreteIndices;
                 assert false : error;
                 listener.onFailure(new IllegalStateException(error));
                 return;
@@ -181,6 +176,7 @@ public void execute(
                 RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
                 List.of(),
                 configuration,
+                foldContext,
                 null,
                 null
             );
@@ -194,7 +190,7 @@ public void execute(
                 return;
             }
         } else {
-            if (concreteIndexNames.isEmpty()) {
+            if (clusterToConcreteIndices.values().stream().allMatch(v -> v.indices().length == 0)) {
                 var error = "expected concrete indices with data node plan but got empty; data node plan " + dataNodePlan;
                 assert false : error;
                 listener.onFailure(new IllegalStateException(error));
@@ -233,6 +229,7 @@ public void execute(
                         RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
                         List.of(),
                         configuration,
+                        foldContext,
                         exchangeSource,
                         null
                     ),
@@ -268,42 +265,6 @@ public void execute(
         }
     }
 
-    private Set<String> selectConcreteIndices(Map<String, OriginalIndices> clusterToConcreteIndices, Set<String> indexesToIgnore) {
-        Set<String> concreteIndexNames = new HashSet<>();
-        clusterToConcreteIndices.forEach((clusterAlias, concreteIndices) -> {
-            for (String index : concreteIndices.indices()) {
-                if (indexesToIgnore.contains(index) == false) {
-                    concreteIndexNames.add(index);
-                }
-            }
-        });
-        return concreteIndexNames;
-    }
-
-    private Set<String> findLookupIndexNames(PhysicalPlan physicalPlan) {
-        Set<String> lookupIndexNames = new HashSet<>();
-        // When planning JOIN on the coordinator node: "LookupJoinExec.lookup()->FragmentExec.fragment()->EsRelation.index()"
-        physicalPlan.forEachDown(
-            LookupJoinExec.class,
-            lookupJoinExec -> lookupJoinExec.lookup()
-                .forEachDown(
-                    FragmentExec.class,
-                    frag -> frag.fragment().forEachDown(EsRelation.class, esRelation -> lookupIndexNames.add(esRelation.index().name()))
-                )
-        );
-        // When planning JOIN on the data node: "FragmentExec.fragment()->Join.right()->EsRelation.index()"
-        // TODO this only works for LEFT join, so we still need to support RIGHT join
-        physicalPlan.forEachDown(
-            FragmentExec.class,
-            fragmentExec -> fragmentExec.fragment()
-                .forEachDown(
-                    Join.class,
-                    join -> join.right().forEachDown(EsRelation.class, esRelation -> lookupIndexNames.add(esRelation.index().name()))
-                )
-        );
-        return lookupIndexNames;
-    }
-
     // For queries like: FROM logs* | LIMIT 0 (including cross-cluster LIMIT 0 queries)
     private static void updateShardCountForCoordinatorOnlyQuery(EsqlExecutionInfo execInfo) {
         if (execInfo.isCrossClusterSearch()) {
@@ -401,19 +362,22 @@ private void startComputeOnDataNodes(
                             exchangeSource.addRemoteSink(remoteSink, true, queryPragmas.concurrentExchangeClients(), ActionListener.noop());
                             ActionListener<ComputeResponse> computeResponseListener = computeListener.acquireCompute(clusterAlias);
                             var dataNodeListener = ActionListener.runBefore(computeResponseListener, () -> l.onResponse(null));
+                            final boolean sameNode = transportService.getLocalNode().getId().equals(node.connection.getNode().getId());
+                            var dataNodeRequest = new DataNodeRequest(
+                                childSessionId,
+                                configuration,
+                                clusterAlias,
+                                node.shardIds,
+                                node.aliasFilters,
+                                dataNodePlan,
+                                originalIndices.indices(),
+                                originalIndices.indicesOptions(),
+                                sameNode == false && queryPragmas.nodeLevelReduction()
+                            );
                             transportService.sendChildRequest(
                                 node.connection,
                                 DATA_ACTION_NAME,
-                                new DataNodeRequest(
-                                    childSessionId,
-                                    configuration,
-                                    clusterAlias,
-                                    node.shardIds,
-                                    node.aliasFilters,
-                                    dataNodePlan,
-                                    originalIndices.indices(),
-                                    originalIndices.indicesOptions()
-                                ),
+                                dataNodeRequest,
                                 parentTask,
                                 TransportRequestOptions.EMPTY,
                                 new ActionListenerResponseHandler<>(dataNodeListener, ComputeResponse::new, esqlExecutor)
@@ -500,16 +464,16 @@ public SourceProvider createSourceProvider() {
                 context.exchangeSink(),
                 enrichLookupService,
                 lookupFromIndexService,
-                new EsPhysicalOperationProviders(contexts, searchService.getIndicesService().getAnalysis())
+                new EsPhysicalOperationProviders(context.foldCtx(), contexts, searchService.getIndicesService().getAnalysis())
             );
 
             LOGGER.debug("Received physical plan:\n{}", plan);
 
-            plan = PlannerUtils.localPlan(context.searchExecutionContexts(), context.configuration, plan);
+            plan = PlannerUtils.localPlan(context.searchExecutionContexts(), context.configuration, context.foldCtx(), plan);
             // the planner will also set the driver parallelism in LocalExecutionPlanner.LocalExecutionPlan (used down below)
             // it's doing this in the planning of EsQueryExec (the source of the data)
             // see also EsPhysicalOperationProviders.sourcePhysicalOperation
-            LocalExecutionPlanner.LocalExecutionPlan localExecutionPlan = planner.plan(plan);
+            LocalExecutionPlanner.LocalExecutionPlan localExecutionPlan = planner.plan(context.foldCtx(), plan);
             if (LOGGER.isDebugEnabled()) {
                 LOGGER.debug("Local execution plan:\n{}", localExecutionPlan.describe());
             }
@@ -755,7 +719,15 @@ public void onFailure(Exception e) {
             };
             acquireSearchContexts(clusterAlias, shardIds, configuration, request.aliasFilters(), ActionListener.wrap(searchContexts -> {
                 assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH, ESQL_WORKER_THREAD_POOL_NAME);
-                var computeContext = new ComputeContext(sessionId, clusterAlias, searchContexts, configuration, null, exchangeSink);
+                var computeContext = new ComputeContext(
+                    sessionId,
+                    clusterAlias,
+                    searchContexts,
+                    configuration,
+                    configuration.newFoldContext(),
+                    null,
+                    exchangeSink
+                );
                 runCompute(parentTask, computeContext, request.plan(), batchListener);
             }, batchListener::onFailure));
         }
@@ -806,6 +778,7 @@ private void runComputeOnDataNode(
                     request.clusterAlias(),
                     List.of(),
                     request.configuration(),
+                    new FoldContext(request.pragmas().foldLimit().getBytes()),
                     exchangeSource,
                     externalSink
                 ),
@@ -846,7 +819,7 @@ public void messageReceived(DataNodeRequest request, TransportChannel channel, T
             final ActionListener<ComputeResponse> listener = new ChannelActionListener<>(channel);
             final PhysicalPlan reductionPlan;
             if (request.plan() instanceof ExchangeSinkExec plan) {
-                reductionPlan = reductionPlan(plan, request.pragmas().nodeLevelReduction());
+                reductionPlan = reductionPlan(plan, request.runNodeLevelReduction());
             } else {
                 listener.onFailure(new IllegalStateException("expected exchange sink for a remote compute; got " + request.plan()));
                 return;
@@ -860,7 +833,8 @@ public void messageReceived(DataNodeRequest request, TransportChannel channel, T
                 request.aliasFilters(),
                 request.plan(),
                 request.indices(),
-                request.indicesOptions()
+                request.indicesOptions(),
+                request.runNodeLevelReduction()
             );
             try (var computeListener = ComputeListener.create(transportService, (CancellableTask) task, listener)) {
                 runComputeOnDataNode((CancellableTask) task, sessionId, reductionPlan, request, computeListener);
@@ -940,7 +914,15 @@ void runComputeOnRemoteCluster(
             exchangeSink.addCompletionListener(computeListener.acquireAvoid());
             runCompute(
                 parentTask,
-                new ComputeContext(localSessionId, clusterAlias, List.of(), configuration, exchangeSource, exchangeSink),
+                new ComputeContext(
+                    localSessionId,
+                    clusterAlias,
+                    List.of(),
+                    configuration,
+                    configuration.newFoldContext(),
+                    exchangeSource,
+                    exchangeSink
+                ),
                 coordinatorPlan,
                 computeListener.acquireCompute(clusterAlias)
             );
@@ -964,6 +946,7 @@ record ComputeContext(
         String clusterAlias,
         List<SearchContext> searchContexts,
         Configuration configuration,
+        FoldContext foldCtx,
         ExchangeSourceHandler exchangeSource,
         ExchangeSinkHandler exchangeSink
     ) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequest.java
index b6fa82360d1e8..0f113ab78efc0 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequest.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequest.java
@@ -54,6 +54,7 @@ final class DataNodeRequest extends TransportRequest implements IndicesRequest.R
     private List<ShardId> shardIds;
     private String[] indices;
     private final IndicesOptions indicesOptions;
+    private final boolean runNodeLevelReduction;
 
     DataNodeRequest(
         String sessionId,
@@ -63,7 +64,8 @@ final class DataNodeRequest extends TransportRequest implements IndicesRequest.R
         Map<Index, AliasFilter> aliasFilters,
         PhysicalPlan plan,
         String[] indices,
-        IndicesOptions indicesOptions
+        IndicesOptions indicesOptions,
+        boolean runNodeLevelReduction
     ) {
         this.sessionId = sessionId;
         this.configuration = configuration;
@@ -73,6 +75,7 @@ final class DataNodeRequest extends TransportRequest implements IndicesRequest.R
         this.plan = plan;
         this.indices = indices;
         this.indicesOptions = indicesOptions;
+        this.runNodeLevelReduction = runNodeLevelReduction;
     }
 
     DataNodeRequest(StreamInput in) throws IOException {
@@ -97,6 +100,11 @@ final class DataNodeRequest extends TransportRequest implements IndicesRequest.R
             this.indices = shardIds.stream().map(ShardId::getIndexName).distinct().toArray(String[]::new);
             this.indicesOptions = IndicesOptions.strictSingleIndexNoExpandForbidClosed();
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_ENABLE_NODE_LEVEL_REDUCTION)) {
+            this.runNodeLevelReduction = in.readBoolean();
+        } else {
+            this.runNodeLevelReduction = false;
+        }
     }
 
     @Override
@@ -114,6 +122,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeStringArray(indices);
             indicesOptions.writeIndicesOptions(out);
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_ENABLE_NODE_LEVEL_REDUCTION)) {
+            out.writeBoolean(runNodeLevelReduction);
+        }
     }
 
     @Override
@@ -186,6 +197,10 @@ PhysicalPlan plan() {
         return plan;
     }
 
+    boolean runNodeLevelReduction() {
+        return runNodeLevelReduction;
+    }
+
     @Override
     public String getDescription() {
         return "shards=" + shardIds + " plan=" + plan;
@@ -209,11 +224,22 @@ public boolean equals(Object o) {
             && plan.equals(request.plan)
             && getParentTask().equals(request.getParentTask())
             && Arrays.equals(indices, request.indices)
-            && indicesOptions.equals(request.indicesOptions);
+            && indicesOptions.equals(request.indicesOptions)
+            && runNodeLevelReduction == request.runNodeLevelReduction;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(sessionId, configuration, clusterAlias, shardIds, aliasFilters, plan, Arrays.hashCode(indices), indicesOptions);
+        return Objects.hash(
+            sessionId,
+            configuration,
+            clusterAlias,
+            shardIds,
+            aliasFilters,
+            plan,
+            Arrays.hashCode(indices),
+            indicesOptions,
+            runNodeLevelReduction
+        );
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java
index a347a6947bf67..89b4231a999d4 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java
@@ -24,129 +24,16 @@
  * examples below are *just* used for that. Don't make more of those - add them
  * to {@link EsqlCapabilities} instead.
  * <p>
- *     NOTE: You can't remove a feature now and probably never will be able to.
+ *     NOTE: You can only remove features on major version boundaries.
  *     Only add more of these if you need a fast CPU level check.
  * </p>
  */
 public class EsqlFeatures implements FeatureSpecification {
-    /**
-     * Introduction of {@code MV_SORT}, {@code MV_SLICE}, and {@code MV_ZIP}.
-     * Added in #106095.
-     */
-    private static final NodeFeature MV_SORT = new NodeFeature("esql.mv_sort");
-
-    /**
-     * When we disabled some broken optimizations around {@code nullable}.
-     * Fixed in #105691.
-     */
-    private static final NodeFeature DISABLE_NULLABLE_OPTS = new NodeFeature("esql.disable_nullable_opts");
-
-    /**
-     * Introduction of {@code ST_X} and {@code ST_Y}. Added in #105768.
-     */
-    private static final NodeFeature ST_X_Y = new NodeFeature("esql.st_x_y");
-
-    /**
-     * Changed precision of {@code geo_point} and {@code cartesian_point} fields, by loading from source into WKB. Done in #103691.
-     */
-    private static final NodeFeature SPATIAL_POINTS_FROM_SOURCE = new NodeFeature("esql.spatial_points_from_source");
-
-    /**
-     * Support for loading {@code geo_shape} and {@code cartesian_shape} fields. Done in #104269.
-     */
-    private static final NodeFeature SPATIAL_SHAPES = new NodeFeature("esql.spatial_shapes");
-
-    /**
-     * Support for spatial aggregation {@code ST_CENTROID}. Done in #104269.
-     */
-    private static final NodeFeature ST_CENTROID_AGG = new NodeFeature("esql.st_centroid_agg");
-
-    /**
-     * Support for spatial aggregation {@code ST_INTERSECTS}. Done in #104907.
-     */
-    private static final NodeFeature ST_INTERSECTS = new NodeFeature("esql.st_intersects");
-
-    /**
-     * Support for spatial aggregation {@code ST_CONTAINS} and {@code ST_WITHIN}. Done in #106503.
-     */
-    private static final NodeFeature ST_CONTAINS_WITHIN = new NodeFeature("esql.st_contains_within");
-
-    /**
-     * Support for spatial aggregation {@code ST_DISJOINT}. Done in #107007.
-     */
-    private static final NodeFeature ST_DISJOINT = new NodeFeature("esql.st_disjoint");
-
-    /**
-     * The introduction of the {@code VALUES} agg.
-     */
-    private static final NodeFeature AGG_VALUES = new NodeFeature("esql.agg_values");
-
-    /**
-     * Does ESQL support async queries.
-     */
-    public static final NodeFeature ASYNC_QUERY = new NodeFeature("esql.async_query");
-
-    /**
-     * Does ESQL support FROM OPTIONS?
-     */
-    @Deprecated
-    public static final NodeFeature FROM_OPTIONS = new NodeFeature("esql.from_options");
-
-    /**
-     * Cast string literals to a desired data type.
-     */
-    public static final NodeFeature STRING_LITERAL_AUTO_CASTING = new NodeFeature("esql.string_literal_auto_casting");
-
-    /**
-     * Base64 encoding and decoding functions.
-     */
-    public static final NodeFeature BASE64_DECODE_ENCODE = new NodeFeature("esql.base64_decode_encode");
-
-    /**
-     * Support for the :: casting operator
-     */
-    public static final NodeFeature CASTING_OPERATOR = new NodeFeature("esql.casting_operator");
-
-    /**
-     * Blocks can be labelled with {@link org.elasticsearch.compute.data.Block.MvOrdering#SORTED_ASCENDING} for optimizations.
-     */
-    public static final NodeFeature MV_ORDERING_SORTED_ASCENDING = new NodeFeature("esql.mv_ordering_sorted_ascending");
-
-    /**
-     * Support for metrics counter fields
-     */
-    public static final NodeFeature METRICS_COUNTER_FIELDS = new NodeFeature("esql.metrics_counter_fields");
-
-    /**
-     * Cast string literals to a desired data type for IN predicate and more types for BinaryComparison.
-     */
-    public static final NodeFeature STRING_LITERAL_AUTO_CASTING_EXTENDED = new NodeFeature("esql.string_literal_auto_casting_extended");
-
-    /**
-     * Support for metadata fields.
-     */
-    public static final NodeFeature METADATA_FIELDS = new NodeFeature("esql.metadata_fields");
-
-    /**
-     * Support for timespan units abbreviations
-     */
-    public static final NodeFeature TIMESPAN_ABBREVIATIONS = new NodeFeature("esql.timespan_abbreviations");
-
-    /**
-     * Support metrics counter types
-     */
-    public static final NodeFeature COUNTER_TYPES = new NodeFeature("esql.counter_types");
-
     /**
      * Support metrics syntax
      */
     public static final NodeFeature METRICS_SYNTAX = new NodeFeature("esql.metrics_syntax");
 
-    /**
-     * Internal resolve_fields API for ES|QL
-     */
-    public static final NodeFeature RESOLVE_FIELDS_API = new NodeFeature("esql.resolve_fields_api");
-
     private Set<NodeFeature> snapshotBuildFeatures() {
         assert Build.current().isSnapshot() : Build.current();
         return Set.of(METRICS_SYNTAX);
@@ -154,30 +41,7 @@ private Set<NodeFeature> snapshotBuildFeatures() {
 
     @Override
     public Set<NodeFeature> getFeatures() {
-        Set<NodeFeature> features = Set.of(
-            ASYNC_QUERY,
-            AGG_VALUES,
-            BASE64_DECODE_ENCODE,
-            MV_SORT,
-            DISABLE_NULLABLE_OPTS,
-            ST_X_Y,
-            FROM_OPTIONS,
-            SPATIAL_POINTS_FROM_SOURCE,
-            SPATIAL_SHAPES,
-            ST_CENTROID_AGG,
-            ST_INTERSECTS,
-            ST_CONTAINS_WITHIN,
-            ST_DISJOINT,
-            STRING_LITERAL_AUTO_CASTING,
-            CASTING_OPERATOR,
-            MV_ORDERING_SORTED_ASCENDING,
-            METRICS_COUNTER_FIELDS,
-            STRING_LITERAL_AUTO_CASTING_EXTENDED,
-            METADATA_FIELDS,
-            TIMESPAN_ABBREVIATIONS,
-            COUNTER_TYPES,
-            RESOLVE_FIELDS_API
-        );
+        Set<NodeFeature> features = Set.of();
         if (Build.current().isSnapshot()) {
             return Collections.unmodifiableSet(Sets.union(features, snapshotBuildFeatures()));
         } else {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
index 67948fe717f2f..b79dda900f39c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
@@ -61,6 +61,7 @@
 import org.elasticsearch.xpack.esql.action.RestEsqlGetAsyncResultAction;
 import org.elasticsearch.xpack.esql.action.RestEsqlQueryAction;
 import org.elasticsearch.xpack.esql.enrich.EnrichLookupOperator;
+import org.elasticsearch.xpack.esql.enrich.LookupFromIndexOperator;
 import org.elasticsearch.xpack.esql.execution.PlanExecutor;
 import org.elasticsearch.xpack.esql.expression.ExpressionWritables;
 import org.elasticsearch.xpack.esql.plan.PlanWritables;
@@ -192,6 +193,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         entries.add(SingleValueQuery.ENTRY);
         entries.add(AsyncOperator.Status.ENTRY);
         entries.add(EnrichLookupOperator.Status.ENTRY);
+        entries.add(LookupFromIndexOperator.Status.ENTRY);
 
         entries.addAll(BlockWritables.getNamedWriteables());
         entries.addAll(ExpressionWritables.getNamedWriteables());
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/QueryPragmas.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/QueryPragmas.java
index e68debe1ab2a6..2443c3f2cda62 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/QueryPragmas.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/QueryPragmas.java
@@ -12,12 +12,14 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.compute.lucene.DataPartitioning;
 import org.elasticsearch.compute.operator.Driver;
 import org.elasticsearch.compute.operator.DriverStatus;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -55,7 +57,9 @@ public final class QueryPragmas implements Writeable {
 
     public static final Setting<Integer> MAX_CONCURRENT_SHARDS_PER_NODE = Setting.intSetting("max_concurrent_shards_per_node", 10, 1, 100);
 
-    public static final Setting<Boolean> NODE_LEVEL_REDUCTION = Setting.boolSetting("node_level_reduction", false);
+    public static final Setting<Boolean> NODE_LEVEL_REDUCTION = Setting.boolSetting("node_level_reduction", true);
+
+    public static final Setting<ByteSizeValue> FOLD_LIMIT = Setting.memorySizeSetting("fold_limit", "5%");
 
     public static final QueryPragmas EMPTY = new QueryPragmas(Settings.EMPTY);
 
@@ -134,6 +138,17 @@ public boolean nodeLevelReduction() {
         return NODE_LEVEL_REDUCTION.get(settings);
     }
 
+    /**
+     * The maximum amount of memory we can use for {@link Expression#fold} during planing. This
+     * defaults to 5% of memory available on the current node. If this method is called on the
+     * coordinating node, this is 5% of the coordinating node's memory. If it's called on a data
+     * node, it's 5% of the data node. That's an <strong>exciting</strong> inconsistency. But it's
+     * important. Bigger nodes have more space to do folding.
+     */
+    public ByteSizeValue foldLimit() {
+        return FOLD_LIMIT.get(settings);
+    }
+
     public boolean isEmpty() {
         return settings.isEmpty();
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java
index 8dee3478b3b64..61ac67674b252 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java
@@ -28,10 +28,6 @@ final class ReinitializingSourceProvider implements SourceProvider {
     private PerThreadSourceProvider perThreadProvider;
     private final Supplier<SourceProvider> sourceProviderFactory;
 
-    // Keeping track of last seen doc and if current doc is before last seen doc then source provider is initialized:
-    // (when source mode is synthetic then _source is read from doc values and doc values don't support going backwards)
-    private int lastSeenDocId;
-
     ReinitializingSourceProvider(Supplier<SourceProvider> sourceProviderFactory) {
         this.sourceProviderFactory = sourceProviderFactory;
     }
@@ -40,15 +36,25 @@ final class ReinitializingSourceProvider implements SourceProvider {
     public Source getSource(LeafReaderContext ctx, int doc) throws IOException {
         var currentThread = Thread.currentThread();
         PerThreadSourceProvider provider = perThreadProvider;
-        if (provider == null || provider.creatingThread != currentThread || doc < lastSeenDocId) {
+        if (provider == null || provider.creatingThread != currentThread || doc < provider.lastSeenDocId) {
             provider = new PerThreadSourceProvider(sourceProviderFactory.get(), currentThread);
             this.perThreadProvider = provider;
         }
-        lastSeenDocId = doc;
+        provider.lastSeenDocId = doc;
         return provider.source.getSource(ctx, doc);
     }
 
-    private record PerThreadSourceProvider(SourceProvider source, Thread creatingThread) {
+    private static final class PerThreadSourceProvider {
+        final SourceProvider source;
+        final Thread creatingThread;
+        // Keeping track of last seen doc and if current doc is before last seen doc then source provider is initialized:
+        // (when source mode is synthetic then _source is read from doc values and doc values don't support going backwards)
+        int lastSeenDocId;
+
+        private PerThreadSourceProvider(SourceProvider source, Thread creatingThread) {
+            this.source = source;
+            this.creatingThread = creatingThread;
+        }
 
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java
index 76bfb95d07926..84173eeecc060 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java
@@ -9,9 +9,12 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
+import org.elasticsearch.action.admin.cluster.stats.CCSUsage;
+import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -19,16 +22,20 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.operator.exchange.ExchangeService;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.usage.UsageService;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.async.AsyncExecutionId;
+import org.elasticsearch.xpack.esql.VerificationException;
 import org.elasticsearch.xpack.esql.action.ColumnInfoImpl;
 import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo;
 import org.elasticsearch.xpack.esql.action.EsqlQueryAction;
@@ -36,12 +43,14 @@
 import org.elasticsearch.xpack.esql.action.EsqlQueryResponse;
 import org.elasticsearch.xpack.esql.action.EsqlQueryTask;
 import org.elasticsearch.xpack.esql.core.async.AsyncTaskManagementService;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.enrich.EnrichLookupService;
 import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolver;
 import org.elasticsearch.xpack.esql.enrich.LookupFromIndexService;
 import org.elasticsearch.xpack.esql.execution.PlanExecutor;
 import org.elasticsearch.xpack.esql.session.Configuration;
 import org.elasticsearch.xpack.esql.session.EsqlSession.PlanRunner;
+import org.elasticsearch.xpack.esql.session.QueryBuilderResolver;
 import org.elasticsearch.xpack.esql.session.Result;
 
 import java.io.IOException;
@@ -50,6 +59,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN;
 
@@ -68,6 +78,8 @@ public class TransportEsqlQueryAction extends HandledTransportAction<EsqlQueryRe
     private final LookupFromIndexService lookupFromIndexService;
     private final AsyncTaskManagementService<EsqlQueryRequest, EsqlQueryResponse, EsqlQueryTask> asyncTaskManagementService;
     private final RemoteClusterService remoteClusterService;
+    private final QueryBuilderResolver queryBuilderResolver;
+    private final UsageService usageService;
 
     @Inject
     @SuppressWarnings("this-escape")
@@ -82,8 +94,9 @@ public TransportEsqlQueryAction(
         BigArrays bigArrays,
         BlockFactory blockFactory,
         Client client,
-        NamedWriteableRegistry registry
-
+        NamedWriteableRegistry registry,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        UsageService usageService
     ) {
         // TODO replace SAME when removing workaround for https://github.com/elastic/elasticsearch/issues/97916
         super(EsqlQueryAction.NAME, transportService, actionFilters, EsqlQueryRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE);
@@ -121,6 +134,8 @@ public TransportEsqlQueryAction(
             bigArrays
         );
         this.remoteClusterService = transportService.getRemoteClusterService();
+        this.queryBuilderResolver = new QueryBuilderResolver(searchService, clusterService, transportService, indexNameExpressionResolver);
+        this.usageService = usageService;
     }
 
     @Override
@@ -175,11 +190,13 @@ private void innerExecute(Task task, EsqlQueryRequest request, ActionListener<Es
         // async-query uses EsqlQueryTask, so pull the EsqlExecutionInfo out of the task
         // sync query uses CancellableTask which does not have EsqlExecutionInfo, so create one
         EsqlExecutionInfo executionInfo = getOrCreateExecutionInfo(task, request);
+        FoldContext foldCtx = configuration.newFoldContext();
         PlanRunner planRunner = (plan, resultListener) -> computeService.execute(
             sessionId,
             (CancellableTask) task,
             plan,
             configuration,
+            foldCtx,
             executionInfo,
             resultListener
         );
@@ -187,12 +204,71 @@ private void innerExecute(Task task, EsqlQueryRequest request, ActionListener<Es
             request,
             sessionId,
             configuration,
+            foldCtx,
             enrichPolicyResolver,
             executionInfo,
             remoteClusterService,
             planRunner,
-            listener.map(result -> toResponse(task, request, configuration, result))
+            queryBuilderResolver,
+            ActionListener.wrap(result -> {
+                recordCCSTelemetry(task, executionInfo, request, null);
+                listener.onResponse(toResponse(task, request, configuration, result));
+            }, ex -> {
+                recordCCSTelemetry(task, executionInfo, request, ex);
+                listener.onFailure(ex);
+            })
         );
+
+    }
+
+    private void recordCCSTelemetry(Task task, EsqlExecutionInfo executionInfo, EsqlQueryRequest request, @Nullable Exception exception) {
+        if (executionInfo.isCrossClusterSearch() == false) {
+            return;
+        }
+
+        CCSUsage.Builder usageBuilder = new CCSUsage.Builder();
+        usageBuilder.setClientFromTask(task);
+        if (exception != null) {
+            if (exception instanceof VerificationException ve) {
+                CCSUsageTelemetry.Result failureType = classifyVerificationException(ve);
+                if (failureType != CCSUsageTelemetry.Result.UNKNOWN) {
+                    usageBuilder.setFailure(failureType);
+                } else {
+                    usageBuilder.setFailure(exception);
+                }
+            } else {
+                usageBuilder.setFailure(exception);
+            }
+        }
+        var took = executionInfo.overallTook();
+        if (took != null) {
+            usageBuilder.took(took.getMillis());
+        }
+        if (request.async()) {
+            usageBuilder.setFeature(CCSUsageTelemetry.ASYNC_FEATURE);
+        }
+
+        AtomicInteger remotesCount = new AtomicInteger();
+        executionInfo.getClusters().forEach((clusterAlias, cluster) -> {
+            if (cluster.getStatus() == EsqlExecutionInfo.Cluster.Status.SKIPPED) {
+                usageBuilder.skippedRemote(clusterAlias);
+            } else {
+                usageBuilder.perClusterUsage(clusterAlias, cluster.getTook());
+            }
+            if (clusterAlias.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) == false) {
+                remotesCount.getAndIncrement();
+            }
+        });
+        assert remotesCount.get() > 0 : "Got cross-cluster search telemetry without any remote clusters";
+        usageBuilder.setRemotesCount(remotesCount.get());
+        usageService.getEsqlUsageHolder().updateUsage(usageBuilder.build());
+    }
+
+    private CCSUsageTelemetry.Result classifyVerificationException(VerificationException exception) {
+        if (exception.getDetailedMessage().contains("Unknown index")) {
+            return CCSUsageTelemetry.Result.NOT_FOUND;
+        }
+        return CCSUsageTelemetry.Result.UNKNOWN;
     }
 
     private EsqlExecutionInfo getOrCreateExecutionInfo(Task task, EsqlQueryRequest request) {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQuery.java
similarity index 97%
rename from x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQuery.java
rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQuery.java
index e6b6dc20c951a..1614b4f455456 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQuery.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQuery.java
@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.core.querydsl.query;
+package org.elasticsearch.xpack.esql.querydsl.query;
 
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.core.Booleans;
@@ -12,6 +12,7 @@
 import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 
 import java.util.Collections;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MultiMatchQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQuery.java
similarity index 95%
rename from x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MultiMatchQuery.java
rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQuery.java
index 71e3cb9fd494a..84524bad29e08 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MultiMatchQuery.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQuery.java
@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.core.querydsl.query;
+package org.elasticsearch.xpack.esql.querydsl.query;
 
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.core.Booleans;
@@ -12,8 +12,9 @@
 import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MultiMatchQueryPredicate;
+import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.predicate.fulltext.MultiMatchQueryPredicate;
 
 import java.util.Map;
 import java.util.Objects;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java
index 997f3265803f7..059ad175468a7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.compute.data.BlockStreamInput;
 import org.elasticsearch.xpack.esql.Column;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
 
 import java.io.IOException;
@@ -183,6 +184,13 @@ public long getQueryStartTimeNanos() {
         return queryStartTimeNanos;
     }
 
+    /**
+     * Create a new {@link FoldContext} with the limit configured in the {@link QueryPragmas}.
+     */
+    public FoldContext newFoldContext() {
+        return new FoldContext(pragmas.foldLimit().getBytes());
+    }
+
     /**
      * Tables specified in the request.
      */
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
index 83480f6651abf..b10f766babb36 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.TriFunction;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.compute.data.Block;
@@ -40,6 +39,7 @@
 import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedStar;
@@ -77,10 +77,12 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
@@ -112,6 +114,7 @@ public interface PlanRunner {
     private final PhysicalPlanOptimizer physicalPlanOptimizer;
     private final PlanningMetrics planningMetrics;
     private final IndicesExpressionGrouper indicesExpressionGrouper;
+    private final QueryBuilderResolver queryBuilderResolver;
 
     public EsqlSession(
         String sessionId,
@@ -124,7 +127,8 @@ public EsqlSession(
         Mapper mapper,
         Verifier verifier,
         PlanningMetrics planningMetrics,
-        IndicesExpressionGrouper indicesExpressionGrouper
+        IndicesExpressionGrouper indicesExpressionGrouper,
+        QueryBuilderResolver queryBuilderResolver
     ) {
         this.sessionId = sessionId;
         this.configuration = configuration;
@@ -138,6 +142,7 @@ public EsqlSession(
         this.physicalPlanOptimizer = new PhysicalPlanOptimizer(new PhysicalOptimizerContext(configuration));
         this.planningMetrics = planningMetrics;
         this.indicesExpressionGrouper = indicesExpressionGrouper;
+        this.queryBuilderResolver = queryBuilderResolver;
     }
 
     public String sessionId() {
@@ -157,7 +162,16 @@ public void execute(EsqlQueryRequest request, EsqlExecutionInfo executionInfo, P
             new EsqlSessionCCSUtils.CssPartialErrorsActionListener(executionInfo, listener) {
                 @Override
                 public void onResponse(LogicalPlan analyzedPlan) {
-                    executeOptimizedPlan(request, executionInfo, planRunner, optimizedPlan(analyzedPlan), listener);
+                    try {
+                        var optimizedPlan = optimizedPlan(analyzedPlan);
+                        queryBuilderResolver.resolveQueryBuilders(
+                            optimizedPlan,
+                            listener,
+                            (newPlan, next) -> executeOptimizedPlan(request, executionInfo, planRunner, newPlan, next)
+                        );
+                    } catch (Exception e) {
+                        listener.onFailure(e);
+                    }
                 }
             }
         );
@@ -282,10 +296,10 @@ public void analyzedPlan(
             return;
         }
 
-        TriFunction<IndexResolution, IndexResolution, EnrichResolution, LogicalPlan> analyzeAction = (indices, lookupIndices, policies) -> {
+        Function<PreAnalysisResult, LogicalPlan> analyzeAction = (l) -> {
             planningMetrics.gatherPreAnalysisMetrics(parsed);
             Analyzer analyzer = new Analyzer(
-                new AnalyzerContext(configuration, functionRegistry, indices, lookupIndices, policies),
+                new AnalyzerContext(configuration, functionRegistry, l.indices, l.lookupIndices, l.enrichResolution),
                 verifier
             );
             LogicalPlan plan = analyzer.analyze(parsed);
@@ -295,116 +309,88 @@ public void analyzedPlan(
 
         PreAnalyzer.PreAnalysis preAnalysis = preAnalyzer.preAnalyze(parsed);
         var unresolvedPolicies = preAnalysis.enriches.stream()
-            .map(e -> new EnrichPolicyResolver.UnresolvedPolicy((String) e.policyName().fold(), e.mode()))
+            .map(
+                e -> new EnrichPolicyResolver.UnresolvedPolicy(
+                    (String) e.policyName().fold(FoldContext.small() /* TODO remove me*/),
+                    e.mode()
+                )
+            )
             .collect(Collectors.toSet());
         final List<TableInfo> indices = preAnalysis.indices;
 
-        EsqlSessionCCSUtils.checkForCcsLicense(indices, indicesExpressionGrouper, verifier.licenseState());
+        EsqlSessionCCSUtils.checkForCcsLicense(executionInfo, indices, indicesExpressionGrouper, verifier.licenseState());
 
-        // TODO: make a separate call for lookup indices
         final Set<String> targetClusters = enrichPolicyResolver.groupIndicesPerCluster(
             indices.stream().flatMap(t -> Arrays.stream(Strings.commaDelimitedListToStringArray(t.id().index()))).toArray(String[]::new)
         ).keySet();
 
-        SubscribableListener.<EnrichResolution>newForked(l -> enrichPolicyResolver.resolvePolicies(targetClusters, unresolvedPolicies, l))
-            .<ListenerResult>andThen((l, enrichResolution) -> {
-                // we need the match_fields names from enrich policies and THEN, with an updated list of fields, we call field_caps API
-                var enrichMatchFields = enrichResolution.resolvedEnrichPolicies()
-                    .stream()
-                    .map(ResolvedEnrichPolicy::matchField)
-                    .collect(Collectors.toSet());
-                // get the field names from the parsed plan combined with the ENRICH match fields from the ENRICH policy
-                var fieldNames = fieldNames(parsed, enrichMatchFields);
-                ListenerResult listenerResult = new ListenerResult(null, null, enrichResolution, fieldNames);
-
-                // first resolve the lookup indices, then the main indices
-                preAnalyzeLookupIndices(preAnalysis.lookupIndices, listenerResult, l);
-            })
-            .<ListenerResult>andThen((l, listenerResult) -> {
-                // resolve the main indices
-                preAnalyzeIndices(preAnalysis.indices, executionInfo, listenerResult, requestFilter, l);
-            })
-            .<ListenerResult>andThen((l, listenerResult) -> {
-                // TODO in follow-PR (for skip_unavailable handling of missing concrete indexes) add some tests for
-                // invalid index resolution to updateExecutionInfo
-                if (listenerResult.indices.isValid()) {
-                    // CCS indices and skip_unavailable cluster values can stop the analysis right here
-                    if (analyzeCCSIndices(executionInfo, targetClusters, unresolvedPolicies, listenerResult, logicalPlanListener, l))
-                        return;
-                }
-                // whatever tuple we have here (from CCS-special handling or from the original pre-analysis), pass it on to the next step
-                l.onResponse(listenerResult);
-            })
-            .<ListenerResult>andThen((l, listenerResult) -> {
-                // first attempt (maybe the only one) at analyzing the plan
-                analyzeAndMaybeRetry(analyzeAction, requestFilter, listenerResult, logicalPlanListener, l);
-            })
-            .<ListenerResult>andThen((l, listenerResult) -> {
-                assert requestFilter != null : "The second pre-analysis shouldn't take place when there is no index filter in the request";
-
-                // "reset" execution information for all ccs or non-ccs (local) clusters, since we are performing the indices
-                // resolving one more time (the first attempt failed and the query had a filter)
-                for (String clusterAlias : executionInfo.clusterAliases()) {
-                    executionInfo.swapCluster(clusterAlias, (k, v) -> null);
-                }
-
-                // here the requestFilter is set to null, performing the pre-analysis after the first step failed
-                preAnalyzeIndices(preAnalysis.indices, executionInfo, listenerResult, null, l);
-            })
-            .<LogicalPlan>andThen((l, listenerResult) -> {
-                assert requestFilter != null : "The second analysis shouldn't take place when there is no index filter in the request";
-                LOGGER.debug("Analyzing the plan (second attempt, without filter)");
-                LogicalPlan plan;
-                try {
-                    plan = analyzeAction.apply(listenerResult.indices, listenerResult.lookupIndices, listenerResult.enrichResolution);
-                } catch (Exception e) {
-                    l.onFailure(e);
-                    return;
-                }
-                LOGGER.debug("Analyzed plan (second attempt, without filter):\n{}", plan);
-                l.onResponse(plan);
-            })
-            .addListener(logicalPlanListener);
-    }
+        var listener = SubscribableListener.<EnrichResolution>newForked(
+            l -> enrichPolicyResolver.resolvePolicies(targetClusters, unresolvedPolicies, l)
+        ).<PreAnalysisResult>andThen((l, enrichResolution) -> resolveFieldNames(parsed, enrichResolution, l));
+        // first resolve the lookup indices, then the main indices
+        for (TableInfo lookupIndex : preAnalysis.lookupIndices) {
+            listener = listener.andThen((l, preAnalysisResult) -> { preAnalyzeLookupIndex(lookupIndex, preAnalysisResult, l); });
+        }
+        listener.<PreAnalysisResult>andThen((l, result) -> {
+            // resolve the main indices
+            preAnalyzeIndices(preAnalysis.indices, executionInfo, result, requestFilter, l);
+        }).<PreAnalysisResult>andThen((l, result) -> {
+            // TODO in follow-PR (for skip_unavailable handling of missing concrete indexes) add some tests for
+            // invalid index resolution to updateExecutionInfo
+            if (result.indices.isValid()) {
+                // CCS indices and skip_unavailable cluster values can stop the analysis right here
+                if (analyzeCCSIndices(executionInfo, targetClusters, unresolvedPolicies, result, logicalPlanListener, l)) return;
+            }
+            // whatever tuple we have here (from CCS-special handling or from the original pre-analysis), pass it on to the next step
+            l.onResponse(result);
+        }).<PreAnalysisResult>andThen((l, result) -> {
+            // first attempt (maybe the only one) at analyzing the plan
+            analyzeAndMaybeRetry(analyzeAction, requestFilter, result, logicalPlanListener, l);
+        }).<PreAnalysisResult>andThen((l, result) -> {
+            assert requestFilter != null : "The second pre-analysis shouldn't take place when there is no index filter in the request";
+
+            // "reset" execution information for all ccs or non-ccs (local) clusters, since we are performing the indices
+            // resolving one more time (the first attempt failed and the query had a filter)
+            for (String clusterAlias : executionInfo.clusterAliases()) {
+                executionInfo.swapCluster(clusterAlias, (k, v) -> null);
+            }
 
-    private void preAnalyzeLookupIndices(List<TableInfo> indices, ListenerResult listenerResult, ActionListener<ListenerResult> listener) {
-        if (indices.size() > 1) {
-            // Note: JOINs on more than one index are not yet supported
-            listener.onFailure(new MappingException("More than one LOOKUP JOIN is not supported"));
-        } else if (indices.size() == 1) {
-            TableInfo tableInfo = indices.get(0);
-            TableIdentifier table = tableInfo.id();
-            // call the EsqlResolveFieldsAction (field-caps) to resolve indices and get field types
-            indexResolver.resolveAsMergedMapping(
-                table.index(),
-                Set.of("*"), // TODO: for LOOKUP JOIN, this currently declares all lookup index fields relevant and might fetch too many.
-                null,
-                listener.map(indexResolution -> listenerResult.withLookupIndexResolution(indexResolution))
-            );
-            // TODO: Verify that the resolved index actually has indexMode: "lookup"
-        } else {
+            // here the requestFilter is set to null, performing the pre-analysis after the first step failed
+            preAnalyzeIndices(preAnalysis.indices, executionInfo, result, null, l);
+        }).<LogicalPlan>andThen((l, result) -> {
+            assert requestFilter != null : "The second analysis shouldn't take place when there is no index filter in the request";
+            LOGGER.debug("Analyzing the plan (second attempt, without filter)");
+            LogicalPlan plan;
             try {
-                // No lookup indices specified
-                listener.onResponse(
-                    new ListenerResult(
-                        listenerResult.indices,
-                        IndexResolution.invalid("[none specified]"),
-                        listenerResult.enrichResolution,
-                        listenerResult.fieldNames
-                    )
-                );
-            } catch (Exception ex) {
-                listener.onFailure(ex);
+                plan = analyzeAction.apply(result);
+            } catch (Exception e) {
+                l.onFailure(e);
+                return;
             }
-        }
+            LOGGER.debug("Analyzed plan (second attempt, without filter):\n{}", plan);
+            l.onResponse(plan);
+        }).addListener(logicalPlanListener);
+    }
+
+    private void preAnalyzeLookupIndex(TableInfo tableInfo, PreAnalysisResult result, ActionListener<PreAnalysisResult> listener) {
+        TableIdentifier table = tableInfo.id();
+        Set<String> fieldNames = result.wildcardJoinIndices().contains(table.index()) ? IndexResolver.ALL_FIELDS : result.fieldNames;
+        // call the EsqlResolveFieldsAction (field-caps) to resolve indices and get field types
+        indexResolver.resolveAsMergedMapping(
+            table.index(),
+            fieldNames,
+            null,
+            listener.map(indexResolution -> result.addLookupIndexResolution(table.index(), indexResolution))
+        );
+        // TODO: Verify that the resolved index actually has indexMode: "lookup"
     }
 
     private void preAnalyzeIndices(
         List<TableInfo> indices,
         EsqlExecutionInfo executionInfo,
-        ListenerResult listenerResult,
+        PreAnalysisResult result,
         QueryBuilder requestFilter,
-        ActionListener<ListenerResult> listener
+        ActionListener<PreAnalysisResult> listener
     ) {
         // TODO we plan to support joins in the future when possible, but for now we'll just fail early if we see one
         if (indices.size() > 1) {
@@ -412,7 +398,7 @@ private void preAnalyzeIndices(
             listener.onFailure(new MappingException("Queries with multiple indices are not supported"));
         } else if (indices.size() == 1) {
             // known to be unavailable from the enrich policy API call
-            Map<String, Exception> unavailableClusters = listenerResult.enrichResolution.getUnavailableClusters();
+            Map<String, Exception> unavailableClusters = result.enrichResolution.getUnavailableClusters();
             TableInfo tableInfo = indices.get(0);
             TableIdentifier table = tableInfo.id();
 
@@ -445,34 +431,20 @@ private void preAnalyzeIndices(
             String indexExpressionToResolve = EsqlSessionCCSUtils.createIndexExpressionFromAvailableClusters(executionInfo);
             if (indexExpressionToResolve.isEmpty()) {
                 // if this was a pure remote CCS request (no local indices) and all remotes are offline, return an empty IndexResolution
-                listener.onResponse(
-                    new ListenerResult(
-                        IndexResolution.valid(new EsIndex(table.index(), Map.of(), Map.of())),
-                        listenerResult.lookupIndices,
-                        listenerResult.enrichResolution,
-                        listenerResult.fieldNames
-                    )
-                );
+                listener.onResponse(result.withIndexResolution(IndexResolution.valid(new EsIndex(table.index(), Map.of(), Map.of()))));
             } else {
                 // call the EsqlResolveFieldsAction (field-caps) to resolve indices and get field types
                 indexResolver.resolveAsMergedMapping(
                     indexExpressionToResolve,
-                    listenerResult.fieldNames,
+                    result.fieldNames,
                     requestFilter,
-                    listener.map(indexResolution -> listenerResult.withIndexResolution(indexResolution))
+                    listener.map(indexResolution -> result.withIndexResolution(indexResolution))
                 );
             }
         } else {
             try {
                 // occurs when dealing with local relations (row a = 1)
-                listener.onResponse(
-                    new ListenerResult(
-                        IndexResolution.invalid("[none specified]"),
-                        listenerResult.lookupIndices,
-                        listenerResult.enrichResolution,
-                        listenerResult.fieldNames
-                    )
-                );
+                listener.onResponse(result.withIndexResolution(IndexResolution.invalid("[none specified]")));
             } catch (Exception ex) {
                 listener.onFailure(ex);
             }
@@ -483,11 +455,11 @@ private boolean analyzeCCSIndices(
         EsqlExecutionInfo executionInfo,
         Set<String> targetClusters,
         Set<EnrichPolicyResolver.UnresolvedPolicy> unresolvedPolicies,
-        ListenerResult listenerResult,
+        PreAnalysisResult result,
         ActionListener<LogicalPlan> logicalPlanListener,
-        ActionListener<ListenerResult> l
+        ActionListener<PreAnalysisResult> l
     ) {
-        IndexResolution indexResolution = listenerResult.indices;
+        IndexResolution indexResolution = result.indices;
         EsqlSessionCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution);
         EsqlSessionCCSUtils.updateExecutionInfoWithUnavailableClusters(executionInfo, indexResolution.unavailableClusters());
         if (executionInfo.isCrossClusterSearch() && executionInfo.getClusterStateCount(EsqlExecutionInfo.Cluster.Status.RUNNING) == 0) {
@@ -509,7 +481,7 @@ private boolean analyzeCCSIndices(
             enrichPolicyResolver.resolvePolicies(
                 newClusters,
                 unresolvedPolicies,
-                l.map(enrichResolution -> listenerResult.withEnrichResolution(enrichResolution))
+                l.map(enrichResolution -> result.withEnrichResolution(enrichResolution))
             );
             return true;
         }
@@ -517,11 +489,11 @@ private boolean analyzeCCSIndices(
     }
 
     private static void analyzeAndMaybeRetry(
-        TriFunction<IndexResolution, IndexResolution, EnrichResolution, LogicalPlan> analyzeAction,
+        Function<PreAnalysisResult, LogicalPlan> analyzeAction,
         QueryBuilder requestFilter,
-        ListenerResult listenerResult,
+        PreAnalysisResult result,
         ActionListener<LogicalPlan> logicalPlanListener,
-        ActionListener<ListenerResult> l
+        ActionListener<PreAnalysisResult> l
     ) {
         LogicalPlan plan = null;
         var filterPresentMessage = requestFilter == null ? "without" : "with";
@@ -529,7 +501,7 @@ private static void analyzeAndMaybeRetry(
         LOGGER.debug("Analyzing the plan ({} attempt, {} filter)", attemptMessage, filterPresentMessage);
 
         try {
-            plan = analyzeAction.apply(listenerResult.indices, listenerResult.lookupIndices, listenerResult.enrichResolution);
+            plan = analyzeAction.apply(result);
         } catch (Exception e) {
             if (e instanceof VerificationException ve) {
                 LOGGER.debug(
@@ -544,7 +516,7 @@ private static void analyzeAndMaybeRetry(
                 } else {
                     // interested only in a VerificationException, but this time we are taking out the index filter
                     // to try and make the index resolution work without any index filtering. In the next step... to be continued
-                    l.onResponse(listenerResult);
+                    l.onResponse(result);
                 }
             } else {
                 // if the query failed with any other type of exception, then just pass the exception back to the user
@@ -557,10 +529,24 @@ private static void analyzeAndMaybeRetry(
         logicalPlanListener.onResponse(plan);
     }
 
-    static Set<String> fieldNames(LogicalPlan parsed, Set<String> enrichPolicyMatchFields) {
+    private static void resolveFieldNames(LogicalPlan parsed, EnrichResolution enrichResolution, ActionListener<PreAnalysisResult> l) {
+        try {
+            // we need the match_fields names from enrich policies and THEN, with an updated list of fields, we call field_caps API
+            var enrichMatchFields = enrichResolution.resolvedEnrichPolicies()
+                .stream()
+                .map(ResolvedEnrichPolicy::matchField)
+                .collect(Collectors.toSet());
+            // get the field names from the parsed plan combined with the ENRICH match fields from the ENRICH policy
+            l.onResponse(fieldNames(parsed, enrichMatchFields, new PreAnalysisResult(enrichResolution)));
+        } catch (Exception ex) {
+            l.onFailure(ex);
+        }
+    }
+
+    static PreAnalysisResult fieldNames(LogicalPlan parsed, Set<String> enrichPolicyMatchFields, PreAnalysisResult result) {
         if (false == parsed.anyMatch(plan -> plan instanceof Aggregate || plan instanceof Project)) {
             // no explicit columns selection, for example "from employees"
-            return IndexResolver.ALL_FIELDS;
+            return result.withFieldNames(IndexResolver.ALL_FIELDS);
         }
 
         Holder<Boolean> projectAll = new Holder<>(false);
@@ -571,7 +557,7 @@ static Set<String> fieldNames(LogicalPlan parsed, Set<String> enrichPolicyMatchF
             projectAll.set(true);
         });
         if (projectAll.get()) {
-            return IndexResolver.ALL_FIELDS;
+            return result.withFieldNames(IndexResolver.ALL_FIELDS);
         }
 
         AttributeSet references = new AttributeSet();
@@ -579,6 +565,7 @@ static Set<String> fieldNames(LogicalPlan parsed, Set<String> enrichPolicyMatchF
         // ie "from test | eval lang = languages + 1 | keep *l" should consider both "languages" and "*l" as valid fields to ask for
         AttributeSet keepCommandReferences = new AttributeSet();
         AttributeSet keepJoinReferences = new AttributeSet();
+        Set<String> wildcardJoinIndices = new java.util.HashSet<>();
 
         parsed.forEachDown(p -> {// go over each plan top-down
             if (p instanceof RegexExtract re) { // for Grok and Dissect
@@ -596,10 +583,16 @@ static Set<String> fieldNames(LogicalPlan parsed, Set<String> enrichPolicyMatchF
                 enrichRefs.removeIf(attr -> attr instanceof EmptyAttribute);
                 references.addAll(enrichRefs);
             } else if (p instanceof LookupJoin join) {
-                keepJoinReferences.addAll(join.config().matchFields());  // TODO: why is this empty
                 if (join.config().type() instanceof JoinTypes.UsingJoinType usingJoinType) {
                     keepJoinReferences.addAll(usingJoinType.columns());
                 }
+                if (keepCommandReferences.isEmpty()) {
+                    // No KEEP commands after the JOIN, so we need to mark this index for "*" field resolution
+                    wildcardJoinIndices.add(((UnresolvedRelation) join.right()).table().index());
+                } else {
+                    // Keep commands can reference the join columns with names that shadow aliases, so we block their removal
+                    keepJoinReferences.addAll(keepCommandReferences);
+                }
             } else {
                 references.addAll(p.references());
                 if (p instanceof UnresolvedRelation ur && ur.indexMode() == IndexMode.TIME_SERIES) {
@@ -634,6 +627,10 @@ static Set<String> fieldNames(LogicalPlan parsed, Set<String> enrichPolicyMatchF
         });
         // Add JOIN ON column references afterward to avoid Alias removal
         references.addAll(keepJoinReferences);
+        // If any JOIN commands need wildcard field-caps calls, persist the index names
+        if (wildcardJoinIndices.isEmpty() == false) {
+            result = result.withWildcardJoinIndices(wildcardJoinIndices);
+        }
 
         // remove valid metadata attributes because they will be filtered out by the IndexResolver anyway
         // otherwise, in some edge cases, we will fail to ask for "*" (all fields) instead
@@ -642,12 +639,12 @@ static Set<String> fieldNames(LogicalPlan parsed, Set<String> enrichPolicyMatchF
 
         if (fieldNames.isEmpty() && enrichPolicyMatchFields.isEmpty()) {
             // there cannot be an empty list of fields, we'll ask the simplest and lightest one instead: _index
-            return IndexResolver.INDEX_METADATA_FIELD;
+            return result.withFieldNames(IndexResolver.INDEX_METADATA_FIELD);
         } else {
             fieldNames.addAll(subfields(fieldNames));
             fieldNames.addAll(enrichPolicyMatchFields);
             fieldNames.addAll(subfields(enrichPolicyMatchFields));
-            return fieldNames;
+            return result.withFieldNames(fieldNames);
         }
     }
 
@@ -706,22 +703,36 @@ public PhysicalPlan optimizedPhysicalPlan(LogicalPlan optimizedPlan) {
         return plan;
     }
 
-    private record ListenerResult(
+    record PreAnalysisResult(
         IndexResolution indices,
-        IndexResolution lookupIndices,
+        Map<String, IndexResolution> lookupIndices,
         EnrichResolution enrichResolution,
-        Set<String> fieldNames
+        Set<String> fieldNames,
+        Set<String> wildcardJoinIndices
     ) {
-        ListenerResult withEnrichResolution(EnrichResolution newEnrichResolution) {
-            return new ListenerResult(indices(), lookupIndices(), newEnrichResolution, fieldNames());
+        PreAnalysisResult(EnrichResolution newEnrichResolution) {
+            this(null, new HashMap<>(), newEnrichResolution, Set.of(), Set.of());
+        }
+
+        PreAnalysisResult withEnrichResolution(EnrichResolution newEnrichResolution) {
+            return new PreAnalysisResult(indices(), lookupIndices(), newEnrichResolution, fieldNames(), wildcardJoinIndices());
         }
 
-        ListenerResult withIndexResolution(IndexResolution newIndexResolution) {
-            return new ListenerResult(newIndexResolution, lookupIndices(), enrichResolution(), fieldNames());
+        PreAnalysisResult withIndexResolution(IndexResolution newIndexResolution) {
+            return new PreAnalysisResult(newIndexResolution, lookupIndices(), enrichResolution(), fieldNames(), wildcardJoinIndices());
         }
 
-        ListenerResult withLookupIndexResolution(IndexResolution newIndexResolution) {
-            return new ListenerResult(indices(), newIndexResolution, enrichResolution(), fieldNames());
+        PreAnalysisResult addLookupIndexResolution(String index, IndexResolution newIndexResolution) {
+            lookupIndices.put(index, newIndexResolution);
+            return this;
         }
-    };
+
+        PreAnalysisResult withFieldNames(Set<String> newFields) {
+            return new PreAnalysisResult(indices(), lookupIndices(), enrichResolution(), newFields, wildcardJoinIndices());
+        }
+
+        public PreAnalysisResult withWildcardJoinIndices(Set<String> wildcardJoinIndices) {
+            return new PreAnalysisResult(indices(), lookupIndices(), enrichResolution(), fieldNames(), wildcardJoinIndices);
+        }
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java
index 662572c466511..f8670a8e6d053 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java
@@ -211,6 +211,10 @@ static void updateExecutionInfoWithClustersWithNoMatchingIndices(EsqlExecutionIn
          * Mark it as SKIPPED with 0 shards searched and took=0.
          */
         for (String c : clustersWithNoMatchingIndices) {
+            if (executionInfo.getCluster(c).getStatus() == EsqlExecutionInfo.Cluster.Status.SKIPPED) {
+                // if cluster was already marked SKIPPED during enrich policy resolution, do not overwrite
+                continue;
+            }
             final String indexExpression = executionInfo.getCluster(c).getIndexExpression();
             if (missingIndicesIsFatal(c, executionInfo)) {
                 String error = Strings.format(
@@ -308,6 +312,7 @@ static void updateExecutionInfoAtEndOfPlanning(EsqlExecutionInfo execInfo) {
      * @throws org.elasticsearch.ElasticsearchStatusException if the license is not valid (or present) for ES|QL CCS search.
      */
     public static void checkForCcsLicense(
+        EsqlExecutionInfo executionInfo,
         List<TableInfo> indices,
         IndicesExpressionGrouper indicesGrouper,
         XPackLicenseState licenseState
@@ -326,6 +331,17 @@ public static void checkForCcsLicense(
             // check if it is a cross-cluster query
             if (groupedIndices.size() > 1 || groupedIndices.containsKey(RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY) == false) {
                 if (EsqlLicenseChecker.isCcsAllowed(licenseState) == false) {
+                    // initialize the cluster entries in EsqlExecutionInfo before throwing the invalid license error
+                    // so that the CCS telemetry handler can recognize that this error is CCS-related
+                    for (Map.Entry<String, OriginalIndices> entry : groupedIndices.entrySet()) {
+                        executionInfo.swapCluster(
+                            entry.getKey(),
+                            (k, v) -> new EsqlExecutionInfo.Cluster(
+                                entry.getKey(),
+                                Strings.arrayToCommaDelimitedString(entry.getValue().indices())
+                            )
+                        );
+                    }
                     throw EsqlLicenseChecker.invalidLicenseForCcsException(licenseState);
                 }
             }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/QueryBuilderResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/QueryBuilderResolver.java
new file mode 100644
index 0000000000000..ab4a9b02e41fa
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/QueryBuilderResolver.java
@@ -0,0 +1,168 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.session;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ResolvedIndices;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.Rewriteable;
+import org.elasticsearch.search.SearchService;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.esql.core.util.Holder;
+import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction;
+import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
+import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.planner.PlannerUtils;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiConsumer;
+
+/**
+ * Some {@link FullTextFunction} implementations such as {@link org.elasticsearch.xpack.esql.expression.function.fulltext.Match}
+ * will be translated to a {@link QueryBuilder} that require a rewrite phase on the coordinator.
+ * {@link QueryBuilderResolver#resolveQueryBuilders(LogicalPlan, ActionListener, BiConsumer)} will rewrite the plan by replacing
+ * {@link FullTextFunction} expression with new ones that hold rewritten {@link QueryBuilder}s.
+ */
+public class QueryBuilderResolver {
+    private final SearchService searchService;
+    private final ClusterService clusterService;
+    private final TransportService transportService;
+    private final IndexNameExpressionResolver indexNameExpressionResolver;
+
+    public QueryBuilderResolver(
+        SearchService searchService,
+        ClusterService clusterService,
+        TransportService transportService,
+        IndexNameExpressionResolver indexNameExpressionResolver
+    ) {
+        this.searchService = searchService;
+        this.clusterService = clusterService;
+        this.transportService = transportService;
+        this.indexNameExpressionResolver = indexNameExpressionResolver;
+    }
+
+    public void resolveQueryBuilders(
+        LogicalPlan plan,
+        ActionListener<Result> listener,
+        BiConsumer<LogicalPlan, ActionListener<Result>> callback
+    ) {
+        if (plan.optimized() == false) {
+            listener.onFailure(new IllegalStateException("Expected optimized plan before query builder rewrite."));
+            return;
+        }
+
+        Set<FullTextFunction> unresolved = fullTextFunctions(plan);
+        Set<String> indexNames = indexNames(plan);
+
+        if (indexNames == null || indexNames.isEmpty() || unresolved.isEmpty()) {
+            callback.accept(plan, listener);
+            return;
+        }
+        QueryRewriteContext ctx = queryRewriteContext(indexNames);
+        FullTextFunctionsRewritable rewritable = new FullTextFunctionsRewritable(unresolved);
+        Rewriteable.rewriteAndFetch(rewritable, ctx, new ActionListener<FullTextFunctionsRewritable>() {
+            @Override
+            public void onResponse(FullTextFunctionsRewritable fullTextFunctionsRewritable) {
+                try {
+                    LogicalPlan newPlan = planWithResolvedQueryBuilders(plan, fullTextFunctionsRewritable.results());
+                    callback.accept(newPlan, listener);
+                } catch (Exception e) {
+                    onFailure(e);
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                listener.onFailure(e);
+            }
+        });
+    }
+
+    private Set<FullTextFunction> fullTextFunctions(LogicalPlan plan) {
+        Set<FullTextFunction> functions = new HashSet<>();
+        plan.forEachExpressionDown(FullTextFunction.class, func -> functions.add(func));
+        return functions;
+    }
+
+    public Set<String> indexNames(LogicalPlan plan) {
+        Holder<Set<String>> indexNames = new Holder<>();
+
+        plan.forEachDown(EsRelation.class, esRelation -> { indexNames.set(esRelation.index().concreteIndices()); });
+
+        return indexNames.get();
+    }
+
+    public LogicalPlan planWithResolvedQueryBuilders(LogicalPlan plan, Map<FullTextFunction, QueryBuilder> newQueryBuilders) {
+        LogicalPlan newPlan = plan.transformExpressionsDown(FullTextFunction.class, m -> {
+            if (newQueryBuilders.keySet().contains(m)) {
+                return m.replaceQueryBuilder(newQueryBuilders.get(m));
+            }
+            return m;
+        });
+        // The given plan was already analyzed and optimized, so we set the resulted plan to optimized as well.
+        newPlan.setOptimized();
+        return newPlan;
+    }
+
+    private QueryRewriteContext queryRewriteContext(Set<String> indexNames) {
+        ResolvedIndices resolvedIndices = ResolvedIndices.resolveWithIndexNamesAndOptions(
+            indexNames.toArray(String[]::new),
+            IndexResolver.FIELD_CAPS_INDICES_OPTIONS,
+            clusterService.state(),
+            indexNameExpressionResolver,
+            transportService.getRemoteClusterService(),
+            System.currentTimeMillis()
+        );
+
+        return searchService.getRewriteContext(() -> System.currentTimeMillis(), resolvedIndices, null);
+    }
+
+    private class FullTextFunctionsRewritable implements Rewriteable<FullTextFunctionsRewritable> {
+
+        private final Map<FullTextFunction, QueryBuilder> queryBuilderMap;
+
+        FullTextFunctionsRewritable(Map<FullTextFunction, QueryBuilder> queryBuilderMap) {
+            this.queryBuilderMap = queryBuilderMap;
+        }
+
+        FullTextFunctionsRewritable(Set<FullTextFunction> functions) {
+            this.queryBuilderMap = new HashMap<>();
+
+            for (FullTextFunction func : functions) {
+                queryBuilderMap.put(func, func.asQuery(PlannerUtils.TRANSLATOR_HANDLER).asBuilder());
+            }
+        }
+
+        @Override
+        public FullTextFunctionsRewritable rewrite(QueryRewriteContext ctx) throws IOException {
+            Map<FullTextFunction, QueryBuilder> results = new HashMap<>();
+
+            boolean hasChanged = false;
+            for (var entry : queryBuilderMap.entrySet()) {
+                var initial = entry.getValue();
+                var rewritten = initial.rewrite(ctx);
+                hasChanged |= rewritten != initial;
+
+                results.put(entry.getKey(), rewritten);
+            }
+
+            return hasChanged ? new FullTextFunctionsRewritable(results) : this;
+        }
+
+        public Map<FullTextFunction, QueryBuilder> results() {
+            return queryBuilderMap;
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java
index 6ba2d8451f956..95ee6ab337bd6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java
@@ -13,11 +13,14 @@
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.common.time.DateFormatters;
+import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.Converter;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -51,7 +54,6 @@
 import java.time.Period;
 import java.time.ZoneId;
 import java.time.temporal.ChronoField;
-import java.time.temporal.TemporalAccessor;
 import java.time.temporal.TemporalAmount;
 import java.util.List;
 import java.util.Locale;
@@ -200,6 +202,9 @@ public static Converter converterFor(DataType from, DataType to) {
             if (to == DataType.DATETIME) {
                 return EsqlConverter.STRING_TO_DATETIME;
             }
+            if (to == DATE_NANOS) {
+                return EsqlConverter.STRING_TO_DATE_NANOS;
+            }
             if (to == DataType.IP) {
                 return EsqlConverter.STRING_TO_IP;
             }
@@ -235,9 +240,9 @@ public static Converter converterFor(DataType from, DataType to) {
         return null;
     }
 
-    public static TemporalAmount foldToTemporalAmount(Expression field, String sourceText, DataType expectedType) {
+    public static TemporalAmount foldToTemporalAmount(FoldContext ctx, Expression field, String sourceText, DataType expectedType) {
         if (field.foldable()) {
-            Object v = field.fold();
+            Object v = field.fold(ctx);
             if (v instanceof BytesRef b) {
                 try {
                     return EsqlDataTypeConverter.parseTemporalAmount(b.utf8ToString(), expectedType);
@@ -514,13 +519,12 @@ public static long dateTimeToLong(String dateTime, DateFormatter formatter) {
     }
 
     public static long dateNanosToLong(String dateNano) {
-        return dateNanosToLong(dateNano, DateFormatter.forPattern("strict_date_optional_time_nanos"));
+        return dateNanosToLong(dateNano, DEFAULT_DATE_NANOS_FORMATTER);
     }
 
     public static long dateNanosToLong(String dateNano, DateFormatter formatter) {
-        TemporalAccessor parsed = formatter.parse(dateNano);
-        long nanos = parsed.getLong(ChronoField.INSTANT_SECONDS) * 1_000_000_000 + parsed.getLong(ChronoField.NANO_OF_SECOND);
-        return nanos;
+        Instant parsed = DateFormatters.from(formatter.parse(dateNano)).toInstant();
+        return DateUtils.toLong(parsed);
     }
 
     public static String dateTimeToString(long dateTime) {
@@ -639,6 +643,7 @@ public enum EsqlConverter implements Converter {
         STRING_TO_TIME_DURATION(x -> EsqlDataTypeConverter.parseTemporalAmount(x, DataType.TIME_DURATION)),
         STRING_TO_CHRONO_FIELD(EsqlDataTypeConverter::stringToChrono),
         STRING_TO_DATETIME(x -> EsqlDataTypeConverter.dateTimeToLong((String) x)),
+        STRING_TO_DATE_NANOS(x -> EsqlDataTypeConverter.dateNanosToLong((String) x)),
         STRING_TO_IP(x -> EsqlDataTypeConverter.stringToIP((String) x)),
         STRING_TO_VERSION(x -> EsqlDataTypeConverter.stringToVersion((String) x)),
         STRING_TO_DOUBLE(x -> EsqlDataTypeConverter.stringToDouble((String) x)),
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
index c11ef8615eb72..6cbf910906aaf 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
@@ -55,8 +55,10 @@
 import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
 import org.elasticsearch.xpack.esql.analysis.PreAnalyzer;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.type.InvalidMappedField;
 import org.elasticsearch.xpack.esql.enrich.EnrichLookupService;
 import org.elasticsearch.xpack.esql.enrich.LookupFromIndexService;
 import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy;
@@ -97,11 +99,15 @@
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.xpack.esql.CsvSpecReader.specParser;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.ExpectedResults;
@@ -235,7 +241,10 @@ public final void test() throws Throwable {
              * The csv tests support all but a few features. The unsupported features
              * are tested in integration tests.
              */
-            assumeFalse("metadata fields aren't supported", testCase.requiredCapabilities.contains(cap(EsqlFeatures.METADATA_FIELDS)));
+            assumeFalse(
+                "metadata fields aren't supported",
+                testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.METADATA_FIELDS.capabilityName())
+            );
             assumeFalse(
                 "enrich can't load fields in csv tests",
                 testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.ENRICH_LOAD.capabilityName())
@@ -245,10 +254,6 @@ public final void test() throws Throwable {
                 testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.MATCH_OPERATOR_COLON.capabilityName())
             );
             assumeFalse("can't load metrics in csv tests", testCase.requiredCapabilities.contains(cap(EsqlFeatures.METRICS_SYNTAX)));
-            assumeFalse(
-                "multiple indices aren't supported",
-                testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.UNION_TYPES.capabilityName())
-            );
             assumeFalse(
                 "can't use QSTR function in csv tests",
                 testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.QSTR_FUNCTION.capabilityName())
@@ -263,12 +268,16 @@ public final void test() throws Throwable {
             );
             assumeFalse(
                 "lookup join disabled for csv tests",
-                testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.JOIN_LOOKUP_V5.capabilityName())
+                testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.JOIN_LOOKUP_V10.capabilityName())
             );
             assumeFalse(
                 "can't use TERM function in csv tests",
                 testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.TERM_FUNCTION.capabilityName())
             );
+            assumeFalse(
+                "CSV tests cannot correctly handle the field caps change",
+                testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.SEMANTIC_TEXT_FIELD_CAPS.capabilityName())
+            );
             if (Build.current().isSnapshot()) {
                 assertThat(
                     "Capability is not included in the enabled list capabilities on a snapshot build. Spelling mistake?",
@@ -317,43 +326,92 @@ private void doTest() throws Exception {
         } finally {
             Releasables.close(() -> Iterators.map(actualResults.pages().iterator(), p -> p::releaseBlocks));
             // Give the breaker service some time to clear in case we got results before the rest of the driver had cleaned up
-            assertBusy(() -> assertThat(bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L)));
+            assertBusy(
+                () -> assertThat(
+                    "Not all circuits were cleaned up",
+                    bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST).getUsed(),
+                    equalTo(0L)
+                )
+            );
         }
     }
 
     protected void assertResults(ExpectedResults expected, ActualResults actual, boolean ignoreOrder, Logger logger) {
-        CsvAssert.assertResults(expected, actual, ignoreOrder, logger);
         /*
-         * Comment the assertion above and enable the next two lines to see the results returned by ES without any assertions being done.
+         * Enable the next two lines to see the results returned by ES.
          * This is useful when creating a new test or trying to figure out what are the actual results.
          */
         // CsvTestUtils.logMetaData(actual.columnNames(), actual.columnTypes(), LOGGER);
         // CsvTestUtils.logData(actual.values(), LOGGER);
+
+        CsvAssert.assertResults(expected, actual, ignoreOrder, logger);
     }
 
-    private static IndexResolution loadIndexResolution(String mappingName, String indexName, Map<String, String> typeMapping) {
-        var mapping = new TreeMap<>(loadMapping(mappingName));
-        if ((typeMapping == null || typeMapping.isEmpty()) == false) {
-            for (var entry : typeMapping.entrySet()) {
-                if (mapping.containsKey(entry.getKey())) {
-                    DataType dataType = DataType.fromTypeName(entry.getValue());
-                    EsField field = mapping.get(entry.getKey());
-                    EsField editedField = new EsField(field.getName(), dataType, field.getProperties(), field.isAggregatable());
-                    mapping.put(entry.getKey(), editedField);
-                }
+    private static IndexResolution loadIndexResolution(CsvTestsDataLoader.MultiIndexTestDataset datasets) {
+        var indexNames = datasets.datasets().stream().map(CsvTestsDataLoader.TestDataset::indexName);
+        Map<String, IndexMode> indexModes = indexNames.collect(Collectors.toMap(x -> x, x -> IndexMode.STANDARD));
+        List<MappingPerIndex> mappings = datasets.datasets()
+            .stream()
+            .map(ds -> new MappingPerIndex(ds.indexName(), createMappingForIndex(ds)))
+            .toList();
+        return IndexResolution.valid(new EsIndex(datasets.indexPattern(), mergeMappings(mappings), indexModes));
+    }
+
+    private static Map<String, EsField> createMappingForIndex(CsvTestsDataLoader.TestDataset dataset) {
+        var mapping = new TreeMap<>(loadMapping(dataset.mappingFileName()));
+        if (dataset.typeMapping() == null) {
+            return mapping;
+        }
+        for (var entry : dataset.typeMapping().entrySet()) {
+            if (mapping.containsKey(entry.getKey())) {
+                DataType dataType = DataType.fromTypeName(entry.getValue());
+                EsField field = mapping.get(entry.getKey());
+                EsField editedField = new EsField(field.getName(), dataType, field.getProperties(), field.isAggregatable());
+                mapping.put(entry.getKey(), editedField);
+            }
+        }
+        return mapping;
+    }
+
+    record MappingPerIndex(String index, Map<String, EsField> mapping) {}
+
+    private static Map<String, EsField> mergeMappings(List<MappingPerIndex> mappingsPerIndex) {
+        Map<String, Map<String, EsField>> columnNamesToFieldByIndices = new HashMap<>();
+        for (var mappingPerIndex : mappingsPerIndex) {
+            for (var entry : mappingPerIndex.mapping().entrySet()) {
+                String columnName = entry.getKey();
+                EsField field = entry.getValue();
+                columnNamesToFieldByIndices.computeIfAbsent(columnName, k -> new HashMap<>()).put(mappingPerIndex.index(), field);
+            }
+        }
+
+        return columnNamesToFieldByIndices.entrySet()
+            .stream()
+            .collect(Collectors.toMap(Map.Entry::getKey, e -> mergeFields(e.getKey(), e.getValue())));
+    }
+
+    private static EsField mergeFields(String index, Map<String, EsField> columnNameToField) {
+        var indexFields = columnNameToField.values();
+        if (indexFields.stream().distinct().count() > 1) {
+            var typesToIndices = new HashMap<String, Set<String>>();
+            for (var typeToIndex : columnNameToField.entrySet()) {
+                typesToIndices.computeIfAbsent(typeToIndex.getValue().getDataType().typeName(), k -> new HashSet<>())
+                    .add(typeToIndex.getKey());
             }
+            return new InvalidMappedField(index, typesToIndices);
+        } else {
+            return indexFields.iterator().next();
         }
-        return IndexResolution.valid(new EsIndex(indexName, mapping, Map.of(indexName, IndexMode.STANDARD)));
     }
 
     private static EnrichResolution loadEnrichPolicies() {
         EnrichResolution enrichResolution = new EnrichResolution();
         for (CsvTestsDataLoader.EnrichConfig policyConfig : CsvTestsDataLoader.ENRICH_POLICIES) {
             EnrichPolicy policy = loadEnrichPolicyMapping(policyConfig.policyFileName());
-            CsvTestsDataLoader.TestsDataset sourceIndex = CSV_DATASET_MAP.get(policy.getIndices().get(0));
+            CsvTestsDataLoader.TestDataset sourceIndex = CSV_DATASET_MAP.get(policy.getIndices().get(0));
             // this could practically work, but it's wrong:
             // EnrichPolicyResolution should contain the policy (system) index, not the source index
-            EsIndex esIndex = loadIndexResolution(sourceIndex.mappingFileName(), sourceIndex.indexName(), null).get();
+            EsIndex esIndex = loadIndexResolution(CsvTestsDataLoader.MultiIndexTestDataset.of(sourceIndex.withTypeMapping(Map.of()))).get();
             var concreteIndices = Map.of(RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY, Iterables.get(esIndex.concreteIndices(), 0));
             enrichResolution.addResolvedPolicy(
                 policyConfig.policyName(),
@@ -381,8 +439,8 @@ private static EnrichPolicy loadEnrichPolicyMapping(String policyFileName) {
         }
     }
 
-    private LogicalPlan analyzedPlan(LogicalPlan parsed, CsvTestsDataLoader.TestsDataset dataset) {
-        var indexResolution = loadIndexResolution(dataset.mappingFileName(), dataset.indexName(), dataset.typeMapping());
+    private LogicalPlan analyzedPlan(LogicalPlan parsed, CsvTestsDataLoader.MultiIndexTestDataset datasets) {
+        var indexResolution = loadIndexResolution(datasets);
         var enrichPolicies = loadEnrichPolicies();
         var analyzer = new Analyzer(new AnalyzerContext(configuration, functionRegistry, indexResolution, enrichPolicies), TEST_VERIFIER);
         LogicalPlan plan = analyzer.analyze(parsed);
@@ -391,7 +449,7 @@ private LogicalPlan analyzedPlan(LogicalPlan parsed, CsvTestsDataLoader.TestsDat
         return plan;
     }
 
-    private static CsvTestsDataLoader.TestsDataset testsDataset(LogicalPlan parsed) {
+    private static CsvTestsDataLoader.MultiIndexTestDataset testDatasets(LogicalPlan parsed) {
         var preAnalysis = new PreAnalyzer().preAnalyze(parsed);
         var indices = preAnalysis.indices;
         if (indices.isEmpty()) {
@@ -399,13 +457,13 @@ private static CsvTestsDataLoader.TestsDataset testsDataset(LogicalPlan parsed)
              * If the data set doesn't matter we'll just grab one we know works.
              * Employees is fine.
              */
-            return CSV_DATASET_MAP.get("employees");
+            return CsvTestsDataLoader.MultiIndexTestDataset.of(CSV_DATASET_MAP.get("employees"));
         } else if (preAnalysis.indices.size() > 1) {
             throw new IllegalArgumentException("unexpected index resolution to multiple entries [" + preAnalysis.indices.size() + "]");
         }
 
         String indexName = indices.get(0).id().index();
-        List<CsvTestsDataLoader.TestsDataset> datasets = new ArrayList<>();
+        List<CsvTestsDataLoader.TestDataset> datasets = new ArrayList<>();
         if (indexName.endsWith("*")) {
             String indexPrefix = indexName.substring(0, indexName.length() - 1);
             for (var entry : CSV_DATASET_MAP.entrySet()) {
@@ -414,26 +472,38 @@ private static CsvTestsDataLoader.TestsDataset testsDataset(LogicalPlan parsed)
                 }
             }
         } else {
-            var dataset = CSV_DATASET_MAP.get(indexName);
-            datasets.add(dataset);
+            for (String index : indexName.split(",")) {
+                var dataset = CSV_DATASET_MAP.get(index);
+                if (dataset == null) {
+                    throw new IllegalArgumentException("unknown CSV dataset for table [" + index + "]");
+                }
+                datasets.add(dataset);
+            }
         }
         if (datasets.isEmpty()) {
             throw new IllegalArgumentException("unknown CSV dataset for table [" + indexName + "]");
         }
-        // TODO: Support multiple datasets
-        return datasets.get(0);
+        return new CsvTestsDataLoader.MultiIndexTestDataset(indexName, datasets);
     }
 
-    private static TestPhysicalOperationProviders testOperationProviders(CsvTestsDataLoader.TestsDataset dataset) throws Exception {
-        var testData = loadPageFromCsv(CsvTests.class.getResource("/" + dataset.dataFileName()), dataset.typeMapping());
-        return new TestPhysicalOperationProviders(testData.v1(), testData.v2());
+    private static TestPhysicalOperationProviders testOperationProviders(
+        FoldContext foldCtx,
+        CsvTestsDataLoader.MultiIndexTestDataset datasets
+    ) throws Exception {
+        var indexPages = new ArrayList<TestPhysicalOperationProviders.IndexPage>();
+        for (CsvTestsDataLoader.TestDataset dataset : datasets.datasets()) {
+            var testData = loadPageFromCsv(CsvTests.class.getResource("/data/" + dataset.dataFileName()), dataset.typeMapping());
+            indexPages.add(new TestPhysicalOperationProviders.IndexPage(dataset.indexName(), testData.v1(), testData.v2()));
+        }
+        return TestPhysicalOperationProviders.create(foldCtx, indexPages);
     }
 
     private ActualResults executePlan(BigArrays bigArrays) throws Exception {
         LogicalPlan parsed = parser.createStatement(testCase.query);
-        var testDataset = testsDataset(parsed);
-        LogicalPlan analyzed = analyzedPlan(parsed, testDataset);
+        var testDatasets = testDatasets(parsed);
+        LogicalPlan analyzed = analyzedPlan(parsed, testDatasets);
 
+        FoldContext foldCtx = FoldContext.small();
         EsqlSession session = new EsqlSession(
             getTestName(),
             configuration,
@@ -441,20 +511,21 @@ private ActualResults executePlan(BigArrays bigArrays) throws Exception {
             null,
             null,
             functionRegistry,
-            new LogicalPlanOptimizer(new LogicalOptimizerContext(configuration)),
+            new LogicalPlanOptimizer(new LogicalOptimizerContext(configuration, foldCtx)),
             mapper,
             TEST_VERIFIER,
             new PlanningMetrics(),
-            null
+            null,
+            EsqlTestUtils.MOCK_QUERY_BUILDER_RESOLVER
         );
-        TestPhysicalOperationProviders physicalOperationProviders = testOperationProviders(testDataset);
+        TestPhysicalOperationProviders physicalOperationProviders = testOperationProviders(foldCtx, testDatasets);
 
         PlainActionFuture<ActualResults> listener = new PlainActionFuture<>();
 
         session.executeOptimizedPlan(
             new EsqlQueryRequest(),
             new EsqlExecutionInfo(randomBoolean()),
-            planRunner(bigArrays, physicalOperationProviders),
+            planRunner(bigArrays, foldCtx, physicalOperationProviders),
             session.optimizedPlan(analyzed),
             listener.delegateFailureAndWrap(
                 // Wrap so we can capture the warnings in the calling thread
@@ -514,12 +585,13 @@ private void assertWarnings(List<String> warnings) {
         testCase.assertWarnings(false).assertWarnings(normalized);
     }
 
-    PlanRunner planRunner(BigArrays bigArrays, TestPhysicalOperationProviders physicalOperationProviders) {
-        return (physicalPlan, listener) -> executeSubPlan(bigArrays, physicalOperationProviders, physicalPlan, listener);
+    PlanRunner planRunner(BigArrays bigArrays, FoldContext foldCtx, TestPhysicalOperationProviders physicalOperationProviders) {
+        return (physicalPlan, listener) -> executeSubPlan(bigArrays, foldCtx, physicalOperationProviders, physicalPlan, listener);
     }
 
     void executeSubPlan(
         BigArrays bigArrays,
+        FoldContext foldCtx,
         TestPhysicalOperationProviders physicalOperationProviders,
         PhysicalPlan physicalPlan,
         ActionListener<Result> listener
@@ -565,12 +637,17 @@ void executeSubPlan(
 
         // replace fragment inside the coordinator plan
         List<Driver> drivers = new ArrayList<>();
-        LocalExecutionPlan coordinatorNodeExecutionPlan = executionPlanner.plan(new OutputExec(coordinatorPlan, collectedPages::add));
+        LocalExecutionPlan coordinatorNodeExecutionPlan = executionPlanner.plan(
+            foldCtx,
+            new OutputExec(coordinatorPlan, collectedPages::add)
+        );
         drivers.addAll(coordinatorNodeExecutionPlan.createDrivers(getTestName()));
         if (dataNodePlan != null) {
             var searchStats = new DisabledSearchStats();
-            var logicalTestOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, searchStats));
-            var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer(new LocalPhysicalOptimizerContext(configuration, searchStats));
+            var logicalTestOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, foldCtx, searchStats));
+            var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer(
+                new LocalPhysicalOptimizerContext(configuration, foldCtx, searchStats)
+            );
 
             var csvDataNodePhysicalPlan = PlannerUtils.localPlan(dataNodePlan, logicalTestOptimizer, physicalTestOptimizer);
             exchangeSource.addRemoteSink(
@@ -581,7 +658,7 @@ void executeSubPlan(
                     throw new AssertionError("expected no failure", e);
                 })
             );
-            LocalExecutionPlan dataNodeExecutionPlan = executionPlanner.plan(csvDataNodePhysicalPlan);
+            LocalExecutionPlan dataNodeExecutionPlan = executionPlanner.plan(foldCtx, csvDataNodePhysicalPlan);
 
             drivers.addAll(dataNodeExecutionPlan.createDrivers(getTestName()));
             Randomness.shuffle(drivers);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
index 35364089127cc..4b9ae0d1692e5 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
@@ -204,7 +204,7 @@ private Page randomPage(List<ColumnInfoImpl> columns) {
                 case BOOLEAN -> ((BooleanBlock.Builder) builder).appendBoolean(randomBoolean());
                 case UNSUPPORTED -> ((BytesRefBlock.Builder) builder).appendNull();
                 // TODO - add a random instant thing here?
-                case DATE_NANOS -> ((LongBlock.Builder) builder).appendLong(randomLong());
+                case DATE_NANOS -> ((LongBlock.Builder) builder).appendLong(randomNonNegativeLong());
                 case VERSION -> ((BytesRefBlock.Builder) builder).appendBytesRef(new Version(randomIdentifier()).toBytesRef());
                 case GEO_POINT -> ((BytesRefBlock.Builder) builder).appendBytesRef(GEO.asWkb(GeometryTestUtils.randomPoint()));
                 case CARTESIAN_POINT -> ((BytesRefBlock.Builder) builder).appendBytesRef(CARTESIAN.asWkb(ShapeTestUtils.randomPoint()));
@@ -529,7 +529,7 @@ public void testChunkResponseSizeColumnar() {
         try (EsqlQueryResponse resp = randomResponseAsync(true, null, true)) {
             int columnCount = resp.pages().get(0).getBlockCount();
             int bodySize = resp.pages().stream().mapToInt(p -> p.getPositionCount() * p.getBlockCount()).sum() + columnCount * 2;
-            assertChunkCount(resp, r -> 7 + sizeClusterDetails + bodySize); // is_running
+            assertChunkCount(resp, r -> 6 + sizeClusterDetails + bodySize); // is_running
         }
     }
 
@@ -541,7 +541,7 @@ public void testChunkResponseSizeRows() {
         }
         try (EsqlQueryResponse resp = randomResponseAsync(false, null, true)) {
             int bodySize = resp.pages().stream().mapToInt(Page::getPositionCount).sum();
-            assertChunkCount(resp, r -> 7 + sizeClusterDetails + bodySize);
+            assertChunkCount(resp, r -> 6 + sizeClusterDetails + bodySize);
         }
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java
index 5e79e40b7e938..d4e786a9d9bb0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java
@@ -46,6 +46,10 @@ public static Analyzer analyzer(IndexResolution indexResolution) {
         return analyzer(indexResolution, TEST_VERIFIER);
     }
 
+    public static Analyzer analyzer(IndexResolution indexResolution, Map<String, IndexResolution> lookupResolution) {
+        return analyzer(indexResolution, lookupResolution, TEST_VERIFIER);
+    }
+
     public static Analyzer analyzer(IndexResolution indexResolution, Verifier verifier) {
         return new Analyzer(
             new AnalyzerContext(
@@ -59,6 +63,19 @@ public static Analyzer analyzer(IndexResolution indexResolution, Verifier verifi
         );
     }
 
+    public static Analyzer analyzer(IndexResolution indexResolution, Map<String, IndexResolution> lookupResolution, Verifier verifier) {
+        return new Analyzer(
+            new AnalyzerContext(
+                EsqlTestUtils.TEST_CFG,
+                new EsqlFunctionRegistry(),
+                indexResolution,
+                lookupResolution,
+                defaultEnrichResolution()
+            ),
+            verifier
+        );
+    }
+
     public static Analyzer analyzer(IndexResolution indexResolution, Verifier verifier, Configuration config) {
         return new Analyzer(
             new AnalyzerContext(config, new EsqlFunctionRegistry(), indexResolution, defaultLookupResolution(), defaultEnrichResolution()),
@@ -111,7 +128,7 @@ public static IndexResolution loadMapping(String resource, String indexName, Ind
     }
 
     public static IndexResolution loadMapping(String resource, String indexName) {
-        EsIndex test = new EsIndex(indexName, EsqlTestUtils.loadMapping(resource));
+        EsIndex test = new EsIndex(indexName, EsqlTestUtils.loadMapping(resource), Map.of(indexName, IndexMode.STANDARD));
         return IndexResolution.valid(test);
     }
 
@@ -123,8 +140,8 @@ public static IndexResolution expandedDefaultIndexResolution() {
         return loadMapping("mapping-default.json", "test");
     }
 
-    public static IndexResolution defaultLookupResolution() {
-        return loadMapping("mapping-languages.json", "languages_lookup", IndexMode.LOOKUP);
+    public static Map<String, IndexResolution> defaultLookupResolution() {
+        return Map.of("languages_lookup", loadMapping("mapping-languages.json", "languages_lookup", IndexMode.LOOKUP));
     }
 
     public static EnrichResolution defaultEnrichResolution() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
index cfff245b19244..2df6e30e96081 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
@@ -1076,7 +1077,7 @@ public void testImplicitLimit() {
             from test
             """);
         var limit = as(plan, Limit.class);
-        assertThat(limit.limit().fold(), equalTo(DEFAULT_LIMIT));
+        assertThat(as(limit.limit(), Literal.class).value(), equalTo(DEFAULT_LIMIT));
         as(limit.child(), EsRelation.class);
     }
 
@@ -1084,7 +1085,7 @@ public void testImplicitMaxLimitAfterLimit() {
         for (int i = -1; i <= 1; i++) {
             var plan = analyze("from test | limit " + (MAX_LIMIT + i));
             var limit = as(plan, Limit.class);
-            assertThat(limit.limit().fold(), equalTo(MAX_LIMIT));
+            assertThat(as(limit.limit(), Literal.class).value(), equalTo(MAX_LIMIT));
             limit = as(limit.child(), Limit.class);
             as(limit.child(), EsRelation.class);
         }
@@ -1101,7 +1102,7 @@ public void testImplicitMaxLimitAfterLimitAndNonLimit() {
         for (int i = -1; i <= 1; i++) {
             var plan = analyze("from test | limit " + (MAX_LIMIT + i) + " | eval s = salary * 10 | where s > 0");
             var limit = as(plan, Limit.class);
-            assertThat(limit.limit().fold(), equalTo(MAX_LIMIT));
+            assertThat(as(limit.limit(), Literal.class).value(), equalTo(MAX_LIMIT));
             var filter = as(limit.child(), Filter.class);
             var eval = as(filter.child(), Eval.class);
             limit = as(eval.child(), Limit.class);
@@ -1113,7 +1114,7 @@ public void testImplicitDefaultLimitAfterLimitAndBreaker() {
         for (var breaker : List.of("stats c = count(salary) by last_name", "sort salary")) {
             var plan = analyze("from test | limit 100000 | " + breaker);
             var limit = as(plan, Limit.class);
-            assertThat(limit.limit().fold(), equalTo(MAX_LIMIT));
+            assertThat(as(limit.limit(), Literal.class).value(), equalTo(MAX_LIMIT));
         }
     }
 
@@ -1121,7 +1122,7 @@ public void testImplicitDefaultLimitAfterBreakerAndNonBreakers() {
         for (var breaker : List.of("stats c = count(salary) by last_name", "eval c = salary | sort c")) {
             var plan = analyze("from test | " + breaker + " | eval cc = c * 10 | where cc > 0");
             var limit = as(plan, Limit.class);
-            assertThat(limit.limit().fold(), equalTo(DEFAULT_LIMIT));
+            assertThat(as(limit.limit(), Literal.class).value(), equalTo(DEFAULT_LIMIT));
         }
     }
 
@@ -1427,7 +1428,7 @@ public void testEmptyEsRelationOnLimitZeroWithCount() throws IOException {
         var plan = analyzeWithEmptyFieldCapsResponse(query);
         var limit = as(plan, Limit.class);
         limit = as(limit.child(), Limit.class);
-        assertThat(limit.limit().fold(), equalTo(0));
+        assertThat(as(limit.limit(), Literal.class).value(), equalTo(0));
         var orderBy = as(limit.child(), OrderBy.class);
         var agg = as(orderBy.child(), Aggregate.class);
         assertEmptyEsRelation(agg.child());
@@ -1442,7 +1443,7 @@ public void testEmptyEsRelationOnConstantEvalAndKeep() throws IOException {
         var plan = analyzeWithEmptyFieldCapsResponse(query);
         var limit = as(plan, Limit.class);
         limit = as(limit.child(), Limit.class);
-        assertThat(limit.limit().fold(), equalTo(2));
+        assertThat(as(limit.limit(), Literal.class).value(), equalTo(2));
         var project = as(limit.child(), EsqlProject.class);
         var eval = as(project.child(), Eval.class);
         assertEmptyEsRelation(eval.child());
@@ -1459,7 +1460,7 @@ public void testEmptyEsRelationOnConstantEvalAndStats() throws IOException {
         var agg = as(limit.child(), Aggregate.class);
         var eval = as(agg.child(), Eval.class);
         limit = as(eval.child(), Limit.class);
-        assertThat(limit.limit().fold(), equalTo(10));
+        assertThat(as(limit.limit(), Literal.class).value(), equalTo(10));
         assertEmptyEsRelation(limit.child());
     }
 
@@ -2053,10 +2054,10 @@ public void testLookup() {
         }
         LogicalPlan plan = analyze(query);
         var limit = as(plan, Limit.class);
-        assertThat(limit.limit().fold(), equalTo(1000));
+        assertThat(as(limit.limit(), Literal.class).value(), equalTo(1000));
 
         var lookup = as(limit.child(), Lookup.class);
-        assertThat(lookup.tableName().fold(), equalTo("int_number_names"));
+        assertThat(as(lookup.tableName(), Literal.class).value(), equalTo("int_number_names"));
         assertMap(lookup.matchFields().stream().map(Object::toString).toList(), matchesList().item(startsWith("int{r}")));
         assertThat(
             lookup.localRelation().output().stream().map(Object::toString).toList(),
@@ -2139,7 +2140,7 @@ public void testLookupMatchTypeWrong() {
     }
 
     public void testLookupJoinUnknownIndex() {
-        assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled());
+        assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
 
         String errorMessage = "Unknown index [foobar]";
         IndexResolution missingLookupIndex = IndexResolution.invalid(errorMessage);
@@ -2149,7 +2150,7 @@ public void testLookupJoinUnknownIndex() {
                 EsqlTestUtils.TEST_CFG,
                 new EsqlFunctionRegistry(),
                 analyzerDefaultMapping(),
-                missingLookupIndex,
+                Map.of("foobar", missingLookupIndex),
                 defaultEnrichResolution()
             ),
             TEST_VERIFIER
@@ -2168,7 +2169,7 @@ public void testLookupJoinUnknownIndex() {
     }
 
     public void testLookupJoinUnknownField() {
-        assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled());
+        assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
 
         String query = "FROM test | LOOKUP JOIN languages_lookup ON last_name";
         String errorMessage = "1:45: Unknown column [last_name] in right side of join";
@@ -2190,6 +2191,65 @@ public void testLookupJoinUnknownField() {
         assertThat(e.getMessage(), containsString(errorMessage3 + "right side of join"));
     }
 
+    public void testMultipleLookupJoinsGiveDifferentAttributes() {
+        assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+
+        // The field attributes that get contributed by different LOOKUP JOIN commands must have different name ids,
+        // even if they have the same names. Otherwise, things like dependency analysis - like in PruneColumns - cannot work based on
+        // name ids and shadowing semantics proliferate into all kinds of optimizer code.
+
+        String query = "FROM test"
+            + "| EVAL language_code = languages"
+            + "| LOOKUP JOIN languages_lookup ON language_code"
+            + "| LOOKUP JOIN languages_lookup ON language_code";
+        LogicalPlan analyzedPlan = analyze(query);
+
+        List<AttributeSet> lookupFields = new ArrayList<>();
+        List<Set<String>> lookupFieldNames = new ArrayList<>();
+        analyzedPlan.forEachUp(EsRelation.class, esRelation -> {
+            if (esRelation.indexMode() == IndexMode.LOOKUP) {
+                lookupFields.add(esRelation.outputSet());
+                lookupFieldNames.add(esRelation.outputSet().stream().map(NamedExpression::name).collect(Collectors.toSet()));
+            }
+        });
+
+        assertEquals(lookupFieldNames.size(), 2);
+        assertEquals(lookupFieldNames.get(0), lookupFieldNames.get(1));
+
+        assertEquals(lookupFields.size(), 2);
+        AttributeSet intersection = lookupFields.get(0).intersect(lookupFields.get(1));
+        assertEquals(AttributeSet.EMPTY, intersection);
+    }
+
+    public void testLookupJoinIndexMode() {
+        assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+
+        var indexResolution = AnalyzerTestUtils.expandedDefaultIndexResolution();
+        var lookupResolution = AnalyzerTestUtils.defaultLookupResolution();
+        var indexResolutionAsLookup = Map.of("test", indexResolution);
+        var lookupResolutionAsIndex = lookupResolution.get("languages_lookup");
+
+        analyze("FROM test | EVAL language_code = languages | LOOKUP JOIN languages_lookup ON language_code");
+        analyze(
+            "FROM languages_lookup | LOOKUP JOIN languages_lookup ON language_code",
+            AnalyzerTestUtils.analyzer(lookupResolutionAsIndex, lookupResolution)
+        );
+
+        VerificationException e = expectThrows(
+            VerificationException.class,
+            () -> analyze(
+                "FROM languages_lookup | EVAL languages = language_code | LOOKUP JOIN test ON languages",
+                AnalyzerTestUtils.analyzer(lookupResolutionAsIndex, indexResolutionAsLookup)
+            )
+        );
+        assertThat(e.getMessage(), containsString("1:70: invalid [test] resolution in lookup mode to an index in [standard] mode"));
+        e = expectThrows(
+            VerificationException.class,
+            () -> analyze("FROM test | LOOKUP JOIN test ON languages", AnalyzerTestUtils.analyzer(indexResolution, indexResolutionAsLookup))
+        );
+        assertThat(e.getMessage(), containsString("1:25: invalid [test] resolution in lookup mode to an index in [standard] mode"));
+    }
+
     public void testImplicitCasting() {
         var e = expectThrows(VerificationException.class, () -> analyze("""
              from test | eval x = concat("2024", "-04", "-01") + 1 day
@@ -2276,7 +2336,7 @@ public void testCoalesceWithMixedNumericTypes() {
         projection = as(projections.get(3), ReferenceAttribute.class);
         assertEquals(projection.name(), "w");
         assertEquals(projection.dataType(), DataType.DOUBLE);
-        assertThat(limit.limit().fold(), equalTo(1000));
+        assertThat(as(limit.limit(), Literal.class).value(), equalTo(1000));
     }
 
     public void testNamedParamsForIdentifiers() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java
index 68529e99c6b1b..180e32fb7c15d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
 import org.elasticsearch.xpack.esql.LoadMapping;
-import org.elasticsearch.xpack.esql.core.ParsingException;
+import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
@@ -20,6 +20,7 @@
 import org.elasticsearch.xpack.esql.index.EsIndex;
 import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
+import org.elasticsearch.xpack.esql.parser.ParsingException;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.plan.logical.Row;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter;
@@ -49,27 +50,27 @@ public class ParsingTests extends ESTestCase {
     );
 
     public void testCaseFunctionInvalidInputs() {
-        assertEquals("1:23: error building [case]: expects at least two arguments", error("row a = 1 | eval x = case()"));
-        assertEquals("1:23: error building [case]: expects at least two arguments", error("row a = 1 | eval x = case(a)"));
-        assertEquals("1:23: error building [case]: expects at least two arguments", error("row a = 1 | eval x = case(1)"));
+        assertEquals("1:22: error building [case]: expects at least two arguments", error("row a = 1 | eval x = case()"));
+        assertEquals("1:22: error building [case]: expects at least two arguments", error("row a = 1 | eval x = case(a)"));
+        assertEquals("1:22: error building [case]: expects at least two arguments", error("row a = 1 | eval x = case(1)"));
     }
 
     public void testConcatFunctionInvalidInputs() {
-        assertEquals("1:23: error building [concat]: expects at least two arguments", error("row a = 1 | eval x = concat()"));
-        assertEquals("1:23: error building [concat]: expects at least two arguments", error("row a = 1 | eval x = concat(a)"));
-        assertEquals("1:23: error building [concat]: expects at least two arguments", error("row a = 1 | eval x = concat(1)"));
+        assertEquals("1:22: error building [concat]: expects at least two arguments", error("row a = 1 | eval x = concat()"));
+        assertEquals("1:22: error building [concat]: expects at least two arguments", error("row a = 1 | eval x = concat(a)"));
+        assertEquals("1:22: error building [concat]: expects at least two arguments", error("row a = 1 | eval x = concat(1)"));
     }
 
     public void testCoalesceFunctionInvalidInputs() {
-        assertEquals("1:23: error building [coalesce]: expects at least one argument", error("row a = 1 | eval x = coalesce()"));
+        assertEquals("1:22: error building [coalesce]: expects at least one argument", error("row a = 1 | eval x = coalesce()"));
     }
 
     public void testGreatestFunctionInvalidInputs() {
-        assertEquals("1:23: error building [greatest]: expects at least one argument", error("row a = 1 | eval x = greatest()"));
+        assertEquals("1:22: error building [greatest]: expects at least one argument", error("row a = 1 | eval x = greatest()"));
     }
 
     public void testLeastFunctionInvalidInputs() {
-        assertEquals("1:23: error building [least]: expects at least one argument", error("row a = 1 | eval x = least()"));
+        assertEquals("1:22: error building [least]: expects at least one argument", error("row a = 1 | eval x = least()"));
     }
 
     /**
@@ -108,7 +109,47 @@ public void testTooBigQuery() {
         while (query.length() < EsqlParser.MAX_LENGTH) {
             query.append(", a = CONCAT(a, a)");
         }
-        assertEquals("-1:0: ESQL statement is too large [1000011 characters > 1000000]", error(query.toString()));
+        assertEquals("-1:-1: ESQL statement is too large [1000011 characters > 1000000]", error(query.toString()));
+    }
+
+    public void testJoinOnConstant() {
+        assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+        assertEquals(
+            "1:55: JOIN ON clause only supports fields at the moment, found [123]",
+            error("row languages = 1, gender = \"f\" | lookup join test on 123")
+        );
+        assertEquals(
+            "1:55: JOIN ON clause only supports fields at the moment, found [\"abc\"]",
+            error("row languages = 1, gender = \"f\" | lookup join test on \"abc\"")
+        );
+        assertEquals(
+            "1:55: JOIN ON clause only supports fields at the moment, found [false]",
+            error("row languages = 1, gender = \"f\" | lookup join test on false")
+        );
+    }
+
+    public void testJoinOnMultipleFields() {
+        assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+        assertEquals(
+            "1:35: JOIN ON clause only supports one field at the moment, found [2]",
+            error("row languages = 1, gender = \"f\" | lookup join test on gender, languages")
+        );
+    }
+
+    public void testJoinTwiceOnTheSameField() {
+        assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+        assertEquals(
+            "1:35: JOIN ON clause only supports one field at the moment, found [2]",
+            error("row languages = 1, gender = \"f\" | lookup join test on languages, languages")
+        );
+    }
+
+    public void testJoinTwiceOnTheSameField_TwoLookups() {
+        assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+        assertEquals(
+            "1:80: JOIN ON clause only supports one field at the moment, found [2]",
+            error("row languages = 1, gender = \"f\" | lookup join test on languages | eval x = 1 | lookup join test on gender, gender")
+        );
     }
 
     private String functionName(EsqlFunctionRegistry registry, Expression functionCall) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
index 4b916106165fb..d78c4bfa21ced 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
@@ -945,6 +945,33 @@ public void testPeriodAndDurationInEval() {
 
     public void testFilterNonBoolField() {
         assertEquals("1:19: Condition expression needs to be boolean, found [INTEGER]", error("from test | where emp_no"));
+
+        assertEquals(
+            "1:19: Condition expression needs to be boolean, found [KEYWORD]",
+            error("from test | where concat(first_name, \"foobar\")")
+        );
+    }
+
+    public void testFilterNullField() {
+        // `where null` should return empty result set
+        query("from test | where null");
+
+        // Value null of type `BOOLEAN`
+        query("from test | where null::boolean");
+
+        // Provide `NULL` type in `EVAL`
+        query("from t | EVAL x = null | where x");
+
+        // `to_string(null)` is of `KEYWORD` type null, resulting in `to_string(null) == "abc"` being of `BOOLEAN`
+        query("from t | where to_string(null) == \"abc\"");
+
+        // Other DataTypes can contain null values
+        assertEquals("1:19: Condition expression needs to be boolean, found [KEYWORD]", error("from test | where null::string"));
+        assertEquals("1:19: Condition expression needs to be boolean, found [INTEGER]", error("from test | where null::integer"));
+        assertEquals(
+            "1:45: Condition expression needs to be boolean, found [DATETIME]",
+            error("from test | EVAL x = null::datetime | where x")
+        );
     }
 
     public void testFilterDateConstant() {
@@ -1166,12 +1193,14 @@ public void testMatchInsideEval() throws Exception {
     public void testMatchFilter() throws Exception {
         assertEquals(
             "1:19: Invalid condition [first_name:\"Anna\" or starts_with(first_name, \"Anne\")]. "
-                + "[:] operator can't be used as part of an or condition",
+                + "Full text functions can be used in an OR condition, "
+                + "but only if just full text functions are used in the OR condition",
             error("from test | where first_name:\"Anna\" or starts_with(first_name, \"Anne\")")
         );
 
         assertEquals(
-            "1:51: Invalid condition [first_name:\"Anna\" OR new_salary > 100]. " + "[:] operator can't be used as part of an or condition",
+            "1:51: Invalid condition [first_name:\"Anna\" OR new_salary > 100]. Full text functions can be"
+                + " used in an OR condition, but only if just full text functions are used in the OR condition",
             error("from test | eval new_salary = salary + 10 | where first_name:\"Anna\" OR new_salary > 100")
         );
     }
@@ -1259,9 +1288,6 @@ public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception {
     }
 
     public void testKqlFunctionsNotAllowedAfterCommands() throws Exception {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         // Source commands
         assertEquals("1:13: [KQL] function cannot be used after SHOW", error("show info | where kql(\"8.16.0\")"));
         assertEquals("1:17: [KQL] function cannot be used after ROW", error("row a= \"Anna\" | where kql(\"Anna\")"));
@@ -1319,9 +1345,6 @@ public void testQueryStringFunctionOnlyAllowedInWhere() throws Exception {
     }
 
     public void testKqlFunctionOnlyAllowedInWhere() throws Exception {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         assertEquals("1:9: [KQL] function is only supported in WHERE commands", error("row a = kql(\"Anna\")"));
         checkFullTextFunctionsOnlyAllowedInWhere("KQL", "kql(\"Anna\")", "function");
     }
@@ -1373,9 +1396,6 @@ public void testQueryStringFunctionArgNotNullOrConstant() throws Exception {
     }
 
     public void testKqlFunctionArgNotNullOrConstant() throws Exception {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         assertEquals(
             "1:19: argument of [kql(first_name)] must be a constant, received [first_name]",
             error("from test | where kql(first_name)")
@@ -1389,9 +1409,6 @@ public void testQueryStringWithDisjunctions() {
     }
 
     public void testKqlFunctionWithDisjunctions() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         checkWithDisjunctions("KQL", "kql(\"first_name: Anna\")", "function");
     }
 
@@ -1409,56 +1426,59 @@ public void testMatchOperatorWithDisjunctions() {
     }
 
     private void checkWithDisjunctions(String functionName, String functionInvocation, String functionType) {
+        String expression = functionInvocation + " or length(first_name) > 12";
+        checkdisjunctionError("1:19", expression, functionName, functionType);
+        expression = "(" + functionInvocation + " or first_name is not null) or (length(first_name) > 12 and match(last_name, \"Smith\"))";
+        checkdisjunctionError("1:19", expression, functionName, functionType);
+        expression = functionInvocation + " or (last_name is not null and first_name is null)";
+        checkdisjunctionError("1:19", expression, functionName, functionType);
+    }
+
+    private void checkdisjunctionError(String position, String expression, String functionName, String functionType) {
         assertEquals(
             LoggerMessageFormat.format(
                 null,
-                "1:19: Invalid condition [{} or length(first_name) > 12]. "
-                    + "[{}] "
-                    + functionType
-                    + " can't be used as part of an or condition",
-                functionInvocation,
-                functionName
-            ),
-            error("from test | where " + functionInvocation + " or length(first_name) > 12")
-        );
-        assertEquals(
-            LoggerMessageFormat.format(
-                null,
-                "1:19: Invalid condition [({} and first_name is not null) or (length(first_name) > 12 and first_name is null)]. "
-                    + "[{}] "
-                    + functionType
-                    + " can't be used as part of an or condition",
-                functionInvocation,
-                functionName
-            ),
-            error(
-                "from test | where ("
-                    + functionInvocation
-                    + " and first_name is not null) or (length(first_name) > 12 and first_name is null)"
-            )
-        );
-        assertEquals(
-            LoggerMessageFormat.format(
-                null,
-                "1:19: Invalid condition [({} and first_name is not null) or first_name is null]. "
-                    + "[{}] "
-                    + functionType
-                    + " can't be used as part of an or condition",
-                functionInvocation,
-                functionName
+                "{}: Invalid condition [{}]. Full text functions can be used in an OR condition, "
+                    + "but only if just full text functions are used in the OR condition",
+                position,
+                expression
             ),
-            error("from test | where (" + functionInvocation + " and first_name is not null) or first_name is null")
+            error("from test | where " + expression)
         );
     }
 
+    public void testFullTextFunctionsDisjunctions() {
+        checkWithFullTextFunctionsDisjunctions("MATCH", "match(last_name, \"Smith\")", "function");
+        checkWithFullTextFunctionsDisjunctions(":", "last_name : \"Smith\"", "operator");
+        checkWithFullTextFunctionsDisjunctions("QSTR", "qstr(\"last_name: Smith\")", "function");
+        checkWithFullTextFunctionsDisjunctions("KQL", "kql(\"last_name: Smith\")", "function");
+    }
+
+    private void checkWithFullTextFunctionsDisjunctions(String functionName, String functionInvocation, String functionType) {
+
+        String expression = functionInvocation + " or length(first_name) > 10";
+        checkdisjunctionError("1:19", expression, functionName, functionType);
+
+        expression = "match(last_name, \"Anneke\") or (" + functionInvocation + " and length(first_name) > 10)";
+        checkdisjunctionError("1:19", expression, functionName, functionType);
+
+        expression = "("
+            + functionInvocation
+            + " and length(first_name) > 0) or (match(last_name, \"Anneke\") and length(first_name) > 10)";
+        checkdisjunctionError("1:19", expression, functionName, functionType);
+
+        query("from test | where " + functionInvocation + " or match(first_name, \"Anna\")");
+        query("from test | where " + functionInvocation + " or not match(first_name, \"Anna\")");
+        query("from test | where (" + functionInvocation + " or match(first_name, \"Anna\")) and length(first_name) > 10");
+        query("from test | where (" + functionInvocation + " or match(first_name, \"Anna\")) and match(last_name, \"Smith\")");
+        query("from test | where " + functionInvocation + " or (match(first_name, \"Anna\") and match(last_name, \"Smith\"))");
+    }
+
     public void testQueryStringFunctionWithNonBooleanFunctions() {
         checkFullTextFunctionsWithNonBooleanFunctions("QSTR", "qstr(\"first_name: Anna\")", "function");
     }
 
     public void testKqlFunctionWithNonBooleanFunctions() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         checkFullTextFunctionsWithNonBooleanFunctions("KQL", "kql(\"first_name: Anna\")", "function");
     }
 
@@ -1964,7 +1984,7 @@ public void testSortByAggregate() {
     }
 
     public void testLookupJoinDataTypeMismatch() {
-        assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled());
+        assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
 
         query("FROM test | EVAL language_code = languages | LOOKUP JOIN languages_lookup ON language_code");
 
@@ -1975,7 +1995,11 @@ public void testLookupJoinDataTypeMismatch() {
     }
 
     private void query(String query) {
-        defaultAnalyzer.analyze(parser.createStatement(query));
+        query(query, defaultAnalyzer);
+    }
+
+    private void query(String query, Analyzer analyzer) {
+        analyzer.analyze(parser.createStatement(query));
     }
 
     private String error(String query) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapperTests.java
new file mode 100644
index 0000000000000..828f9e061686b
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapperTests.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.evaluator.mapper;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
+import org.hamcrest.Matchers;
+
+public class EvaluatorMapperTests extends ESTestCase {
+    public void testFoldCompletesWithPlentyOfMemory() {
+        Add add = new Add(
+            Source.synthetic("shouldn't break"),
+            new Literal(Source.EMPTY, 1, DataType.INTEGER),
+            new Literal(Source.EMPTY, 3, DataType.INTEGER)
+        );
+        assertEquals(add.fold(new FoldContext(100)), 4);
+    }
+
+    public void testFoldBreaksWithLittleMemory() {
+        Add add = new Add(
+            Source.synthetic("should break"),
+            new Literal(Source.EMPTY, 1, DataType.INTEGER),
+            new Literal(Source.EMPTY, 3, DataType.INTEGER)
+        );
+        Exception e = expectThrows(FoldContext.FoldTooMuchMemoryException.class, () -> add.fold(new FoldContext(10)));
+        assertThat(
+            e.getMessage(),
+            Matchers.equalTo(
+                "line -1:-1: Folding query used more than 10b. "
+                    + "The expression that pushed past the limit is [should break] which needed 32b."
+            )
+        );
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java
index 6dd0c5fe88afd..050293e58c19d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java
@@ -14,10 +14,15 @@
 import org.elasticsearch.xpack.esql.plan.AbstractNodeSerializationTests;
 
 public abstract class AbstractExpressionSerializationTests<T extends Expression> extends AbstractNodeSerializationTests<T> {
+
     public static Expression randomChild() {
         return ReferenceAttributeTests.randomReferenceAttribute(false);
     }
 
+    public static Expression mutateExpression(Expression expression) {
+        return randomValueOtherThan(expression, AbstractExpressionSerializationTests::randomChild);
+    }
+
     @Override
     protected final NamedWriteableRegistry getNamedWriteableRegistry() {
         return new NamedWriteableRegistry(ExpressionWritables.getNamedWriteables());
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java
index c086245d6fd61..87ea6315d4f3b 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
@@ -40,6 +41,7 @@
 import java.util.stream.IntStream;
 
 import static org.elasticsearch.compute.data.BlockUtils.toJavaObject;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
@@ -263,12 +265,12 @@ private void evaluate(Expression evaluableExpression) {
         assertTrue(evaluableExpression.foldable());
 
         if (testCase.foldingExceptionClass() != null) {
-            Throwable t = expectThrows(testCase.foldingExceptionClass(), evaluableExpression::fold);
+            Throwable t = expectThrows(testCase.foldingExceptionClass(), () -> evaluableExpression.fold(FoldContext.small()));
             assertThat(t.getMessage(), equalTo(testCase.foldingExceptionMessage()));
             return;
         }
 
-        Object result = evaluableExpression.fold();
+        Object result = evaluableExpression.fold(FoldContext.small());
         // Decode unsigned longs into BigIntegers
         if (testCase.expectedType() == DataType.UNSIGNED_LONG && result != null) {
             result = NumericUtils.unsignedLongAsBigInteger((Long) result);
@@ -289,7 +291,7 @@ private void resolveExpression(Expression expression, Consumer<Expression> onAgg
         expression = resolveSurrogates(expression);
 
         // As expressions may be composed of multiple functions, we need to fold nulls bottom-up
-        expression = expression.transformUp(e -> new FoldNull().rule(e));
+        expression = expression.transformUp(e -> new FoldNull().rule(e, unboundLogicalOptimizerContext()));
         assertThat(expression.dataType(), equalTo(testCase.expectedType()));
 
         Expression.TypeResolution resolution = expression.typeResolved();
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
index c609eb3a7ad41..1cf087cf55ccd 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull;
@@ -44,6 +45,7 @@
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
+import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
 import org.elasticsearch.xpack.esql.expression.function.fulltext.Match;
 import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Greatest;
 import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
@@ -97,6 +99,7 @@
 import static java.util.Map.entry;
 import static org.elasticsearch.compute.data.BlockUtils.toJavaObject;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.endsWith;
@@ -191,7 +194,6 @@ protected static List<TestCaseSupplier> anyNullIsNull(
         ExpectedType expectedType,
         ExpectedEvaluatorToString evaluatorToString
     ) {
-        typesRequired(testCaseSuppliers);
         List<TestCaseSupplier> suppliers = new ArrayList<>(testCaseSuppliers.size());
         suppliers.addAll(testCaseSuppliers);
 
@@ -274,7 +276,7 @@ protected static List<TestCaseSupplier> anyNullIsNull(
     }
 
     @FunctionalInterface
-    protected interface PositionalErrorMessageSupplier {
+    public interface PositionalErrorMessageSupplier {
         /**
          * This interface defines functions to supply error messages for incorrect types in specific positions. Functions which have
          * the same type requirements for all positions can simplify this with a lambda returning a string constant.
@@ -291,7 +293,9 @@ protected interface PositionalErrorMessageSupplier {
     /**
      * Adds test cases containing unsupported parameter types that assert
      * that they throw type errors.
+     * @deprecated make a subclass of {@link ErrorsForCasesWithoutExamplesTestCase} instead
      */
+    @Deprecated
     protected static List<TestCaseSupplier> errorsForCasesWithoutExamples(
         List<TestCaseSupplier> testCaseSuppliers,
         PositionalErrorMessageSupplier positionalErrorMessageSupplier
@@ -331,11 +335,14 @@ protected interface TypeErrorMessageSupplier {
         String apply(boolean includeOrdinal, List<Set<DataType>> validPerPosition, List<DataType> types);
     }
 
+    /**
+     * @deprecated make a subclass of {@link ErrorsForCasesWithoutExamplesTestCase} instead
+     */
+    @Deprecated
     protected static List<TestCaseSupplier> errorsForCasesWithoutExamples(
         List<TestCaseSupplier> testCaseSuppliers,
         TypeErrorMessageSupplier typeErrorMessageSupplier
     ) {
-        typesRequired(testCaseSuppliers);
         List<TestCaseSupplier> suppliers = new ArrayList<>(testCaseSuppliers.size());
         suppliers.addAll(testCaseSuppliers);
 
@@ -346,7 +353,7 @@ protected static List<TestCaseSupplier> errorsForCasesWithoutExamples(
             .map(s -> s.types().size())
             .collect(Collectors.toSet())
             .stream()
-            .flatMap(count -> allPermutations(count))
+            .flatMap(AbstractFunctionTestCase::allPermutations)
             .filter(types -> valid.contains(types) == false)
             /*
              * Skip any cases with more than one null. Our tests don't generate
@@ -366,10 +373,6 @@ private static List<DataType> append(List<DataType> orig, DataType extra) {
         return longer;
     }
 
-    protected static Stream<DataType> representable() {
-        return DataType.types().stream().filter(DataType::isRepresentable);
-    }
-
     protected static TestCaseSupplier typeErrorSupplier(
         boolean includeOrdinal,
         List<Set<DataType>> validPerPosition,
@@ -398,7 +401,7 @@ protected static TestCaseSupplier typeErrorSupplier(
         );
     }
 
-    private static List<Set<DataType>> validPerPosition(Set<List<DataType>> valid) {
+    static List<Set<DataType>> validPerPosition(Set<List<DataType>> valid) {
         int max = valid.stream().mapToInt(List::size).max().getAsInt();
         List<Set<DataType>> result = new ArrayList<>(max);
         for (int i = 0; i < max; i++) {
@@ -530,14 +533,28 @@ protected final Expression buildLiteralExpression(TestCaseSupplier.TestCase test
         return build(testCase.getSource(), testCase.getDataAsLiterals());
     }
 
+    public static EvaluatorMapper.ToEvaluator toEvaluator() {
+        return new EvaluatorMapper.ToEvaluator() {
+            @Override
+            public ExpressionEvaluator.Factory apply(Expression expression) {
+                return evaluator(expression);
+            }
+
+            @Override
+            public FoldContext foldCtx() {
+                return FoldContext.small();
+            }
+        };
+    }
+
     /**
      * Convert an {@link Expression} tree into a {@link ExpressionEvaluator.Factory}
      * for {@link ExpressionEvaluator}s in the same way as our planner.
      */
     public static ExpressionEvaluator.Factory evaluator(Expression e) {
-        e = new FoldNull().rule(e);
+        e = new FoldNull().rule(e, unboundLogicalOptimizerContext());
         if (e.foldable()) {
-            e = new Literal(e.source(), e.fold(), e.dataType());
+            e = new Literal(e.source(), e.fold(FoldContext.small()), e.dataType());
         }
         Layout.Builder builder = new Layout.Builder();
         buildLayout(builder, e);
@@ -545,7 +562,7 @@ public static ExpressionEvaluator.Factory evaluator(Expression e) {
         if (resolution.unresolved()) {
             throw new AssertionError("expected resolved " + resolution.message());
         }
-        return EvalMapper.toEvaluator(e, builder.build());
+        return EvalMapper.toEvaluator(FoldContext.small(), e, builder.build());
     }
 
     protected final Page row(List<Object> values) {
@@ -1327,17 +1344,6 @@ public void allMemoryReleased() {
         }
     }
 
-    /**
-     * Validate that we know the types for all the test cases already created
-     * @param suppliers - list of suppliers before adding in the illegal type combinations
-     */
-    protected static void typesRequired(List<TestCaseSupplier> suppliers) {
-        String bad = suppliers.stream().filter(s -> s.types() == null).map(s -> s.name()).collect(Collectors.joining("\n"));
-        if (bad.equals("") == false) {
-            throw new IllegalArgumentException("types required but not found for these tests:\n" + bad);
-        }
-    }
-
     /**
      * Returns true if the current test case is for an aggregation function.
      * <p>
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java
index 958b706a3f260..944515e54af75 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.indices.CrankyCircuitBreakerService;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.optimizer.rules.logical.FoldNull;
@@ -38,6 +39,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -58,7 +60,11 @@ public abstract class AbstractScalarFunctionTestCase extends AbstractFunctionTes
      * </p>
      *
      * @param entirelyNullPreservesType See {@link #anyNullIsNull(boolean, List)}
+     * @deprecated use {@link #parameterSuppliersFromTypedDataWithDefaultChecksNoErrors}
+     *             and make a subclass of {@link ErrorsForCasesWithoutExamplesTestCase}.
+     *             It's a <strong>long</strong> faster.
      */
+    @Deprecated
     protected static Iterable<Object[]> parameterSuppliersFromTypedDataWithDefaultChecks(
         boolean entirelyNullPreservesType,
         List<TestCaseSupplier> suppliers,
@@ -72,6 +78,23 @@ protected static Iterable<Object[]> parameterSuppliersFromTypedDataWithDefaultCh
         );
     }
 
+    /**
+     * Converts a list of test cases into a list of parameter suppliers.
+     * Also, adds a default set of extra test cases.
+     * <p>
+     *     Use if possible, as this method may get updated with new checks in the future.
+     * </p>
+     *
+     * @param entirelyNullPreservesType See {@link #anyNullIsNull(boolean, List)}
+     */
+    protected static Iterable<Object[]> parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(
+        // TODO remove after removing parameterSuppliersFromTypedDataWithDefaultChecks rename this to that.
+        boolean entirelyNullPreservesType,
+        List<TestCaseSupplier> suppliers
+    ) {
+        return parameterSuppliersFromTypedData(anyNullIsNull(entirelyNullPreservesType, randomizeBytesRefsOffset(suppliers)));
+    }
+
     /**
      * Converts a list of test cases into a list of parameter suppliers.
      * Also, adds a default set of extra test cases.
@@ -111,7 +134,7 @@ public final void testEvaluate() {
         if (resolution.unresolved()) {
             throw new AssertionError("expected resolved " + resolution.message());
         }
-        expression = new FoldNull().rule(expression);
+        expression = new FoldNull().rule(expression, unboundLogicalOptimizerContext());
         assertThat(expression.dataType(), equalTo(testCase.expectedType()));
         logger.info("Result type: " + expression.dataType());
 
@@ -342,11 +365,11 @@ public void testFold() {
             return;
         }
         assertFalse("expected resolved", expression.typeResolved().unresolved());
-        Expression nullOptimized = new FoldNull().rule(expression);
+        Expression nullOptimized = new FoldNull().rule(expression, unboundLogicalOptimizerContext());
         assertThat(nullOptimized.dataType(), equalTo(testCase.expectedType()));
         assertTrue(nullOptimized.foldable());
         if (testCase.foldingExceptionClass() == null) {
-            Object result = nullOptimized.fold();
+            Object result = nullOptimized.fold(FoldContext.small());
             // Decode unsigned longs into BigIntegers
             if (testCase.expectedType() == DataType.UNSIGNED_LONG && result != null) {
                 result = NumericUtils.unsignedLongAsBigInteger((Long) result);
@@ -359,48 +382,15 @@ public void testFold() {
                 assertWarnings(testCase.getExpectedWarnings());
             }
         } else {
-            Throwable t = expectThrows(testCase.foldingExceptionClass(), nullOptimized::fold);
+            Throwable t = expectThrows(testCase.foldingExceptionClass(), () -> nullOptimized.fold(FoldContext.small()));
             assertThat(t.getMessage(), equalTo(testCase.foldingExceptionMessage()));
         }
     }
 
-    public static String errorMessageStringForBinaryOperators(
-        boolean includeOrdinal,
-        List<Set<DataType>> validPerPosition,
-        List<DataType> types,
-        PositionalErrorMessageSupplier positionalErrorMessageSupplier
-    ) {
-        try {
-            return typeErrorMessage(includeOrdinal, validPerPosition, types, positionalErrorMessageSupplier);
-        } catch (IllegalStateException e) {
-            // This means all the positional args were okay, so the expected error is from the combination
-            if (types.get(0).equals(DataType.UNSIGNED_LONG)) {
-                return "first argument of [] is [unsigned_long] and second is ["
-                    + types.get(1).typeName()
-                    + "]. [unsigned_long] can only be operated on together with another [unsigned_long]";
-
-            }
-            if (types.get(1).equals(DataType.UNSIGNED_LONG)) {
-                return "first argument of [] is ["
-                    + types.get(0).typeName()
-                    + "] and second is [unsigned_long]. [unsigned_long] can only be operated on together with another [unsigned_long]";
-            }
-            return "first argument of [] is ["
-                + (types.get(0).isNumeric() ? "numeric" : types.get(0).typeName())
-                + "] so second argument must also be ["
-                + (types.get(0).isNumeric() ? "numeric" : types.get(0).typeName())
-                + "] but was ["
-                + types.get(1).typeName()
-                + "]";
-
-        }
-    }
-
     /**
      * Adds test cases containing unsupported parameter types that immediately fail.
      */
     protected static List<TestCaseSupplier> failureForCasesWithoutExamples(List<TestCaseSupplier> testCaseSuppliers) {
-        typesRequired(testCaseSuppliers);
         List<TestCaseSupplier> suppliers = new ArrayList<>(testCaseSuppliers.size());
         suppliers.addAll(testCaseSuppliers);
 
@@ -431,7 +421,7 @@ protected static TestCaseSupplier arithmeticExceptionOverflowCase(
         String typeNameOverflow = dataType.typeName().toLowerCase(Locale.ROOT) + " overflow";
         return new TestCaseSupplier(
             "<" + typeNameOverflow + ">",
-            List.of(dataType),
+            List.of(dataType, dataType),
             () -> new TestCaseSupplier.TestCase(
                 List.of(
                     new TestCaseSupplier.TypedData(lhsSupplier.get(), dataType, "lhs"),
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java
index 98f36d339976c..e507640c7b23c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java
@@ -15,20 +15,24 @@
 import org.elasticsearch.license.internal.XPackLicenseStatus;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
+import org.elasticsearch.xpack.esql.LicenseAware;
 import org.elasticsearch.xpack.esql.VerificationException;
 import org.elasticsearch.xpack.esql.analysis.Analyzer;
 import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
 import org.elasticsearch.xpack.esql.analysis.Verifier;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.function.Function;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
+import org.elasticsearch.xpack.esql.plan.logical.Limit;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.stats.Metrics;
 
 import java.util.List;
+import java.util.Objects;
 
 import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyzerDefaultMapping;
 import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.defaultEnrichResolution;
@@ -44,25 +48,28 @@ public void testLicense() {
             final LicensedFeature functionLicenseFeature = random().nextBoolean()
                 ? LicensedFeature.momentary("test", "license", functionLicense)
                 : LicensedFeature.persistent("test", "license", functionLicense);
-            final EsqlFunctionRegistry.FunctionBuilder builder = (source, expression, cfg) -> {
-                final LicensedFunction licensedFunction = new LicensedFunction(source);
-                licensedFunction.setLicensedFeature(functionLicenseFeature);
-                return licensedFunction;
-            };
             for (License.OperationMode operationMode : License.OperationMode.values()) {
                 if (License.OperationMode.TRIAL != operationMode && License.OperationMode.compare(operationMode, functionLicense) < 0) {
                     // non-compliant license
-                    final VerificationException ex = expectThrows(VerificationException.class, () -> analyze(builder, operationMode));
-                    assertThat(ex.getMessage(), containsString("current license is non-compliant for function [license()]"));
+                    final VerificationException ex = expectThrows(
+                        VerificationException.class,
+                        () -> analyze(operationMode, functionLicenseFeature)
+                    );
+                    assertThat(ex.getMessage(), containsString("current license is non-compliant for [license()]"));
+                    assertThat(ex.getMessage(), containsString("current license is non-compliant for [LicensedLimit]"));
                 } else {
                     // compliant license
-                    assertNotNull(analyze(builder, operationMode));
+                    assertNotNull(analyze(operationMode, functionLicenseFeature));
                 }
             }
         }
     }
 
-    private LogicalPlan analyze(EsqlFunctionRegistry.FunctionBuilder builder, License.OperationMode operationMode) {
+    private LogicalPlan analyze(License.OperationMode operationMode, LicensedFeature functionLicenseFeature) {
+        final EsqlFunctionRegistry.FunctionBuilder builder = (source, expression, cfg) -> new LicensedFunction(
+            source,
+            functionLicenseFeature
+        );
         final FunctionDefinition def = EsqlFunctionRegistry.def(LicensedFunction.class, builder, "license");
         final EsqlFunctionRegistry registry = new EsqlFunctionRegistry(def) {
             @Override
@@ -70,7 +77,15 @@ public EsqlFunctionRegistry snapshotRegistry() {
                 return this;
             }
         };
-        return analyzer(registry, operationMode).analyze(parser.createStatement(esql));
+
+        var plan = parser.createStatement(esql);
+        plan = plan.transformDown(
+            Limit.class,
+            l -> Objects.equals(l.limit().fold(FoldContext.small()), 10)
+                ? new LicensedLimit(l.source(), l.limit(), l.child(), functionLicenseFeature)
+                : l
+        );
+        return analyzer(registry, operationMode).analyze(plan);
     }
 
     private static Analyzer analyzer(EsqlFunctionRegistry registry, License.OperationMode operationMode) {
@@ -88,25 +103,18 @@ private static XPackLicenseState getLicenseState(License.OperationMode operation
 
     // It needs to be public because we run validation on it via reflection in org.elasticsearch.xpack.esql.tree.EsqlNodeSubclassTests.
     // This test prevents to add the license as constructor parameter too.
-    public static class LicensedFunction extends Function {
+    public static class LicensedFunction extends Function implements LicenseAware {
 
-        private LicensedFeature licensedFeature;
+        private final LicensedFeature licensedFeature;
 
-        public LicensedFunction(Source source) {
+        public LicensedFunction(Source source, LicensedFeature licensedFeature) {
             super(source, List.of());
-        }
-
-        void setLicensedFeature(LicensedFeature licensedFeature) {
             this.licensedFeature = licensedFeature;
         }
 
         @Override
-        public boolean checkLicense(XPackLicenseState state) {
-            if (licensedFeature instanceof LicensedFeature.Momentary momentary) {
-                return momentary.check(state);
-            } else {
-                return licensedFeature.checkWithoutTracking(state);
-            }
+        public boolean licenseCheck(XPackLicenseState state) {
+            return checkLicense(state, licensedFeature);
         }
 
         @Override
@@ -121,7 +129,7 @@ public Expression replaceChildren(List<Expression> newChildren) {
 
         @Override
         protected NodeInfo<? extends Expression> info() {
-            return NodeInfo.create(this);
+            return NodeInfo.create(this, LicensedFunction::new, licensedFeature);
         }
 
         @Override
@@ -135,4 +143,39 @@ public void writeTo(StreamOutput out) {
         }
     }
 
+    public static class LicensedLimit extends Limit implements LicenseAware {
+
+        private final LicensedFeature licensedFeature;
+
+        public LicensedLimit(Source source, Expression limit, LogicalPlan child, LicensedFeature licensedFeature) {
+            super(source, limit, child);
+            this.licensedFeature = licensedFeature;
+        }
+
+        @Override
+        public boolean licenseCheck(XPackLicenseState state) {
+            return checkLicense(state, licensedFeature);
+        }
+
+        @Override
+        public Limit replaceChild(LogicalPlan newChild) {
+            return new LicensedLimit(source(), limit(), newChild, licensedFeature);
+        }
+
+        @Override
+        protected NodeInfo<Limit> info() {
+            return NodeInfo.create(this, LicensedLimit::new, limit(), child(), licensedFeature);
+        }
+
+        @Override
+        public String sourceText() {
+            return "LicensedLimit";
+        }
+    }
+
+    private static boolean checkLicense(XPackLicenseState state, LicensedFeature licensedFeature) {
+        return licensedFeature instanceof LicensedFeature.Momentary momentary
+            ? momentary.check(state)
+            : licensedFeature.checkWithoutTracking(state);
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ErrorsForCasesWithoutExamplesTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ErrorsForCasesWithoutExamplesTestCase.java
new file mode 100644
index 0000000000000..7269abad07297
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ErrorsForCasesWithoutExamplesTestCase.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.hamcrest.Matcher;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
+import static org.hamcrest.Matchers.greaterThan;
+
+public abstract class ErrorsForCasesWithoutExamplesTestCase extends ESTestCase {
+    protected abstract List<TestCaseSupplier> cases();
+
+    /**
+     * Build the expression being tested, for the given source and list of arguments.  Test classes need to implement this
+     * to have something to test.
+     *
+     * @param source the source
+     * @param args   arg list from the test case, should match the length expected
+     * @return an expression for evaluating the function being tested on the given arguments
+     */
+    protected abstract Expression build(Source source, List<Expression> args);
+
+    protected abstract Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature);
+
+    protected final List<TestCaseSupplier> paramsToSuppliers(Iterable<Object[]> cases) {
+        List<TestCaseSupplier> result = new ArrayList<>();
+        for (Object[] c : cases) {
+            if (c.length != 1) {
+                throw new IllegalArgumentException("weird layout for test cases");
+            }
+            TestCaseSupplier supplier = (TestCaseSupplier) c[0];
+            result.add(supplier);
+        }
+        return result;
+    }
+
+    public final void test() {
+        int checked = 0;
+        List<TestCaseSupplier> cases = cases();
+        Set<List<DataType>> valid = cases.stream().map(TestCaseSupplier::types).collect(Collectors.toSet());
+        List<Set<DataType>> validPerPosition = AbstractFunctionTestCase.validPerPosition(valid);
+        Iterable<List<DataType>> missingSignatures = missingSignatures(cases, valid)::iterator;
+        for (List<DataType> signature : missingSignatures) {
+            logger.debug("checking {}", signature);
+            List<Expression> args = new ArrayList<>(signature.size());
+            for (DataType type : signature) {
+                args.add(randomLiteral(type));
+            }
+            Expression expression = build(Source.synthetic(sourceForSignature(signature)), args);
+            assertTrue("expected unresolved " + expression, expression.typeResolved().unresolved());
+            assertThat(expression.typeResolved().message(), expectedTypeErrorMatcher(validPerPosition, signature));
+            checked++;
+        }
+        logger.info("checked {} signatures", checked);
+        assertNumberOfCheckedSignatures(checked);
+    }
+
+    /**
+     * Assert the number of checked signature. Generally shouldn't be overridden but
+     * can be to assert that, for example, there weren't any unsupported signatures.
+     */
+    protected void assertNumberOfCheckedSignatures(int checked) {
+        assertThat("didn't check any signatures", checked, greaterThan(0));
+    }
+
+    private Stream<List<DataType>> missingSignatures(List<TestCaseSupplier> cases, Set<List<DataType>> valid) {
+        return cases.stream()
+            .map(s -> s.types().size())
+            .collect(Collectors.toSet())
+            .stream()
+            .flatMap(AbstractFunctionTestCase::allPermutations)
+            .filter(types -> valid.contains(types) == false)
+            /*
+             * Skip any cases with more than one null. Our tests don't generate
+             * the full combinatorial explosions of all nulls - just a single null.
+             * Hopefully <null>, <null> cases will function the same as <null>, <valid>
+             * cases.
+             */
+            .filter(types -> types.stream().filter(t -> t == DataType.NULL).count() <= 1);
+    }
+
+    protected static String sourceForSignature(List<DataType> signature) {
+        StringBuilder source = new StringBuilder();
+        for (DataType type : signature) {
+            if (false == source.isEmpty()) {
+                source.append(", ");
+            }
+            source.append(type.typeName());
+        }
+        return source.toString();
+    }
+
+    /**
+     * Build the expected error message for an invalid type signature.
+     */
+    protected static String typeErrorMessage(
+        boolean includeOrdinal,
+        List<Set<DataType>> validPerPosition,
+        List<DataType> signature,
+        AbstractFunctionTestCase.PositionalErrorMessageSupplier expectedTypeSupplier
+    ) {
+        int badArgPosition = -1;
+        for (int i = 0; i < signature.size(); i++) {
+            if (validPerPosition.get(i).contains(signature.get(i)) == false) {
+                badArgPosition = i;
+                break;
+            }
+        }
+        if (badArgPosition == -1) {
+            throw new IllegalStateException(
+                "Can't generate error message for these types, you probably need a custom error message function"
+            );
+        }
+        String ordinal = includeOrdinal ? TypeResolutions.ParamOrdinal.fromIndex(badArgPosition).name().toLowerCase(Locale.ROOT) + " " : "";
+        String source = sourceForSignature(signature);
+        String expectedTypeString = expectedTypeSupplier.apply(validPerPosition.get(badArgPosition), badArgPosition);
+        String name = signature.get(badArgPosition).typeName();
+        return ordinal + "argument of [" + source + "] must be [" + expectedTypeString + "], found value [] type [" + name + "]";
+    }
+
+    protected static String errorMessageStringForBinaryOperators(
+        List<Set<DataType>> validPerPosition,
+        List<DataType> signature,
+        AbstractFunctionTestCase.PositionalErrorMessageSupplier positionalErrorMessageSupplier
+    ) {
+        try {
+            return typeErrorMessage(true, validPerPosition, signature, positionalErrorMessageSupplier);
+        } catch (IllegalStateException e) {
+            String source = sourceForSignature(signature);
+            // This means all the positional args were okay, so the expected error is from the combination
+            if (signature.get(0).equals(DataType.UNSIGNED_LONG)) {
+                return "first argument of ["
+                    + source
+                    + "] is [unsigned_long] and second is ["
+                    + signature.get(1).typeName()
+                    + "]. [unsigned_long] can only be operated on together with another [unsigned_long]";
+
+            }
+            if (signature.get(1).equals(DataType.UNSIGNED_LONG)) {
+                return "first argument of ["
+                    + source
+                    + "] is ["
+                    + signature.get(0).typeName()
+                    + "] and second is [unsigned_long]. [unsigned_long] can only be operated on together with another [unsigned_long]";
+            }
+            return "first argument of ["
+                + source
+                + "] is ["
+                + (signature.get(0).isNumeric() ? "numeric" : signature.get(0).typeName())
+                + "] so second argument must also be ["
+                + (signature.get(0).isNumeric() ? "numeric" : signature.get(0).typeName())
+                + "] but was ["
+                + signature.get(1).typeName()
+                + "]";
+
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistryTests.java
index 801bd8700d014..50cbbdf4a9338 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistryTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistryTests.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.esql.core.ParsingException;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction;
@@ -19,6 +18,7 @@
 import org.elasticsearch.xpack.esql.core.tree.SourceTests;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction;
+import org.elasticsearch.xpack.esql.parser.ParsingException;
 import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.io.IOException;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
index 575c80a52ae9b..9bf063518d4ba 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
@@ -110,13 +110,11 @@ public static List<TestCaseSupplier> stringCases(
     @Override
     public TestCase get() {
         TestCase supplied = supplier.get();
-        if (types != null) {
-            for (int i = 0; i < types.size(); i++) {
-                if (supplied.getData().get(i).type() != types.get(i)) {
-                    throw new IllegalStateException(
-                        name + ": supplier/data type mismatch " + supplied.getData().get(i).type() + "/" + types.get(i)
-                    );
-                }
+        for (int i = 0; i < types.size(); i++) {
+            if (supplied.getData().get(i).type() != types.get(i)) {
+                throw new IllegalStateException(
+                    name + ": supplier/data type mismatch " + supplied.getData().get(i).type() + "/" + types.get(i)
+                );
             }
         }
         return supplied;
@@ -1009,6 +1007,17 @@ public static List<TypedDataSupplier> dateCases() {
         return dateCases(Long.MIN_VALUE, Long.MAX_VALUE);
     }
 
+    /**
+     * Generate cases for {@link DataType#DATETIME}.
+     * <p>
+     *     For multi-row parameters, see {@link MultiRowTestCaseSupplier#dateCases}.
+     * </p>
+     * Helper function for if you want to specify your min and max range as dates instead of longs.
+     */
+    public static List<TypedDataSupplier> dateCases(Instant min, Instant max) {
+        return dateCases(min.toEpochMilli(), max.toEpochMilli());
+    }
+
     /**
      * Generate cases for {@link DataType#DATETIME}.
      * <p>
@@ -1047,6 +1056,19 @@ public static List<TypedDataSupplier> dateCases(long min, long max) {
         return cases;
     }
 
+    /**
+     *
+     * @return randomized valid date formats
+     */
+    public static List<TypedDataSupplier> dateFormatCases() {
+        return List.of(
+            new TypedDataSupplier("<format as KEYWORD>", () -> new BytesRef(ESTestCase.randomDateFormatterPattern()), DataType.KEYWORD),
+            new TypedDataSupplier("<format as TEXT>", () -> new BytesRef(ESTestCase.randomDateFormatterPattern()), DataType.TEXT),
+            new TypedDataSupplier("<format as KEYWORD>", () -> new BytesRef("yyyy"), DataType.KEYWORD),
+            new TypedDataSupplier("<format as TEXT>", () -> new BytesRef("yyyy"), DataType.TEXT)
+        );
+    }
+
     /**
      * Generate cases for {@link DataType#DATE_NANOS}.
      *
@@ -1390,7 +1412,7 @@ public static final class TestCase {
 
         /**
          * Warnings that are added by calling {@link AbstractFunctionTestCase#evaluator}
-         * or {@link Expression#fold()} on the expression built by this.
+         * or {@link Expression#fold} on the expression built by this.
          */
         private final String[] expectedBuildEvaluatorWarnings;
 
@@ -1520,7 +1542,7 @@ public String[] getExpectedWarnings() {
 
         /**
          * Warnings that are added by calling {@link AbstractFunctionTestCase#evaluator}
-         * or {@link Expression#fold()} on the expression built by this.
+         * or {@link Expression#fold} on the expression built by this.
          */
         public String[] getExpectedBuildEvaluatorWarnings() {
             return expectedBuildEvaluatorWarnings;
@@ -1602,7 +1624,7 @@ public TestCase withWarning(String warning) {
 
         /**
          * Warnings that are added by calling {@link AbstractFunctionTestCase#evaluator}
-         * or {@link Expression#fold()} on the expression built by this.
+         * or {@link Expression#fold} on the expression built by this.
          */
         public TestCase withBuildEvaluatorWarning(String warning) {
             return new TestCase(
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtentTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtentTests.java
index a1faa537ba052..225e10f99c853 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtentTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtentTests.java
@@ -17,11 +17,11 @@
 import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor;
 import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor.WrapLongitude;
 import org.elasticsearch.geometry.utils.WellKnownBinary;
+import org.elasticsearch.test.hamcrest.RectangleMatcher;
+import org.elasticsearch.test.hamcrest.WellKnownBinaryBytesRefMatcher;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.expression.RectangleMatcher;
-import org.elasticsearch.xpack.esql.expression.WellKnownBinaryBytesRefMatcher;
 import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier;
@@ -82,20 +82,7 @@ private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier
                 List.of(fieldTypedData),
                 "SpatialExtent[field=Attribute[channel=0]]",
                 expectedType,
-                new WellKnownBinaryBytesRefMatcher<>(
-                    RectangleMatcher.closeTo(
-                        new Rectangle(
-                            // Since we use integers locally which are later decoded to doubles, all computation is effectively done using
-                            // floats, not doubles.
-                            (float) result.getMinX(),
-                            (float) result.getMaxX(),
-                            (float) result.getMaxY(),
-                            (float) result.getMinY()
-                        ),
-                        1e-3,
-                        pointType
-                    )
-                )
+                new WellKnownBinaryBytesRefMatcher<>(RectangleMatcher.closeToFloat(result, 1e-3, pointType.encoder()))
             );
         });
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java
index d97be6b169eef..118bf56c5d6b9 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java
@@ -10,21 +10,14 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.junit.BeforeClass;
 
 import java.util.List;
 import java.util.function.Supplier;
 
 public class KqlTests extends NoneFieldFullTextFunctionTestCase {
-    @BeforeClass
-    protected static void ensureKqlFunctionEnabled() {
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-    }
-
     public KqlTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
         super(testCaseSupplier);
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeErrorTests.java
new file mode 100644
index 0000000000000..f674f9b2c3d72
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.grouping;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class CategorizeErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(CategorizeTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Categorize(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeTests.java
index 68bafd2af9960..dfdfd82d08afe 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeTests.java
@@ -51,7 +51,7 @@ public static Iterable<Object[]> parameters() {
                 )
             );
         }
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseErrorTests.java
new file mode 100644
index 0000000000000..28e1b846660c0
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseErrorTests.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.conditional;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class CaseErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(CaseTests.parameters()).stream()
+            // Take only the shorter signatures because we don't have error generation for longer cases
+            // TODO handle longer signatures
+            .filter(c -> c.types().size() < 4)
+            .toList();
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Case(source, args.get(0), args.subList(1, args.size()));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        if (signature.get(0) != DataType.BOOLEAN && signature.get(0) != DataType.NULL) {
+            return typeErrorMessage(signature, 0, "boolean");
+        }
+        DataType mainType = signature.get(1).noText();
+        for (int i = 2; i < signature.size(); i++) {
+            if (i % 2 == 0 && i != signature.size() - 1) {
+                // condition
+                if (signature.get(i) != DataType.BOOLEAN && signature.get(i) != DataType.NULL) {
+                    return typeErrorMessage(signature, i, "boolean");
+                }
+            } else {
+                // value
+                if (signature.get(i).noText() != mainType) {
+                    return typeErrorMessage(signature, i, mainType.typeName());
+                }
+            }
+        }
+        throw new IllegalStateException("can't find bad arg for " + signature);
+    }
+
+    private static Matcher<String> typeErrorMessage(List<DataType> signature, int badArgPosition, String expectedTypeString) {
+        String ordinal = TypeResolutions.ParamOrdinal.fromIndex(badArgPosition).name().toLowerCase(Locale.ROOT);
+        String sig = sourceForSignature(signature);
+        String name = signature.get(badArgPosition).typeName();
+        return equalTo(ordinal + " argument of [" + sig + "] must be [" + expectedTypeString + "], found value [] type [" + name + "]");
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseExtraTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseExtraTests.java
index de84086e3cb4e..911878a645b42 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseExtraTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseExtraTests.java
@@ -19,9 +19,11 @@
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.junit.After;
 
@@ -67,7 +69,7 @@ public void testPartialFoldDropsFirstFalse() {
         );
         assertThat(c.foldable(), equalTo(false));
         assertThat(
-            c.partiallyFold(),
+            c.partiallyFold(FoldContext.small()),
             equalTo(new Case(Source.synthetic("case"), field("last_cond", DataType.BOOLEAN), List.of(field("last", DataType.LONG))))
         );
     }
@@ -80,7 +82,7 @@ public void testPartialFoldMv() {
         );
         assertThat(c.foldable(), equalTo(false));
         assertThat(
-            c.partiallyFold(),
+            c.partiallyFold(FoldContext.small()),
             equalTo(new Case(Source.synthetic("case"), field("last_cond", DataType.BOOLEAN), List.of(field("last", DataType.LONG))))
         );
     }
@@ -92,7 +94,7 @@ public void testPartialFoldNoop() {
             List.of(field("first", DataType.LONG), field("last", DataType.LONG))
         );
         assertThat(c.foldable(), equalTo(false));
-        assertThat(c.partiallyFold(), sameInstance(c));
+        assertThat(c.partiallyFold(FoldContext.small()), sameInstance(c));
     }
 
     public void testPartialFoldFirst() {
@@ -102,7 +104,7 @@ public void testPartialFoldFirst() {
             List.of(field("first", DataType.LONG), field("last", DataType.LONG))
         );
         assertThat(c.foldable(), equalTo(false));
-        assertThat(c.partiallyFold(), equalTo(field("first", DataType.LONG)));
+        assertThat(c.partiallyFold(FoldContext.small()), equalTo(field("first", DataType.LONG)));
     }
 
     public void testPartialFoldFirstAfterKeepingUnknown() {
@@ -118,7 +120,7 @@ public void testPartialFoldFirstAfterKeepingUnknown() {
         );
         assertThat(c.foldable(), equalTo(false));
         assertThat(
-            c.partiallyFold(),
+            c.partiallyFold(FoldContext.small()),
             equalTo(
                 new Case(
                     Source.synthetic("case"),
@@ -141,7 +143,7 @@ public void testPartialFoldSecond() {
             )
         );
         assertThat(c.foldable(), equalTo(false));
-        assertThat(c.partiallyFold(), equalTo(field("second", DataType.LONG)));
+        assertThat(c.partiallyFold(FoldContext.small()), equalTo(field("second", DataType.LONG)));
     }
 
     public void testPartialFoldSecondAfterDroppingFalse() {
@@ -156,7 +158,7 @@ public void testPartialFoldSecondAfterDroppingFalse() {
             )
         );
         assertThat(c.foldable(), equalTo(false));
-        assertThat(c.partiallyFold(), equalTo(field("second", DataType.LONG)));
+        assertThat(c.partiallyFold(FoldContext.small()), equalTo(field("second", DataType.LONG)));
     }
 
     public void testPartialFoldLast() {
@@ -171,7 +173,7 @@ public void testPartialFoldLast() {
             )
         );
         assertThat(c.foldable(), equalTo(false));
-        assertThat(c.partiallyFold(), equalTo(field("last", DataType.LONG)));
+        assertThat(c.partiallyFold(FoldContext.small()), equalTo(field("last", DataType.LONG)));
     }
 
     public void testPartialFoldLastAfterKeepingUnknown() {
@@ -187,7 +189,7 @@ public void testPartialFoldLastAfterKeepingUnknown() {
         );
         assertThat(c.foldable(), equalTo(false));
         assertThat(
-            c.partiallyFold(),
+            c.partiallyFold(FoldContext.small()),
             equalTo(
                 new Case(
                     Source.synthetic("case"),
@@ -203,7 +205,7 @@ public void testEvalCase() {
             DriverContext driverContext = driverContext();
             Page page = new Page(driverContext.blockFactory().newConstantIntBlockWith(0, 1));
             try (
-                EvalOperator.ExpressionEvaluator eval = caseExpr.toEvaluator(AbstractFunctionTestCase::evaluator).get(driverContext);
+                EvalOperator.ExpressionEvaluator eval = caseExpr.toEvaluator(AbstractFunctionTestCase.toEvaluator()).get(driverContext);
                 Block block = eval.eval(page)
             ) {
                 return toJavaObject(block, 0);
@@ -216,7 +218,7 @@ public void testEvalCase() {
     public void testFoldCase() {
         testCase(caseExpr -> {
             assertTrue(caseExpr.foldable());
-            return caseExpr.fold();
+            return caseExpr.fold(FoldContext.small());
         });
     }
 
@@ -265,22 +267,31 @@ public void testCaseWithIncompatibleTypes() {
     public void testCaseIsLazy() {
         Case caseExpr = caseExpr(true, 1, true, 2);
         DriverContext driveContext = driverContext();
-        EvalOperator.ExpressionEvaluator evaluator = caseExpr.toEvaluator(child -> {
-            Object value = child.fold();
-            if (value != null && value.equals(2)) {
-                return dvrCtx -> new EvalOperator.ExpressionEvaluator() {
-                    @Override
-                    public Block eval(Page page) {
-                        fail("Unexpected evaluation of 4th argument");
-                        return null;
-                    }
+        EvaluatorMapper.ToEvaluator toEvaluator = new EvaluatorMapper.ToEvaluator() {
+            @Override
+            public EvalOperator.ExpressionEvaluator.Factory apply(Expression expression) {
+                Object value = expression.fold(FoldContext.small());
+                if (value != null && value.equals(2)) {
+                    return dvrCtx -> new EvalOperator.ExpressionEvaluator() {
+                        @Override
+                        public Block eval(Page page) {
+                            fail("Unexpected evaluation of 4th argument");
+                            return null;
+                        }
 
-                    @Override
-                    public void close() {}
-                };
+                        @Override
+                        public void close() {}
+                    };
+                }
+                return AbstractFunctionTestCase.evaluator(expression);
             }
-            return AbstractFunctionTestCase.evaluator(child);
-        }).get(driveContext);
+
+            @Override
+            public FoldContext foldCtx() {
+                return FoldContext.small();
+            }
+        };
+        EvalOperator.ExpressionEvaluator evaluator = caseExpr.toEvaluator(toEvaluator).get(driveContext);
         Page page = new Page(driveContext.blockFactory().newConstantIntBlockWith(0, 1));
         try (Block block = evaluator.eval(page)) {
             assertEquals(1, toJavaObject(block, 0));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java
index 93322ab0e35a7..23a0f2307171c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
@@ -25,7 +25,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Locale;
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
@@ -90,10 +89,6 @@ public static Iterable<Object[]> parameters() {
                 )
             );
         }
-        suppliers = errorsForCasesWithoutExamples(
-            suppliers,
-            (includeOrdinal, validPerPosition, types) -> typeErrorMessage(includeOrdinal, types)
-        );
 
         for (DataType type : TYPES) {
             fourAndFiveArgs(suppliers, true, randomSingleValuedCondition(), 0, type, List.of());
@@ -785,7 +780,7 @@ public void testFancyFolding() {
             return;
         }
         assertThat(e.foldable(), equalTo(true));
-        Object result = e.fold();
+        Object result = e.fold(FoldContext.small());
         if (testCase.getExpectedBuildEvaluatorWarnings() != null) {
             assertWarnings(testCase.getExpectedBuildEvaluatorWarnings());
         }
@@ -805,18 +800,18 @@ public void testPartialFold() {
         }
         Case c = (Case) buildFieldExpression(testCase);
         if (extra().expectedPartialFold == null) {
-            assertThat(c.partiallyFold(), sameInstance(c));
+            assertThat(c.partiallyFold(FoldContext.small()), sameInstance(c));
             return;
         }
         if (extra().expectedPartialFold.size() == 1) {
-            assertThat(c.partiallyFold(), equalTo(extra().expectedPartialFold.get(0).asField()));
+            assertThat(c.partiallyFold(FoldContext.small()), equalTo(extra().expectedPartialFold.get(0).asField()));
             return;
         }
         Case expected = build(
             Source.synthetic("expected"),
             extra().expectedPartialFold.stream().map(TestCaseSupplier.TypedData::asField).toList()
         );
-        assertThat(c.partiallyFold(), equalTo(expected));
+        assertThat(c.partiallyFold(FoldContext.small()), equalTo(expected));
     }
 
     private static Function<TestCaseSupplier.TestCase, TestCaseSupplier.TestCase> addWarnings(List<String> warnings) {
@@ -852,31 +847,4 @@ protected Matcher<Object> allNullsMatcher() {
         }
         return super.allNullsMatcher();
     }
-
-    private static String typeErrorMessage(boolean includeOrdinal, List<DataType> types) {
-        if (types.get(0) != DataType.BOOLEAN && types.get(0) != DataType.NULL) {
-            return typeErrorMessage(includeOrdinal, types, 0, "boolean");
-        }
-        DataType mainType = types.get(1).noText();
-        for (int i = 2; i < types.size(); i++) {
-            if (i % 2 == 0 && i != types.size() - 1) {
-                // condition
-                if (types.get(i) != DataType.BOOLEAN && types.get(i) != DataType.NULL) {
-                    return typeErrorMessage(includeOrdinal, types, i, "boolean");
-                }
-            } else {
-                // value
-                if (types.get(i).noText() != mainType) {
-                    return typeErrorMessage(includeOrdinal, types, i, mainType.typeName());
-                }
-            }
-        }
-        throw new IllegalStateException("can't find bad arg for " + types);
-    }
-
-    private static String typeErrorMessage(boolean includeOrdinal, List<DataType> types, int badArgPosition, String expectedTypeString) {
-        String ordinal = includeOrdinal ? TypeResolutions.ParamOrdinal.fromIndex(badArgPosition).name().toLowerCase(Locale.ROOT) + " " : "";
-        String name = types.get(badArgPosition).typeName();
-        return ordinal + "argument of [] must be [" + expectedTypeString + "], found value [" + name + "] type [" + name + "]";
-    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64ErrorTests.java
new file mode 100644
index 0000000000000..390995781d343
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64ErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class FromBase64ErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(FromBase64Tests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new FromBase64(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java
index 60901e2a8214f..8f518d60cd478 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java
@@ -47,7 +47,7 @@ public static Iterable<Object[]> parameters() {
             }));
         }
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64ErrorTests.java
new file mode 100644
index 0000000000000..3621050039246
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64ErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToBase64ErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToBase64Tests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToBase64(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java
index 6e6ff7bf52fce..dddf27a9ff56c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java
@@ -48,7 +48,7 @@ public static Iterable<Object[]> parameters() {
             }));
         }
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanErrorTests.java
new file mode 100644
index 0000000000000..3556a09db2319
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToBooleanErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToBooleanTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToBoolean(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "boolean or numeric or string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java
index 73837c23ff640..93a833f29cd0b 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java
@@ -80,7 +80,7 @@ public static Iterable<Object[]> parameters() {
             emptyList()
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "boolean or numeric or string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointErrorTests.java
new file mode 100644
index 0000000000000..1933a1ba739ad
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToCartesianPointErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToCartesianPointTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToCartesianPoint(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "cartesian_point or string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java
index 9548b0476f3c2..a4c31445f86b5 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java
@@ -72,7 +72,7 @@ public static Iterable<Object[]> parameters() {
             );
         }
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "cartesian_point or string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeErrorTests.java
new file mode 100644
index 0000000000000..226bf623305ee
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToCartesianShapeErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToCartesianShapeTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToCartesianShape(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "cartesian_point or cartesian_shape or string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java
index 5784c3559b10e..9f8eb52de9500 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java
@@ -73,7 +73,7 @@ public static Iterable<Object[]> parameters() {
             );
         }
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "cartesian_point or cartesian_shape or string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosErrorTests.java
new file mode 100644
index 0000000000000..b3a59c9f39ac5
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosErrorTests.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToDateNanosErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToDateNanosTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToDateNanos(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(
+            typeErrorMessage(
+                false,
+                validPerPosition,
+                signature,
+                (v, p) -> "date_nanos or datetime or double or long or string or unsigned_long"
+            )
+        );
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosSerializationTests.java
new file mode 100644
index 0000000000000..4fb39bb8d8da2
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosSerializationTests.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.AbstractUnaryScalarSerializationTests;
+
+public class ToDateNanosSerializationTests extends AbstractUnaryScalarSerializationTests<ToDateNanos> {
+    @Override
+    protected ToDateNanos create(Source source, Expression child) {
+        return new ToDateNanos(source, child);
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java
index 7459abf29410d..dc523bc247215 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java
@@ -132,11 +132,7 @@ public static Iterable<Object[]> parameters() {
                         : ("failed to parse date field [" + bytesRef.utf8ToString() + "] with format [strict_date_optional_time_nanos]"))
             )
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(
-            true,
-            suppliers,
-            (v, p) -> "date_nanos or datetime or double or long or string or unsigned_long"
-        );
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodErrorTests.java
new file mode 100644
index 0000000000000..3998fabb15743
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToDatePeriodErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToDatePeriodTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToDatePeriod(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "date_period or string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodTests.java
index ef71f404eda02..8060326365d5c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodTests.java
@@ -71,9 +71,7 @@ public static Iterable<Object[]> parameters() {
                 }));
             }
         }
-        return parameterSuppliersFromTypedData(
-            errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), (v, p) -> "date_period or string")
-        );
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeErrorTests.java
new file mode 100644
index 0000000000000..e0ee9a84dde5a
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToDatetimeErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToDatetimeTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToDatetime(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "date_nanos or datetime or numeric or string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java
index 43b889baf5306..98c6e0ea8adc7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java
@@ -178,7 +178,7 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "date_nanos or datetime or numeric or string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     private static String randomDateString(long from, long to) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesErrorTests.java
new file mode 100644
index 0000000000000..c19802a3887db
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToDegreesErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToDegreesTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToDegrees(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "numeric"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java
index 4d52a470edb87..cc780624b80a7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java
@@ -89,7 +89,7 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleErrorTests.java
new file mode 100644
index 0000000000000..091193b147c54
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleErrorTests.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToDoubleErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToDoubleTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToDouble(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(
+            typeErrorMessage(
+                false,
+                validPerPosition,
+                signature,
+                (v, p) -> "boolean or counter_double or counter_integer or counter_long or datetime or numeric or string"
+            )
+        );
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java
index 6fcd61bf15d0d..c7e6b4f0f3800 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java
@@ -141,11 +141,7 @@ public static Iterable<Object[]> parameters() {
             List.of()
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(
-            true,
-            suppliers,
-            (v, p) -> "boolean or counter_double or counter_integer or counter_long or datetime or numeric or string"
-        );
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointErrorTests.java
new file mode 100644
index 0000000000000..13b585a0478fc
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToGeoPointErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToGeoPointTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToGeoPoint(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "geo_point or string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java
index 445c4b8f15d10..5fdd83d853c75 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java
@@ -66,7 +66,7 @@ public static Iterable<Object[]> parameters() {
             );
         }
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "geo_point or string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeErrorTests.java
new file mode 100644
index 0000000000000..9e6f81e649d84
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToGeoShapeErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToGeoShapeTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToGeoShape(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "geo_point or geo_shape or string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java
index 38d871b17e8ba..9ac823553491f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java
@@ -66,7 +66,7 @@ public static Iterable<Object[]> parameters() {
                 List.of()
             );
         }
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "geo_point or geo_shape or string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPErrorTests.java
new file mode 100644
index 0000000000000..31dd36041a6fa
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToIPErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToIPTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToIP(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "ip or string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java
index 66358b7ba7440..22517ae995969 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java
@@ -61,9 +61,7 @@ public static Iterable<Object[]> parameters() {
             bytesRef -> parseIP(((BytesRef) bytesRef).utf8ToString()),
             emptyList()
         );
-
-        // add null as parameter
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "ip or string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerErrorTests.java
new file mode 100644
index 0000000000000..6ab9c0be50a73
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerErrorTests.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToIntegerErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToIntegerTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToInteger(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(
+            typeErrorMessage(false, validPerPosition, signature, (v, p) -> "boolean or counter_integer or datetime or numeric or string")
+        );
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java
index 6a3f7022c9d3e..0bd2aa24e80de 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java
@@ -271,11 +271,7 @@ public static Iterable<Object[]> parameters() {
             List.of()
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(
-            true,
-            suppliers,
-            (v, p) -> "boolean or counter_integer or datetime or numeric or string"
-        );
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongErrorTests.java
new file mode 100644
index 0000000000000..cc93e4f050796
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongErrorTests.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToLongErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToLongTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToLong(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(
+            typeErrorMessage(
+                false,
+                validPerPosition,
+                signature,
+                (v, p) -> "boolean or counter_integer or counter_long or date_nanos or datetime or numeric or string"
+            )
+        );
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java
index 49daeba43f13d..c2d7565c3e6f4 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java
@@ -236,11 +236,7 @@ public static Iterable<Object[]> parameters() {
             l -> ((Integer) l).longValue(),
             List.of()
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(
-            true,
-            suppliers,
-            (v, p) -> "boolean or counter_integer or counter_long or date_nanos or datetime or numeric or string"
-        );
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansErrorTests.java
new file mode 100644
index 0000000000000..a132448aa38e7
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToRadiansErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToRadiansTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToRadians(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "numeric"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java
index 6e99e720ad5ea..92a5bdac99db3 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java
@@ -70,7 +70,7 @@ public static Iterable<Object[]> parameters() {
             List.of()
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringErrorTests.java
new file mode 100644
index 0000000000000..31088139b8911
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringErrorTests.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToStringErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToStringTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToString(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> {
+            /*
+             * In general ToString should support all signatures. While building a
+             * new type you may we to temporarily remove this.
+             */
+            throw new UnsupportedOperationException("all signatures should be supported");
+        }));
+    }
+
+    @Override
+    protected void assertNumberOfCheckedSignatures(int checked) {
+        /*
+         * In general ToString should support all signatures. While building a
+         * new type you may we to temporarily relax this.
+         */
+        assertThat("all signatures should be supported", checked, equalTo(0));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java
index 3b30e4b353ae5..c89f8c34b8456 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java
@@ -141,12 +141,9 @@ public static Iterable<Object[]> parameters() {
             v -> new BytesRef(v.toString()),
             List.of()
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> typeErrorString);
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
-    private static String typeErrorString =
-        "boolean or cartesian_point or cartesian_shape or datetime or geo_point or geo_shape or ip or numeric or string or version";
-
     @Override
     protected Expression build(Source source, List<Expression> args) {
         return new ToString(source, args.get(0));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationErrorTests.java
new file mode 100644
index 0000000000000..c44212ed74b7e
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToTimeDurationErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToTimeDurationTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToTimeDuration(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "time_duration or string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationTests.java
index 00555f5c668cd..980eed1af9109 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationTests.java
@@ -70,9 +70,7 @@ public static Iterable<Object[]> parameters() {
                 }));
             }
         }
-        return parameterSuppliersFromTypedData(
-            errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), (v, p) -> "time_duration or string")
-        );
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongErrorTests.java
new file mode 100644
index 0000000000000..9fbc036feaa3c
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToUnsignedLongErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToUnsignedLongTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToUnsignedLong(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "boolean or datetime or numeric or string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java
index ca48bb029f223..adf104e8db5b4 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java
@@ -246,7 +246,7 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "boolean or datetime or numeric or string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionLongErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionLongErrorTests.java
new file mode 100644
index 0000000000000..9dde42cb1265b
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionLongErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToVersionLongErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToVersionTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToVersion(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string or version"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java
index 57f11331818dc..3a3026d1db675 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java
@@ -60,7 +60,7 @@ public static Iterable<Object[]> parameters() {
             );
         }
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string or version");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffErrorTests.java
new file mode 100644
index 0000000000000..a3a808de277d7
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffErrorTests.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.date;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class DateDiffErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(DateDiffTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new DateDiff(source, args.get(0), args.get(1), args.get(2));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, i) -> {
+            if (i == 0) {
+                return "string";
+            }
+            return "datetime";
+        }));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java
index 4dbdfd1e56854..b4a37b0297571 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java
@@ -71,48 +71,6 @@ public static Iterable<Object[]> parameters() {
                 )
             )
         );
-        suppliers.add(
-            new TestCaseSupplier(
-                "Date Diff Error Type unit",
-                List.of(DataType.INTEGER, DataType.DATETIME, DataType.DATETIME),
-                () -> TestCaseSupplier.TestCase.typeError(
-                    List.of(
-                        new TestCaseSupplier.TypedData(new BytesRef("seconds"), DataType.INTEGER, "unit"),
-                        new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
-                        new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
-                    ),
-                    "first argument of [] must be [string], found value [unit] type [integer]"
-                )
-            )
-        );
-        suppliers.add(
-            new TestCaseSupplier(
-                "Date Diff Error Type startTimestamp",
-                List.of(DataType.TEXT, DataType.INTEGER, DataType.DATETIME),
-                () -> TestCaseSupplier.TestCase.typeError(
-                    List.of(
-                        new TestCaseSupplier.TypedData(new BytesRef("minutes"), DataType.TEXT, "unit"),
-                        new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.INTEGER, "startTimestamp"),
-                        new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
-                    ),
-                    "second argument of [] must be [datetime], found value [startTimestamp] type [integer]"
-                )
-            )
-        );
-        suppliers.add(
-            new TestCaseSupplier(
-                "Date Diff Error Type endTimestamp",
-                List.of(DataType.TEXT, DataType.DATETIME, DataType.INTEGER),
-                () -> TestCaseSupplier.TestCase.typeError(
-                    List.of(
-                        new TestCaseSupplier.TypedData(new BytesRef("minutes"), DataType.TEXT, "unit"),
-                        new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
-                        new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.INTEGER, "endTimestamp")
-                    ),
-                    "third argument of [] must be [datetime], found value [endTimestamp] type [integer]"
-                )
-            )
-        );
         suppliers.add(new TestCaseSupplier("Date Diff In Year - 1", List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME), () -> {
             ZonedDateTime zdtStart2 = ZonedDateTime.parse("2023-12-12T00:01:01Z");
             ZonedDateTime zdtEnd2 = ZonedDateTime.parse("2024-12-12T00:01:01Z");
@@ -141,7 +99,7 @@ public static Iterable<Object[]> parameters() {
                 equalTo(0)
             );
         }));
-        return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers));
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     public void testDateDiffFunction() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java
new file mode 100644
index 0000000000000..d5b9a06c8738e
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.date;
+
+import org.elasticsearch.xpack.esql.EsqlTestUtils;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class DateExtractErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(DateExtractTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new DateExtract(source, args.get(0), args.get(1), EsqlTestUtils.TEST_CFG);
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) {
+            case 0 -> "string";
+            case 1 -> "datetime";
+            default -> "";
+        }));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java
index db375171b7224..cd27ce511b317 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -39,7 +40,7 @@ public DateExtractTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> te
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
-        return parameterSuppliersFromTypedDataWithDefaultChecks(
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(
             true,
             List.of(
                 new TestCaseSupplier(
@@ -84,12 +85,7 @@ public static Iterable<Object[]> parameters() {
                         )
                         .withFoldingException(InvalidArgumentException.class, "invalid date field for []: not a unit")
                 )
-            ),
-            (v, p) -> switch (p) {
-                case 0 -> "string";
-                case 1 -> "datetime";
-                default -> "";
-            }
+            )
         );
     }
 
@@ -104,7 +100,7 @@ public void testAllChronoFields() {
                 EsqlTestUtils.TEST_CFG
             );
 
-            assertThat(instance.fold(), is(date.getLong(value)));
+            assertThat(instance.fold(FoldContext.small()), is(date.getLong(value)));
             assertThat(
                 DateExtract.process(epochMilli, new BytesRef(value.name()), EsqlTestUtils.TEST_CFG.zoneId()),
                 is(date.getLong(value))
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java
new file mode 100644
index 0000000000000..a5e6514b3e02c
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.date;
+
+import org.elasticsearch.xpack.esql.EsqlTestUtils;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class DateFormatErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(DateFormatTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new DateFormat(source, args.get(0), args.size() == 2 ? args.get(1) : null, EsqlTestUtils.TEST_CFG);
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        // Single argument version
+        String source = sourceForSignature(signature);
+        String name = signature.get(0).typeName();
+        if (signature.size() == 1) {
+            return equalTo("first argument of [" + source + "] must be [datetime], found value [] type [" + name + "]");
+        }
+        // Two argument version
+        // Handle the weird case where we're calling the two argument version with the date first instead of the format.
+        if (signature.get(0).isDate()) {
+            return equalTo("first argument of [" + source + "] must be [string], found value [] type [" + name + "]");
+        }
+        return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) {
+            case 0 -> "string";
+            case 1 -> "datetime";
+            default -> "";
+        }));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java
index d132c2ef2b365..3dd1f3e629da4 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java
@@ -11,18 +11,21 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractConfigurationFunctionTestCase;
 import org.elasticsearch.xpack.esql.session.Configuration;
+import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter;
 
+import java.time.Instant;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Supplier;
 
-import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.matchesPattern;
 
 public class DateFormatTests extends AbstractConfigurationFunctionTestCase {
     public DateFormatTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
@@ -31,44 +34,35 @@ public DateFormatTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> tes
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
-        return parameterSuppliersFromTypedDataWithDefaultChecks(
-            true,
-            List.of(
-                new TestCaseSupplier(
-                    List.of(DataType.KEYWORD, DataType.DATETIME),
-                    () -> new TestCaseSupplier.TestCase(
-                        List.of(
-                            new TestCaseSupplier.TypedData(new BytesRef("yyyy"), DataType.KEYWORD, "formatter"),
-                            new TestCaseSupplier.TypedData(1687944333000L, DataType.DATETIME, "val")
-                        ),
-                        "DateFormatEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], locale=en_US]",
-                        DataType.KEYWORD,
-                        equalTo(BytesRefs.toBytesRef("2023"))
-                    )
+        List<TestCaseSupplier> suppliers = new ArrayList<>();
+        // Formatter supplied cases
+        suppliers.addAll(
+            TestCaseSupplier.forBinaryNotCasting(
+                (format, value) -> new BytesRef(
+                    DateFormatter.forPattern(((BytesRef) format).utf8ToString()).formatMillis(((Instant) value).toEpochMilli())
                 ),
-                new TestCaseSupplier(
-                    List.of(DataType.TEXT, DataType.DATETIME),
-                    () -> new TestCaseSupplier.TestCase(
-                        List.of(
-                            new TestCaseSupplier.TypedData(new BytesRef("yyyy"), DataType.TEXT, "formatter"),
-                            new TestCaseSupplier.TypedData(1687944333000L, DataType.DATETIME, "val")
-                        ),
-                        "DateFormatEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], locale=en_US]",
-                        DataType.KEYWORD,
-                        equalTo(BytesRefs.toBytesRef("2023"))
-                    )
-                )
-            ),
-            (v, p) -> switch (p) {
-                case 0 -> "string";
-                case 1 -> "datetime";
-                default -> "";
-            }
+                DataType.KEYWORD,
+                TestCaseSupplier.dateFormatCases(),
+                TestCaseSupplier.dateCases(Instant.parse("1900-01-01T00:00:00.00Z"), Instant.parse("9999-12-31T00:00:00.00Z")),
+                matchesPattern("DateFormatEvaluator\\[val=Attribute\\[channel=1], formatter=Attribute\\[(channel=0|\\w+)], locale=en_US]"),
+                (lhs, rhs) -> List.of(),
+                false
+            )
         );
+        // Default formatter cases
+        TestCaseSupplier.unary(
+            suppliers,
+            "DateFormatConstantEvaluator[val=Attribute[channel=0], formatter=format[strict_date_optional_time] locale[]]",
+            TestCaseSupplier.dateCases(Instant.parse("1900-01-01T00:00:00.00Z"), Instant.parse("9999-12-31T00:00:00.00Z")),
+            DataType.KEYWORD,
+            (value) -> new BytesRef(EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER.formatMillis(((Instant) value).toEpochMilli())),
+            List.of()
+        );
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
     protected Expression buildWithConfiguration(Source source, List<Expression> args, Configuration configuration) {
-        return new DateFormat(source, args.get(0), args.get(1), configuration);
+        return new DateFormat(source, args.get(0), args.size() == 2 ? args.get(1) : null, configuration);
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseErrorTests.java
new file mode 100644
index 0000000000000..21d9b5fb00537
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.date;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class DateParseErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(DateParseTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new DateParse(source, args.get(0), args.size() > 1 ? args.get(1) : null);
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, i) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java
index 8da01fc1989ba..b51866c0cc8a7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
 import java.util.List;
+import java.util.Locale;
 import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -35,7 +36,7 @@ public DateParseTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> test
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
-        return parameterSuppliersFromTypedDataWithDefaultChecks(
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(
             true,
             List.of(
                 new TestCaseSupplier(
@@ -46,11 +47,26 @@ public static Iterable<Object[]> parameters() {
                             new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.KEYWORD, "first"),
                             new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         equalTo(1683244800000L)
                     )
                 ),
+                new TestCaseSupplier("Timezoned Case", List.of(DataType.KEYWORD, DataType.KEYWORD), () -> {
+                    long ts_sec = 1657585450L; // 2022-07-12T00:24:10Z
+                    int hours = randomIntBetween(0, 23);
+                    String date = String.format(Locale.ROOT, "12/Jul/2022:%02d:24:10 +0900", hours);
+                    long expected_ts = (ts_sec + (hours - 9) * 3600L) * 1000L;
+                    return new TestCaseSupplier.TestCase(
+                        List.of(
+                            new TestCaseSupplier.TypedData(new BytesRef("dd/MMM/yyyy:HH:mm:ss Z"), DataType.KEYWORD, "first"),
+                            new TestCaseSupplier.TypedData(new BytesRef(date), DataType.KEYWORD, "second")
+                        ),
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
+                        DataType.DATETIME,
+                        equalTo(expected_ts)
+                    );
+                }),
                 new TestCaseSupplier(
                     "With Text",
                     List.of(DataType.KEYWORD, DataType.TEXT),
@@ -59,7 +75,7 @@ public static Iterable<Object[]> parameters() {
                             new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.KEYWORD, "first"),
                             new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.TEXT, "second")
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         equalTo(1683244800000L)
                     )
@@ -72,7 +88,7 @@ public static Iterable<Object[]> parameters() {
                             new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.TEXT, "first"),
                             new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.TEXT, "second")
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         equalTo(1683244800000L)
                     )
@@ -85,7 +101,7 @@ public static Iterable<Object[]> parameters() {
                             new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.TEXT, "first"),
                             new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         equalTo(1683244800000L)
                     )
@@ -98,7 +114,7 @@ public static Iterable<Object[]> parameters() {
                             new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
 
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         is(nullValue())
                     ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
@@ -118,7 +134,7 @@ public static Iterable<Object[]> parameters() {
                             new TestCaseSupplier.TypedData(new BytesRef("not a date"), DataType.KEYWORD, "second")
 
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         is(nullValue())
                     ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
@@ -127,8 +143,7 @@ public static Iterable<Object[]> parameters() {
                                 + "failed to parse date field [not a date] with format [yyyy-MM-dd]"
                         )
                 )
-            ),
-            (v, p) -> "string"
+            )
         );
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncErrorTests.java
new file mode 100644
index 0000000000000..9494888f9314f
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncErrorTests.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.date;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class DateTruncErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(DateTruncTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new DateTrunc(source, args.get(0), args.get(1));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) {
+            case 0 -> "dateperiod or timeduration";
+            case 1 -> "date_nanos or datetime";
+            default -> null;
+        }));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java
index 93970f64df845..50de64ff8b173 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java
@@ -56,11 +56,7 @@ public static Iterable<Object[]> parameters() {
         suppliers.addAll(ofDuration(Duration.ofSeconds(30), ts, "2023-02-17T10:25:30.00Z"));
         suppliers.add(randomSecond());
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) {
-            case 0 -> "dateperiod or timeduration";
-            case 1 -> "date_nanos or datetime";
-            default -> null;
-        });
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     private static List<TestCaseSupplier> ofDatePeriod(Period period, long value, String expectedDate) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java
index ce6ee1702ee66..c667747a8ba75 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java
@@ -32,7 +32,8 @@ public NowTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSu
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
-        return parameterSuppliersFromTypedData(
+        return parameterSuppliersFromTypedDataWithDefaultChecks(
+            true,
             List.of(
                 new TestCaseSupplier(
                     "Now Test",
@@ -44,7 +45,8 @@ public static Iterable<Object[]> parameters() {
                         equalTo(EsqlTestUtils.TEST_CFG.now().toInstant().toEpochMilli())
                     )
                 )
-            )
+            ),
+            (valid, position) -> ""
         );
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchErrorTests.java
new file mode 100644
index 0000000000000..319adbc00569e
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchErrorTests.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.ip;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class CIDRMatchErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(CIDRMatchTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new CIDRMatch(source, args.get(0), List.of(args.get(1)));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) {
+            case 0 -> "ip";
+            case 1 -> "string";
+            default -> "";
+        }));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java
index e777a0ce587e0..5047a3a4ab118 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java
@@ -84,11 +84,7 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) {
-            case 0 -> "ip";
-            case 1 -> "string";
-            default -> "";
-        });
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixErrorTests.java
new file mode 100644
index 0000000000000..1db58c225475e
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixErrorTests.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.ip;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class IpPrefixErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(IpPrefixTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new IpPrefix(source, args.get(0), args.get(1), args.size() == 3 ? args.get(2) : null);
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) {
+            case 0 -> "ip";
+            case 1, 2 -> "integer";
+            default -> "";
+        }));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java
index 5209d042b6408..08b517b54553d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java
@@ -106,11 +106,7 @@ public static Iterable<Object[]> parameters() {
             })
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) {
-            case 0 -> "ip";
-            case 1, 2 -> "integer";
-            default -> "";
-        });
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java
index 278c9123e30b1..6531e7bee90ab 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java
@@ -56,7 +56,7 @@ public static Iterable<Object[]> parameters() {
                 )
             )
         );
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java
index 04fec5a20b438..410dc61ec5fa6 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java
@@ -56,7 +56,7 @@ public static Iterable<Object[]> parameters() {
                 )
             )
         );
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2ErrorTests.java
new file mode 100644
index 0000000000000..75fa802067931
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2ErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.math;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class Atan2ErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(Atan2Tests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Atan2(source, args.get(0), args.get(1));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, i) -> "numeric"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java
index c475a75699da7..d7463be25e886 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java
@@ -36,7 +36,7 @@ public static Iterable<Object[]> parameters() {
             Double.POSITIVE_INFINITY,
             List.of()
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java
index bfe35a08b8ba1..d702e28baf9d8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java
@@ -72,7 +72,7 @@ public static Iterable<Object[]> parameters() {
         );
         suppliers = anyNullIsNull(true, suppliers);
 
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java
index 50ed71262e5df..f3922a355180d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java
@@ -29,14 +29,22 @@ public ETests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupp
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
-        return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("E Test", List.of(DataType.INTEGER), () -> {
-            return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(1, DataType.INTEGER, "foo")),
-                "LiteralsEvaluator[lit=2.718281828459045]",
-                DataType.DOUBLE,
-                equalTo(Math.E)
-            );
-        })));
+        return parameterSuppliersFromTypedDataWithDefaultChecks(
+            true,
+            List.of(
+                new TestCaseSupplier(
+                    "E Test",
+                    List.of(),
+                    () -> new TestCaseSupplier.TestCase(
+                        List.of(),
+                        "LiteralsEvaluator[lit=2.718281828459045]",
+                        DataType.DOUBLE,
+                        equalTo(Math.E)
+                    )
+                )
+            ),
+            (v, p) -> ""
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java
index d42f4ffde0609..bc5faf1b2560d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java
@@ -75,7 +75,7 @@ public static Iterable<Object[]> parameters() {
 
         suppliers = anyNullIsNull(true, suppliers);
 
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java
index 44ad4547481d6..7942320656f3f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java
@@ -124,7 +124,7 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java
index 671cffe9e7f9e..0ee277dbcadb2 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java
@@ -187,11 +187,7 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        // Add null cases before the rest of the error cases, so messages are correct.
-        suppliers = anyNullIsNull(true, suppliers);
-
-        // Negative cases
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java
index e93bc7b43d98e..79742952dbf59 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java
@@ -29,14 +29,22 @@ public PiTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSup
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
-        return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Pi Test", List.of(DataType.INTEGER), () -> {
-            return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(1, DataType.INTEGER, "foo")),
-                "LiteralsEvaluator[lit=3.141592653589793]",
-                DataType.DOUBLE,
-                equalTo(Math.PI)
-            );
-        })));
+        return parameterSuppliersFromTypedDataWithDefaultChecks(
+            true,
+            List.of(
+                new TestCaseSupplier(
+                    "Pi Test",
+                    List.of(),
+                    () -> new TestCaseSupplier.TestCase(
+                        List.of(),
+                        "LiteralsEvaluator[lit=3.141592653589793]",
+                        DataType.DOUBLE,
+                        equalTo(Math.PI)
+                    )
+                )
+            ),
+            (v, p) -> "numeric"
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java
index 9d8b87bab8878..2fc139a5458c3 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java
@@ -77,7 +77,7 @@ public static Iterable<Object[]> parameters() {
                 )
             )
         );
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java
index 8c612e5e664e0..4bf1351969d79 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java
@@ -72,7 +72,7 @@ public static Iterable<Object[]> parameters() {
 
         suppliers = anyNullIsNull(true, suppliers);
 
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinErrorTests.java
new file mode 100644
index 0000000000000..4637f8a85096a
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.math;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class SinErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(SinTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Sin(source, args.getFirst());
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, i) -> "numeric"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java
index 990356f8df6de..fba95289e3e8e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java
@@ -33,7 +33,7 @@ public static Iterable<Object[]> parameters() {
             Double.POSITIVE_INFINITY,
             List.of()
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java
index 23f2adc6c02e0..7cba5d6d57d45 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java
@@ -109,7 +109,7 @@ public static Iterable<Object[]> parameters() {
                 "Line -1:-1: java.lang.ArithmeticException: Square root of negative"
             )
         );
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java
index 1a622b25b3353..40e66333f953e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java
@@ -29,14 +29,22 @@ public TauTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSu
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
-        return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Tau Test", List.of(DataType.INTEGER), () -> {
-            return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(1, DataType.INTEGER, "foo")),
-                "LiteralsEvaluator[lit=6.283185307179586]",
-                DataType.DOUBLE,
-                equalTo(Tau.TAU)
-            );
-        })));
+        return parameterSuppliersFromTypedDataWithDefaultChecks(
+            true,
+            List.of(
+                new TestCaseSupplier(
+                    "Tau Test",
+                    List.of(),
+                    () -> new TestCaseSupplier.TestCase(
+                        List.of(),
+                        "LiteralsEvaluator[lit=6.283185307179586]",
+                        DataType.DOUBLE,
+                        equalTo(Tau.TAU)
+                    )
+                )
+            ),
+            (v, p) -> "numeric"
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumTests.java
index 156fc4bfe7c36..0c905b28ac931 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumTests.java
@@ -32,22 +32,20 @@ public MvPSeriesWeightedSumTests(@Name("TestCase") Supplier<TestCaseSupplier.Tes
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> cases = new ArrayList<>();
+
         doubles(cases);
 
-        cases = randomizeBytesRefsOffset(cases);
-        cases = anyNullIsNull(
-            cases,
+        return parameterSuppliersFromTypedDataWithDefaultChecks(
             (nullPosition, nullValueDataType, original) -> nullValueDataType == DataType.NULL ? DataType.NULL : original.expectedType(),
             (nullPosition, nullData, original) -> {
                 if (nullData.isForceLiteral()) {
                     return equalTo("LiteralsEvaluator[lit=null]");
                 }
                 return nullData.type() == DataType.NULL ? equalTo("LiteralsEvaluator[lit=null]") : original;
-            }
+            },
+            cases,
+            (valid, position) -> "double"
         );
-        cases = errorsForCasesWithoutExamples(cases, (valid, position) -> "double");
-
-        return parameterSuppliersFromTypedData(cases);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java
index e5f240c811bd0..5d21c135e467e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java
@@ -18,9 +18,11 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matchers;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 import java.util.function.Supplier;
 
@@ -42,6 +44,12 @@ public static Iterable<Object[]> parameters() {
         doubles(suppliers);
         bytesRefs(suppliers);
         nulls(suppliers);
+
+        suppliers = anyNullIsNull(
+            suppliers,
+            (nullPosition, nullValueDataType, original) -> nullPosition == 0 ? nullValueDataType : original.expectedType(),
+            (nullPosition, nullData, original) -> nullData.isForceLiteral() ? Matchers.equalTo("LiteralsEvaluator[lit=null]") : original
+        );
         return parameterSuppliersFromTypedData(suppliers);
     }
 
@@ -143,75 +151,22 @@ private static void doubles(List<TestCaseSupplier> suppliers) {
     }
 
     private static void bytesRefs(List<TestCaseSupplier> suppliers) {
-        suppliers.add(new TestCaseSupplier(List.of(DataType.KEYWORD, DataType.KEYWORD), () -> {
-            List<Object> field = randomList(1, 10, () -> randomLiteral(DataType.KEYWORD).value());
-            BytesRef order = new BytesRef("DESC");
-            return new TestCaseSupplier.TestCase(
-                List.of(
-                    new TestCaseSupplier.TypedData(field, DataType.KEYWORD, "field"),
-                    new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
-                ),
-                "MvSortBytesRef[field=Attribute[channel=0], order=false]",
-                DataType.KEYWORD,
-                equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted(Collections.reverseOrder()).toList())
-            );
-        }));
-
-        suppliers.add(new TestCaseSupplier(List.of(DataType.TEXT, DataType.KEYWORD), () -> {
-            List<Object> field = randomList(1, 10, () -> randomLiteral(DataType.TEXT).value());
-            BytesRef order = new BytesRef("ASC");
-            return new TestCaseSupplier.TestCase(
-                List.of(
-                    new TestCaseSupplier.TypedData(field, DataType.TEXT, "field"),
-                    new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
-                ),
-                "MvSortBytesRef[field=Attribute[channel=0], order=true]",
-                DataType.TEXT,
-                equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList())
-            );
-        }));
-
-        suppliers.add(new TestCaseSupplier(List.of(DataType.SEMANTIC_TEXT, DataType.KEYWORD), () -> {
-            List<Object> field = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value());
-            BytesRef order = new BytesRef("ASC");
-            return new TestCaseSupplier.TestCase(
-                List.of(
-                    new TestCaseSupplier.TypedData(field, DataType.SEMANTIC_TEXT, "field"),
-                    new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
-                ),
-                "MvSortBytesRef[field=Attribute[channel=0], order=true]",
-                DataType.SEMANTIC_TEXT,
-                equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList())
-            );
-        }));
-
-        suppliers.add(new TestCaseSupplier(List.of(DataType.IP, DataType.KEYWORD), () -> {
-            List<Object> field = randomList(1, 10, () -> randomLiteral(DataType.IP).value());
-            BytesRef order = new BytesRef("DESC");
-            return new TestCaseSupplier.TestCase(
-                List.of(
-                    new TestCaseSupplier.TypedData(field, DataType.IP, "field"),
-                    new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
-                ),
-                "MvSortBytesRef[field=Attribute[channel=0], order=false]",
-                DataType.IP,
-                equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted(Collections.reverseOrder()).toList())
-            );
-        }));
-
-        suppliers.add(new TestCaseSupplier(List.of(DataType.VERSION, DataType.KEYWORD), () -> {
-            List<Object> field = randomList(1, 10, () -> randomLiteral(DataType.VERSION).value());
-            BytesRef order = new BytesRef("ASC");
-            return new TestCaseSupplier.TestCase(
-                List.of(
-                    new TestCaseSupplier.TypedData(field, DataType.VERSION, "field"),
-                    new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
-                ),
-                "MvSortBytesRef[field=Attribute[channel=0], order=true]",
-                DataType.VERSION,
-                equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList())
-            );
-        }));
+        for (DataType type : List.of(DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT, DataType.IP, DataType.VERSION)) {
+            suppliers.add(new TestCaseSupplier(List.of(type, DataType.KEYWORD), () -> {
+                List<BytesRef> field = randomList(1, 10, () -> (BytesRef) randomLiteral(type).value());
+                boolean order = randomBoolean();
+                var sortedData = field.stream().sorted(order ? Comparator.naturalOrder() : Comparator.reverseOrder()).toList();
+                return new TestCaseSupplier.TestCase(
+                    List.of(
+                        new TestCaseSupplier.TypedData(field, type, "field"),
+                        new TestCaseSupplier.TypedData(new BytesRef(order ? "ASC" : "DESC"), DataType.KEYWORD, "order").forceLiteral()
+                    ),
+                    "MvSortBytesRef[field=Attribute[channel=0], order=" + order + "]",
+                    type,
+                    equalTo(sortedData.size() == 1 ? sortedData.get(0) : sortedData)
+                );
+            }));
+        }
     }
 
     private static void nulls(List<TestCaseSupplier> suppliers) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java
index 26d113359a6f8..89b148144fc83 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java
@@ -65,7 +65,7 @@ public static Iterable<Object[]> parameters() {
             data.add(asLongUnsigned(UNSIGNED_LONG_MAX));
             return data;
         }));
-        return parameterSuppliersFromTypedData(anyNullIsNull(false, cases));
+        return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases, (v, p) -> "numeric");
     }
 
     private static TestCaseSupplier arithmeticExceptionCase(DataType dataType, Supplier<Object> dataSupplier) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java
index 797c99992815e..688341ebaa2b7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -187,19 +188,27 @@ public void testCoalesceIsLazy() {
         Layout.Builder builder = new Layout.Builder();
         buildLayout(builder, exp);
         Layout layout = builder.build();
-        EvaluatorMapper.ToEvaluator toEvaluator = child -> {
-            if (child == evil) {
-                return dvrCtx -> new EvalOperator.ExpressionEvaluator() {
-                    @Override
-                    public Block eval(Page page) {
-                        throw new AssertionError("shouldn't be called");
-                    }
-
-                    @Override
-                    public void close() {}
-                };
+        EvaluatorMapper.ToEvaluator toEvaluator = new EvaluatorMapper.ToEvaluator() {
+            @Override
+            public EvalOperator.ExpressionEvaluator.Factory apply(Expression expression) {
+                if (expression == evil) {
+                    return dvrCtx -> new EvalOperator.ExpressionEvaluator() {
+                        @Override
+                        public Block eval(Page page) {
+                            throw new AssertionError("shouldn't be called");
+                        }
+
+                        @Override
+                        public void close() {}
+                    };
+                }
+                return EvalMapper.toEvaluator(FoldContext.small(), expression, layout);
+            }
+
+            @Override
+            public FoldContext foldCtx() {
+                return FoldContext.small();
             }
-            return EvalMapper.toEvaluator(child, layout);
         };
         try (
             EvalOperator.ExpressionEvaluator eval = exp.toEvaluator(toEvaluator).get(driverContext());
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java
index d069f7ffe2298..b1271b3a5b45e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java
@@ -67,7 +67,7 @@ static Iterable<Object[]> parameters(String name, boolean trimLeading, boolean t
                 }));
             }
         }
-        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(false, suppliers);
     }
 
     private static TestCaseSupplier.TestCase testCase(String name, DataType type, String data, String expected) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthErrorTests.java
new file mode 100644
index 0000000000000..23411d486901c
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class BitLengthErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(BitLengthTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new BitLength(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java
index bce4328a08abf..6f7c81a102996 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java
@@ -40,7 +40,7 @@ public static Iterable<Object[]> parameters() {
             }
         }
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthErrorTests.java
new file mode 100644
index 0000000000000..629639485f784
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ByteLengthErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ByteLengthTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ByteLength(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthTests.java
index 866b8e0cd8da3..dec96a6fb554e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthTests.java
@@ -42,7 +42,7 @@ public static Iterable<Object[]> parameters() {
         cases.addAll(makeTestCases("3 bytes, 1 code point", () -> "☕", 3));
         cases.addAll(makeTestCases("6 bytes, 2 code points", () -> "❗️", 6));
         cases.addAll(makeTestCases("100 random alpha", () -> randomAlphaOfLength(100), 100));
-        return parameterSuppliersFromTypedDataWithDefaultChecks(ENTIRELY_NULL_PRESERVES_TYPE, cases, (v, p) -> "string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(ENTIRELY_NULL_PRESERVES_TYPE, cases);
     }
 
     private static List<TestCaseSupplier> makeTestCases(String title, Supplier<String> text, int expectedByteLength) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithErrorTests.java
new file mode 100644
index 0000000000000..12067fa88b86b
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class EndsWithErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(EndsWithTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new EndsWith(source, args.get(0), args.get(1));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java
index 1b2e9c41cb25c..c41b1e14257ee 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java
@@ -73,7 +73,7 @@ public static Iterable<Object[]> parameters() {
                 );
             }
         }
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (valid, position) -> "string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     private static TestCaseSupplier.TestCase testCase(
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashErrorTests.java
new file mode 100644
index 0000000000000..63ef07d333eb8
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class HashErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(HashTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Hash(source, args.get(0), args.get(1));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashSerializationTests.java
new file mode 100644
index 0000000000000..f21105c2c8bca
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashSerializationTests.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests;
+
+import java.io.IOException;
+
+public class HashSerializationTests extends AbstractExpressionSerializationTests<Hash> {
+
+    @Override
+    protected Hash createTestInstance() {
+        return new Hash(randomSource(), randomChild(), randomChild());
+    }
+
+    @Override
+    protected Hash mutateInstance(Hash instance) throws IOException {
+        return randomBoolean()
+            ? new Hash(instance.source(), mutateExpression(instance.algorithm()), instance.input())
+            : new Hash(instance.source(), instance.algorithm(), mutateExpression(instance.input()));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashStaticTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashStaticTests.java
new file mode 100644
index 0000000000000..871bec7c06804
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashStaticTests.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.junit.After;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase.evaluator;
+import static org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase.field;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.startsWith;
+
+public class HashStaticTests extends ESTestCase {
+
+    public void testInvalidAlgorithmLiteral() {
+        Source source = new Source(0, 0, "hast(\"invalid\", input)");
+        DriverContext driverContext = driverContext();
+        InvalidArgumentException e = expectThrows(
+            InvalidArgumentException.class,
+            () -> evaluator(
+                new Hash(source, new Literal(source, new BytesRef("invalid"), DataType.KEYWORD), field("input", DataType.KEYWORD))
+            ).get(driverContext)
+        );
+        assertThat(e.getMessage(), startsWith("invalid algorithm for [hast(\"invalid\", input)]: invalid MessageDigest not available"));
+    }
+
+    /**
+     * The following fields and methods were borrowed from AbstractScalarFunctionTestCase
+     */
+    private final List<CircuitBreaker> breakers = Collections.synchronizedList(new ArrayList<>());
+
+    private DriverContext driverContext() {
+        BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, ByteSizeValue.ofMb(256)).withCircuitBreaking();
+        CircuitBreaker breaker = bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST);
+        breakers.add(breaker);
+        return new DriverContext(bigArrays, new BlockFactory(breaker, bigArrays));
+    }
+
+    @After
+    public void allMemoryReleased() {
+        for (CircuitBreaker breaker : breakers) {
+            assertThat(breaker.getUsed(), equalTo(0L));
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java
new file mode 100644
index 0000000000000..fe62b6d7b4f1b
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+
+public class HashTests extends AbstractScalarFunctionTestCase {
+
+    public HashTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        List<TestCaseSupplier> cases = new ArrayList<>();
+        for (String algorithm : List.of("MD5", "SHA", "SHA-224", "SHA-256", "SHA-384", "SHA-512")) {
+            cases.addAll(createTestCases(algorithm));
+        }
+        cases.add(new TestCaseSupplier("Invalid algorithm", List.of(DataType.KEYWORD, DataType.KEYWORD), () -> {
+            var input = randomAlphaOfLength(10);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(new BytesRef("invalid"), DataType.KEYWORD, "algorithm"),
+                    new TestCaseSupplier.TypedData(new BytesRef(input), DataType.KEYWORD, "input")
+                ),
+                "HashEvaluator[algorithm=Attribute[channel=0], input=Attribute[channel=1]]",
+                DataType.KEYWORD,
+                is(nullValue())
+            ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
+                .withWarning("Line -1:-1: java.security.NoSuchAlgorithmException: invalid MessageDigest not available")
+                .withFoldingException(InvalidArgumentException.class, "invalid algorithm for []: invalid MessageDigest not available");
+        }));
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases);
+    }
+
+    private static List<TestCaseSupplier> createTestCases(String algorithm) {
+        return List.of(
+            createTestCase(algorithm, false, DataType.KEYWORD, DataType.KEYWORD),
+            createTestCase(algorithm, false, DataType.KEYWORD, DataType.TEXT),
+            createTestCase(algorithm, false, DataType.TEXT, DataType.KEYWORD),
+            createTestCase(algorithm, false, DataType.TEXT, DataType.TEXT),
+            createTestCase(algorithm, true, DataType.KEYWORD, DataType.KEYWORD),
+            createTestCase(algorithm, true, DataType.KEYWORD, DataType.TEXT),
+            createTestCase(algorithm, true, DataType.TEXT, DataType.KEYWORD),
+            createTestCase(algorithm, true, DataType.TEXT, DataType.TEXT)
+        );
+    }
+
+    private static TestCaseSupplier createTestCase(String algorithm, boolean forceLiteral, DataType algorithmType, DataType inputType) {
+        return new TestCaseSupplier(algorithm, List.of(algorithmType, inputType), () -> {
+            var input = randomFrom(TestCaseSupplier.stringCases(inputType)).get();
+            return new TestCaseSupplier.TestCase(
+                List.of(createTypedData(algorithm, forceLiteral, algorithmType, "algorithm"), input),
+                forceLiteral
+                    ? "HashConstantEvaluator[algorithm=" + algorithm + ", input=Attribute[channel=0]]"
+                    : "HashEvaluator[algorithm=Attribute[channel=0], input=Attribute[channel=1]]",
+                DataType.KEYWORD,
+                equalTo(new BytesRef(hash(algorithm, BytesRefs.toString(input.data()))))
+            );
+        });
+    }
+
+    static void addHashFunctionTestCases(List<TestCaseSupplier> cases, String algorithm) {
+        TestCaseSupplier.forUnaryStrings(
+            cases,
+            "HashConstantEvaluator[algorithm=" + algorithm + ", input=Attribute[channel=0]]",
+            DataType.KEYWORD,
+            input -> new BytesRef(HashTests.hash(algorithm, BytesRefs.toString(input))),
+            List.of()
+        );
+    }
+
+    private static TestCaseSupplier.TypedData createTypedData(String value, boolean forceLiteral, DataType type, String name) {
+        var data = new TestCaseSupplier.TypedData(new BytesRef(value), type, name);
+        return forceLiteral ? data.forceLiteral() : data;
+    }
+
+    static String hash(String algorithm, String input) {
+        try {
+            return HexFormat.of().formatHex(MessageDigest.getInstance(algorithm).digest(input.getBytes(StandardCharsets.UTF_8)));
+        } catch (NoSuchAlgorithmException e) {
+            throw new IllegalArgumentException("Unknown algorithm: " + algorithm);
+        }
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Hash(source, args.get(0), args.get(1));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimErrorTests.java
new file mode 100644
index 0000000000000..8b3a20606eb93
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class LTrimErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(LTrimTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new LTrim(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftErrorTests.java
new file mode 100644
index 0000000000000..44b9c13e1baa6
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftErrorTests.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class LeftErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(LeftTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Left(source, args.get(0), args.get(1));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) {
+            case 0 -> "string";
+            case 1 -> "integer";
+            default -> "";
+        }));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java
index e4e54a9e0935f..2a194a39d34e5 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java
@@ -181,11 +181,7 @@ public static Iterable<Object[]> parameters() {
             );
         }));
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) {
-            case 0 -> "string";
-            case 1 -> "integer";
-            default -> "";
-        });
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     private static String unicodeLeftSubstring(String str, int length) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthErrorTests.java
new file mode 100644
index 0000000000000..ca8f2cb208209
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class LengthErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(LengthTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Length(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java
index ba4c8c8ce1ea4..e4d8292fc2cb7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.hamcrest.Matcher;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -49,7 +48,7 @@ public static Iterable<Object[]> parameters() {
         cases.addAll(makeTestCases("6 bytes, 2 code points", () -> "❗️", 2));
         cases.addAll(makeTestCases("100 random alpha", () -> randomAlphaOfLength(100), 100));
         cases.addAll(makeTestCases("100 random code points", () -> randomUnicodeOfCodepointLength(100), 100));
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases);
     }
 
     private static List<TestCaseSupplier> makeTestCases(String title, Supplier<String> text, int expectedLength) {
@@ -87,10 +86,6 @@ private static List<TestCaseSupplier> makeTestCases(String title, Supplier<Strin
         );
     }
 
-    private Matcher<Object> resultsMatcher(List<TestCaseSupplier.TypedData> typedData) {
-        return equalTo(UnicodeUtil.codePointCount((BytesRef) typedData.get(0).data()));
-    }
-
     @Override
     protected Expression build(Source source, List<Expression> args) {
         return new Length(source, args.get(0));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateErrorTests.java
new file mode 100644
index 0000000000000..a9ee74a029374
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateErrorTests.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class LocateErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(LocateTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Locate(source, args.get(0), args.get(1), args.size() < 3 ? null : args.get(2));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> {
+            if (p == 0 || p == 1) {
+                return "string";
+            }
+            return "integer";
+        }));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java
index a10f97c45aa04..e58819beb803c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java
@@ -76,13 +76,6 @@ public static Iterable<Object[]> parameters() {
             }
         }
 
-        suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), (v, p) -> {
-            if (p == 0 || p == 1) {
-                return "string";
-            }
-            return "integer";
-        });
-
         // Here follows some non-randomized examples that we want to cover on every run
         suppliers.add(supplier("a tiger", "a t", null, 1));
         suppliers.add(supplier("a tiger", "a", null, 1));
@@ -133,7 +126,7 @@ public static Iterable<Object[]> parameters() {
         suppliers.add(supplier("🐱Meow!🐶Woof!", "Meow!🐶Woof!", 0, 2));
         suppliers.add(supplier("🐱Meow!🐶Woof!", "eow!🐶Woof!", 0, 3));
 
-        return parameterSuppliersFromTypedData(suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5ErrorTests.java
new file mode 100644
index 0000000000000..701cef748c932
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5ErrorTests.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class Md5ErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(Md5Tests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Md5(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5SerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5SerializationTests.java
new file mode 100644
index 0000000000000..666e5a3ea5d58
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5SerializationTests.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests;
+
+import java.io.IOException;
+
+public class Md5SerializationTests extends AbstractExpressionSerializationTests<Md5> {
+
+    @Override
+    protected Md5 createTestInstance() {
+        return new Md5(randomSource(), randomChild());
+    }
+
+    @Override
+    protected Md5 mutateInstance(Md5 instance) throws IOException {
+        return new Md5(instance.source(), mutateExpression(instance.field()));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java
new file mode 100644
index 0000000000000..3c0a2067a81b0
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+public class Md5Tests extends AbstractScalarFunctionTestCase {
+
+    public Md5Tests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        List<TestCaseSupplier> cases = new ArrayList<>();
+        HashTests.addHashFunctionTestCases(cases, "MD5");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases);
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Md5(source, args.get(0));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java
index 4f8adf3abaae6..6c41552a9fc52 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java
@@ -12,6 +12,7 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -159,8 +160,8 @@ protected Expression build(Source source, List<Expression> args) {
         Expression expression = args.get(0);
         Literal pattern = (Literal) args.get(1);
         Literal caseInsensitive = args.size() > 2 ? (Literal) args.get(2) : null;
-        String patternString = ((BytesRef) pattern.fold()).utf8ToString();
-        boolean caseInsensitiveBool = caseInsensitive != null ? (boolean) caseInsensitive.fold() : false;
+        String patternString = ((BytesRef) pattern.fold(FoldContext.small())).utf8ToString();
+        boolean caseInsensitiveBool = caseInsensitive != null ? (boolean) caseInsensitive.fold(FoldContext.small()) : false;
         logger.info("pattern={} caseInsensitive={}", patternString, caseInsensitiveBool);
 
         return caseInsensitiveBool
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimErrorTests.java
new file mode 100644
index 0000000000000..38ab08345428b
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class RTrimErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(RTrimTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new RTrim(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java
index 8b4ea066fdccb..5eb654b0d8235 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java
@@ -122,13 +122,11 @@ public static Iterable<Object[]> parameters() {
                 .withFoldingException(IllegalArgumentException.class, "Number parameter cannot be negative, found [" + number + "]");
         }));
 
-        cases = anyNullIsNull(true, cases);
-        cases = errorsForCasesWithoutExamples(cases, (v, p) -> switch (p) {
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> switch (p) {
             case 0 -> "string";
             case 1 -> "integer";
             default -> "";
         });
-        return parameterSuppliersFromTypedData(cases);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceErrorTests.java
new file mode 100644
index 0000000000000..ed1cbda5c3de0
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ReplaceErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ReplaceTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Replace(source, args.get(0), args.get(1), args.get(2));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java
index bf1325854f1a2..dfc4db228f8e6 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java
@@ -103,7 +103,7 @@ public static Iterable<Object[]> parameters() {
                     "Unclosed character class near index 0\n[\n^".replaceAll("\n", System.lineSeparator())
                 );
         }));
-        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(false, suppliers);
     }
 
     private static TestCaseSupplier fixedCase(String name, String str, String oldStr, String newStr, String result) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java
index 58d52cc02b548..397fb8064626c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java
@@ -39,7 +39,7 @@ public static Iterable<Object[]> parameters() {
             }
         }
 
-        return parameterSuppliersFromTypedData(suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightErrorTests.java
new file mode 100644
index 0000000000000..3b90a26fb908f
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightErrorTests.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class RightErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(RightTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Right(source, args.get(0), args.get(1));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) {
+            case 0 -> "string";
+            case 1 -> "integer";
+            default -> "";
+        }));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java
index bf93ef42ed6ad..1a65407e538c2 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java
@@ -179,11 +179,7 @@ public static Iterable<Object[]> parameters() {
                 equalTo(new BytesRef(unicodeRightSubstring(text, length)))
             );
         }));
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) {
-            case 0 -> "string";
-            case 1 -> "integer";
-            default -> throw new IllegalStateException("bad parameter number");
-        });
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     private static String unicodeRightSubstring(String str, int length) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1ErrorTests.java
new file mode 100644
index 0000000000000..613aabe658ceb
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1ErrorTests.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class Sha1ErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(Sha1Tests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Sha1(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1SerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1SerializationTests.java
new file mode 100644
index 0000000000000..12c9cc7580de9
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1SerializationTests.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests;
+
+import java.io.IOException;
+
+public class Sha1SerializationTests extends AbstractExpressionSerializationTests<Sha1> {
+
+    @Override
+    protected Sha1 createTestInstance() {
+        return new Sha1(randomSource(), randomChild());
+    }
+
+    @Override
+    protected Sha1 mutateInstance(Sha1 instance) throws IOException {
+        return new Sha1(instance.source(), mutateExpression(instance.field()));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java
new file mode 100644
index 0000000000000..06c23f186d4c8
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+public class Sha1Tests extends AbstractScalarFunctionTestCase {
+
+    public Sha1Tests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        List<TestCaseSupplier> cases = new ArrayList<>();
+        HashTests.addHashFunctionTestCases(cases, "SHA1");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases);
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Sha1(source, args.get(0));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256ErrorTests.java
new file mode 100644
index 0000000000000..a9678346f8508
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256ErrorTests.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class Sha256ErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(Sha256Tests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Sha256(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256SerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256SerializationTests.java
new file mode 100644
index 0000000000000..2f209fef25d36
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256SerializationTests.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests;
+
+import java.io.IOException;
+
+public class Sha256SerializationTests extends AbstractExpressionSerializationTests<Sha256> {
+
+    @Override
+    protected Sha256 createTestInstance() {
+        return new Sha256(randomSource(), randomChild());
+    }
+
+    @Override
+    protected Sha256 mutateInstance(Sha256 instance) throws IOException {
+        return new Sha256(instance.source(), mutateExpression(instance.field()));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java
new file mode 100644
index 0000000000000..0a136e3214082
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+public class Sha256Tests extends AbstractScalarFunctionTestCase {
+
+    public Sha256Tests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        List<TestCaseSupplier> cases = new ArrayList<>();
+        HashTests.addHashFunctionTestCases(cases, "SHA256");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases);
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Sha256(source, args.get(0));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceErrorTests.java
new file mode 100644
index 0000000000000..bee85a645743b
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class SpaceErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(SpaceTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Space(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "integer"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceTests.java
index 308ce2c9d932f..40c9b33609a31 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceTests.java
@@ -71,7 +71,7 @@ public static Iterable<Object[]> parameters() {
                 .withFoldingException(IllegalArgumentException.class, "Creating strings longer than [" + max + "] bytes is not supported");
         }));
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "integer");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitErrorTests.java
new file mode 100644
index 0000000000000..c113f52a1824a
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class SplitErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(SplitTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Split(source, args.get(0), args.get(1));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java
index 098be8e1fda37..c66812ccdd1df 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java
@@ -63,7 +63,7 @@ public static Iterable<Object[]> parameters() {
                 }));
             }
         }
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithErrorTests.java
new file mode 100644
index 0000000000000..9ca061997e762
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class StartsWithErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(StartsWithTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new StartsWith(source, args.get(0), args.get(1));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java
index 60ed3b05ad642..789059fb7b6ba 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java
@@ -51,7 +51,7 @@ public static Iterable<Object[]> parameters() {
                 }));
             }
         }
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (valid, position) -> "string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringErrorTests.java
new file mode 100644
index 0000000000000..245f4e2254e50
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringErrorTests.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class SubstringErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(SubstringTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Substring(source, args.get(0), args.get(1), args.size() < 3 ? null : args.get(2));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) {
+            case 0 -> "string";
+            case 1, 2 -> "integer";
+            default -> "";
+        }));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java
index ae8a2a1840dfb..289cc0129c4e9 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java
@@ -36,7 +36,7 @@ public SubstringTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> test
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
-        return parameterSuppliersFromTypedDataWithDefaultChecks(
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(
             true,
             List.of(new TestCaseSupplier("Substring basic test", List.of(DataType.KEYWORD, DataType.INTEGER, DataType.INTEGER), () -> {
                 int start = between(1, 8);
@@ -105,12 +105,7 @@ public static Iterable<Object[]> parameters() {
                         equalTo(new BytesRef(""))
                     );
                 })
-            ),
-            (v, p) -> switch (p) {
-                case 0 -> "string";
-                case 1, 2 -> "integer";
-                default -> "";
-            }
+            )
         );
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerErrorTests.java
new file mode 100644
index 0000000000000..5744f340627c6
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerErrorTests.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.EsqlTestUtils;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToLowerErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToLowerTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToLower(source, args.get(0), EsqlTestUtils.TEST_CFG);
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java
index 69dbe023bde66..f779dd038454d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -47,16 +48,14 @@ public static Iterable<Object[]> parameters() {
         suppliers.add(supplier("text unicode", DataType.TEXT, () -> randomUnicodeOfLengthBetween(1, 10)));
         suppliers.add(supplier("semantic_text ascii", DataType.SEMANTIC_TEXT, () -> randomAlphaOfLengthBetween(1, 10)));
         suppliers.add(supplier("semantic_text unicode", DataType.SEMANTIC_TEXT, () -> randomUnicodeOfLengthBetween(1, 10)));
-
-        // add null as parameter
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     public void testRandomLocale() {
         String testString = randomAlphaOfLength(10);
         Configuration cfg = randomLocaleConfig();
         ToLower func = new ToLower(Source.EMPTY, new Literal(Source.EMPTY, testString, DataType.KEYWORD), cfg);
-        assertThat(BytesRefs.toBytesRef(testString.toLowerCase(cfg.locale())), equalTo(func.fold()));
+        assertThat(BytesRefs.toBytesRef(testString.toLowerCase(cfg.locale())), equalTo(func.fold(FoldContext.small())));
     }
 
     private Configuration randomLocaleConfig() {
@@ -83,7 +82,7 @@ protected Expression buildWithConfiguration(Source source, List<Expression> args
     private static TestCaseSupplier supplier(String name, DataType type, Supplier<String> valueSupplier) {
         return new TestCaseSupplier(name, List.of(type), () -> {
             List<TestCaseSupplier.TypedData> values = new ArrayList<>();
-            String expectedToString = "ToLowerEvaluator[val=Attribute[channel=0], locale=en_US]";
+            String expectedToString = "ChangeCaseEvaluator[val=Attribute[channel=0], locale=en_US, caseType=LOWER]";
 
             String value = valueSupplier.get();
             values.add(new TestCaseSupplier.TypedData(new BytesRef(value), type, "0"));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperErrorTests.java
new file mode 100644
index 0000000000000..94c6dc6e4a29a
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperErrorTests.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.EsqlTestUtils;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ToUpperErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(ToUpperTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new ToUpper(source, args.get(0), EsqlTestUtils.TEST_CFG);
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java
index 33d6f929503b3..3957c2e1fb2c0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -47,16 +48,14 @@ public static Iterable<Object[]> parameters() {
         suppliers.add(supplier("text unicode", DataType.TEXT, () -> randomUnicodeOfLengthBetween(1, 10)));
         suppliers.add(supplier("semantic_text ascii", DataType.SEMANTIC_TEXT, () -> randomAlphaOfLengthBetween(1, 10)));
         suppliers.add(supplier("semantic_text unicode", DataType.SEMANTIC_TEXT, () -> randomUnicodeOfLengthBetween(1, 10)));
-
-        // add null as parameter
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     public void testRandomLocale() {
         String testString = randomAlphaOfLength(10);
         Configuration cfg = randomLocaleConfig();
         ToUpper func = new ToUpper(Source.EMPTY, new Literal(Source.EMPTY, testString, DataType.KEYWORD), cfg);
-        assertThat(BytesRefs.toBytesRef(testString.toUpperCase(cfg.locale())), equalTo(func.fold()));
+        assertThat(BytesRefs.toBytesRef(testString.toUpperCase(cfg.locale())), equalTo(func.fold(FoldContext.small())));
     }
 
     private Configuration randomLocaleConfig() {
@@ -83,7 +82,7 @@ protected Expression buildWithConfiguration(Source source, List<Expression> args
     private static TestCaseSupplier supplier(String name, DataType type, Supplier<String> valueSupplier) {
         return new TestCaseSupplier(name, List.of(type), () -> {
             List<TestCaseSupplier.TypedData> values = new ArrayList<>();
-            String expectedToString = "ToUpperEvaluator[val=Attribute[channel=0], locale=en_US]";
+            String expectedToString = "ChangeCaseEvaluator[val=Attribute[channel=0], locale=en_US, caseType=UPPER]";
 
             String value = valueSupplier.get();
             values.add(new TestCaseSupplier.TypedData(new BytesRef(value), type, "0"));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimErrorTests.java
new file mode 100644
index 0000000000000..dd6de5ba9f404
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class TrimErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(TrimTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Trim(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java
index eed2c7379e9e1..6626ac50d60b5 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java
@@ -12,6 +12,7 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -78,8 +79,8 @@ protected Expression build(Source source, List<Expression> args) {
         Literal pattern = (Literal) args.get(1);
         if (args.size() > 2) {
             Literal caseInsensitive = (Literal) args.get(2);
-            assertThat(caseInsensitive.fold(), equalTo(false));
+            assertThat(caseInsensitive.fold(FoldContext.small()), equalTo(false));
         }
-        return new WildcardLike(source, expression, new WildcardPattern(((BytesRef) pattern.fold()).utf8ToString()));
+        return new WildcardLike(source, expression, new WildcardPattern(((BytesRef) pattern.fold(FoldContext.small())).utf8ToString()));
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/AbstractFulltextSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/AbstractFulltextSerializationTests.java
similarity index 88%
rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/AbstractFulltextSerializationTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/AbstractFulltextSerializationTests.java
index 370cfaf67fe0f..abd46f4b2b1aa 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/AbstractFulltextSerializationTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/AbstractFulltextSerializationTests.java
@@ -5,9 +5,8 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.expression.predicate.operator.fulltext;
+package org.elasticsearch.xpack.esql.expression.predicate.fulltext;
 
-import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.FullTextPredicate;
 import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests;
 
 import java.util.HashMap;
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/FullTextUtilsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/FullTextUtilsTests.java
similarity index 79%
rename from x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/FullTextUtilsTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/FullTextUtilsTests.java
index c6358b4682a79..46bafe5ebae9c 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/FullTextUtilsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/FullTextUtilsTests.java
@@ -4,11 +4,11 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.core.expression.predicate.fulltext;
+package org.elasticsearch.xpack.esql.expression.predicate.fulltext;
 
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.esql.core.ParsingException;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.parser.ParsingException;
 
 import java.util.Map;
 
@@ -28,15 +28,15 @@ public void testColonDelimited() {
 
     public void testColonDelimitedErrorString() {
         ParsingException e = expectThrows(ParsingException.class, () -> FullTextUtils.parseSettings("k1=v1;k2v2", source));
-        assertThat(e.getMessage(), is("line 1:3: Cannot parse entry k2v2 in options k1=v1;k2v2"));
+        assertThat(e.getMessage(), is("line 1:2: Cannot parse entry k2v2 in options k1=v1;k2v2"));
         assertThat(e.getLineNumber(), is(1));
-        assertThat(e.getColumnNumber(), is(3));
+        assertThat(e.getColumnNumber(), is(2));
     }
 
     public void testColonDelimitedErrorDuplicate() {
         ParsingException e = expectThrows(ParsingException.class, () -> FullTextUtils.parseSettings("k1=v1;k1=v2", source));
-        assertThat(e.getMessage(), is("line 1:3: Duplicate option k1=v2 detected in options k1=v1;k1=v2"));
+        assertThat(e.getMessage(), is("line 1:2: Duplicate option k1=v2 detected in options k1=v1;k1=v2"));
         assertThat(e.getLineNumber(), is(1));
-        assertThat(e.getColumnNumber(), is(3));
+        assertThat(e.getColumnNumber(), is(2));
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/MatchQuerySerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/MatchQuerySerializationTests.java
similarity index 89%
rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/MatchQuerySerializationTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/MatchQuerySerializationTests.java
index 80a538cf84baa..7781c804a6dfc 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/MatchQuerySerializationTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/MatchQuerySerializationTests.java
@@ -5,9 +5,8 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.expression.predicate.operator.fulltext;
+package org.elasticsearch.xpack.esql.expression.predicate.fulltext;
 
-import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate;
 import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests;
 
 import java.io.IOException;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/MultiMatchQuerySerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/MultiMatchQuerySerializationTests.java
similarity index 92%
rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/MultiMatchQuerySerializationTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/MultiMatchQuerySerializationTests.java
index d4d0f2edc11b1..17843e24a8663 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/MultiMatchQuerySerializationTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/MultiMatchQuerySerializationTests.java
@@ -5,9 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.expression.predicate.operator.fulltext;
-
-import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MultiMatchQueryPredicate;
+package org.elasticsearch.xpack.esql.expression.predicate.fulltext;
 
 import java.io.IOException;
 import java.util.HashMap;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java
index aa4c037e5e961..62cb0bd868ee5 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java
@@ -35,7 +35,6 @@
 import static org.elasticsearch.xpack.esql.core.util.DateUtils.asMillis;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asLongUnsigned;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.startsWith;
 
@@ -247,33 +246,7 @@ public static Iterable<Object[]> parameters() {
             );
         }).toList());
 
-        // Datetime tests are split in two, depending on their permissiveness of null-injection, which cannot happen "automatically" for
-        // Datetime + Period/Duration, since the expression will take the non-null arg's type.
-        suppliers = anyNullIsNull(
-            suppliers,
-            (nullPosition, nullType, original) -> original.expectedType(),
-            (nullPosition, nullData, original) -> nullData.isForceLiteral() ? equalTo("LiteralsEvaluator[lit=null]") : original
-        );
-        suppliers = errorsForCasesWithoutExamples(suppliers, AddTests::addErrorMessageString);
-
-        // Cases that should generate warnings
-        suppliers.add(new TestCaseSupplier("MV", List.of(DataType.INTEGER, DataType.INTEGER), () -> {
-            // Ensure we don't have an overflow
-            int rhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1);
-            int lhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1);
-            int lhs2 = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1);
-            return new TestCaseSupplier.TestCase(
-                List.of(
-                    new TestCaseSupplier.TypedData(List.of(lhs, lhs2), DataType.INTEGER, "lhs"),
-                    new TestCaseSupplier.TypedData(rhs, DataType.INTEGER, "rhs")
-                ),
-                "AddIntsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
-                DataType.INTEGER,
-                is(nullValue())
-            ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
-                .withWarning("Line -1:-1: java.lang.IllegalArgumentException: single-value function encountered multi-value");
-        }));
-        // exact math arithmetic exceptions
+        // Exact math arithmetic exceptions
         suppliers.add(
             arithmeticExceptionOverflowCase(
                 DataType.INTEGER,
@@ -315,6 +288,16 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
+        // Datetime tests are split in two, depending on their permissiveness of null-injection, which cannot happen "automatically" for
+        // Datetime + Period/Duration, since the expression will take the non-null arg's type.
+        suppliers = anyNullIsNull(
+            suppliers,
+            (nullPosition, nullType, original) -> original.expectedType(),
+            (nullPosition, nullData, original) -> nullData.isForceLiteral() ? equalTo("LiteralsEvaluator[lit=null]") : original
+        );
+        suppliers = errorsForCasesWithoutExamples(suppliers, AddTests::addErrorMessageString);
+
+        // Cannot use parameterSuppliersFromTypedDataWithDefaultChecks as error messages are non-trivial
         return parameterSuppliersFromTypedData(suppliers);
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java
index c6b607ded7999..70536cd0036e8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java
@@ -89,8 +89,6 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), DivTests::divErrorMessageString);
-
         // Divide by zero cases - all of these should warn and return null
         TestCaseSupplier.NumericTypeTestConfigs<Number> typeStuff = new TestCaseSupplier.NumericTypeTestConfigs<>(
             new TestCaseSupplier.NumericTypeTestConfig<>(
@@ -163,6 +161,9 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
+        suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), DivTests::divErrorMessageString);
+
+        // Cannot use parameterSuppliersFromTypedDataWithDefaultChecks as error messages are non-trivial
         return parameterSuppliersFromTypedData(suppliers);
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java
index 65335cdeeb4e1..2ef1cbe89e6a2 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java
@@ -76,8 +76,6 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), ModTests::modErrorMessageString);
-
         // Divide by zero cases - all of these should warn and return null
         TestCaseSupplier.NumericTypeTestConfigs<Number> typeStuff = new TestCaseSupplier.NumericTypeTestConfigs<>(
             new TestCaseSupplier.NumericTypeTestConfig<>(
@@ -150,6 +148,9 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
+        suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), ModTests::modErrorMessageString);
+
+        // Cannot use parameterSuppliersFromTypedDataWithDefaultChecks as error messages are non-trivial
         return parameterSuppliersFromTypedData(suppliers);
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java
index a2ca331cb261e..12b685eaeeb8d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java
@@ -18,6 +18,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asLongUnsigned;
@@ -124,7 +125,20 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers));
+        suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), MulTests::mulErrorMessageString);
+
+        // Cannot use parameterSuppliersFromTypedDataWithDefaultChecks as error messages are non-trivial
+        return parameterSuppliersFromTypedData(suppliers);
+    }
+
+    private static String mulErrorMessageString(boolean includeOrdinal, List<Set<DataType>> validPerPosition, List<DataType> types) {
+        try {
+            return typeErrorMessage(includeOrdinal, validPerPosition, types, (a, b) -> "numeric");
+        } catch (IllegalStateException e) {
+            // This means all the positional args were okay, so the expected error is from the combination
+            return "[*] has arguments with incompatible types [" + types.get(0).typeName() + "] and [" + types.get(1).typeName() + "]";
+
+        }
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegErrorTests.java
new file mode 100644
index 0000000000000..fcb877f15e41c
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegErrorTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class NegErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(NegTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Neg(source, args.get(0));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "numeric, date_period or time_duration"));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java
index 5bfaccfbd9347..15860d35539e0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java
@@ -12,6 +12,7 @@
 
 import org.elasticsearch.xpack.esql.VerificationException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -110,7 +111,7 @@ public static Iterable<Object[]> parameters() {
                 equalTo(arg.negated())
             ).withoutEvaluator();
         })));
-        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "numeric, date_period or time_duration");
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(false, suppliers);
     }
 
     @Override
@@ -154,7 +155,7 @@ public void testEdgeCases() {
 
     private Object foldTemporalAmount(Object val) {
         Neg neg = new Neg(Source.EMPTY, new Literal(Source.EMPTY, val, typeOf(val)));
-        return neg.fold();
+        return neg.fold(FoldContext.small());
     }
 
     private static DataType typeOf(Object val) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java
index 3a7d4e6482d72..600db229e90a3 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java
@@ -24,6 +24,7 @@
 import java.time.ZonedDateTime;
 import java.time.temporal.TemporalAmount;
 import java.util.List;
+import java.util.Set;
 import java.util.function.BinaryOperator;
 import java.util.function.Supplier;
 import java.util.function.ToLongBiFunction;
@@ -33,8 +34,6 @@
 import static org.elasticsearch.xpack.esql.core.util.DateUtils.asMillis;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.ZERO_AS_UNSIGNED_LONG;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.startsWith;
 
 public class SubTests extends AbstractScalarFunctionTestCase {
@@ -245,7 +244,8 @@ public static Iterable<Object[]> parameters() {
                 "SubUnsignedLongsEvaluator"
             )
         );
-        suppliers = anyNullIsNull(suppliers, (nullPosition, nullValueDataType, original) -> {
+
+        suppliers = errorsForCasesWithoutExamples(anyNullIsNull(suppliers, (nullPosition, nullValueDataType, original) -> {
             if (nullValueDataType == DataType.NULL) {
                 return original.getData().get(nullPosition == 0 ? 1 : 0).type();
             }
@@ -255,28 +255,30 @@ public static Iterable<Object[]> parameters() {
                 return equalTo("LiteralsEvaluator[lit=null]");
             }
             return original;
-        });
-
-        suppliers.add(new TestCaseSupplier("MV", List.of(DataType.INTEGER, DataType.INTEGER), () -> {
-            // Ensure we don't have an overflow
-            int rhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1);
-            int lhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1);
-            int lhs2 = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1);
-            return new TestCaseSupplier.TestCase(
-                List.of(
-                    new TestCaseSupplier.TypedData(List.of(lhs, lhs2), DataType.INTEGER, "lhs"),
-                    new TestCaseSupplier.TypedData(rhs, DataType.INTEGER, "rhs")
-                ),
-                "SubIntsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
-                DataType.INTEGER,
-                is(nullValue())
-            ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
-                .withWarning("Line -1:-1: java.lang.IllegalArgumentException: single-value function encountered multi-value");
-        }));
+        }), SubTests::subErrorMessageString);
 
+        // Cannot use parameterSuppliersFromTypedDataWithDefaultChecks as error messages are non-trivial
         return parameterSuppliersFromTypedData(suppliers);
     }
 
+    private static String subErrorMessageString(boolean includeOrdinal, List<Set<DataType>> validPerPosition, List<DataType> types) {
+        if (types.get(1) == DataType.DATETIME) {
+            if (types.get(0).isNumeric() || DataType.isMillisOrNanos(types.get(0))) {
+                return "[-] has arguments with incompatible types [" + types.get(0).typeName() + "] and [datetime]";
+            }
+            if (DataType.isNull(types.get(0))) {
+                return "[-] arguments are in unsupported order: cannot subtract a [DATETIME] value [datetime] from a [NULL] amount [null]";
+            }
+        }
+
+        try {
+            return typeErrorMessage(includeOrdinal, validPerPosition, types, (a, b) -> "date_nanos, datetime or numeric");
+        } catch (IllegalStateException e) {
+            // This means all the positional args were okay, so the expected error is from the combination
+            return "[-] has arguments with incompatible types [" + types.get(0).typeName() + "] and [" + types.get(1).typeName() + "]";
+        }
+    }
+
     @Override
     protected Expression build(Source source, List<Expression> args) {
         return new Sub(source, args.get(0), args.get(1));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java
new file mode 100644
index 0000000000000..cecb6a2987fd4
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class EqualsErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(EqualsTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Equals(source, args.get(0), args.get(1));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(errorMessageStringForBinaryOperators(validPerPosition, signature, (l, p) -> TYPE_ERROR));
+    }
+
+    private static final String TYPE_ERROR =
+        "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long,"
+            + " semantic_text, text, unsigned_long or version";
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java
index 6666eb8adab61..4b9f21187f425 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java
@@ -236,18 +236,9 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedData(
-            errorsForCasesWithoutExamples(
-                anyNullIsNull(true, suppliers),
-                (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(o, v, t, (l, p) -> typeErrorString)
-            )
-        );
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
-    private static String typeErrorString =
-        "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long,"
-            + " semantic_text, text, unsigned_long or version";
-
     @Override
     protected Expression build(Source source, List<Expression> args) {
         return new Equals(source, args.get(0), args.get(1));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanErrorTests.java
new file mode 100644
index 0000000000000..5b56642bb6241
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanErrorTests.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class GreaterThanErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(GreaterThanTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new GreaterThan(source, args.get(0), args.get(1));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(
+            errorMessageStringForBinaryOperators(
+                validPerPosition,
+                signature,
+                (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version"
+            )
+        );
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualErrorTests.java
new file mode 100644
index 0000000000000..904affebd3b5e
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualErrorTests.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class GreaterThanOrEqualErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(GreaterThanOrEqualTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new GreaterThanOrEqual(source, args.get(0), args.get(1));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(
+            errorMessageStringForBinaryOperators(
+                validPerPosition,
+                signature,
+                (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version"
+            )
+        );
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java
index 0fbd49abd885b..45cfef518d39a 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java
@@ -158,17 +158,7 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedData(
-            errorsForCasesWithoutExamples(
-                anyNullIsNull(true, suppliers),
-                (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(
-                    o,
-                    v,
-                    t,
-                    (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version"
-                )
-            )
-        );
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java
index ccc66df60fb3f..6ab55ddbcf403 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java
@@ -172,17 +172,7 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedData(
-            errorsForCasesWithoutExamples(
-                anyNullIsNull(true, suppliers),
-                (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(
-                    o,
-                    v,
-                    t,
-                    (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version"
-                )
-            )
-        );
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java
index b004adca351ab..80f67ec8e5e3a 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.geo.GeometryTestUtils;
 import org.elasticsearch.geo.ShapeTestUtils;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -48,27 +49,27 @@ public InTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSup
 
     public void testInWithContainedValue() {
         In in = new In(EMPTY, TWO, Arrays.asList(ONE, TWO, THREE));
-        assertTrue((Boolean) in.fold());
+        assertTrue((Boolean) in.fold(FoldContext.small()));
     }
 
     public void testInWithNotContainedValue() {
         In in = new In(EMPTY, THREE, Arrays.asList(ONE, TWO));
-        assertFalse((Boolean) in.fold());
+        assertFalse((Boolean) in.fold(FoldContext.small()));
     }
 
     public void testHandleNullOnLeftValue() {
         In in = new In(EMPTY, NULL, Arrays.asList(ONE, TWO, THREE));
-        assertNull(in.fold());
+        assertNull(in.fold(FoldContext.small()));
         in = new In(EMPTY, NULL, Arrays.asList(ONE, NULL, THREE));
-        assertNull(in.fold());
+        assertNull(in.fold(FoldContext.small()));
 
     }
 
     public void testHandleNullsOnRightValue() {
         In in = new In(EMPTY, THREE, Arrays.asList(ONE, NULL, THREE));
-        assertTrue((Boolean) in.fold());
+        assertTrue((Boolean) in.fold(FoldContext.small()));
         in = new In(EMPTY, ONE, Arrays.asList(TWO, NULL, THREE));
-        assertNull(in.fold());
+        assertNull(in.fold(FoldContext.small()));
     }
 
     private static Literal L(Object value) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsTests.java
index faf0a0d8f418c..6fa1112f23f45 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.of;
@@ -18,37 +19,37 @@
 public class InsensitiveEqualsTests extends ESTestCase {
 
     public void testFold() {
-        assertTrue(insensitiveEquals(l("foo"), l("foo")).fold());
-        assertTrue(insensitiveEquals(l("Foo"), l("foo")).fold());
-        assertTrue(insensitiveEquals(l("Foo"), l("fOO")).fold());
-        assertTrue(insensitiveEquals(l("foo*"), l("foo*")).fold());
-        assertTrue(insensitiveEquals(l("foo*"), l("FOO*")).fold());
-        assertTrue(insensitiveEquals(l("foo?bar"), l("foo?bar")).fold());
-        assertTrue(insensitiveEquals(l("foo?bar"), l("FOO?BAR")).fold());
-        assertFalse(insensitiveEquals(l("Foo"), l("fo*")).fold());
-        assertFalse(insensitiveEquals(l("Fox"), l("fo?")).fold());
-        assertFalse(insensitiveEquals(l("Foo"), l("*OO")).fold());
-        assertFalse(insensitiveEquals(l("BarFooBaz"), l("*O*")).fold());
-        assertFalse(insensitiveEquals(l("BarFooBaz"), l("bar*baz")).fold());
-        assertFalse(insensitiveEquals(l("foo"), l("*")).fold());
+        assertTrue(insensitiveEquals(l("foo"), l("foo")).fold(FoldContext.small()));
+        assertTrue(insensitiveEquals(l("Foo"), l("foo")).fold(FoldContext.small()));
+        assertTrue(insensitiveEquals(l("Foo"), l("fOO")).fold(FoldContext.small()));
+        assertTrue(insensitiveEquals(l("foo*"), l("foo*")).fold(FoldContext.small()));
+        assertTrue(insensitiveEquals(l("foo*"), l("FOO*")).fold(FoldContext.small()));
+        assertTrue(insensitiveEquals(l("foo?bar"), l("foo?bar")).fold(FoldContext.small()));
+        assertTrue(insensitiveEquals(l("foo?bar"), l("FOO?BAR")).fold(FoldContext.small()));
+        assertFalse(insensitiveEquals(l("Foo"), l("fo*")).fold(FoldContext.small()));
+        assertFalse(insensitiveEquals(l("Fox"), l("fo?")).fold(FoldContext.small()));
+        assertFalse(insensitiveEquals(l("Foo"), l("*OO")).fold(FoldContext.small()));
+        assertFalse(insensitiveEquals(l("BarFooBaz"), l("*O*")).fold(FoldContext.small()));
+        assertFalse(insensitiveEquals(l("BarFooBaz"), l("bar*baz")).fold(FoldContext.small()));
+        assertFalse(insensitiveEquals(l("foo"), l("*")).fold(FoldContext.small()));
 
-        assertFalse(insensitiveEquals(l("foo*bar"), l("foo\\*bar")).fold());
-        assertFalse(insensitiveEquals(l("foo?"), l("foo\\?")).fold());
-        assertFalse(insensitiveEquals(l("foo?bar"), l("foo\\?bar")).fold());
-        assertFalse(insensitiveEquals(l(randomAlphaOfLength(10)), l("*")).fold());
-        assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("???")).fold());
+        assertFalse(insensitiveEquals(l("foo*bar"), l("foo\\*bar")).fold(FoldContext.small()));
+        assertFalse(insensitiveEquals(l("foo?"), l("foo\\?")).fold(FoldContext.small()));
+        assertFalse(insensitiveEquals(l("foo?bar"), l("foo\\?bar")).fold(FoldContext.small()));
+        assertFalse(insensitiveEquals(l(randomAlphaOfLength(10)), l("*")).fold(FoldContext.small()));
+        assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("???")).fold(FoldContext.small()));
 
-        assertFalse(insensitiveEquals(l("foo"), l("bar")).fold());
-        assertFalse(insensitiveEquals(l("foo"), l("ba*")).fold());
-        assertFalse(insensitiveEquals(l("foo"), l("*a*")).fold());
-        assertFalse(insensitiveEquals(l(""), l("bar")).fold());
-        assertFalse(insensitiveEquals(l("foo"), l("")).fold());
-        assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("??")).fold());
-        assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("????")).fold());
+        assertFalse(insensitiveEquals(l("foo"), l("bar")).fold(FoldContext.small()));
+        assertFalse(insensitiveEquals(l("foo"), l("ba*")).fold(FoldContext.small()));
+        assertFalse(insensitiveEquals(l("foo"), l("*a*")).fold(FoldContext.small()));
+        assertFalse(insensitiveEquals(l(""), l("bar")).fold(FoldContext.small()));
+        assertFalse(insensitiveEquals(l("foo"), l("")).fold(FoldContext.small()));
+        assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("??")).fold(FoldContext.small()));
+        assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("????")).fold(FoldContext.small()));
 
-        assertNull(insensitiveEquals(l("foo"), Literal.NULL).fold());
-        assertNull(insensitiveEquals(Literal.NULL, l("foo")).fold());
-        assertNull(insensitiveEquals(Literal.NULL, Literal.NULL).fold());
+        assertNull(insensitiveEquals(l("foo"), Literal.NULL).fold(FoldContext.small()));
+        assertNull(insensitiveEquals(Literal.NULL, l("foo")).fold(FoldContext.small()));
+        assertNull(insensitiveEquals(Literal.NULL, Literal.NULL).fold(FoldContext.small()));
     }
 
     public void testProcess() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanErrorTests.java
new file mode 100644
index 0000000000000..94e3be44ba2f2
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanErrorTests.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class LessThanErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(LessThanTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new LessThan(source, args.get(0), args.get(1));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(
+            errorMessageStringForBinaryOperators(
+                validPerPosition,
+                signature,
+                (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version"
+            )
+        );
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualErrorTests.java
new file mode 100644
index 0000000000000..6074cc7c63db2
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualErrorTests.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class LessThanOrEqualErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(LessThanOrEqualTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new LessThanOrEqual(source, args.get(0), args.get(1));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(
+            errorMessageStringForBinaryOperators(
+                validPerPosition,
+                signature,
+                (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version"
+            )
+        );
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java
index 1e91a65e04c0e..8d48d5812d9da 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java
@@ -158,17 +158,7 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedData(
-            errorsForCasesWithoutExamples(
-                anyNullIsNull(true, suppliers),
-                (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(
-                    o,
-                    v,
-                    t,
-                    (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version"
-                )
-            )
-        );
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java
index 69dc59bac6456..f5f4bae5b2afe 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java
@@ -172,17 +172,7 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedData(
-            errorsForCasesWithoutExamples(
-                anyNullIsNull(true, suppliers),
-                (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(
-                    o,
-                    v,
-                    t,
-                    (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version"
-                )
-            )
-        );
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java
new file mode 100644
index 0000000000000..ed0a477ec613c
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.Matcher;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class NotEqualsErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
+    @Override
+    protected List<TestCaseSupplier> cases() {
+        return paramsToSuppliers(NotEqualsTests.parameters());
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new NotEquals(source, args.get(0), args.get(1));
+    }
+
+    @Override
+    protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
+        return equalTo(errorMessageStringForBinaryOperators(validPerPosition, signature, (l, p) -> TYPE_ERROR));
+    }
+
+    private static final String TYPE_ERROR =
+        "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long, text, "
+            + "unsigned_long or version";
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java
index 7b57b97dfe28e..95001366f1cad 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java
@@ -232,17 +232,9 @@ public static Iterable<Object[]> parameters() {
                 false
             )
         );
-        return parameterSuppliersFromTypedData(
-            errorsForCasesWithoutExamples(
-                anyNullIsNull(true, suppliers),
-                (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(o, v, t, (l, p) -> typeErrorString)
-            )
-        );
-    }
 
-    private static String typeErrorString =
-        "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long, text, "
-            + "unsigned_long or version";
+        return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
+    }
 
     @Override
     protected Expression build(Source source, List<Expression> args) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java
index 82dd5a88ffaf1..8f846edf2b41c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java
@@ -56,7 +56,7 @@ private static Map<String, IndexMode> randomConcreteIndices() {
 
     @Override
     protected Writeable.Reader<EsIndex> instanceReader() {
-        return a -> new EsIndex(new PlanStreamInput(a, a.namedWriteableRegistry(), null));
+        return a -> EsIndex.readFrom(new PlanStreamInput(a, a.namedWriteableRegistry(), null));
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java
index 0c03556241d28..11cd123c731e8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
@@ -70,6 +71,7 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.statsForExistingField;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.statsForMissingField;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
 import static org.hamcrest.Matchers.contains;
@@ -93,7 +95,7 @@ public static void init() {
         mapping = loadMapping("mapping-basic.json");
         EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD));
         IndexResolution getIndexResult = IndexResolution.valid(test);
-        logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG));
+        logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext());
 
         analyzer = new Analyzer(
             new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), getIndexResult, EsqlTestUtils.emptyPolicyResolution()),
@@ -161,7 +163,7 @@ public void testMissingFieldInProject() {
         assertThat(Expressions.names(eval.fields()), contains("last_name"));
         var alias = as(eval.fields().get(0), Alias.class);
         var literal = as(alias.child(), Literal.class);
-        assertThat(literal.fold(), is(nullValue()));
+        assertThat(literal.value(), is(nullValue()));
         assertThat(literal.dataType(), is(DataType.KEYWORD));
 
         var limit = as(eval.child(), Limit.class);
@@ -304,7 +306,7 @@ public void testMissingFieldInEval() {
 
         var alias = as(eval.fields().get(0), Alias.class);
         var literal = as(alias.child(), Literal.class);
-        assertThat(literal.fold(), is(nullValue()));
+        assertThat(literal.value(), is(nullValue()));
         assertThat(literal.dataType(), is(DataType.INTEGER));
 
         var limit = as(eval.child(), Limit.class);
@@ -402,7 +404,7 @@ public void testSparseDocument() throws Exception {
 
         EsIndex index = new EsIndex("large", large, Map.of("large", IndexMode.STANDARD));
         IndexResolution getIndexResult = IndexResolution.valid(index);
-        var logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG));
+        var logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext());
 
         var analyzer = new Analyzer(
             new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), getIndexResult, EsqlTestUtils.emptyPolicyResolution()),
@@ -411,7 +413,7 @@ public void testSparseDocument() throws Exception {
 
         var analyzed = analyzer.analyze(parser.createStatement(query));
         var optimized = logicalOptimizer.optimize(analyzed);
-        var localContext = new LocalLogicalOptimizerContext(EsqlTestUtils.TEST_CFG, searchStats);
+        var localContext = new LocalLogicalOptimizerContext(EsqlTestUtils.TEST_CFG, FoldContext.small(), searchStats);
         var plan = new LocalLogicalPlanOptimizer(localContext).localOptimize(optimized);
 
         var project = as(plan, Project.class);
@@ -423,7 +425,7 @@ public void testSparseDocument() throws Exception {
         var eval = as(project.child(), Eval.class);
         var field = eval.fields().get(0);
         assertThat(Expressions.name(field), is("field005"));
-        assertThat(Alias.unwrap(field).fold(), Matchers.nullValue());
+        assertThat(Alias.unwrap(field).fold(FoldContext.small()), Matchers.nullValue());
     }
 
     // InferIsNotNull
@@ -561,7 +563,7 @@ private LogicalPlan plan(String query) {
     }
 
     private LogicalPlan localPlan(LogicalPlan plan, SearchStats searchStats) {
-        var localContext = new LocalLogicalOptimizerContext(EsqlTestUtils.TEST_CFG, searchStats);
+        var localContext = new LocalLogicalOptimizerContext(EsqlTestUtils.TEST_CFG, FoldContext.small(), searchStats);
         // System.out.println(plan);
         var localPlan = new LocalLogicalPlanOptimizer(localContext).localOptimize(plan);
         // System.out.println(localPlan);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java
index 879a413615202..1536ed7f99fec 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.QueryStringQueryBuilder;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.license.XPackLicenseState;
@@ -35,6 +36,7 @@
 import org.elasticsearch.xpack.esql.analysis.Verifier;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -57,6 +59,7 @@
 import org.elasticsearch.xpack.esql.plan.physical.EvalExec;
 import org.elasticsearch.xpack.esql.plan.physical.ExchangeExec;
 import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec;
+import org.elasticsearch.xpack.esql.plan.physical.FilterExec;
 import org.elasticsearch.xpack.esql.plan.physical.LimitExec;
 import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
@@ -88,6 +91,7 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.as;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning;
 import static org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.StatsType;
 import static org.hamcrest.Matchers.contains;
@@ -405,7 +409,7 @@ public void testMultiCountAllWithFilter() {
     @SuppressWarnings("unchecked")
     public void testSingleCountWithStatsFilter() {
         // an optimizer that filters out the ExtractAggregateCommonFilter rule
-        var logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config)) {
+        var logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext()) {
             @Override
             protected List<Batch<LogicalPlan>> batches() {
                 var oldBatches = super.batches();
@@ -484,7 +488,7 @@ public void testQueryStringFunction() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
         var expected = QueryBuilders.queryStringQuery("last_name: Smith");
         assertThat(query.query().toString(), is(expected.toString()));
     }
@@ -513,7 +517,7 @@ public void testQueryStringFunctionConjunctionWhereOperands() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
 
         Source filterSource = new Source(2, 37, "emp_no > 10000");
         var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource);
@@ -548,7 +552,7 @@ public void testQueryStringFunctionWithFunctionsPushedToLucene() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
 
         Source filterSource = new Source(2, 37, "cidr_match(ip, \"127.0.0.1/32\")");
         var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource);
@@ -583,7 +587,7 @@ public void testQueryStringFunctionMultipleWhereClauses() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
 
         Source filterSource = new Source(3, 8, "emp_no > 10000");
         var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource);
@@ -616,7 +620,7 @@ public void testQueryStringFunctionMultipleQstrClauses() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
 
         var queryStringLeft = QueryBuilders.queryStringQuery("last_name: Smith");
         var queryStringRight = QueryBuilders.queryStringQuery("emp_no: [10010 TO *]");
@@ -645,7 +649,7 @@ public void testMatchFunction() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
         var expected = QueryBuilders.matchQuery("last_name", "Smith").lenient(true);
         assertThat(query.query().toString(), is(expected.toString()));
     }
@@ -674,7 +678,7 @@ public void testMatchFunctionConjunctionWhereOperands() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
 
         Source filterSource = new Source(2, 38, "emp_no > 10000");
         var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource);
@@ -709,7 +713,7 @@ public void testMatchFunctionWithFunctionsPushedToLucene() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
 
         Source filterSource = new Source(2, 32, "cidr_match(ip, \"127.0.0.1/32\")");
         var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource);
@@ -743,7 +747,7 @@ public void testMatchFunctionMultipleWhereClauses() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
 
         Source filterSource = new Source(3, 8, "emp_no > 10000");
         var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource);
@@ -775,7 +779,7 @@ public void testMatchFunctionMultipleMatchClauses() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
 
         var queryStringLeft = QueryBuilders.matchQuery("last_name", "Smith").lenient(true);
         var queryStringRight = QueryBuilders.matchQuery("first_name", "John").lenient(true);
@@ -794,9 +798,6 @@ public void testMatchFunctionMultipleMatchClauses() {
      *       \_EsQueryExec[test], indexMode[standard], query[{"kql":{"query":"last_name: Smith"}}]
      */
     public void testKqlFunction() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         var plan = plannerOptimizer.plan("""
             from test
             | where kql("last_name: Smith")
@@ -807,7 +808,7 @@ public void testKqlFunction() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
         var expected = kqlQueryBuilder("last_name: Smith");
         assertThat(query.query().toString(), is(expected.toString()));
     }
@@ -825,9 +826,6 @@ public void testKqlFunction() {
      *        "boost":1.0}}][_doc{f}#1423], limit[1000], sort[] estimatedRowSize[324]
      */
     public void testKqlFunctionConjunctionWhereOperands() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         String queryText = """
             from test
             | where kql("last_name: Smith") and emp_no > 10010
@@ -839,7 +837,7 @@ public void testKqlFunctionConjunctionWhereOperands() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
 
         Source filterSource = new Source(2, 36, "emp_no > 10000");
         var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource);
@@ -862,9 +860,6 @@ public void testKqlFunctionConjunctionWhereOperands() {
      *       "source":"cidr_match(ip, \"127.0.0.1/32\")@2:38"}}],"boost":1.0}}][_doc{f}#21], limit[1000], sort[] estimatedRowSize[354]
      */
     public void testKqlFunctionWithFunctionsPushedToLucene() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         String queryText = """
             from test
             | where kql("last_name: Smith") and cidr_match(ip, "127.0.0.1/32")
@@ -877,7 +872,7 @@ public void testKqlFunctionWithFunctionsPushedToLucene() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
 
         Source filterSource = new Source(2, 36, "cidr_match(ip, \"127.0.0.1/32\")");
         var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource);
@@ -900,9 +895,6 @@ public void testKqlFunctionWithFunctionsPushedToLucene() {
      *       "boost":1.0}}][_doc{f}#1167], limit[1000], sort[] estimatedRowSize[324]
      */
     public void testKqlFunctionMultipleWhereClauses() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         String queryText = """
             from test
             | where kql("last_name: Smith")
@@ -915,7 +907,7 @@ public void testKqlFunctionMultipleWhereClauses() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
 
         Source filterSource = new Source(3, 8, "emp_no > 10000");
         var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource);
@@ -937,9 +929,6 @@ public void testKqlFunctionMultipleWhereClauses() {
      *       {"kql":{"query":"emp_no > 10010"}}],"boost":1.0}}]
      */
     public void testKqlFunctionMultipleKqlClauses() {
-        // Skip test if the kql function is not enabled.
-        assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled());
-
         String queryText = """
             from test
             | where kql("last_name: Smith") and kql("emp_no > 10010")
@@ -951,7 +940,7 @@ public void testKqlFunctionMultipleKqlClauses() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
 
         var kqlQueryLeft = kqlQueryBuilder("last_name: Smith");
         var kqlQueryRight = kqlQueryBuilder("emp_no > 10010");
@@ -1017,7 +1006,7 @@ public void testIsNotNullPushdownFilter() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
         var expected = QueryBuilders.existsQuery("emp_no");
         assertThat(query.query().toString(), is(expected.toString()));
     }
@@ -1041,7 +1030,7 @@ public void testIsNullPushdownFilter() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
         var expected = QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("emp_no"));
         assertThat(query.query().toString(), is(expected.toString()));
     }
@@ -1448,6 +1437,10 @@ private void checkMatchFunctionPushDown(
         var analyzer = makeAnalyzer("mapping-all-types.json");
         // Check for every possible query data type
         for (DataType fieldDataType : fieldDataTypes) {
+            if (DataType.UNDER_CONSTRUCTION.containsKey(fieldDataType)) {
+                continue;
+            }
+
             var queryValue = randomQueryValue(fieldDataType);
 
             String fieldName = fieldDataType == DataType.DATETIME ? "date" : fieldDataType.name().toLowerCase(Locale.ROOT);
@@ -1481,6 +1474,7 @@ private static Object randomQueryValue(DataType dataType) {
             case KEYWORD -> randomAlphaOfLength(5);
             case IP -> NetworkAddress.format(randomIp(randomBoolean()));
             case TEXT -> randomAlphaOfLength(50);
+            case SEMANTIC_TEXT -> randomAlphaOfLength(5);
             case VERSION -> VersionUtils.randomVersion(random()).toString();
             default -> throw new IllegalArgumentException("Unexpected type: " + dataType);
         };
@@ -1543,6 +1537,46 @@ public void testMultipleMatchFilterPushdown() {
         assertThat(actualLuceneQuery.toString(), is(expectedLuceneQuery.toString()));
     }
 
+    public void testFullTextFunctionsDisjunctionPushdown() {
+        String query = """
+            from test
+            | where (match(first_name, "Anna") or qstr("first_name: Anneke")) and last_name: "Smith"
+            | sort emp_no
+            """;
+        var plan = plannerOptimizer.plan(query);
+        var topNExec = as(plan, TopNExec.class);
+        var exchange = as(topNExec.child(), ExchangeExec.class);
+        var project = as(exchange.child(), ProjectExec.class);
+        var fieldExtract = as(project.child(), FieldExtractExec.class);
+        var actualLuceneQuery = as(fieldExtract.child(), EsQueryExec.class).query();
+        var expectedLuceneQuery = new BoolQueryBuilder().must(
+            new BoolQueryBuilder().should(new MatchQueryBuilder("first_name", "Anna").lenient(true))
+                .should(new QueryStringQueryBuilder("first_name: Anneke"))
+        ).must(new MatchQueryBuilder("last_name", "Smith").lenient(true));
+        assertThat(actualLuceneQuery.toString(), is(expectedLuceneQuery.toString()));
+    }
+
+    public void testFullTextFunctionsDisjunctionWithFiltersPushdown() {
+        String query = """
+            from test
+            | where (first_name:"Anna" or first_name:"Anneke") and length(last_name) > 5
+            | sort emp_no
+            """;
+        var plan = plannerOptimizer.plan(query);
+        var topNExec = as(plan, TopNExec.class);
+        var exchange = as(topNExec.child(), ExchangeExec.class);
+        var project = as(exchange.child(), ProjectExec.class);
+        var fieldExtract = as(project.child(), FieldExtractExec.class);
+        var secondTopNExec = as(fieldExtract.child(), TopNExec.class);
+        var secondFieldExtract = as(secondTopNExec.child(), FieldExtractExec.class);
+        var filterExec = as(secondFieldExtract.child(), FilterExec.class);
+        var thirdFilterExtract = as(filterExec.child(), FieldExtractExec.class);
+        var actualLuceneQuery = as(thirdFilterExtract.child(), EsQueryExec.class).query();
+        var expectedLuceneQuery = new BoolQueryBuilder().should(new MatchQueryBuilder("first_name", "Anna").lenient(true))
+            .should(new MatchQueryBuilder("first_name", "Anneke").lenient(true));
+        assertThat(actualLuceneQuery.toString(), is(expectedLuceneQuery.toString()));
+    }
+
     /**
      * Expecting
      * LimitExec[1000[INTEGER]]
@@ -1567,7 +1601,7 @@ public void testTermFunction() {
         var project = as(exchange.child(), ProjectExec.class);
         var field = as(project.child(), FieldExtractExec.class);
         var query = as(field.child(), EsQueryExec.class);
-        assertThat(query.limit().fold(), is(1000));
+        assertThat(as(query.limit(), Literal.class).value(), is(1000));
         var expected = QueryBuilders.termQuery("last_name", "Smith");
         assertThat(query.query().toString(), is(expected.toString()));
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
index 1d10ebab267ce..2aed259e7ad0b 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
@@ -33,12 +33,14 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
@@ -85,6 +87,7 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.InsensitiveEquals;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
 import org.elasticsearch.xpack.esql.index.EsIndex;
 import org.elasticsearch.xpack.esql.index.IndexResolution;
@@ -126,6 +129,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.function.BiFunction;
 import java.util.function.Function;
@@ -148,6 +152,7 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.localSource;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.referenceAttribute;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning;
 import static org.elasticsearch.xpack.esql.analysis.Analyzer.NO_FIELDS;
 import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyze;
@@ -185,6 +190,7 @@ public class LogicalPlanOptimizerTests extends ESTestCase {
 
     private static EsqlParser parser;
     private static Analyzer analyzer;
+    private static LogicalOptimizerContext logicalOptimizerCtx;
     private static LogicalPlanOptimizer logicalOptimizer;
     private static Map<String, EsField> mapping;
     private static Map<String, EsField> mappingAirports;
@@ -200,7 +206,7 @@ public class LogicalPlanOptimizerTests extends ESTestCase {
     private static Analyzer metricsAnalyzer;
 
     private static class SubstitutionOnlyOptimizer extends LogicalPlanOptimizer {
-        static SubstitutionOnlyOptimizer INSTANCE = new SubstitutionOnlyOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG));
+        static SubstitutionOnlyOptimizer INSTANCE = new SubstitutionOnlyOptimizer(unboundLogicalOptimizerContext());
 
         SubstitutionOnlyOptimizer(LogicalOptimizerContext optimizerContext) {
             super(optimizerContext);
@@ -215,15 +221,11 @@ protected List<Batch<LogicalPlan>> batches() {
     @BeforeClass
     public static void init() {
         parser = new EsqlParser();
-        logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG));
+        logicalOptimizerCtx = unboundLogicalOptimizerContext();
+        logicalOptimizer = new LogicalPlanOptimizer(logicalOptimizerCtx);
         enrichResolution = new EnrichResolution();
         AnalyzerTestUtils.loadEnrichPolicyResolution(enrichResolution, "languages_idx", "id", "languages_idx", "mapping-languages.json");
 
-        var lookupMapping = loadMapping("mapping-languages.json");
-        IndexResolution lookupResolution = IndexResolution.valid(
-            new EsIndex("language_code", lookupMapping, Map.of("language_code", IndexMode.LOOKUP))
-        );
-
         // Most tests used data from the test index, so we load it here, and use it in the plan() function.
         mapping = loadMapping("mapping-basic.json");
         EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD));
@@ -327,7 +329,7 @@ public void testEmptyProjectInStatWithEval() {
         assertThat(exprs.size(), equalTo(1));
         var alias = as(exprs.get(0), Alias.class);
         assertThat(alias.name(), equalTo("x"));
-        assertThat(alias.child().fold(), equalTo(1));
+        assertThat(alias.child().fold(FoldContext.small()), equalTo(1));
     }
 
     /**
@@ -363,11 +365,11 @@ public void testEmptyProjectInStatWithGroupAndEval() {
         assertThat(exprs.size(), equalTo(1));
         var alias = as(exprs.get(0), Alias.class);
         assertThat(alias.name(), equalTo("x"));
-        assertThat(alias.child().fold(), equalTo(1));
+        assertThat(alias.child().fold(FoldContext.small()), equalTo(1));
 
         var filterCondition = as(filter.condition(), GreaterThan.class);
         assertThat(Expressions.name(filterCondition.left()), equalTo("languages"));
-        assertThat(filterCondition.right().fold(), equalTo(1));
+        assertThat(filterCondition.right().fold(FoldContext.small()), equalTo(1));
     }
 
     public void testCombineProjections() {
@@ -627,7 +629,7 @@ public void testReplaceStatsFilteredAggWithEvalSingleAggWithExpression() {
         assertThat(alias.name(), is("sum(salary) + 1 where false"));
         var add = as(alias.child(), Add.class);
         var literal = as(add.right(), Literal.class);
-        assertThat(literal.fold(), is(1));
+        assertThat(literal.value(), is(1));
 
         var limit = as(eval.child(), Limit.class);
         var source = as(limit.child(), LocalRelation.class);
@@ -660,7 +662,7 @@ public void testReplaceStatsFilteredAggWithEvalMixedFilterAndNoFilter() {
 
         var alias = as(eval.fields().getFirst(), Alias.class);
         assertTrue(alias.child().foldable());
-        assertThat(alias.child().fold(), nullValue());
+        assertThat(alias.child().fold(FoldContext.small()), nullValue());
         assertThat(alias.child().dataType(), is(LONG));
 
         alias = as(eval.fields().getLast(), Alias.class);
@@ -697,7 +699,7 @@ public void testReplaceStatsFilteredAggWithEvalFilterFalseAndNull() {
 
         var alias = as(eval.fields().getFirst(), Alias.class);
         assertTrue(alias.child().foldable());
-        assertThat(alias.child().fold(), nullValue());
+        assertThat(alias.child().fold(FoldContext.small()), nullValue());
         assertThat(alias.child().dataType(), is(LONG));
 
         alias = as(eval.fields().get(1), Alias.class);
@@ -705,7 +707,7 @@ public void testReplaceStatsFilteredAggWithEvalFilterFalseAndNull() {
 
         alias = as(eval.fields().getLast(), Alias.class);
         assertTrue(alias.child().foldable());
-        assertThat(alias.child().fold(), nullValue());
+        assertThat(alias.child().fold(FoldContext.small()), nullValue());
         assertThat(alias.child().dataType(), is(LONG));
 
         var limit = as(eval.child(), Limit.class);
@@ -754,7 +756,7 @@ public void testReplaceStatsFilteredAggWithEvalCountDistinctInExpression() {
         assertThat(alias.name(), is("count_distinct(salary + 2) + 3 where false"));
         var add = as(alias.child(), Add.class);
         var literal = as(add.right(), Literal.class);
-        assertThat(literal.fold(), is(3));
+        assertThat(literal.value(), is(3));
 
         var limit = as(eval.child(), Limit.class);
         var source = as(limit.child(), LocalRelation.class);
@@ -790,13 +792,13 @@ public void testReplaceStatsFilteredAggWithEvalSameAggWithAndWithoutFilter() {
         var alias = as(eval.fields().getFirst(), Alias.class);
         assertThat(Expressions.name(alias), containsString("max_a"));
         assertTrue(alias.child().foldable());
-        assertThat(alias.child().fold(), nullValue());
+        assertThat(alias.child().fold(FoldContext.small()), nullValue());
         assertThat(alias.child().dataType(), is(INTEGER));
 
         alias = as(eval.fields().getLast(), Alias.class);
         assertThat(Expressions.name(alias), containsString("min_a"));
         assertTrue(alias.child().foldable());
-        assertThat(alias.child().fold(), nullValue());
+        assertThat(alias.child().fold(FoldContext.small()), nullValue());
         assertThat(alias.child().dataType(), is(INTEGER));
 
         var limit = as(eval.child(), Limit.class);
@@ -935,7 +937,7 @@ public void testExtractStatsCommonFilterUsingJustOneAlias() {
         var gt = as(filter.condition(), GreaterThan.class);
         assertThat(Expressions.name(gt.left()), is("emp_no"));
         assertTrue(gt.right().foldable());
-        assertThat(gt.right().fold(), is(1));
+        assertThat(gt.right().fold(FoldContext.small()), is(1));
 
         var source = as(filter.child(), EsRelation.class);
     }
@@ -1055,7 +1057,7 @@ public void testExtractStatsCommonFilterInConjunction() {
         var gt = as(filter.condition(), GreaterThan.class); // name is "emp_no > 1 + 1"
         assertThat(Expressions.name(gt.left()), is("emp_no"));
         assertTrue(gt.right().foldable());
-        assertThat(gt.right().fold(), is(2));
+        assertThat(gt.right().fold(FoldContext.small()), is(2));
 
         var source = as(filter.child(), EsRelation.class);
     }
@@ -1085,12 +1087,12 @@ public void testExtractStatsCommonFilterInConjunctionWithMultipleCommonConjuncti
         var lt = as(and.left(), LessThan.class);
         assertThat(Expressions.name(lt.left()), is("emp_no"));
         assertTrue(lt.right().foldable());
-        assertThat(lt.right().fold(), is(10));
+        assertThat(lt.right().fold(FoldContext.small()), is(10));
 
         var equals = as(and.right(), Equals.class);
         assertThat(Expressions.name(equals.left()), is("last_name"));
         assertTrue(equals.right().foldable());
-        assertThat(equals.right().fold(), is(BytesRefs.toBytesRef("Doe")));
+        assertThat(equals.right().fold(FoldContext.small()), is(BytesRefs.toBytesRef("Doe")));
 
         var source = as(filter.child(), EsRelation.class);
     }
@@ -1305,7 +1307,7 @@ public void testCombineLimits() {
         var anotherLimit = new Limit(EMPTY, L(limitValues[secondLimit]), oneLimit);
         assertEquals(
             new Limit(EMPTY, L(Math.min(limitValues[0], limitValues[1])), emptySource()),
-            new PushDownAndCombineLimits().rule(anotherLimit)
+            new PushDownAndCombineLimits().rule(anotherLimit, logicalOptimizerCtx)
         );
     }
 
@@ -1324,7 +1326,7 @@ public void testPushdownLimitsPastLeftJoin() {
 
         var limit = new Limit(EMPTY, L(10), join);
 
-        var optimizedPlan = new PushDownAndCombineLimits().rule(limit);
+        var optimizedPlan = new PushDownAndCombineLimits().rule(limit, logicalOptimizerCtx);
 
         assertEquals(join.replaceChildren(limit.replaceChild(join.left()), join.right()), optimizedPlan);
     }
@@ -1342,10 +1344,7 @@ public void testMultipleCombineLimits() {
             var value = i == limitWithMinimum ? minimum : randomIntBetween(100, 1000);
             plan = new Limit(EMPTY, L(value), plan);
         }
-        assertEquals(
-            new Limit(EMPTY, L(minimum), relation),
-            new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG)).optimize(plan)
-        );
+        assertEquals(new Limit(EMPTY, L(minimum), relation), logicalOptimizer.optimize(plan));
     }
 
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/115311")
@@ -1866,7 +1865,7 @@ public void testCopyDefaultLimitPastMvExpand() {
         assertThat(mvExpand.limit(), equalTo(1000));
         var keep = as(mvExpand.child(), EsqlProject.class);
         var limitPastMvExpand = as(keep.child(), Limit.class);
-        assertThat(limitPastMvExpand.limit().fold(), equalTo(1000));
+        assertThat(limitPastMvExpand.limit().fold(FoldContext.small()), equalTo(1000));
         as(limitPastMvExpand.child(), EsRelation.class);
     }
 
@@ -1889,7 +1888,7 @@ public void testDontPushDownLimitPastMvExpand() {
         assertThat(mvExpand.limit(), equalTo(10));
         var project = as(mvExpand.child(), EsqlProject.class);
         var limit = as(project.child(), Limit.class);
-        assertThat(limit.limit().fold(), equalTo(1));
+        assertThat(limit.limit().fold(FoldContext.small()), equalTo(1));
         as(limit.child(), EsRelation.class);
     }
 
@@ -1923,7 +1922,7 @@ public void testMultipleMvExpandWithSortAndLimit() {
 
         var keep = as(plan, EsqlProject.class);
         var topN = as(keep.child(), TopN.class);
-        assertThat(topN.limit().fold(), equalTo(5));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(5));
         assertThat(orderNames(topN), contains("salary"));
         var mvExp = as(topN.child(), MvExpand.class);
         assertThat(mvExp.limit(), equalTo(5));
@@ -1933,7 +1932,7 @@ public void testMultipleMvExpandWithSortAndLimit() {
         mvExp = as(filter.child(), MvExpand.class);
         assertThat(mvExp.limit(), equalTo(10));
         topN = as(mvExp.child(), TopN.class);
-        assertThat(topN.limit().fold(), equalTo(10));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(10));
         filter = as(topN.child(), Filter.class);
         as(filter.child(), EsRelation.class);
     }
@@ -1957,11 +1956,11 @@ public void testPushDownLimitThroughMultipleSort_AfterMvExpand() {
 
         var keep = as(plan, EsqlProject.class);
         var topN = as(keep.child(), TopN.class);
-        assertThat(topN.limit().fold(), equalTo(5));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(5));
         assertThat(orderNames(topN), contains("salary", "first_name"));
         var mvExp = as(topN.child(), MvExpand.class);
         topN = as(mvExp.child(), TopN.class);
-        assertThat(topN.limit().fold(), equalTo(10000));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000));
         assertThat(orderNames(topN), contains("emp_no"));
         as(topN.child(), EsRelation.class);
     }
@@ -1987,14 +1986,14 @@ public void testPushDownLimitThroughMultipleSort_AfterMvExpand2() {
 
         var keep = as(plan, EsqlProject.class);
         var topN = as(keep.child(), TopN.class);
-        assertThat(topN.limit().fold(), equalTo(5));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(5));
         assertThat(orderNames(topN), contains("first_name"));
         topN = as(topN.child(), TopN.class);
-        assertThat(topN.limit().fold(), equalTo(5));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(5));
         assertThat(orderNames(topN), contains("salary"));
         var mvExp = as(topN.child(), MvExpand.class);
         topN = as(mvExp.child(), TopN.class);
-        assertThat(topN.limit().fold(), equalTo(10000));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000));
         assertThat(orderNames(topN), contains("emp_no"));
         as(topN.child(), EsRelation.class);
     }
@@ -2023,11 +2022,11 @@ public void testDontPushDownLimitPastAggregate_AndMvExpand() {
 
         var limit = as(plan, Limit.class);
         var filter = as(limit.child(), Filter.class);
-        assertThat(limit.limit().fold(), equalTo(5));
+        assertThat(limit.limit().fold(FoldContext.small()), equalTo(5));
         var agg = as(filter.child(), Aggregate.class);
         var mvExp = as(agg.child(), MvExpand.class);
         var topN = as(mvExp.child(), TopN.class);
-        assertThat(topN.limit().fold(), equalTo(50));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(50));
         assertThat(orderNames(topN), contains("emp_no"));
         as(topN.child(), EsRelation.class);
     }
@@ -2054,13 +2053,13 @@ public void testPushDown_TheRightLimit_PastMvExpand() {
             | limit 5""");
 
         var limit = as(plan, Limit.class);
-        assertThat(limit.limit().fold(), equalTo(5));
+        assertThat(limit.limit().fold(FoldContext.small()), equalTo(5));
         var filter = as(limit.child(), Filter.class);
         var agg = as(filter.child(), Aggregate.class);
         var mvExp = as(agg.child(), MvExpand.class);
         assertThat(mvExp.limit(), equalTo(50));
         limit = as(mvExp.child(), Limit.class);
-        assertThat(limit.limit().fold(), equalTo(50));
+        assertThat(limit.limit().fold(FoldContext.small()), equalTo(50));
         as(limit.child(), EsRelation.class);
     }
 
@@ -2085,12 +2084,12 @@ public void testPushDownLimit_PastEvalAndMvExpand() {
 
         var keep = as(plan, EsqlProject.class);
         var topN = as(keep.child(), TopN.class);
-        assertThat(topN.limit().fold(), equalTo(5));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(5));
         assertThat(orderNames(topN), contains("salary"));
         var eval = as(topN.child(), Eval.class);
         var mvExp = as(eval.child(), MvExpand.class);
         topN = as(mvExp.child(), TopN.class);
-        assertThat(topN.limit().fold(), equalTo(10000));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000));
         assertThat(orderNames(topN), contains("first_name"));
         as(topN.child(), EsRelation.class);
     }
@@ -2116,7 +2115,7 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedField_ResultT
 
         var keep = as(plan, EsqlProject.class);
         var topN = as(keep.child(), TopN.class);
-        assertThat(topN.limit().fold(), equalTo(1000));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(1000));
         assertThat(orderNames(topN), contains("salary", "first_name"));
         var filter = as(topN.child(), Filter.class);
         assertThat(filter.condition(), instanceOf(And.class));
@@ -2145,7 +2144,7 @@ public void testFilterWithSortBeforeMvExpand() {
         var mvExp = as(plan, MvExpand.class);
         assertThat(mvExp.limit(), equalTo(10));
         var topN = as(mvExp.child(), TopN.class);
-        assertThat(topN.limit().fold(), equalTo(10));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(10));
         assertThat(orderNames(topN), contains("emp_no"));
         var filter = as(topN.child(), Filter.class);
         as(filter.child(), EsRelation.class);
@@ -2170,7 +2169,7 @@ public void testMultiMvExpand_SortDownBelow() {
             | sort first_name""");
 
         var topN = as(plan, TopN.class);
-        assertThat(topN.limit().fold(), equalTo(1000));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(1000));
         assertThat(orderNames(topN), contains("first_name"));
         var mvExpand = as(topN.child(), MvExpand.class);
         var filter = as(mvExpand.child(), Filter.class);
@@ -2202,11 +2201,11 @@ public void testLimitThenSortBeforeMvExpand() {
         assertThat(mvExpand.limit(), equalTo(10000));
         var project = as(mvExpand.child(), EsqlProject.class);
         var topN = as(project.child(), TopN.class);
-        assertThat(topN.limit().fold(), equalTo(7300));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(7300));
         assertThat(orderNames(topN), contains("a"));
         mvExpand = as(topN.child(), MvExpand.class);
         var limit = as(mvExpand.child(), Limit.class);
-        assertThat(limit.limit().fold(), equalTo(7300));
+        assertThat(limit.limit().fold(FoldContext.small()), equalTo(7300));
         as(limit.child(), LocalRelation.class);
     }
 
@@ -2226,7 +2225,7 @@ public void testRemoveUnusedSortBeforeMvExpand_DefaultLimit10000() {
 
         var topN = as(plan, TopN.class);
         assertThat(orderNames(topN), contains("first_name"));
-        assertThat(topN.limit().fold(), equalTo(10000));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000));
         var mvExpand = as(topN.child(), MvExpand.class);
         var topN2 = as(mvExpand.child(), TopN.class); // TODO is it correct? Double-check AddDefaultTopN rule
         as(topN2.child(), EsRelation.class);
@@ -2254,7 +2253,7 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedField() {
 
         var keep = as(plan, EsqlProject.class);
         var topN = as(keep.child(), TopN.class);
-        assertThat(topN.limit().fold(), equalTo(15));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(15));
         assertThat(orderNames(topN), contains("salary", "first_name"));
         var filter = as(topN.child(), Filter.class);
         assertThat(filter.condition(), instanceOf(And.class));
@@ -2262,7 +2261,7 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedField() {
         topN = as(mvExp.child(), TopN.class);
         // the filter acts on first_name (the one used in mv_expand), so the limit 15 is not pushed down past mv_expand
         // instead the default limit is added
-        assertThat(topN.limit().fold(), equalTo(10000));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000));
         assertThat(orderNames(topN), contains("emp_no"));
         as(topN.child(), EsRelation.class);
     }
@@ -2289,7 +2288,7 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilter_NOT_OnExpandedField()
 
         var keep = as(plan, EsqlProject.class);
         var topN = as(keep.child(), TopN.class);
-        assertThat(topN.limit().fold(), equalTo(15));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(15));
         assertThat(orderNames(topN), contains("salary", "first_name"));
         var filter = as(topN.child(), Filter.class);
         assertThat(filter.condition(), instanceOf(And.class));
@@ -2297,7 +2296,7 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilter_NOT_OnExpandedField()
         topN = as(mvExp.child(), TopN.class);
         // the filters after mv_expand do not act on the expanded field values, as such the limit 15 is the one being pushed down
         // otherwise that limit wouldn't have pushed down and the default limit was instead being added by default before mv_expanded
-        assertThat(topN.limit().fold(), equalTo(10000));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000));
         assertThat(orderNames(topN), contains("emp_no"));
         as(topN.child(), EsRelation.class);
     }
@@ -2325,14 +2324,14 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedFieldAlias()
 
         var keep = as(plan, EsqlProject.class);
         var topN = as(keep.child(), TopN.class);
-        assertThat(topN.limit().fold(), equalTo(15));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(15));
         assertThat(orderNames(topN), contains("salary", "first_name"));
         var filter = as(topN.child(), Filter.class);
         assertThat(filter.condition(), instanceOf(And.class));
         var mvExp = as(filter.child(), MvExpand.class);
         topN = as(mvExp.child(), TopN.class);
         // the filter uses an alias ("x") to the expanded field ("first_name"), so the default limit is used and not the one provided
-        assertThat(topN.limit().fold(), equalTo(10000));
+        assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000));
         assertThat(orderNames(topN), contains("gender"));
         as(topN.child(), EsRelation.class);
     }
@@ -2371,7 +2370,7 @@ public void testSortMvExpandLimit() {
         var expand = as(plan, MvExpand.class);
         assertThat(expand.limit(), equalTo(20));
         var topN = as(expand.child(), TopN.class);
-        assertThat(topN.limit().fold(), is(20));
+        assertThat(topN.limit().fold(FoldContext.small()), is(20));
         var row = as(topN.child(), EsRelation.class);
     }
 
@@ -2392,7 +2391,7 @@ public void testWhereMvExpand() {
         var expand = as(plan, MvExpand.class);
         assertThat(expand.limit(), equalTo(1000));
         var limit2 = as(expand.child(), Limit.class);
-        assertThat(limit2.limit().fold(), is(1000));
+        assertThat(limit2.limit().fold(FoldContext.small()), is(1000));
         var row = as(limit2.child(), LocalRelation.class);
     }
 
@@ -2585,7 +2584,7 @@ public void testSimplifyLikeNoWildcard() {
 
         assertTrue(filter.condition() instanceof Equals);
         Equals equals = as(filter.condition(), Equals.class);
-        assertEquals(BytesRefs.toBytesRef("foo"), equals.right().fold());
+        assertEquals(BytesRefs.toBytesRef("foo"), equals.right().fold(FoldContext.small()));
         assertTrue(filter.child() instanceof EsRelation);
     }
 
@@ -2611,7 +2610,7 @@ public void testSimplifyRLikeNoWildcard() {
 
         assertTrue(filter.condition() instanceof Equals);
         Equals equals = as(filter.condition(), Equals.class);
-        assertEquals(BytesRefs.toBytesRef("foo"), equals.right().fold());
+        assertEquals(BytesRefs.toBytesRef("foo"), equals.right().fold(FoldContext.small()));
         assertTrue(filter.child() instanceof EsRelation);
     }
 
@@ -2775,7 +2774,7 @@ public void testEnrich() {
             """);
         var enrich = as(plan, Enrich.class);
         assertTrue(enrich.policyName().resolved());
-        assertThat(enrich.policyName().fold(), is(BytesRefs.toBytesRef("languages_idx")));
+        assertThat(enrich.policyName().fold(FoldContext.small()), is(BytesRefs.toBytesRef("languages_idx")));
         var eval = as(enrich.child(), Eval.class);
         var limit = as(eval.child(), Limit.class);
         as(limit.child(), EsRelation.class);
@@ -2821,7 +2820,7 @@ public void testEnrichNotNullFilter() {
         var filter = as(limit.child(), Filter.class);
         var enrich = as(filter.child(), Enrich.class);
         assertTrue(enrich.policyName().resolved());
-        assertThat(enrich.policyName().fold(), is(BytesRefs.toBytesRef("languages_idx")));
+        assertThat(enrich.policyName().fold(FoldContext.small()), is(BytesRefs.toBytesRef("languages_idx")));
         var eval = as(enrich.child(), Eval.class);
         as(eval.child(), EsRelation.class);
     }
@@ -2942,7 +2941,7 @@ public void testMedianReplacement() {
         var a = as(aggs.get(0), Alias.class);
         var per = as(a.child(), Percentile.class);
         var literal = as(per.percentile(), Literal.class);
-        assertThat((int) QuantileStates.MEDIAN, is(literal.fold()));
+        assertThat((int) QuantileStates.MEDIAN, is(literal.value()));
 
         assertThat(Expressions.names(agg.groupings()), contains("last_name"));
     }
@@ -2951,7 +2950,7 @@ public void testSplittingInWithFoldableValue() {
         FieldAttribute fa = getFieldAttribute("foo");
         In in = new In(EMPTY, ONE, List.of(TWO, THREE, fa, L(null)));
         Or expected = new Or(EMPTY, new In(EMPTY, ONE, List.of(TWO, THREE)), new In(EMPTY, ONE, List.of(fa, L(null))));
-        assertThat(new SplitInWithFoldableValue().rule(in), equalTo(expected));
+        assertThat(new SplitInWithFoldableValue().rule(in, logicalOptimizerCtx), equalTo(expected));
     }
 
     public void testReplaceFilterWithExact() {
@@ -3016,6 +3015,24 @@ public void testPruneChainedEval() {
         var source = as(limit.child(), EsRelation.class);
     }
 
+    public void testPruneChainedEvalNoProjection() {
+        var plan = plan("""
+              from test
+            | eval garbage = salary + 3
+            | eval garbage = emp_no / garbage, garbage = garbage
+            | eval garbage = 1
+            """);
+        var eval = as(plan, Eval.class);
+        var limit = as(eval.child(), Limit.class);
+        var source = as(limit.child(), EsRelation.class);
+
+        assertEquals(1, eval.fields().size());
+        var alias = as(eval.fields().getFirst(), Alias.class);
+        assertEquals(alias.name(), "garbage");
+        var literal = as(alias.child(), Literal.class);
+        assertEquals(1, literal.value());
+    }
+
     /**
      * Expects
      * Limit[1000[INTEGER]]
@@ -3690,7 +3707,7 @@ private <T extends AggregateFunction> void aggFieldName(Expression exp, Class<T>
         var alias = as(exp, Alias.class);
         var af = as(alias.child(), aggType);
         var field = af.field();
-        var name = field.foldable() ? BytesRefs.toString(field.fold()) : Expressions.name(field);
+        var name = field.foldable() ? BytesRefs.toString(field.fold(FoldContext.small())) : Expressions.name(field);
         assertThat(name, is(fieldName));
     }
 
@@ -4102,7 +4119,7 @@ public void testNestedExpressionsWithGroupingKeyInAggs() {
         var value = Alias.unwrap(fields.get(0));
         var math = as(value, Mod.class);
         assertThat(Expressions.name(math.left()), is("emp_no"));
-        assertThat(math.right().fold(), is(2));
+        assertThat(math.right().fold(FoldContext.small()), is(2));
         // languages + emp_no % 2
         var add = as(Alias.unwrap(fields.get(1).canonical()), Add.class);
         if (add.left() instanceof Mod mod) {
@@ -4111,7 +4128,7 @@ public void testNestedExpressionsWithGroupingKeyInAggs() {
         assertThat(Expressions.name(add.left()), is("languages"));
         var mod = as(add.right().canonical(), Mod.class);
         assertThat(Expressions.name(mod.left()), is("emp_no"));
-        assertThat(mod.right().fold(), is(2));
+        assertThat(mod.right().fold(FoldContext.small()), is(2));
     }
 
     /**
@@ -4140,7 +4157,7 @@ public void testNestedExpressionsWithMultiGrouping() {
         var value = Alias.unwrap(fields.get(0).canonical());
         var math = as(value, Mod.class);
         assertThat(Expressions.name(math.left()), is("emp_no"));
-        assertThat(math.right().fold(), is(2));
+        assertThat(math.right().fold(FoldContext.small()), is(2));
         // languages + salary
         var add = as(Alias.unwrap(fields.get(1).canonical()), Add.class);
         assertThat(Expressions.name(add.left()), anyOf(is("languages"), is("salary")));
@@ -4157,7 +4174,7 @@ public void testNestedExpressionsWithMultiGrouping() {
         assertThat(Expressions.name(add3.right()), anyOf(is("salary"), is("languages")));
         // emp_no % 2
         assertThat(Expressions.name(mod.left()), is("emp_no"));
-        assertThat(mod.right().fold(), is(2));
+        assertThat(mod.right().fold(FoldContext.small()), is(2));
     }
 
     /**
@@ -4595,8 +4612,8 @@ public void testCountOfLiteral() {
         var mvCoalesce = as(mul.left(), Coalesce.class);
         assertThat(mvCoalesce.children().size(), equalTo(2));
         var mvCount = as(mvCoalesce.children().get(0), MvCount.class);
-        assertThat(mvCount.fold(), equalTo(2));
-        assertThat(mvCoalesce.children().get(1).fold(), equalTo(0));
+        assertThat(mvCount.fold(FoldContext.small()), equalTo(2));
+        assertThat(mvCoalesce.children().get(1).fold(FoldContext.small()), equalTo(0));
         var count = as(mul.right(), ReferenceAttribute.class);
         assertThat(count.name(), equalTo("$$COUNT$s$0"));
 
@@ -4607,8 +4624,8 @@ public void testCountOfLiteral() {
         var mvCoalesce_expr = as(mul_expr.left(), Coalesce.class);
         assertThat(mvCoalesce_expr.children().size(), equalTo(2));
         var mvCount_expr = as(mvCoalesce_expr.children().get(0), MvCount.class);
-        assertThat(mvCount_expr.fold(), equalTo(1));
-        assertThat(mvCoalesce_expr.children().get(1).fold(), equalTo(0));
+        assertThat(mvCount_expr.fold(FoldContext.small()), equalTo(1));
+        assertThat(mvCoalesce_expr.children().get(1).fold(FoldContext.small()), equalTo(0));
         var count_expr = as(mul_expr.right(), ReferenceAttribute.class);
         assertThat(count_expr.name(), equalTo("$$COUNT$s$0"));
 
@@ -4620,7 +4637,7 @@ public void testCountOfLiteral() {
         assertThat(mvCoalesce_null.children().size(), equalTo(2));
         var mvCount_null = as(mvCoalesce_null.children().get(0), MvCount.class);
         assertThat(mvCount_null.field(), equalTo(NULL));
-        assertThat(mvCoalesce_null.children().get(1).fold(), equalTo(0));
+        assertThat(mvCoalesce_null.children().get(1).fold(FoldContext.small()), equalTo(0));
         var count_null = as(mul_null.right(), ReferenceAttribute.class);
         assertThat(count_null.name(), equalTo("$$COUNT$s$0"));
     }
@@ -4659,7 +4676,7 @@ public void testSumOfLiteral() {
         assertThat(s.name(), equalTo("s"));
         var mul = as(s.child(), Mul.class);
         var mvSum = as(mul.left(), MvSum.class);
-        assertThat(mvSum.fold(), equalTo(3));
+        assertThat(mvSum.fold(FoldContext.small()), equalTo(3));
         var count = as(mul.right(), ReferenceAttribute.class);
         assertThat(count.name(), equalTo("$$COUNT$s$0"));
 
@@ -4668,7 +4685,7 @@ public void testSumOfLiteral() {
         assertThat(s_expr.name(), equalTo("s_expr"));
         var mul_expr = as(s_expr.child(), Mul.class);
         var mvSum_expr = as(mul_expr.left(), MvSum.class);
-        assertThat(mvSum_expr.fold(), equalTo(3.14));
+        assertThat(mvSum_expr.fold(FoldContext.small()), equalTo(3.14));
         var count_expr = as(mul_expr.right(), ReferenceAttribute.class);
         assertThat(count_expr.name(), equalTo("$$COUNT$s$0"));
 
@@ -4817,7 +4834,7 @@ private static void assertAggOfConstExprs(AggOfLiteralTestCase testCase, List<Al
         var s = as(exprs.get(0), Alias.class);
         assertThat(s.source().toString(), containsString(LoggerMessageFormat.format(null, testCase.aggFunctionTemplate, "[1,2]")));
         assertEquals(s.child(), testCase.replacementForConstant.apply(new Literal(EMPTY, List.of(1, 2), INTEGER)));
-        assertEquals(s.child().fold(), testCase.aggMultiValue.apply(new int[] { 1, 2 }));
+        assertEquals(s.child().fold(FoldContext.small()), testCase.aggMultiValue.apply(new int[] { 1, 2 }));
 
         var s_expr = as(exprs.get(1), Alias.class);
         assertThat(s_expr.source().toString(), containsString(LoggerMessageFormat.format(null, testCase.aggFunctionTemplate, "314.0/100")));
@@ -4825,7 +4842,7 @@ private static void assertAggOfConstExprs(AggOfLiteralTestCase testCase, List<Al
             s_expr.child(),
             testCase.replacementForConstant.apply(new Div(EMPTY, new Literal(EMPTY, 314.0, DOUBLE), new Literal(EMPTY, 100, INTEGER)))
         );
-        assertEquals(s_expr.child().fold(), testCase.aggSingleValue.apply(3.14));
+        assertEquals(s_expr.child().fold(FoldContext.small()), testCase.aggSingleValue.apply(3.14));
 
         var s_null = as(exprs.get(2), Alias.class);
         assertThat(s_null.source().toString(), containsString(LoggerMessageFormat.format(null, testCase.aggFunctionTemplate, "null")));
@@ -4911,7 +4928,7 @@ public void testPlanSanityCheck() throws Exception {
     }
 
     public void testPlanSanityCheckWithBinaryPlans() throws Exception {
-        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled());
+        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
 
         var plan = optimizedPlan("""
               FROM test
@@ -4919,8 +4936,7 @@ public void testPlanSanityCheckWithBinaryPlans() throws Exception {
             | LOOKUP JOIN languages_lookup ON language_code
             """);
 
-        var project = as(plan, Project.class);
-        var join = as(project.child(), Join.class);
+        var join = as(plan, Join.class);
 
         var joinWithInvalidLeftPlan = join.replaceChildren(join.right(), join.right());
         IllegalStateException e = expectThrows(IllegalStateException.class, () -> logicalOptimizer.optimize(joinWithInvalidLeftPlan));
@@ -5583,8 +5599,8 @@ private void assertSemanticMatching(String expected, String provided) {
     private static void assertSemanticMatching(Expression fieldAttributeExp, Expression unresolvedAttributeExp) {
         Expression unresolvedUpdated = unresolvedAttributeExp.transformUp(
             LITERALS_ON_THE_RIGHT.expressionToken(),
-            LITERALS_ON_THE_RIGHT::rule
-        ).transformUp(x -> x.foldable() ? new Literal(x.source(), x.fold(), x.dataType()) : x);
+            be -> LITERALS_ON_THE_RIGHT.rule(be, logicalOptimizerCtx)
+        ).transformUp(x -> x.foldable() ? new Literal(x.source(), x.fold(FoldContext.small()), x.dataType()) : x);
 
         List<Expression> resolvedFields = fieldAttributeExp.collectFirstChildren(x -> x instanceof FieldAttribute);
         for (Expression field : resolvedFields) {
@@ -5664,8 +5680,7 @@ public void testSimplifyComparisonArithmeticWithFloatsAndDirectionChange() {
     }
 
     private void assertNullLiteral(Expression expression) {
-        assertEquals(Literal.class, expression.getClass());
-        assertNull(expression.fold());
+        assertNull(as(expression, Literal.class).value());
     }
 
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/108519")
@@ -5723,6 +5738,78 @@ public void testSimplifyComparisonArithmeticSkippedOnFloats() {
         }
     }
 
+    public void testReplaceStringCasingWithInsensitiveEqualsUpperFalse() {
+        var plan = optimizedPlan("FROM test | WHERE TO_UPPER(first_name) == \"VALÜe\"");
+        var local = as(plan, LocalRelation.class);
+        assertThat(local.supplier(), equalTo(LocalSupplier.EMPTY));
+    }
+
+    public void testReplaceStringCasingWithInsensitiveEqualsUpperTrue() {
+        var plan = optimizedPlan("FROM test | WHERE TO_UPPER(first_name) != \"VALÜe\"");
+        var limit = as(plan, Limit.class);
+        var filter = as(limit.child(), Filter.class);
+        var isNotNull = as(filter.condition(), IsNotNull.class);
+        assertThat(Expressions.name(isNotNull.field()), is("first_name"));
+        as(filter.child(), EsRelation.class);
+    }
+
+    public void testReplaceStringCasingWithInsensitiveEqualsLowerFalse() {
+        var plan = optimizedPlan("FROM test | WHERE TO_LOWER(first_name) == \"VALÜe\"");
+        var local = as(plan, LocalRelation.class);
+        assertThat(local.supplier(), equalTo(LocalSupplier.EMPTY));
+    }
+
+    public void testReplaceStringCasingWithInsensitiveEqualsLowerTrue() {
+        var plan = optimizedPlan("FROM test | WHERE TO_LOWER(first_name) != \"VALÜe\"");
+        var limit = as(plan, Limit.class);
+        var filter = as(limit.child(), Filter.class);
+        assertThat(filter.condition(), instanceOf(IsNotNull.class));
+        as(filter.child(), EsRelation.class);
+    }
+
+    public void testReplaceStringCasingWithInsensitiveEqualsEquals() {
+        for (var fn : List.of("TO_LOWER", "TO_UPPER")) {
+            var value = fn.equals("TO_LOWER") ? fn.toLowerCase(Locale.ROOT) : fn.toUpperCase(Locale.ROOT);
+            value += "🐔✈🔥🎉"; // these should not cause folding, they're not in the upper/lower char class
+            var plan = optimizedPlan("FROM test | WHERE " + fn + "(first_name) == \"" + value + "\"");
+            var limit = as(plan, Limit.class);
+            var filter = as(limit.child(), Filter.class);
+            var insensitive = as(filter.condition(), InsensitiveEquals.class);
+            as(insensitive.left(), FieldAttribute.class);
+            var bRef = as(insensitive.right().fold(FoldContext.small()), BytesRef.class);
+            assertThat(bRef.utf8ToString(), is(value));
+            as(filter.child(), EsRelation.class);
+        }
+    }
+
+    public void testReplaceStringCasingWithInsensitiveEqualsNotEquals() {
+        for (var fn : List.of("TO_LOWER", "TO_UPPER")) {
+            var value = fn.equals("TO_LOWER") ? fn.toLowerCase(Locale.ROOT) : fn.toUpperCase(Locale.ROOT);
+            value += "🐔✈🔥🎉"; // these should not cause folding, they're not in the upper/lower char class
+            var plan = optimizedPlan("FROM test | WHERE " + fn + "(first_name) != \"" + value + "\"");
+            var limit = as(plan, Limit.class);
+            var filter = as(limit.child(), Filter.class);
+            var not = as(filter.condition(), Not.class);
+            var insensitive = as(not.field(), InsensitiveEquals.class);
+            as(insensitive.left(), FieldAttribute.class);
+            var bRef = as(insensitive.right().fold(FoldContext.small()), BytesRef.class);
+            assertThat(bRef.utf8ToString(), is(value));
+            as(filter.child(), EsRelation.class);
+        }
+    }
+
+    public void testReplaceStringCasingWithInsensitiveEqualsUnwrap() {
+        var plan = optimizedPlan("FROM test | WHERE TO_UPPER(TO_LOWER(TO_UPPER(first_name))) == \"VALÜ\"");
+        var limit = as(plan, Limit.class);
+        var filter = as(limit.child(), Filter.class);
+        var insensitive = as(filter.condition(), InsensitiveEquals.class);
+        var field = as(insensitive.left(), FieldAttribute.class);
+        assertThat(field.fieldName(), is("first_name"));
+        var bRef = as(insensitive.right().fold(FoldContext.small()), BytesRef.class);
+        assertThat(bRef.utf8ToString(), is("VALÜ"));
+        as(filter.child(), EsRelation.class);
+    }
+
     @Override
     protected List<String> filteredWarnings() {
         return withDefaultLimitWarning(super.filteredWarnings());
@@ -5769,7 +5856,7 @@ public void testLookupSimple() {
         var left = as(join.left(), EsqlProject.class);
         assertThat(left.output().toString(), containsString("int{r}"));
         var limit = as(left.child(), Limit.class);
-        assertThat(limit.limit().fold(), equalTo(1000));
+        assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000));
 
         assertThat(join.config().type(), equalTo(JoinTypes.LEFT));
         assertThat(join.config().matchFields().stream().map(Object::toString).toList(), matchesList().item(startsWith("int{r}")));
@@ -5838,7 +5925,7 @@ public void testLookupStats() {
         }
         var plan = optimizedPlan(query);
         var limit = as(plan, Limit.class);
-        assertThat(limit.limit().fold(), equalTo(1000));
+        assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000));
 
         var agg = as(limit.child(), Aggregate.class);
         assertMap(
@@ -5913,25 +6000,24 @@ public void testLookupStats() {
      *     | \_Limit[1000[INTEGER]]
      *     |  \_Filter[languages{f}#10 > 1[INTEGER]]
      *     |    \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..]
-     *     \_EsRelation[language_code][LOOKUP][language_code{f}#18, language_name{f}#19]
+     *     \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19]
      */
     public void testLookupJoinPushDownFilterOnJoinKeyWithRename() {
-        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled());
+        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
 
         String query = """
               FROM test
             | RENAME languages AS language_code
-            | LOOKUP JOIN language_code ON language_code
+            | LOOKUP JOIN languages_lookup ON language_code
             | WHERE language_code > 1
             """;
         var plan = optimizedPlan(query);
 
-        var project = as(plan, Project.class);
-        var join = as(project.child(), Join.class);
+        var join = as(plan, Join.class);
         assertThat(join.config().type(), equalTo(JoinTypes.LEFT));
-        project = as(join.left(), Project.class);
+        var project = as(join.left(), Project.class);
         var limit = as(project.child(), Limit.class);
-        assertThat(limit.limit().fold(), equalTo(1000));
+        assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000));
         var filter = as(limit.child(), Filter.class);
         // assert that the rename has been undone
         var op = as(filter.condition(), GreaterThan.class);
@@ -5956,27 +6042,26 @@ public void testLookupJoinPushDownFilterOnJoinKeyWithRename() {
      *     | \_Limit[1000[INTEGER]]
      *     |  \_Filter[emp_no{f}#7 > 1[INTEGER]]
      *     |    \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..]
-     *     \_EsRelation[language_code][LOOKUP][language_code{f}#18, language_name{f}#19]
+     *     \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19]
      */
     public void testLookupJoinPushDownFilterOnLeftSideField() {
-        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled());
+        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
 
         String query = """
               FROM test
             | RENAME languages AS language_code
-            | LOOKUP JOIN language_code ON language_code
+            | LOOKUP JOIN languages_lookup ON language_code
             | WHERE emp_no > 1
             """;
 
         var plan = optimizedPlan(query);
 
-        var project = as(plan, Project.class);
-        var join = as(project.child(), Join.class);
+        var join = as(plan, Join.class);
         assertThat(join.config().type(), equalTo(JoinTypes.LEFT));
-        project = as(join.left(), Project.class);
+        var project = as(join.left(), Project.class);
 
         var limit = as(project.child(), Limit.class);
-        assertThat(limit.limit().fold(), equalTo(1000));
+        assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000));
         var filter = as(limit.child(), Filter.class);
         var op = as(filter.condition(), GreaterThan.class);
         var field = as(op.left(), FieldAttribute.class);
@@ -6000,23 +6085,22 @@ public void testLookupJoinPushDownFilterOnLeftSideField() {
      *       |_EsqlProject[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang
      * uages{f}#10 AS language_code, last_name{f}#11, long_noidx{f}#17, salary{f}#12]]
      *       | \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..]
-     *       \_EsRelation[language_code][LOOKUP][language_code{f}#18, language_name{f}#19]
+     *       \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19]
      */
     public void testLookupJoinPushDownDisabledForLookupField() {
-        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled());
+        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
 
         String query = """
               FROM test
             | RENAME languages AS language_code
-            | LOOKUP JOIN language_code ON language_code
+            | LOOKUP JOIN languages_lookup ON language_code
             | WHERE language_name == "English"
             """;
 
         var plan = optimizedPlan(query);
 
-        var project = as(plan, Project.class);
-        var limit = as(project.child(), Limit.class);
-        assertThat(limit.limit().fold(), equalTo(1000));
+        var limit = as(plan, Limit.class);
+        assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000));
 
         var filter = as(limit.child(), Filter.class);
         var op = as(filter.condition(), Equals.class);
@@ -6027,7 +6111,7 @@ public void testLookupJoinPushDownDisabledForLookupField() {
 
         var join = as(filter.child(), Join.class);
         assertThat(join.config().type(), equalTo(JoinTypes.LEFT));
-        project = as(join.left(), Project.class);
+        var project = as(join.left(), Project.class);
 
         var leftRel = as(project.child(), EsRelation.class);
         var rightRel = as(join.right(), EsRelation.class);
@@ -6045,23 +6129,22 @@ public void testLookupJoinPushDownDisabledForLookupField() {
      * guages{f}#11 AS language_code, last_name{f}#12, long_noidx{f}#18, salary{f}#13]]
      *       | \_Filter[emp_no{f}#8 > 1[INTEGER]]
      *       |   \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge..]
-     *       \_EsRelation[language_code][LOOKUP][language_code{f}#19, language_name{f}#20]
+     *       \_EsRelation[languages_lookup][LOOKUP][language_code{f}#19, language_name{f}#20]
      */
     public void testLookupJoinPushDownSeparatedForConjunctionBetweenLeftAndRightField() {
-        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled());
+        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
 
         String query = """
               FROM test
             | RENAME languages AS language_code
-            | LOOKUP JOIN language_code ON language_code
+            | LOOKUP JOIN languages_lookup ON language_code
             | WHERE language_name == "English" AND emp_no > 1
             """;
 
         var plan = optimizedPlan(query);
 
-        var project = as(plan, Project.class);
-        var limit = as(project.child(), Limit.class);
-        assertThat(limit.limit().fold(), equalTo(1000));
+        var limit = as(plan, Limit.class);
+        assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000));
         // filter kept in place, working on the right side
         var filter = as(limit.child(), Filter.class);
         EsqlBinaryComparison op = as(filter.condition(), Equals.class);
@@ -6072,7 +6155,7 @@ public void testLookupJoinPushDownSeparatedForConjunctionBetweenLeftAndRightFiel
 
         var join = as(filter.child(), Join.class);
         assertThat(join.config().type(), equalTo(JoinTypes.LEFT));
-        project = as(join.left(), Project.class);
+        var project = as(join.left(), Project.class);
         // filter pushed down
         filter = as(project.child(), Filter.class);
         op = as(filter.condition(), GreaterThan.class);
@@ -6084,7 +6167,6 @@ public void testLookupJoinPushDownSeparatedForConjunctionBetweenLeftAndRightFiel
 
         var leftRel = as(filter.child(), EsRelation.class);
         var rightRel = as(join.right(), EsRelation.class);
-
     }
 
     /**
@@ -6098,23 +6180,22 @@ public void testLookupJoinPushDownSeparatedForConjunctionBetweenLeftAndRightFiel
      *       |_EsqlProject[[_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, gender{f}#10, hire_date{f}#15, job{f}#16, job.raw{f}#17, lan
      * guages{f}#11 AS language_code, last_name{f}#12, long_noidx{f}#18, salary{f}#13]]
      *       | \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge..]
-     *       \_EsRelation[language_code][LOOKUP][language_code{f}#19, language_name{f}#20]
+     *       \_EsRelation[languages_lookup][LOOKUP][language_code{f}#19, language_name{f}#20]
      */
     public void testLookupJoinPushDownDisabledForDisjunctionBetweenLeftAndRightField() {
-        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled());
+        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
 
         String query = """
               FROM test
             | RENAME languages AS language_code
-            | LOOKUP JOIN language_code ON language_code
+            | LOOKUP JOIN languages_lookup ON language_code
             | WHERE language_name == "English" OR emp_no > 1
             """;
 
         var plan = optimizedPlan(query);
 
-        var project = as(plan, Project.class);
-        var limit = as(project.child(), Limit.class);
-        assertThat(limit.limit().fold(), equalTo(1000));
+        var limit = as(plan, Limit.class);
+        assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000));
 
         var filter = as(limit.child(), Filter.class);
         var or = as(filter.condition(), Or.class);
@@ -6133,7 +6214,7 @@ public void testLookupJoinPushDownDisabledForDisjunctionBetweenLeftAndRightField
 
         var join = as(filter.child(), Join.class);
         assertThat(join.config().type(), equalTo(JoinTypes.LEFT));
-        project = as(join.left(), Project.class);
+        var project = as(join.left(), Project.class);
 
         var leftRel = as(project.child(), EsRelation.class);
         var rightRel = as(join.right(), EsRelation.class);
@@ -6208,7 +6289,7 @@ public void testTranslateMixedAggsWithMathWithoutGrouping() {
         as(addEval.child(), EsRelation.class);
 
         assertThat(Expressions.attribute(mul.left()).id(), equalTo(finalAggs.aggregates().get(1).id()));
-        assertThat(mul.right().fold(), equalTo(1.1));
+        assertThat(mul.right().fold(FoldContext.small()), equalTo(1.1));
 
         assertThat(finalAggs.aggregateType(), equalTo(Aggregate.AggregateType.STANDARD));
         Max maxRate = as(Alias.unwrap(finalAggs.aggregates().get(0)), Max.class);
@@ -6223,7 +6304,7 @@ public void testTranslateMixedAggsWithMathWithoutGrouping() {
         ToPartial toPartialMaxCost = as(Alias.unwrap(aggsByTsid.aggregates().get(1)), ToPartial.class);
         assertThat(Expressions.attribute(toPartialMaxCost.field()).id(), equalTo(addEval.fields().get(0).id()));
         assertThat(Expressions.attribute(add.left()).name(), equalTo("network.cost"));
-        assertThat(add.right().fold(), equalTo(0.2));
+        assertThat(add.right().fold(FoldContext.small()), equalTo(0.2));
     }
 
     public void testTranslateMetricsGroupedByOneDimension() {
@@ -6452,7 +6533,7 @@ METRICS k8s avg(round(1.05 * rate(network.total_bytes_in))) BY bucket(@timestamp
         assertThat(Expressions.attribute(finalAgg.groupings().get(1)).id(), equalTo(aggsByTsid.aggregates().get(1).id()));
 
         assertThat(Expressions.attribute(mul.left()).id(), equalTo(aggsByTsid.aggregates().get(0).id()));
-        assertThat(mul.right().fold(), equalTo(1.05));
+        assertThat(mul.right().fold(FoldContext.small()), equalTo(1.05));
         assertThat(aggsByTsid.aggregateType(), equalTo(Aggregate.AggregateType.METRICS));
         Rate rate = as(Alias.unwrap(aggsByTsid.aggregates().get(0)), Rate.class);
         assertThat(Expressions.attribute(rate.field()).name(), equalTo("network.total_bytes_in"));
@@ -6727,4 +6808,17 @@ public void testMatchFunctionIsNotNullable() {
             containsString("[MATCH] function cannot operate on [text::keyword], which is not a field from an index mapping")
         );
     }
+
+    public void testWhereNull() {
+        var plan = plan("""
+            from test
+            | sort salary
+            | rename emp_no as e, first_name as f
+            | keep salary, e, f
+            | where null
+            | LIMIT 12
+            """);
+        var local = as(plan, LocalRelation.class);
+        assertThat(local.supplier(), equalTo(LocalSupplier.EMPTY));
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
index d0c7a1cd61010..66891210a1e47 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
@@ -14,8 +14,11 @@
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.compute.aggregation.AggregatorMode;
+import org.elasticsearch.compute.operator.exchange.ExchangeSinkHandler;
+import org.elasticsearch.compute.operator.exchange.ExchangeSourceHandler;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.geometry.Circle;
 import org.elasticsearch.geometry.Polygon;
@@ -37,6 +40,7 @@
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.EsqlTestUtils.TestConfigurableSearchStats;
 import org.elasticsearch.xpack.esql.EsqlTestUtils.TestConfigurableSearchStats.Config;
+import org.elasticsearch.xpack.esql.TestBlockFactory;
 import org.elasticsearch.xpack.esql.VerificationException;
 import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.esql.analysis.Analyzer;
@@ -47,6 +51,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
@@ -122,6 +127,8 @@
 import org.elasticsearch.xpack.esql.plan.physical.ProjectExec;
 import org.elasticsearch.xpack.esql.plan.physical.TopNExec;
 import org.elasticsearch.xpack.esql.plan.physical.UnaryExec;
+import org.elasticsearch.xpack.esql.planner.EsPhysicalOperationProviders;
+import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner;
 import org.elasticsearch.xpack.esql.planner.PlannerUtils;
 import org.elasticsearch.xpack.esql.planner.mapper.Mapper;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
@@ -133,12 +140,15 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import static java.util.Arrays.asList;
@@ -155,6 +165,7 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.statsForMissingField;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning;
 import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization;
 import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyze;
@@ -163,13 +174,17 @@
 import static org.elasticsearch.xpack.esql.core.expression.Expressions.names;
 import static org.elasticsearch.xpack.esql.core.expression.function.scalar.FunctionTestUtils.l;
 import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
+import static org.elasticsearch.xpack.esql.core.util.TestUtils.stripThrough;
 import static org.elasticsearch.xpack.esql.parser.ExpressionBuilder.MAX_EXPRESSION_DEPTH;
 import static org.elasticsearch.xpack.esql.parser.LogicalPlanBuilder.MAX_QUERY_DEPTH;
 import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasItem;
@@ -203,6 +218,9 @@ public class PhysicalPlanOptimizerTests extends ESTestCase {
     private TestDataSource airportsNotIndexed;  // Test when spatial field has doc values but is not indexed
     private TestDataSource airportsNotIndexedNorDocValues;  // Test when spatial field is neither indexed nor has doc-values
     private TestDataSource airportsWeb;         // Cartesian point field tests
+    private TestDataSource airportsCityBoundaries;
+    private TestDataSource cartesianMultipolygons; // cartesian_shape field tests
+    private TestDataSource cartesianMultipolygonsNoDocValues; // cartesian_shape field tests but has no doc values
     private TestDataSource countriesBbox;       // geo_shape field tests
     private TestDataSource countriesBboxWeb;    // cartesian_shape field tests
 
@@ -229,7 +247,7 @@ public PhysicalPlanOptimizerTests(String name, Configuration config) {
     @Before
     public void init() {
         parser = new EsqlParser();
-        logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG));
+        logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext());
         physicalPlanOptimizer = new PhysicalPlanOptimizer(new PhysicalOptimizerContext(config));
         EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry();
         mapper = new Mapper();
@@ -272,6 +290,25 @@ public void init() {
             new TestConfigurableSearchStats().exclude(Config.INDEXED, "location").exclude(Config.DOC_VALUES, "location")
         );
         this.airportsWeb = makeTestDataSource("airports_web", "mapping-airports_web.json", functionRegistry, enrichResolution);
+        this.airportsCityBoundaries = makeTestDataSource(
+            "airports_city_boundaries",
+            "mapping-airport_city_boundaries.json",
+            functionRegistry,
+            enrichResolution
+        );
+        this.cartesianMultipolygons = makeTestDataSource(
+            "cartesian_multipolygons",
+            "mapping-cartesian_multipolygons.json",
+            functionRegistry,
+            enrichResolution
+        );
+        this.cartesianMultipolygonsNoDocValues = makeTestDataSource(
+            "cartesian_multipolygons_no_doc_values",
+            "mapping-cartesian_multipolygons_no_doc_values.json",
+            functionRegistry,
+            enrichResolution,
+            new TestConfigurableSearchStats().exclude(Config.DOC_VALUES, "shape")
+        );
         this.countriesBbox = makeTestDataSource("countriesBbox", "mapping-countries_bbox.json", functionRegistry, enrichResolution);
         this.countriesBboxWeb = makeTestDataSource(
             "countriesBboxWeb",
@@ -285,7 +322,7 @@ TestDataSource makeTestDataSource(
         String indexName,
         String mappingFileName,
         EsqlFunctionRegistry functionRegistry,
-        IndexResolution lookupResolution,
+        Map<String, IndexResolution> lookupResolution,
         EnrichResolution enrichResolution,
         SearchStats stats
     ) {
@@ -886,8 +923,8 @@ public void testQueryWithNull() {
 
         var optimized = optimizedPlan(plan);
         var topN = as(optimized, TopNExec.class);
-        // no fields are added after the top n - so 0 here
-        assertThat(topN.estimatedRowSize(), equalTo(0));
+        // all fields + nullsum are loaded in the final TopN
+        assertThat(topN.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES));
 
         var exchange = asRemoteExchange(topN.child());
         var project = as(exchange.child(), ProjectExec.class);
@@ -895,7 +932,7 @@ public void testQueryWithNull() {
         var eval = as(extract.child(), EvalExec.class);
         var source = source(eval.child());
         // All fields loaded
-        assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + 3 * Integer.BYTES + Long.BYTES));
+        assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + 3 * Integer.BYTES + 2 * Integer.BYTES));
     }
 
     public void testPushAndInequalitiesFilter() {
@@ -1082,7 +1119,7 @@ public void testLimit() {
         var fieldExtract = as(project.child(), FieldExtractExec.class);
         var source = source(fieldExtract.child());
         assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES));
-        assertThat(source.limit().fold(), is(10));
+        assertThat(source.limit().fold(FoldContext.small()), is(10));
     }
 
     /**
@@ -1107,8 +1144,8 @@ public void testExtractorForEvalWithoutProject() throws Exception {
         var project = as(exchange.child(), ProjectExec.class);
         var extract = as(project.child(), FieldExtractExec.class);
         var topNLocal = as(extract.child(), TopNExec.class);
-        // two extra ints for forwards and backwards map
-        assertThat(topNLocal.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES * 2));
+        // all fields plus nullsum and shards, segments, docs and two extra ints for forwards and backwards map
+        assertThat(topNLocal.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES + Integer.BYTES * 2 + Integer.BYTES * 3));
 
         var eval = as(topNLocal.child(), EvalExec.class);
         var source = source(eval.child());
@@ -1164,7 +1201,7 @@ public void testPushLimitToSource() {
         var leaves = extract.collectLeaves();
         assertEquals(1, leaves.size());
         var source = as(leaves.get(0), EsQueryExec.class);
-        assertThat(source.limit().fold(), is(10));
+        assertThat(source.limit().fold(FoldContext.small()), is(10));
         // extra ints for doc id and emp_no_10
         assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES * 2));
     }
@@ -1211,7 +1248,7 @@ public void testPushLimitAndFilterToSource() {
 
         var source = source(extract.child());
         assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES * 2));
-        assertThat(source.limit().fold(), is(10));
+        assertThat(source.limit().fold(FoldContext.small()), is(10));
         var rq = as(sv(source.query(), "emp_no"), RangeQueryBuilder.class);
         assertThat(rq.fieldName(), equalTo("emp_no"));
         assertThat(rq.from(), equalTo(0));
@@ -1770,7 +1807,7 @@ public void testPushDownEqualsIgnoreCase() {
         assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES));
 
         QueryBuilder query = source.query();
-        assertNotNull(query);
+        assertNotNull(query); // TODO: verify query
     }
 
     /**
@@ -1805,7 +1842,257 @@ public void testNoPushDownEvalEqualsIgnoreCase() {
         var source = source(extract.child());
 
         QueryBuilder query = source.query();
-        assertNull(query);
+        assertNull(query); // TODO: verify query
+    }
+
+    public void testPushDownEqualsToUpper() {
+        doTestPushDownChangeCase("""
+            from test
+            | where to_upper(first_name) == "FOO"
+            """, """
+            {
+              "esql_single_value" : {
+                "field" : "first_name",
+                "next" : {
+                  "term" : {
+                    "first_name" : {
+                      "value" : "FOO",
+                      "case_insensitive" : true
+                    }
+                  }
+                },
+                "source" : "to_upper(first_name) == \\"FOO\\"@2:9"
+              }
+            }""");
+    }
+
+    /*
+     * LimitExec[1000[INTEGER]]
+     * \_ExchangeExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12,
+     *                languages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8],false]
+     *   \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12,
+     *                  languages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8]]
+     *     \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen..]<[]>
+     *       \_EsQueryExec[test], indexMode[standard], query[{...}}][_doc{f}#25], limit[1000], sort[] estimatedRowSize[332]
+     */
+    private void doTestPushDownChangeCase(String esql, String expected) {
+        var plan = physicalPlan(esql);
+        var optimized = optimizedPlan(plan);
+        var topLimit = as(optimized, LimitExec.class);
+        var exchange = asRemoteExchange(topLimit.child());
+        var project = as(exchange.child(), ProjectExec.class);
+        var extractRest = as(project.child(), FieldExtractExec.class);
+        var source = source(extractRest.child());
+        assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES));
+
+        QueryBuilder query = source.query();
+        assertThat(stripThrough(query.toString()), is(stripThrough(expected)));
+    }
+
+    public void testPushDownEqualsToLower() {
+        doTestPushDownChangeCase("""
+            from test
+            | where to_lower(first_name) == "foo"
+            """, """
+            {
+              "esql_single_value" : {
+                "field" : "first_name",
+                "next" : {
+                  "term" : {
+                    "first_name" : {
+                      "value" : "foo",
+                      "case_insensitive" : true
+                    }
+                  }
+                },
+                "source" : "to_lower(first_name) == \\"foo\\"@2:9"
+              }
+            }""");
+    }
+
+    public void testPushDownNotEqualsToUpper() {
+        doTestPushDownChangeCase("""
+            from test
+            | where to_upper(first_name) != "FOO"
+            """, """
+            {
+              "esql_single_value" : {
+                "field" : "first_name",
+                "next" : {
+                  "bool" : {
+                    "must_not" : [
+                      {
+                        "term" : {
+                          "first_name" : {
+                            "value" : "FOO",
+                            "case_insensitive" : true
+                          }
+                        }
+                      }
+                    ],
+                    "boost" : 1.0
+                  }
+                },
+                "source" : "to_upper(first_name) != \\"FOO\\"@2:9"
+              }
+            }""");
+    }
+
+    public void testPushDownNotEqualsToLower() {
+        doTestPushDownChangeCase("""
+            from test
+            | where to_lower(first_name) != "foo"
+            """, """
+            {
+              "esql_single_value" : {
+                "field" : "first_name",
+                "next" : {
+                  "bool" : {
+                    "must_not" : [
+                      {
+                        "term" : {
+                          "first_name" : {
+                            "value" : "foo",
+                            "case_insensitive" : true
+                          }
+                        }
+                      }
+                    ],
+                    "boost" : 1.0
+                  }
+                },
+                "source" : "to_lower(first_name) != \\"foo\\"@2:9"
+              }
+            }""");
+    }
+
+    public void testPushDownChangeCaseMultiplePredicates() {
+        doTestPushDownChangeCase("""
+            from test
+            | where to_lower(first_name) != "foo" or to_upper(first_name) == "FOO" or emp_no > 10
+            """, """
+            {
+              "bool" : {
+                "should" : [
+                  {
+                    "esql_single_value" : {
+                      "field" : "first_name",
+                      "next" : {
+                        "bool" : {
+                          "must_not" : [
+                            {
+                              "term" : {
+                                "first_name" : {
+                                  "value" : "foo",
+                                  "case_insensitive" : true
+                                }
+                              }
+                            }
+                          ],
+                          "boost" : 1.0
+                        }
+                      },
+                      "source" : "to_lower(first_name) != \\"foo\\"@2:9"
+                    }
+                  },
+                  {
+                    "esql_single_value" : {
+                      "field" : "first_name",
+                      "next" : {
+                        "term" : {
+                          "first_name" : {
+                            "value" : "FOO",
+                            "case_insensitive" : true
+                          }
+                        }
+                      },
+                      "source" : "to_upper(first_name) == \\"FOO\\"@2:42"
+                    }
+                  },
+                  {
+                    "esql_single_value" : {
+                      "field" : "emp_no",
+                      "next" : {
+                        "range" : {
+                          "emp_no" : {
+                            "gt" : 10,
+                            "boost" : 1.0
+                          }
+                        }
+                      },
+                      "source" : "emp_no > 10@2:75"
+                    }
+                  }
+                ],
+                "boost" : 1.0
+              }
+            }
+            """);
+    }
+
+    // same tree as with doTestPushDownChangeCase(), but with a topping EvalExec (for `x`)
+    public void testPushDownChangeCaseThroughEval() {
+        var esql = """
+            from test
+            | eval x = first_name
+            | where to_lower(x) == "foo"
+            """;
+        var plan = physicalPlan(esql);
+        var optimized = optimizedPlan(plan);
+        var eval = as(optimized, EvalExec.class);
+        var topLimit = as(eval.child(), LimitExec.class);
+        var exchange = asRemoteExchange(topLimit.child());
+        var project = as(exchange.child(), ProjectExec.class);
+        var extractRest = as(project.child(), FieldExtractExec.class);
+        var source = source(extractRest.child());
+
+        var expected = """
+            {
+              "esql_single_value" : {
+                "field" : "first_name",
+                "next" : {
+                  "term" : {
+                    "first_name" : {
+                      "value" : "foo",
+                      "case_insensitive" : true
+                    }
+                  }
+                },
+                "source" : "to_lower(x) == \\"foo\\"@3:9"
+              }
+            }""";
+        QueryBuilder query = source.query();
+        assertThat(stripThrough(query.toString()), is(stripThrough(expected)));
+    }
+
+    /*
+     * LimitExec[1000[INTEGER]]
+     * \_ExchangeExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12,
+     *                languages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8],false]
+     *   \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12,
+     *                 languages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8]]
+     *     \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, gender{f}#5, hire_da..]<[]>
+     *       \_LimitExec[1000[INTEGER]]
+     *         \_FilterExec[NOT(INSENSITIVEEQUALS(CONCAT(first_name{f}#4,[66 6f 6f][KEYWORD]),[66 6f 6f][KEYWORD]))]
+     *           \_FieldExtractExec[first_name{f}#4]<[]>
+     *             \_EsQueryExec[test], indexMode[standard], query[][_doc{f}#25], limit[], sort[] estimatedRowSize[332]
+     */
+    public void testNoPushDownChangeCase() {
+        var plan = physicalPlan("""
+            from test
+            | where to_lower(concat(first_name, "foo")) != "foo"
+            """);
+
+        var optimized = optimizedPlan(plan);
+        var topLimit = as(optimized, LimitExec.class);
+        var exchange = asRemoteExchange(topLimit.child());
+        var project = as(exchange.child(), ProjectExec.class);
+        var fieldExtract = as(project.child(), FieldExtractExec.class);
+        var limit = as(fieldExtract.child(), LimitExec.class);
+        var filter = as(limit.child(), FilterExec.class);
+        var fieldExtract2 = as(filter.child(), FieldExtractExec.class);
+        var source = source(fieldExtract2.child());
+        assertThat(source.query(), nullValue());
     }
 
     public void testPushDownNotRLike() {
@@ -2331,7 +2618,7 @@ public void testVerifierOnMissingReferences() {
     }
 
     public void testVerifierOnMissingReferencesWithBinaryPlans() throws Exception {
-        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled());
+        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
 
         // Do not assert serialization:
         // This will have a LookupJoinExec, which is not serializable because it doesn't leave the coordinator.
@@ -2617,14 +2904,14 @@ public void testAvgSurrogateFunctionAfterRenameAndLimit() {
         var eval = as(project.child(), EvalExec.class);
         var limit = as(eval.child(), LimitExec.class);
         assertThat(limit.limit(), instanceOf(Literal.class));
-        assertThat(limit.limit().fold(), equalTo(10000));
+        assertThat(limit.limit().fold(FoldContext.small()), equalTo(10000));
         var aggFinal = as(limit.child(), AggregateExec.class);
         assertThat(aggFinal.getMode(), equalTo(FINAL));
         var aggPartial = as(aggFinal.child(), AggregateExec.class);
         assertThat(aggPartial.getMode(), equalTo(INITIAL));
         limit = as(aggPartial.child(), LimitExec.class);
         assertThat(limit.limit(), instanceOf(Literal.class));
-        assertThat(limit.limit().fold(), equalTo(10));
+        assertThat(limit.limit().fold(FoldContext.small()), equalTo(10));
 
         var exchange = as(limit.child(), ExchangeExec.class);
         project = as(exchange.child(), ProjectExec.class);
@@ -2633,7 +2920,7 @@ public void testAvgSurrogateFunctionAfterRenameAndLimit() {
         var fieldExtract = as(project.child(), FieldExtractExec.class);
         assertThat(Expressions.names(fieldExtract.attributesToExtract()), is(expectedFields));
         var source = source(fieldExtract.child());
-        assertThat(source.limit().fold(), equalTo(10));
+        assertThat(source.limit().fold(FoldContext.small()), equalTo(10));
     }
 
     /**
@@ -2815,12 +3102,13 @@ public void testSpatialTypesAndStatsCentroidUseDocValues() {
             "from airports | eval location = to_geopoint(location) | stats centroid = st_centroid_agg(location)" }) {
             for (boolean withDocValues : new boolean[] { false, true }) {
                 var testData = withDocValues ? airports : airportsNoDocValues;
+                var fieldExtractPreference = withDocValues ? FieldExtractPreference.DOC_VALUES : FieldExtractPreference.NONE;
                 var plan = physicalPlan(query, testData);
 
                 var limit = as(plan, LimitExec.class);
                 var agg = as(limit.child(), AggregateExec.class);
                 // Before optimization the aggregation does not use doc-values
-                assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+                assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
 
                 var exchange = as(agg.child(), ExchangeExec.class);
                 var fragment = as(exchange.child(), FragmentExec.class);
@@ -2832,12 +3120,12 @@ public void testSpatialTypesAndStatsCentroidUseDocValues() {
                 limit = as(optimized, LimitExec.class);
                 agg = as(limit.child(), AggregateExec.class);
                 // Above the exchange (in coordinator) the aggregation is not using doc-values
-                assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+                assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
                 exchange = as(agg.child(), ExchangeExec.class);
                 agg = as(exchange.child(), AggregateExec.class);
                 // below the exchange (in data node) the aggregation is using doc-values
-                assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, withDocValues);
-                assertChildIsGeoPointExtract(withDocValues ? agg : as(agg.child(), FilterExec.class), withDocValues);
+                assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, fieldExtractPreference);
+                assertChildIsGeoPointExtract(withDocValues ? agg : as(agg.child(), FilterExec.class), fieldExtractPreference);
             }
         }
     }
@@ -2876,13 +3164,14 @@ public void testSpatialTypesAndStatsExtentUseDocValues() {
             "from airports | stats extent = st_extent_agg(to_geopoint(location))",
             "from airports | eval location = to_geopoint(location) | stats extent = st_extent_agg(location)" }) {
             for (boolean withDocValues : new boolean[] { false, true }) {
+                var fieldExtractPreference = withDocValues ? FieldExtractPreference.DOC_VALUES : FieldExtractPreference.NONE;
                 var testData = withDocValues ? airports : airportsNoDocValues;
                 var plan = physicalPlan(query, testData);
 
                 var limit = as(plan, LimitExec.class);
                 var agg = as(limit.child(), AggregateExec.class);
                 // Before optimization the aggregation does not use doc-values
-                assertAggregation(agg, "extent", SpatialExtent.class, GEO_POINT, false);
+                assertAggregation(agg, "extent", SpatialExtent.class, GEO_POINT, FieldExtractPreference.NONE);
 
                 var exchange = as(agg.child(), ExchangeExec.class);
                 var fragment = as(exchange.child(), FragmentExec.class);
@@ -2894,12 +3183,12 @@ public void testSpatialTypesAndStatsExtentUseDocValues() {
                 limit = as(optimized, LimitExec.class);
                 agg = as(limit.child(), AggregateExec.class);
                 // Above the exchange (in coordinator) the aggregation is not using doc-values
-                assertAggregation(agg, "extent", SpatialExtent.class, GEO_POINT, false);
+                assertAggregation(agg, "extent", SpatialExtent.class, GEO_POINT, FieldExtractPreference.NONE);
                 exchange = as(agg.child(), ExchangeExec.class);
                 agg = as(exchange.child(), AggregateExec.class);
                 // below the exchange (in data node) the aggregation is using doc-values
-                assertAggregation(agg, "extent", SpatialExtent.class, GEO_POINT, withDocValues);
-                assertChildIsGeoPointExtract(withDocValues ? agg : as(agg.child(), FilterExec.class), withDocValues);
+                assertAggregation(agg, "extent", SpatialExtent.class, GEO_POINT, fieldExtractPreference);
+                assertChildIsGeoPointExtract(withDocValues ? agg : as(agg.child(), FilterExec.class), fieldExtractPreference);
             }
         }
     }
@@ -2908,24 +3197,25 @@ public void testSpatialTypesAndStatsExtentUseDocValues() {
      * Before local optimizations:
      * <code>
      * LimitExec[1000[INTEGER]]
-     *   \_AggregateExec[[],[SPATIALSTEXTENT(location{f}#48,true[BOOLEAN]) AS extent],FINAL,[minNegX{r}#52, minPosX{r}#53, maxNegX{r}#54,
-     * maxPosX{r}#55, maxY{r}#56, minY{r}#57],null]
-     *     \_ExchangeExec[[minNegX{r}#52, minPosX{r}#53, maxNegX{r}#54, maxPosX{r}#55, maxY{r}#56, minY{r}#57],true]
-     *       \_FragmentExec[filter=null, estimatedRowSize=0, reducer=[], fragment=[
-     * Aggregate[STANDARD,[],[SPATIALSTEXTENT(location{f}#48,true[BOOLEAN]) AS extent]]
-     *         \_EsRelation[airports][abbrev{f}#44, city{f}#50, city_location{f}#51, coun..]]]
+     * \_AggregateExec[[],[SPATIALEXTENT(location{f}#70,true[BOOLEAN]) AS extent, SPATIALCENTROID(location{f}#70,true[BOOLEAN]) AS cen
+     * troid],FINAL,[...]]
+     *   \_ExchangeExec[[...]]
+     *     \_FragmentExec[filter=null, estimatedRowSize=0, reducer=[], fragment=[
+     * Aggregate[STANDARD,[],[SPATIALEXTENT(location{f}#70,true[BOOLEAN]) AS extent, SPATIALCENTROID(location{f}#70,true[BOOLEAN]
+     * ) AS centroid]]
+     * \_EsRelation[airports][abbrev{f}#66, city{f}#72, city_location{f}#73, coun..]]]
      * </code>
      * After local optimizations:
      * <code>
      * LimitExec[1000[INTEGER]]
-     *   \_AggregateExec[[],[SPATIALSTEXTENT(location{f}#48,true[BOOLEAN]) AS extent],FINAL,[minNegX{r}#52, minPosX{r}#53, maxNegX{r}#54,
-     * maxPosX{r}#55, maxY{r}#56, minY{r}#57],21]
-     *     \_ExchangeExec[[minNegX{r}#52, minPosX{r}#53, maxNegX{r}#54, maxPosX{r}#55, maxY{r}#56, minY{r}#57],true]
-     *       \_AggregateExec[[],[SPATIALSTEXTENT(location{f}#48,true[BOOLEAN]) AS extent],INITIAL,[
-     * minNegX{r}#73, minPosX{r}#74, maxNegX{rb#75, maxPosX{r}#76, maxY{r}#77, minY{r}#78],21]
-     *         \_FieldExtractExec[location{f}#48][location{f}#48]
-     *           \_EsQueryExec[airports], indexMode[standard], query[{"exists":{"field":"location","boost":1.0}}][
-     * _doc{f}#79], limit[], sort[] estimatedRowSize[25]
+     * \_AggregateExec[[],[SPATIALEXTENT(location{f}#70,true[BOOLEAN]) AS extent, SPATIALCENTROID(location{f}#70,true[BOOLEAN]) AS cen
+     * troid],FINAL,[...]]
+     *   \_ExchangeExec[[...]]
+     *     \_AggregateExec[[],[SPATIALEXTENT(location{f}#70,true[BOOLEAN]) AS extent, SPATIALCENTROID(location{f}#70,true[BOOLEAN]) AS cen
+     * troid],INITIAL,[...]]
+     *       \_FieldExtractExec[location{f}#70][location{f}#70],[]
+     *         \_EsQueryExec[airports], indexMode[standard], query[{"exists":{"field":"location","boost":1.0}}][
+     * _doc{f}#117], limit[], sort[] estimatedRowSize[25]
      * </code>
      * Note the FieldExtractExec has 'location' set for stats: FieldExtractExec[location{f}#9][location{f}#9]
      * <p>
@@ -2937,13 +3227,14 @@ public void testSpatialTypesAndStatsExtentAndCentroidUseDocValues() {
             "from airports | stats extent = st_extent_agg(location), centroid = st_centroid_agg(location)",
             "from airports | stats extent = st_extent_agg(location), centroid = st_centroid_agg(city_location)", }) {
             for (boolean withDocValues : new boolean[] { false, true }) {
+                var fieldExtractPreference = withDocValues ? FieldExtractPreference.DOC_VALUES : FieldExtractPreference.NONE;
                 var testData = withDocValues ? airports : airportsNoDocValues;
                 var plan = physicalPlan(query, testData);
 
                 var limit = as(plan, LimitExec.class);
                 var agg = as(limit.child(), AggregateExec.class);
                 // Before optimization the aggregation does not use doc-values
-                assertAggregation(agg, "extent", SpatialExtent.class, GEO_POINT, false);
+                assertAggregation(agg, "extent", SpatialExtent.class, GEO_POINT, FieldExtractPreference.NONE);
 
                 var exchange = as(agg.child(), ExchangeExec.class);
                 var fragment = as(exchange.child(), FragmentExec.class);
@@ -2955,16 +3246,190 @@ public void testSpatialTypesAndStatsExtentAndCentroidUseDocValues() {
                 limit = as(optimized, LimitExec.class);
                 agg = as(limit.child(), AggregateExec.class);
                 // Above the exchange (in coordinator) the aggregation is not using doc-values
-                assertAggregation(agg, "extent", SpatialExtent.class, GEO_POINT, false);
+                assertAggregation(agg, "extent", SpatialExtent.class, GEO_POINT, FieldExtractPreference.NONE);
                 exchange = as(agg.child(), ExchangeExec.class);
                 agg = as(exchange.child(), AggregateExec.class);
                 // below the exchange (in data node) the aggregation is using doc-values
-                assertAggregation(agg, "extent", SpatialExtent.class, GEO_POINT, withDocValues);
-                assertChildIsGeoPointExtract(withDocValues ? agg : as(agg.child(), FilterExec.class), withDocValues);
+                assertAggregation(agg, "extent", SpatialExtent.class, GEO_POINT, fieldExtractPreference);
+                assertChildIsGeoPointExtract(withDocValues ? agg : as(agg.child(), FilterExec.class), fieldExtractPreference);
             }
         }
     }
 
+    /**
+     * <code>
+     * LimitExec[1000[INTEGER]]
+     * \_AggregateExec[[],[SPATIALEXTENT(city_boundary{f}#10,true[BOOLEAN]) AS extent],FINAL,[
+     *     $$extent$minNegX{r}#11, $$extent$minPosX{r}#12, $$extent$maxNegX{r}#13,
+     *     $$extent$maxPosX{r}#14, $$extent$maxY{r}#15, $$extent$minY{r}#16],200]
+     *   \_ExchangeExec[[
+     *       $$extent$minNegX{r}#11, $$extent$minPosX{r}#12, $$extent$maxNegX{r}#13,
+     *       $$extent$maxPosX{r}#14, $$extent$maxY{r}#15, $$extent$minY{r}#16],true]
+     *     \_AggregateExec[[],[SPATIALEXTENT(city_boundary{f}#10,true[BOOLEAN]) AS extent],INITIAL,[
+     *         $$extent$minNegX{r}#30, $$extent$minPosX{r}#31, $$extent$maxNegX{r}#32,
+     *         $$extent$maxPosX{r}#33, $$extent$maxY{r}#34, $$extent$minY{r}#35],200]
+     *       \_FieldExtractExec[city_boundary{f}#10][],[city_boundary{f}#10]
+     *         \_EsQueryExec[airports_city_boundaries], indexMode[standard], query[
+     *             {"exists":{"field":"city_boundary","boost":1.0}}
+     *           ][_doc{f}#36], limit[], sort[] estimatedRowSize[204]
+     * </code>
+     */
+    public void testSpatialTypesAndStatsExtentOfGeoShapeDoesNotUseBinaryExtraction() {
+        // TODO: When we get geo_shape working with bounds extraction from doc-values, change the name of this test
+        var query = "FROM airports_city_boundaries | STATS extent = ST_EXTENT_AGG(city_boundary)";
+        var testData = airportsCityBoundaries;
+        var plan = physicalPlan(query, testData);
+
+        var limit = as(plan, LimitExec.class);
+        var agg = as(limit.child(), AggregateExec.class);
+        // Before optimization the aggregation does not use extent extraction
+        assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE);
+
+        var exchange = as(agg.child(), ExchangeExec.class);
+        var fragment = as(exchange.child(), FragmentExec.class);
+        var fAgg = as(fragment.fragment(), Aggregate.class);
+        as(fAgg.child(), EsRelation.class);
+
+        // Now optimize the plan and assert the aggregation uses extent extraction
+        var optimized = optimizedPlan(plan, testData.stats);
+        limit = as(optimized, LimitExec.class);
+        agg = as(limit.child(), AggregateExec.class);
+        // Above the exchange (in coordinator) the aggregation is not using doc-values
+        assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE);
+        exchange = as(agg.child(), ExchangeExec.class);
+        agg = as(exchange.child(), AggregateExec.class);
+        // below the exchange (in data node) the aggregation is using a specific
+        assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE);
+        assertChildIsExtractedAs(agg, FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS, GEO_SHAPE);
+    }
+
+    /**
+     * This test verifies that the aggregation does not use spatial bounds extraction when the shape appears in an eval or filter.
+     * TODO: Currently this tests nothing, because geo_shape is not supported anyway for bounds extraction,
+     * but it should be updated when it is supported.
+     */
+    public void testSpatialTypesAndStatsExtentOfShapesNegativeCases() {
+        for (String query : new String[] { """
+            FROM airports_city_boundaries
+            | EVAL prefix = SUBSTRING(TO_STRING(city_boundary), 5)
+            | STATS extent = ST_EXTENT_AGG(city_boundary) BY prefix""", """
+            FROM airports_city_boundaries
+            | WHERE STARTS_WITH(TO_STRING(city_boundary), "MULTIPOLYGON")
+            | STATS extent = ST_EXTENT_AGG(city_boundary)""" }) {
+            var testData = airportsCityBoundaries;
+            var plan = physicalPlan(query, testData);
+
+            var limit = as(plan, LimitExec.class);
+            var agg = as(limit.child(), AggregateExec.class);
+            assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE);
+
+            var optimized = optimizedPlan(plan, testData.stats);
+            limit = as(optimized, LimitExec.class);
+            agg = as(limit.child(), AggregateExec.class);
+            assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE);
+            var exchange = as(agg.child(), ExchangeExec.class);
+            agg = as(exchange.child(), AggregateExec.class);
+            assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE);
+            var exec = agg.child() instanceof FieldExtractExec ? agg : as(agg.child(), UnaryExec.class);
+            assertChildIsExtractedAs(exec, FieldExtractPreference.NONE, GEO_SHAPE);
+        }
+    }
+
+    /**
+     * Test cartesian_shape bounds extraction occurs when the shape has doc-values and not otherwise.
+     */
+    public void testSpatialTypesAndStatsExtentOfCartesianShapesWithAndWithoutDocValues() {
+        for (boolean hasDocValues : new boolean[] { true, false }) {
+            var query = """
+                FROM cartesian_multipolygons \
+                | STATS extent = ST_EXTENT_AGG(shape)""";
+            var testData = hasDocValues ? cartesianMultipolygons : cartesianMultipolygonsNoDocValues;
+            var fieldExtractPreference = hasDocValues ? FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS : FieldExtractPreference.NONE;
+            var plan = physicalPlan(query, testData);
+
+            var limit = as(plan, LimitExec.class);
+            var agg = as(limit.child(), AggregateExec.class);
+            assertAggregation(agg, "extent", SpatialExtent.class, CARTESIAN_SHAPE, FieldExtractPreference.NONE);
+
+            var optimized = optimizedPlan(plan, testData.stats);
+            limit = as(optimized, LimitExec.class);
+            agg = as(limit.child(), AggregateExec.class);
+            // For cartesian_shape extraction, we extract bounds from doc-values directly into a BBOX encoded as BytesRef,
+            // so the aggregation does not need to know about it.
+            assertAggregation(agg, "extent", SpatialExtent.class, CARTESIAN_SHAPE, FieldExtractPreference.NONE);
+            var exchange = as(agg.child(), ExchangeExec.class);
+            agg = as(exchange.child(), AggregateExec.class);
+            assertAggregation(
+                agg,
+                "extent",
+                "hasDocValues:" + hasDocValues,
+                SpatialExtent.class,
+                CARTESIAN_SHAPE,
+                FieldExtractPreference.NONE
+            );
+            var exec = agg.child() instanceof FieldExtractExec ? agg : as(agg.child(), UnaryExec.class);
+            // For cartesian_shape, the bounds extraction is done in the FieldExtractExec, so it does need to know about this
+            assertChildIsExtractedAs(exec, fieldExtractPreference, CARTESIAN_SHAPE);
+        }
+    }
+
+    /**
+     * Before local optimizations:
+     * <code>
+     * LimitExec[1000[INTEGER]]
+     * \_AggregateExec[[],[SPATIALEXTENT(city_boundary{f}#13,true[BOOLEAN]) AS extent, SPATIALCENTROID(city_location{f}#12,true[BOOLEA
+     * N]) AS centroid],...]
+     *   \_ExchangeExec[[..]]
+     *     \_FragmentExec[filter=null, estimatedRowSize=0, reducer=[], fragment=[...]]
+     *       \_EsRelation[airports_city_boundaries][abbrev{f}#8, airport{f}#9, city{f}#11, city_boundar..]
+     * </code>
+     * After local optimizations:
+     * <code>
+     * LimitExec[1000[INTEGER]]
+     *   \_AggregateExec[[],[SPATIALSTEXTENT(location{f}#48,true[BOOLEAN]) AS extent],FINAL,[minNegX{r}#52, minPosX{r}#53, maxNegX{r}#54,
+     * maxPosX{r}#55, maxY{r}#56, minY{r}#57],21]
+     *     \_ExchangeExec[[minNegX{r}#52, minPosX{r}#53, maxNegX{r}#54, maxPosX{r}#55, maxY{r}#56, minY{r}#57],true]
+     *       \_AggregateExec[[],[SPATIALSTEXTENT(location{f}#48,true[BOOLEAN]) AS extent],INITIAL,[
+     * minNegX{r}#73, minPosX{r}#74, maxNegX{rb#75, maxPosX{r}#76, maxY{r}#77, minY{r}#78],21]
+     *         \_FieldExtractExec[location{f}#48][location{f}#48]
+     *           \_EsQueryExec[airports], indexMode[standard], query[{"exists":{"field":"location","boost":1.0}}][
+     * _doc{f}#79], limit[], sort[] estimatedRowSize[25]
+     * </code>
+     */
+    public void testMixedSpatialBoundsAndPointsExtracted() {
+        var query = """
+            FROM airports_city_boundaries \
+            | STATS extent = ST_EXTENT_AGG(city_boundary), centroid = ST_CENTROID_AGG(city_location)""";
+        var testData = airportsCityBoundaries;
+        var plan = physicalPlan(query, testData);
+
+        var limit = as(plan, LimitExec.class);
+        var agg = as(limit.child(), AggregateExec.class);
+        // Before optimization the aggregation does not use doc-values
+        assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
+
+        var exchange = as(agg.child(), ExchangeExec.class);
+        var fragment = as(exchange.child(), FragmentExec.class);
+        var fAgg = as(fragment.fragment(), Aggregate.class);
+        as(fAgg.child(), EsRelation.class);
+
+        // Now optimize the plan and assert the aggregation uses both doc-values and bounds extraction
+        var optimized = optimizedPlan(plan, testData.stats);
+        limit = as(optimized, LimitExec.class);
+        agg = as(limit.child(), AggregateExec.class);
+        // Above the exchange (in coordinator) the aggregation is not field-optimized.
+        assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
+        exchange = as(agg.child(), ExchangeExec.class);
+        agg = as(exchange.child(), AggregateExec.class);
+        // below the exchange (in data node) the aggregation is field optimized.
+        assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE);
+        var fieldExtractExec = as(agg.child(), FieldExtractExec.class);
+        assertThat(fieldExtractExec.boundsAttributes().stream().map(a -> a.sourceText()).toList(), equalTo(List.of("city_boundary")));
+        assertThat(fieldExtractExec.docValuesAttributes().stream().map(a -> a.sourceText()).toList(), equalTo(List.of("city_location")));
+    }
+
     /**
      * This test does not have real index fields, and therefor asserts that doc-values field extraction does NOT occur.
      * Before local optimizations:
@@ -2996,11 +3461,11 @@ public void testSpatialTypesAndStatsUseDocValuesNestedLiteral() {
         var agg = as(limit.child(), AggregateExec.class);
         assertThat("Aggregation is FINAL", agg.getMode(), equalTo(FINAL));
         assertThat("No groupings in aggregation", agg.groupings().size(), equalTo(0));
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
         agg = as(agg.child(), AggregateExec.class);
         assertThat("Aggregation is PARTIAL", agg.getMode(), equalTo(INITIAL));
         assertThat("No groupings in aggregation", agg.groupings().size(), equalTo(0));
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
         var eval = as(agg.child(), EvalExec.class);
         as(eval.child(), LocalSourceExec.class);
 
@@ -3010,11 +3475,11 @@ public void testSpatialTypesAndStatsUseDocValuesNestedLiteral() {
         agg = as(limit.child(), AggregateExec.class);
         assertThat("Aggregation is FINAL", agg.getMode(), equalTo(FINAL));
         assertThat("No groupings in aggregation", agg.groupings().size(), equalTo(0));
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
         agg = as(agg.child(), AggregateExec.class);
         assertThat("Aggregation is PARTIAL", agg.getMode(), equalTo(INITIAL));
         assertThat("No groupings in aggregation", agg.groupings().size(), equalTo(0));
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
         eval = as(agg.child(), EvalExec.class);
         as(eval.child(), LocalSourceExec.class);
     }
@@ -3051,7 +3516,7 @@ public void testSpatialTypesAndStatsUseDocValuesMultiAggregations() {
         assertThat("No groupings in aggregation", agg.groupings().size(), equalTo(0));
         // Before optimization the aggregation does not use doc-values
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
 
         var exchange = as(agg.child(), ExchangeExec.class);
         var fragment = as(exchange.child(), FragmentExec.class);
@@ -3064,14 +3529,14 @@ public void testSpatialTypesAndStatsUseDocValuesMultiAggregations() {
         agg = as(limit.child(), AggregateExec.class);
         // Above the exchange (in coordinator) the aggregation is not using doc-values
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
         exchange = as(agg.child(), ExchangeExec.class);
         agg = as(exchange.child(), AggregateExec.class);
         assertThat("Aggregation is PARTIAL", agg.getMode(), equalTo(INITIAL));
         // below the exchange (in data node) the aggregation is using doc-values
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, true);
-        assertChildIsGeoPointExtract(agg, true);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.DOC_VALUES);
+        assertChildIsGeoPointExtract(agg, FieldExtractPreference.DOC_VALUES);
     }
 
     /**
@@ -3112,8 +3577,8 @@ public void testSpatialTypesAndStatsUseDocValuesMultiSpatialAggregations() {
         assertThat("No groupings in aggregation", agg.groupings().size(), equalTo(0));
         // Before optimization the aggregation does not use doc-values
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "airports", SpatialCentroid.class, GEO_POINT, false);
-        assertAggregation(agg, "cities", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "airports", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
+        assertAggregation(agg, "cities", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
 
         var exchange = as(agg.child(), ExchangeExec.class);
         var fragment = as(exchange.child(), FragmentExec.class);
@@ -3126,16 +3591,16 @@ public void testSpatialTypesAndStatsUseDocValuesMultiSpatialAggregations() {
         agg = as(limit.child(), AggregateExec.class);
         // Above the exchange (in coordinator) the aggregation is not using doc-values
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "airports", SpatialCentroid.class, GEO_POINT, false);
-        assertAggregation(agg, "cities", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "airports", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
+        assertAggregation(agg, "cities", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
         exchange = as(agg.child(), ExchangeExec.class);
         agg = as(exchange.child(), AggregateExec.class);
         assertThat("Aggregation is PARTIAL", agg.getMode(), equalTo(INITIAL));
         // below the exchange (in data node) the aggregation is using doc-values
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "airports", SpatialCentroid.class, GEO_POINT, true);
-        assertAggregation(agg, "cities", SpatialCentroid.class, GEO_POINT, true);
-        assertChildIsGeoPointExtract(agg, true);
+        assertAggregation(agg, "airports", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.DOC_VALUES);
+        assertAggregation(agg, "cities", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.DOC_VALUES);
+        assertChildIsGeoPointExtract(agg, FieldExtractPreference.DOC_VALUES);
     }
 
     /**
@@ -3173,7 +3638,7 @@ public void testSpatialTypesAndStatsUseDocValuesMultiAggregationsFiltered() {
         assertThat("No groupings in aggregation", agg.groupings().size(), equalTo(0));
         // Before optimization the aggregation does not use doc-values
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
 
         var exchange = as(agg.child(), ExchangeExec.class);
         var fragment = as(exchange.child(), FragmentExec.class);
@@ -3188,14 +3653,14 @@ public void testSpatialTypesAndStatsUseDocValuesMultiAggregationsFiltered() {
         agg = as(limit.child(), AggregateExec.class);
         // Above the exchange (in coordinator) the aggregation is not using doc-values
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
         exchange = as(agg.child(), ExchangeExec.class);
         agg = as(exchange.child(), AggregateExec.class);
         assertThat("Aggregation is PARTIAL", agg.getMode(), equalTo(INITIAL));
         // below the exchange (in data node) the aggregation is using doc-values
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, true);
-        var source = assertChildIsGeoPointExtract(agg, true);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.DOC_VALUES);
+        var source = assertChildIsGeoPointExtract(agg, FieldExtractPreference.DOC_VALUES);
         var qb = as(source.query(), SingleValueQuery.Builder.class);
         assertThat("Expected predicate to be passed to Lucene query", qb.source().text(), equalTo("scalerank == 9"));
     }
@@ -3227,6 +3692,7 @@ public void testSpatialTypesAndStatsUseDocValuesMultiAggregationsFiltered() {
     public void testSpatialTypesAndStatsUseDocValuesMultiAggregationsGrouped() {
         for (boolean useDocValues : new boolean[] { false }) {
             var testData = useDocValues ? airports : airportsNoDocValues;
+            var fieldExtractPreference = useDocValues ? FieldExtractPreference.DOC_VALUES : FieldExtractPreference.NONE;
             var plan = this.physicalPlan("""
                 FROM airports
                 | STATS centroid=ST_CENTROID_AGG(location), count=COUNT() BY scalerank
@@ -3239,7 +3705,7 @@ public void testSpatialTypesAndStatsUseDocValuesMultiAggregationsGrouped() {
             assertThat(att.name(), equalTo("scalerank"));
             // Before optimization the aggregation does not use doc-values
             assertAggregation(agg, "count", Count.class);
-            assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+            assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
 
             var exchange = as(agg.child(), ExchangeExec.class);
             var fragment = as(exchange.child(), FragmentExec.class);
@@ -3254,7 +3720,7 @@ public void testSpatialTypesAndStatsUseDocValuesMultiAggregationsGrouped() {
             assertThat(att.name(), equalTo("scalerank"));
             // Above the exchange (in coordinator) the aggregation is not using doc-values
             assertAggregation(agg, "count", Count.class);
-            assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+            assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
             exchange = as(agg.child(), ExchangeExec.class);
             agg = as(exchange.child(), AggregateExec.class);
             assertThat("Aggregation is PARTIAL", agg.getMode(), equalTo(INITIAL));
@@ -3262,8 +3728,8 @@ public void testSpatialTypesAndStatsUseDocValuesMultiAggregationsGrouped() {
             assertThat(att.name(), equalTo("scalerank"));
             // below the exchange (in data node) the aggregation is using doc-values
             assertAggregation(agg, "count", Count.class);
-            assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, useDocValues);
-            assertChildIsGeoPointExtract(agg, useDocValues);
+            assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, fieldExtractPreference);
+            assertChildIsGeoPointExtract(agg, fieldExtractPreference);
         }
     }
 
@@ -3304,19 +3770,19 @@ public void testSpatialTypesAndStatsUseDocValuesMultiAggregationsGroupedAggregat
         assertThat("Aggregation is FINAL", agg.getMode(), equalTo(FINAL));
         assertThat("No groupings in aggregation", agg.groupings().size(), equalTo(0));
         assertAggregation(agg, "count", Sum.class);
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
         agg = as(agg.child(), AggregateExec.class);
         assertThat("Aggregation is PARTIAL", agg.getMode(), equalTo(INITIAL));
         assertThat("No groupings in aggregation", agg.groupings().size(), equalTo(0));
         assertAggregation(agg, "count", Sum.class);
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
         agg = as(agg.child(), AggregateExec.class);
         assertThat("Aggregation is FINAL", agg.getMode(), equalTo(FINAL));
         assertThat("One grouping in aggregation", agg.groupings().size(), equalTo(1));
         var att = as(agg.groupings().get(0), Attribute.class);
         assertThat(att.name(), equalTo("scalerank"));
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
 
         var exchange = as(agg.child(), ExchangeExec.class);
         var fragment = as(exchange.child(), FragmentExec.class);
@@ -3330,19 +3796,19 @@ public void testSpatialTypesAndStatsUseDocValuesMultiAggregationsGroupedAggregat
         assertThat("Aggregation is FINAL", agg.getMode(), equalTo(FINAL));
         assertThat("No groupings in aggregation", agg.groupings().size(), equalTo(0));
         assertAggregation(agg, "count", Sum.class);
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
         agg = as(agg.child(), AggregateExec.class);
         assertThat("Aggregation is PARTIAL", agg.getMode(), equalTo(INITIAL));
         assertThat("No groupings in aggregation", agg.groupings().size(), equalTo(0));
         assertAggregation(agg, "count", Sum.class);
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
         agg = as(agg.child(), AggregateExec.class);
         assertThat("Aggregation is FINAL", agg.getMode(), equalTo(FINAL));
         assertThat("One grouping in aggregation", agg.groupings().size(), equalTo(1));
         att = as(agg.groupings().get(0), Attribute.class);
         assertThat(att.name(), equalTo("scalerank"));
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
         exchange = as(agg.child(), ExchangeExec.class);
         agg = as(exchange.child(), AggregateExec.class);
         assertThat("One grouping in aggregation", agg.groupings().size(), equalTo(1));
@@ -3351,8 +3817,8 @@ public void testSpatialTypesAndStatsUseDocValuesMultiAggregationsGroupedAggregat
         // below the exchange (in data node) the aggregation is using doc-values
         assertThat("Aggregation is PARTIAL", agg.getMode(), equalTo(INITIAL));
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, true);
-        assertChildIsGeoPointExtract(agg, true);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.DOC_VALUES);
+        assertChildIsGeoPointExtract(agg, FieldExtractPreference.DOC_VALUES);
     }
 
     /**
@@ -3392,7 +3858,7 @@ public void testEnrichBeforeSpatialAggregationSupportsDocValues() {
         var limit = as(plan, LimitExec.class);
         var agg = as(limit.child(), AggregateExec.class);
         // Before optimization the aggregation does not use doc-values
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
 
         var exchange = as(agg.child(), ExchangeExec.class);
         var fragment = as(exchange.child(), FragmentExec.class);
@@ -3408,16 +3874,16 @@ public void testEnrichBeforeSpatialAggregationSupportsDocValues() {
         limit = as(optimized, LimitExec.class);
         agg = as(limit.child(), AggregateExec.class);
         // Above the exchange (in coordinator) the aggregation is not using doc-values
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
         exchange = as(agg.child(), ExchangeExec.class);
         agg = as(exchange.child(), AggregateExec.class);
         // below the exchange (in data node) the aggregation is using doc-values
-        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, true);
+        assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.DOC_VALUES);
         var enrichExec = as(agg.child(), EnrichExec.class);
         assertThat(enrichExec.mode(), equalTo(Enrich.Mode.ANY));
         assertThat(enrichExec.concreteIndices(), equalTo(Map.of("", "airport_city_boundaries")));
         assertThat(enrichExec.enrichFields().size(), equalTo(3));
-        assertChildIsGeoPointExtract(enrichExec, true);
+        assertChildIsGeoPointExtract(enrichExec, FieldExtractPreference.DOC_VALUES);
     }
 
     /**
@@ -3717,7 +4183,7 @@ public void testPushDownSpatialRelatesStringToSourceAndUseDocValuesForCentroid()
             assertThat("No groupings in aggregation", agg.groupings().size(), equalTo(0));
             // Before optimization the aggregation does not use doc-values
             assertAggregation(agg, "count", Count.class);
-            assertAggregation(agg, "centroid", SpatialCentroid.class, test.locationType(), false);
+            assertAggregation(agg, "centroid", SpatialCentroid.class, test.locationType(), FieldExtractPreference.NONE);
             var exchange = as(agg.child(), ExchangeExec.class);
             var fragment = as(exchange.child(), FragmentExec.class);
             var fAgg = as(fragment.fragment(), Aggregate.class);
@@ -3730,15 +4196,15 @@ public void testPushDownSpatialRelatesStringToSourceAndUseDocValuesForCentroid()
             agg = as(limit.child(), AggregateExec.class);
             // Above the exchange (in coordinator) the aggregation is not using doc-values
             assertAggregation(agg, "count", Count.class);
-            assertAggregation(agg, "centroid", SpatialCentroid.class, test.locationType(), false);
+            assertAggregation(agg, "centroid", SpatialCentroid.class, test.locationType(), FieldExtractPreference.NONE);
             exchange = as(agg.child(), ExchangeExec.class);
             agg = as(exchange.child(), AggregateExec.class);
             assertThat("Aggregation is PARTIAL", agg.getMode(), equalTo(INITIAL));
             // below the exchange (in data node) the aggregation is using doc-values
             assertAggregation(agg, "count", Count.class);
-            assertAggregation(agg, "centroid", SpatialCentroid.class, test.locationType(), true);
+            assertAggregation(agg, "centroid", SpatialCentroid.class, test.locationType(), FieldExtractPreference.DOC_VALUES);
             if (test.canPushToSource) {
-                var source = assertChildIsExtractedAsDocValues(agg, true, test.locationType());
+                var source = assertChildIsExtractedAs(agg, FieldExtractPreference.DOC_VALUES, test.locationType());
                 var condition = as(source.query(), SpatialRelatesQuery.ShapeQueryBuilder.class);
                 assertThat("Geometry field name: " + test.predicate(), condition.fieldName(), equalTo("location"));
                 assertThat("Spatial relationship: " + test.predicate(), condition.relation(), equalTo(test.relationship()));
@@ -3804,6 +4270,7 @@ public void testPushSpatialIntersectsStringToSourceAndUseDocValuesForCentroid()
 
             for (boolean isIndexed : new boolean[] { true, false }) {
                 for (boolean useDocValues : new boolean[] { true, false }) {
+                    var fieldExtractPreference = useDocValues ? FieldExtractPreference.DOC_VALUES : FieldExtractPreference.NONE;
                     var testData = useDocValues
                         ? (isIndexed ? airports : airportsNotIndexed)
                         : (isIndexed ? airportsNoDocValues : airportsNotIndexedNorDocValues);
@@ -3813,7 +4280,7 @@ public void testPushSpatialIntersectsStringToSourceAndUseDocValuesForCentroid()
                     assertThat("No groupings in aggregation", agg.groupings().size(), equalTo(0));
                     // Before optimization the aggregation does not use doc-values
                     assertAggregation(agg, "count", Count.class);
-                    assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+                    assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
 
                     var exchange = as(agg.child(), ExchangeExec.class);
                     var fragment = as(exchange.child(), FragmentExec.class);
@@ -3827,15 +4294,15 @@ public void testPushSpatialIntersectsStringToSourceAndUseDocValuesForCentroid()
                     agg = as(limit.child(), AggregateExec.class);
                     // Above the exchange (in coordinator) the aggregation is not using doc-values
                     assertAggregation(agg, "count", Count.class);
-                    assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+                    assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
                     exchange = as(agg.child(), ExchangeExec.class);
                     agg = as(exchange.child(), AggregateExec.class);
                     assertThat("Aggregation is PARTIAL", agg.getMode(), equalTo(INITIAL));
                     // below the exchange (in data node) the aggregation is using doc-values
                     assertAggregation(agg, "count", Count.class);
-                    assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, useDocValues);
+                    assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, fieldExtractPreference);
                     if (isIndexed) {
-                        var source = assertChildIsGeoPointExtract(agg, useDocValues);
+                        var source = assertChildIsGeoPointExtract(agg, fieldExtractPreference);
                         // Query is pushed to lucene if field is indexed (and does not require doc-values or isAggregatable)
                         var condition = as(source.query(), SpatialRelatesQuery.ShapeQueryBuilder.class);
                         assertThat("Geometry field name", condition.fieldName(), equalTo("location"));
@@ -3926,7 +4393,7 @@ AND ST_INTERSECTS(TO_GEOSHAPE("POLYGON((42 14, 43 14, 43 15, 42 15, 42 14))"), l
             assertThat("No groupings in aggregation", agg.groupings().size(), equalTo(0));
             // Before optimization the aggregation does not use doc-values
             assertAggregation(agg, "count", Count.class);
-            assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+            assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
 
             var exchange = as(agg.child(), ExchangeExec.class);
             var fragment = as(exchange.child(), FragmentExec.class);
@@ -3942,14 +4409,14 @@ AND ST_INTERSECTS(TO_GEOSHAPE("POLYGON((42 14, 43 14, 43 15, 42 15, 42 14))"), l
             agg = as(limit.child(), AggregateExec.class);
             // Above the exchange (in coordinator) the aggregation is not using doc-values
             assertAggregation(agg, "count", Count.class);
-            assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, false);
+            assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
             exchange = as(agg.child(), ExchangeExec.class);
             agg = as(exchange.child(), AggregateExec.class);
             assertThat("Aggregation is PARTIAL", agg.getMode(), equalTo(INITIAL));
             // below the exchange (in data node) the aggregation is using doc-values
             assertAggregation(agg, "count", Count.class);
-            assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, true);
-            var source = assertChildIsGeoPointExtract(agg, true);
+            assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.DOC_VALUES);
+            var source = assertChildIsGeoPointExtract(agg, FieldExtractPreference.DOC_VALUES);
             var booleanQuery = as(source.query(), BoolQueryBuilder.class);
             assertThat("Expected boolean query of three predicates", booleanQuery.must().size(), equalTo(3));
             var condition = as(booleanQuery.must().get(1), SpatialRelatesQuery.ShapeQueryBuilder.class);
@@ -4000,8 +4467,8 @@ public void testIntersectsOnTwoPointFieldAndBothCentroidUsesDocValues() {
         assertThat("No groupings in aggregation", agg.groupings().size(), equalTo(0));
         // Before optimization the aggregation does not use doc-values
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "location", SpatialCentroid.class, GEO_POINT, false);
-        assertAggregation(agg, "city_location", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "location", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
+        assertAggregation(agg, "city_location", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
 
         var exchange = as(agg.child(), ExchangeExec.class);
         var fragment = as(exchange.child(), FragmentExec.class);
@@ -4015,15 +4482,15 @@ public void testIntersectsOnTwoPointFieldAndBothCentroidUsesDocValues() {
         agg = as(limit.child(), AggregateExec.class);
         // Above the exchange (in coordinator) the aggregation is not using doc-values
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "location", SpatialCentroid.class, GEO_POINT, false);
-        assertAggregation(agg, "city_location", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "location", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
+        assertAggregation(agg, "city_location", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
         exchange = as(agg.child(), ExchangeExec.class);
         agg = as(exchange.child(), AggregateExec.class);
         assertThat("Aggregation is PARTIAL", agg.getMode(), equalTo(INITIAL));
         // below the exchange (in data node) the aggregation is using doc-values
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "location", SpatialCentroid.class, GEO_POINT, true);
-        assertAggregation(agg, "city_location", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "location", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.DOC_VALUES);
+        assertAggregation(agg, "city_location", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
         var filterExec = as(agg.child(), FilterExec.class);
         var extract = as(filterExec.child(), FieldExtractExec.class);
         assertFieldExtractionWithDocValues(extract, GEO_POINT, "location");
@@ -4048,7 +4515,7 @@ public void testIntersectsOnTwoPointFieldAndOneCentroidUsesDocValues() {
             // Before optimization the aggregation does not use doc-values
             assertAggregation(agg, "count", Count.class);
             var aggFieldName = findSingleAggregation(agg, "location", "city_location");
-            assertAggregation(agg, aggFieldName, SpatialCentroid.class, GEO_POINT, false);
+            assertAggregation(agg, aggFieldName, SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
 
             var exchange = as(agg.child(), ExchangeExec.class);
             var fragment = as(exchange.child(), FragmentExec.class);
@@ -4062,13 +4529,13 @@ public void testIntersectsOnTwoPointFieldAndOneCentroidUsesDocValues() {
             agg = as(limit.child(), AggregateExec.class);
             // Above the exchange (in coordinator) the aggregation is not using doc-values
             assertAggregation(agg, "count", Count.class);
-            assertAggregation(agg, aggFieldName, SpatialCentroid.class, GEO_POINT, false);
+            assertAggregation(agg, aggFieldName, SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
             exchange = as(agg.child(), ExchangeExec.class);
             agg = as(exchange.child(), AggregateExec.class);
             assertThat("Aggregation is PARTIAL", agg.getMode(), equalTo(INITIAL));
             // below the exchange (in data node) the aggregation is using doc-values
             assertAggregation(agg, "count", Count.class);
-            assertAggregation(agg, aggFieldName, SpatialCentroid.class, GEO_POINT, true);
+            assertAggregation(agg, aggFieldName, SpatialCentroid.class, GEO_POINT, FieldExtractPreference.DOC_VALUES);
             var filterExec = as(agg.child(), FilterExec.class);
             var extract = as(filterExec.child(), FieldExtractExec.class);
             assertFieldExtractionWithDocValues(extract, GEO_POINT, aggFieldName);
@@ -4090,8 +4557,8 @@ AND ST_INTERSECTS(city_location, TO_GEOSHAPE("POLYGON((42 14, 43 14, 43 15, 42 1
         assertThat("No groupings in aggregation", agg.groupings().size(), equalTo(0));
         // Before optimization the aggregation does not use doc-values
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "location", SpatialCentroid.class, GEO_POINT, false);
-        assertAggregation(agg, "city_location", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "location", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
+        assertAggregation(agg, "city_location", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
 
         var exchange = as(agg.child(), ExchangeExec.class);
         var fragment = as(exchange.child(), FragmentExec.class);
@@ -4107,15 +4574,15 @@ AND ST_INTERSECTS(city_location, TO_GEOSHAPE("POLYGON((42 14, 43 14, 43 15, 42 1
         agg = as(limit.child(), AggregateExec.class);
         // Above the exchange (in coordinator) the aggregation is not using doc-values
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "location", SpatialCentroid.class, GEO_POINT, false);
-        assertAggregation(agg, "city_location", SpatialCentroid.class, GEO_POINT, false);
+        assertAggregation(agg, "location", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
+        assertAggregation(agg, "city_location", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE);
         exchange = as(agg.child(), ExchangeExec.class);
         agg = as(exchange.child(), AggregateExec.class);
         assertThat("Aggregation is PARTIAL", agg.getMode(), equalTo(INITIAL));
         // below the exchange (in data node) the aggregation is using doc-values
         assertAggregation(agg, "count", Count.class);
-        assertAggregation(agg, "location", SpatialCentroid.class, GEO_POINT, true);
-        assertAggregation(agg, "city_location", SpatialCentroid.class, GEO_POINT, true);
+        assertAggregation(agg, "location", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.DOC_VALUES);
+        assertAggregation(agg, "city_location", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.DOC_VALUES);
         var extract = as(agg.child(), FieldExtractExec.class);
         assertFieldExtractionWithDocValues(extract, GEO_POINT, "location", "city_location");
         var source = source(extract.child());
@@ -4384,15 +4851,15 @@ public void testPushSpatialDistanceMultiEvalToSource() {
         var alias1 = as(eval.fields().get(0), Alias.class);
         assertThat(alias1.name(), is("poi"));
         var poi = as(alias1.child(), Literal.class);
-        assertThat(poi.fold(), instanceOf(BytesRef.class));
+        assertThat(poi.value(), instanceOf(BytesRef.class));
         var alias2 = as(eval.fields().get(1), Alias.class);
         assertThat(alias2.name(), is("distance"));
         var stDistance = as(alias2.child(), StDistance.class);
         var location = as(stDistance.left(), FieldAttribute.class);
         assertThat(location.fieldName(), is("location"));
         var poiRef = as(stDistance.right(), Literal.class);
-        assertThat(poiRef.fold(), instanceOf(BytesRef.class));
-        assertThat(poiRef.fold().toString(), is(poi.fold().toString()));
+        assertThat(poiRef.value(), instanceOf(BytesRef.class));
+        assertThat(poiRef.value().toString(), is(poi.value().toString()));
 
         // Validate the filter condition
         var and = as(filter.condition(), And.class);
@@ -4679,7 +5146,7 @@ public void testPushSpatialDistanceEvalWithSimpleStatsToSource() {
         var evalExec = as(aggExec2.child(), EvalExec.class);
         var stDistance = as(evalExec.fields().get(0).child(), StDistance.class);
         assertThat("Expect distance function to expect doc-values", stDistance.leftDocValues(), is(false));
-        var source = assertChildIsGeoPointExtract(evalExec, false);
+        var source = assertChildIsGeoPointExtract(evalExec, FieldExtractPreference.NONE);
 
         // No sort is pushed down
         assertThat(source.limit(), nullValue());
@@ -4794,7 +5261,7 @@ public void testPushSpatialDistanceEvalWithStatsToSource() {
         var evalExec = as(aggExec2.child(), EvalExec.class);
         var stDistance = as(evalExec.fields().get(0).child(), StDistance.class);
         assertThat("Expect distance function to expect doc-values", stDistance.leftDocValues(), is(true));
-        var source = assertChildIsGeoPointExtract(evalExec, true);
+        var source = assertChildIsGeoPointExtract(evalExec, FieldExtractPreference.DOC_VALUES);
 
         // No sort is pushed down
         assertThat(source.limit(), nullValue());
@@ -5740,15 +6207,15 @@ public void testPushCompoundTopNDistanceWithCompoundFilterAndCompoundEvalToSourc
         var alias1 = as(evalExec.fields().get(0), Alias.class);
         assertThat(alias1.name(), is("poi"));
         var poi = as(alias1.child(), Literal.class);
-        assertThat(poi.fold(), instanceOf(BytesRef.class));
+        assertThat(poi.value(), instanceOf(BytesRef.class));
         var alias2 = as(evalExec.fields().get(1), Alias.class);
         assertThat(alias2.name(), is("distance"));
         var stDistance = as(alias2.child(), StDistance.class);
         var location = as(stDistance.left(), FieldAttribute.class);
         assertThat(location.fieldName(), is("location"));
         var poiRef = as(stDistance.right(), Literal.class);
-        assertThat(poiRef.fold(), instanceOf(BytesRef.class));
-        assertThat(poiRef.fold().toString(), is(poi.fold().toString()));
+        assertThat(poiRef.value(), instanceOf(BytesRef.class));
+        assertThat(poiRef.value().toString(), is(poi.value().toString()));
         extract = as(evalExec.child(), FieldExtractExec.class);
         assertThat(names(extract.attributesToExtract()), contains("location"));
         var source = source(extract.child());
@@ -5829,7 +6296,7 @@ public void testPushCompoundTopNDistanceWithDeeplyNestedCompoundEvalToSource() {
         var alias1 = as(evalExec.fields().get(0), Alias.class);
         assertThat(alias1.name(), is("poi"));
         var poi = as(alias1.child(), Literal.class);
-        assertThat(poi.fold(), instanceOf(BytesRef.class));
+        assertThat(poi.value(), instanceOf(BytesRef.class));
         var alias4 = as(evalExec.fields().get(3), Alias.class);
         assertThat(alias4.name(), is("loc2"));
         as(alias4.child(), FieldAttribute.class);
@@ -5842,8 +6309,8 @@ public void testPushCompoundTopNDistanceWithDeeplyNestedCompoundEvalToSource() {
         var refLocation = as(stDistance.left(), ReferenceAttribute.class);
         assertThat(refLocation.name(), is("loc3"));
         var poiRef = as(stDistance.right(), Literal.class);
-        assertThat(poiRef.fold(), instanceOf(BytesRef.class));
-        assertThat(poiRef.fold().toString(), is(poi.fold().toString()));
+        assertThat(poiRef.value(), instanceOf(BytesRef.class));
+        assertThat(poiRef.value().toString(), is(poi.value().toString()));
         var alias7 = as(evalExec.fields().get(6), Alias.class);
         assertThat(alias7.name(), is("distance"));
         as(alias7.child(), ReferenceAttribute.class);
@@ -5926,15 +6393,15 @@ public void testPushCompoundTopNDistanceWithCompoundFilterAndNestedCompoundEvalT
         var alias1 = as(evalExec.fields().get(0), Alias.class);
         assertThat(alias1.name(), is("poi"));
         var poi = as(alias1.child(), Literal.class);
-        assertThat(poi.fold(), instanceOf(BytesRef.class));
+        assertThat(poi.value(), instanceOf(BytesRef.class));
         var alias2 = as(evalExec.fields().get(1), Alias.class);
         assertThat(alias2.name(), is("distance"));
         var stDistance = as(alias2.child(), StDistance.class);
         var location = as(stDistance.left(), FieldAttribute.class);
         assertThat(location.fieldName(), is("location"));
         var poiRef = as(stDistance.right(), Literal.class);
-        assertThat(poiRef.fold(), instanceOf(BytesRef.class));
-        assertThat(poiRef.fold().toString(), is(poi.fold().toString()));
+        assertThat(poiRef.value(), instanceOf(BytesRef.class));
+        assertThat(poiRef.value().toString(), is(poi.value().toString()));
         extract = as(evalExec.child(), FieldExtractExec.class);
         assertThat(names(extract.attributesToExtract()), contains("location"));
         var source = source(extract.child());
@@ -6466,7 +6933,7 @@ public void testManyEnrich() {
             var fragment = as(exchange.child(), FragmentExec.class);
             var partialTopN = as(fragment.fragment(), TopN.class);
             var enrich2 = as(partialTopN.child(), Enrich.class);
-            assertThat(BytesRefs.toString(enrich2.policyName().fold()), equalTo("departments"));
+            assertThat(BytesRefs.toString(enrich2.policyName().fold(FoldContext.small())), equalTo("departments"));
             assertThat(enrich2.mode(), equalTo(Enrich.Mode.ANY));
             var eval = as(enrich2.child(), Eval.class);
             as(eval.child(), EsRelation.class);
@@ -6492,7 +6959,7 @@ public void testManyEnrich() {
             var fragment = as(exchange.child(), FragmentExec.class);
             var partialTopN = as(fragment.fragment(), TopN.class);
             var enrich2 = as(partialTopN.child(), Enrich.class);
-            assertThat(BytesRefs.toString(enrich2.policyName().fold()), equalTo("departments"));
+            assertThat(BytesRefs.toString(enrich2.policyName().fold(FoldContext.small())), equalTo("departments"));
             assertThat(enrich2.mode(), equalTo(Enrich.Mode.ANY));
             var eval = as(enrich2.child(), Eval.class);
             as(eval.child(), EsRelation.class);
@@ -6833,6 +7300,299 @@ public void testLookupThenTopN() {
         );
     }
 
+    public void testLookupJoinFieldLoading() throws Exception {
+        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+
+        TestDataSource data = dataSetWithLookupIndices(Map.of("lookup_index", List.of("first_name", "foo", "bar", "baz")));
+
+        String query = """
+              FROM test
+            | LOOKUP JOIN lookup_index ON first_name
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("foo", "bar", "baz")));
+
+        query = """
+              FROM test
+            | LOOKUP JOIN lookup_index ON first_name
+            | KEEP b*
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("bar", "baz")));
+
+        query = """
+              FROM test
+            | LOOKUP JOIN lookup_index ON first_name
+            | DROP b*
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("foo")));
+
+        query = """
+              FROM test
+            | LOOKUP JOIN lookup_index ON first_name
+            | EVAL bar = 10
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("foo", "baz")));
+
+        query = """
+              FROM test
+            | LOOKUP JOIN lookup_index ON first_name
+            | RENAME bar AS foobar
+            | KEEP f*
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("foo", "bar")));
+
+        query = """
+              FROM test
+            | LOOKUP JOIN lookup_index ON first_name
+            | STATS count_distinct(foo) BY bar
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("foo", "bar")), true);
+
+        query = """
+              FROM test
+            | LOOKUP JOIN lookup_index ON first_name
+            | MV_EXPAND foo
+            | KEEP foo
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("foo")));
+
+        query = """
+              FROM test
+            | LOOKUP JOIN lookup_index ON first_name
+            | MV_EXPAND foo
+            | DROP foo
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("foo", "bar", "baz")));
+
+        query = """
+              FROM lookup_index
+            | LOOKUP JOIN lookup_index ON first_name
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("foo", "bar", "baz")));
+
+        query = """
+              FROM lookup_index
+            | LOOKUP JOIN lookup_index ON first_name
+            | KEEP foo
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("foo")));
+    }
+
+    public void testLookupJoinFieldLoadingTwoLookups() throws Exception {
+        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+
+        TestDataSource data = dataSetWithLookupIndices(
+            Map.of(
+                "lookup_index1",
+                List.of("first_name", "foo", "bar", "baz"),
+                "lookup_index2",
+                List.of("first_name", "foo", "bar2", "baz2")
+            )
+        );
+
+        String query = """
+              FROM test
+            | LOOKUP JOIN lookup_index1 ON first_name
+            | LOOKUP JOIN lookup_index2 ON first_name
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("bar", "baz"), Set.of("foo", "bar2", "baz2")));
+
+        query = """
+              FROM test
+            | LOOKUP JOIN lookup_index1 ON first_name
+            | LOOKUP JOIN lookup_index2 ON first_name
+            | DROP foo
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("bar", "baz"), Set.of("bar2", "baz2")));
+
+        query = """
+              FROM test
+            | LOOKUP JOIN lookup_index1 ON first_name
+            | LOOKUP JOIN lookup_index2 ON first_name
+            | KEEP b*
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("bar", "baz"), Set.of("bar2", "baz2")));
+
+        query = """
+              FROM test
+            | LOOKUP JOIN lookup_index1 ON first_name
+            | LOOKUP JOIN lookup_index2 ON first_name
+            | DROP baz*
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("bar"), Set.of("foo", "bar2")));
+
+        query = """
+              FROM test
+            | LOOKUP JOIN lookup_index1 ON first_name
+            | EVAL foo = to_upper(foo)
+            | LOOKUP JOIN lookup_index2 ON first_name
+            | EVAL foo = to_lower(foo)
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("bar", "baz"), Set.of("foo", "bar2", "baz2")));
+    }
+
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/119082")
+    public void testLookupJoinFieldLoadingTwoLookupsProjectInBetween() throws Exception {
+        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+
+        TestDataSource data = dataSetWithLookupIndices(
+            Map.of(
+                "lookup_index1",
+                List.of("first_name", "foo", "bar", "baz"),
+                "lookup_index2",
+                List.of("first_name", "foo", "bar2", "baz2")
+            )
+        );
+
+        String query = """
+              FROM test
+            | LOOKUP JOIN lookup_index1 ON first_name
+            | RENAME foo AS foo1
+            | LOOKUP JOIN lookup_index2 ON first_name
+            | DROP b*
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("foo"), Set.of("foo")));
+
+        query = """
+              FROM test
+            | LOOKUP JOIN lookup_index1 ON first_name
+            | DROP bar
+            | LOOKUP JOIN lookup_index2 ON first_name
+            | DROP b*
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("foo"), Set.of("foo")));
+
+        query = """
+              FROM test
+            | LOOKUP JOIN lookup_index1 ON first_name
+            | KEEP first_name, b*
+            | LOOKUP JOIN lookup_index2 ON first_name
+            | DROP bar*
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of("baz"), Set.of("foo", "baz2")));
+    }
+
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/118778")
+    public void testLookupJoinFieldLoadingDropAllFields() throws Exception {
+        assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+
+        TestDataSource data = dataSetWithLookupIndices(Map.of("lookup_index", List.of("first_name", "foo", "bar", "baz")));
+
+        String query = """
+              FROM test
+            | LOOKUP JOIN lookup_index ON first_name
+            | DROP foo, b*
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of()));
+
+        query = """
+              FROM test
+            | LOOKUP JOIN lookup_index ON first_name
+            | LOOKUP JOIN lookup_index ON first_name
+            """;
+        assertLookupJoinFieldNames(query, data, List.of(Set.of(), Set.of("foo", "bar", "baz")));
+    }
+
+    private void assertLookupJoinFieldNames(String query, TestDataSource data, List<Set<String>> expectedFieldNames) {
+        assertLookupJoinFieldNames(query, data, expectedFieldNames, false);
+    }
+
+    private void assertLookupJoinFieldNames(
+        String query,
+        TestDataSource data,
+        List<Set<String>> expectedFieldNames,
+        boolean useDataNodePlan
+    ) {
+        // Do not assert serialization:
+        // This will have a LookupJoinExec, which is not serializable because it doesn't leave the coordinator.
+        var plan = physicalPlan(query, data, false);
+
+        var physicalOperations = physicalOperationsFromPhysicalPlan(plan, useDataNodePlan);
+
+        List<Set<String>> fields = findFieldNamesInLookupJoinDescription(physicalOperations);
+
+        assertEquals(expectedFieldNames.size(), fields.size());
+        for (int i = 0; i < expectedFieldNames.size(); i++) {
+            assertEquals(expectedFieldNames.get(i), fields.get(i));
+        }
+    }
+
+    private TestDataSource dataSetWithLookupIndices(Map<String, Collection<String>> indexNameToFieldNames) {
+        Map<String, IndexResolution> lookupIndices = new HashMap<>();
+
+        for (Map.Entry<String, Collection<String>> entry : indexNameToFieldNames.entrySet()) {
+            String lookupIndexName = entry.getKey();
+            Map<String, EsField> lookup_fields = fields(entry.getValue());
+
+            EsIndex lookupIndex = new EsIndex(lookupIndexName, lookup_fields, Map.of(lookupIndexName, IndexMode.LOOKUP));
+            lookupIndices.put(lookupIndexName, IndexResolution.valid(lookupIndex));
+        }
+
+        return makeTestDataSource(
+            "test",
+            "mapping-basic.json",
+            new EsqlFunctionRegistry(),
+            lookupIndices,
+            setupEnrichResolution(),
+            TEST_SEARCH_STATS
+        );
+    }
+
+    private Map<String, EsField> fields(Collection<String> fieldNames) {
+        Map<String, EsField> fields = new HashMap<>();
+
+        for (String fieldName : fieldNames) {
+            fields.put(fieldName, new EsField(fieldName, DataType.KEYWORD, Map.of(), false));
+        }
+
+        return fields;
+    }
+
+    private LocalExecutionPlanner.LocalExecutionPlan physicalOperationsFromPhysicalPlan(PhysicalPlan plan, boolean useDataNodePlan) {
+        // The TopN needs an estimated row size for the planner to work
+        var plans = PlannerUtils.breakPlanBetweenCoordinatorAndDataNode(EstimatesRowSize.estimateRowSize(0, plan), config);
+        plan = useDataNodePlan ? plans.v2() : plans.v1();
+        plan = PlannerUtils.localPlan(List.of(), config, FoldContext.small(), plan);
+        LocalExecutionPlanner planner = new LocalExecutionPlanner(
+            "test",
+            "",
+            null,
+            BigArrays.NON_RECYCLING_INSTANCE,
+            TestBlockFactory.getNonBreakingInstance(),
+            Settings.EMPTY,
+            config,
+            new ExchangeSourceHandler(10, null, null),
+            new ExchangeSinkHandler(null, 10, () -> 10),
+            null,
+            null,
+            new EsPhysicalOperationProviders(FoldContext.small(), List.of(), null)
+        );
+
+        return planner.plan(FoldContext.small(), plan);
+    }
+
+    private List<Set<String>> findFieldNamesInLookupJoinDescription(LocalExecutionPlanner.LocalExecutionPlan physicalOperations) {
+
+        String[] descriptionLines = physicalOperations.describe().split("\\r?\\n|\\r");
+
+        // Capture the inside of "...load_fields=[field{f}#19, other_field{f}#20]".
+        String insidePattern = "[^\\]]*";
+        Pattern expected = Pattern.compile("\\\\_LookupOperator.*load_fields=\\[(" + insidePattern + ")].*");
+
+        List<Set<String>> results = new ArrayList<>();
+        for (String line : descriptionLines) {
+            var matcher = expected.matcher(line);
+            if (matcher.find()) {
+                String allFields = matcher.group(1);
+                Set<String> loadedFields = Arrays.stream(allFields.split(","))
+                    .map(name -> name.trim().split("\\{f}#")[0])
+                    .collect(Collectors.toSet());
+                results.add(loadedFields);
+            }
+        }
+
+        return results;
+    }
+
     public void testScore() {
         assumeTrue("'METADATA _score' is disabled", EsqlCapabilities.Cap.METADATA_SCORE.isEnabled());
         var plan = physicalPlan("""
@@ -6893,6 +7653,31 @@ public void testScoreTopN() {
         assertTrue(esRelation.output().stream().anyMatch(a -> a.name().equals(MetadataAttribute.SCORE) && a instanceof MetadataAttribute));
     }
 
+    public void testReductionPlanForTopN() {
+        int limit = between(1, 100);
+        var plan = physicalPlan(String.format(Locale.ROOT, """
+            FROM test
+            | sort emp_no
+            | LIMIT %d
+            """, limit));
+        Tuple<PhysicalPlan, PhysicalPlan> plans = PlannerUtils.breakPlanBetweenCoordinatorAndDataNode(plan, config);
+        PhysicalPlan reduction = PlannerUtils.reductionPlan(plans.v2());
+        TopNExec reductionTopN = as(reduction, TopNExec.class);
+        assertThat(reductionTopN.estimatedRowSize(), equalTo(allFieldRowSize));
+        assertThat(reductionTopN.limit().fold(FoldContext.small()), equalTo(limit));
+    }
+
+    public void testReductionPlanForAggs() {
+        var plan = physicalPlan("""
+            FROM test
+            | stats x = sum(salary) BY first_name
+            """);
+        Tuple<PhysicalPlan, PhysicalPlan> plans = PlannerUtils.breakPlanBetweenCoordinatorAndDataNode(plan, config);
+        PhysicalPlan reduction = PlannerUtils.reductionPlan(plans.v2());
+        AggregateExec reductionAggs = as(reduction, AggregateExec.class);
+        assertThat(reductionAggs.estimatedRowSize(), equalTo(58)); // double and keyword
+    }
+
     @SuppressWarnings("SameParameterValue")
     private static void assertFilterCondition(
         Filter filter,
@@ -6907,17 +7692,35 @@ private static void assertFilterCondition(
         assertThat("Expected filter value", value.value(), equalTo(expected));
     }
 
-    private EsQueryExec assertChildIsGeoPointExtract(UnaryExec parent, boolean useDocValues) {
-        return assertChildIsExtractedAsDocValues(parent, useDocValues, GEO_POINT);
+    private EsQueryExec assertChildIsGeoPointExtract(UnaryExec parent, FieldExtractPreference fieldExtractPreference) {
+        return assertChildIsExtractedAs(parent, fieldExtractPreference, GEO_POINT);
     }
 
-    private EsQueryExec assertChildIsExtractedAsDocValues(UnaryExec parent, boolean useDocValues, DataType dataType) {
+    private static EsQueryExec assertChildIsExtractedAs(
+        UnaryExec parent,
+        FieldExtractPreference fieldExtractPreference,
+        DataType dataType
+    ) {
         var extract = as(parent.child(), FieldExtractExec.class);
+        switch (fieldExtractPreference) {
+            case NONE -> {
+                assertThat(extract.docValuesAttributes(), is(empty()));
+                assertThat(extract.boundsAttributes(), is(empty()));
+            }
+            case DOC_VALUES -> {
+                assertThat(extract.docValuesAttributes(), is(not(empty())));
+                assertThat(extract.boundsAttributes(), is(empty()));
+            }
+            case EXTRACT_SPATIAL_BOUNDS -> {
+                assertThat(extract.docValuesAttributes(), is(empty()));
+                assertThat(extract.boundsAttributes(), is(not(empty())));
+            }
+        }
         assertTrue(
-            "Expect field attribute to be extracted as " + (useDocValues ? "doc-values" : "source"),
+            "Expect field attribute to be extracted as " + fieldExtractPreference,
             extract.attributesToExtract()
                 .stream()
-                .allMatch(attr -> extract.hasDocValuesAttribute(attr) == useDocValues && attr.dataType() == dataType)
+                .allMatch(attr -> extract.fieldExtractPreference(attr) == fieldExtractPreference && attr.dataType() == dataType)
         );
         return source(extract.child());
     }
@@ -6927,17 +7730,24 @@ private static void assertAggregation(
         String aliasName,
         Class<? extends AggregateFunction> aggClass,
         DataType fieldType,
-        boolean useDocValues
+        FieldExtractPreference fieldExtractPreference
+    ) {
+        assertAggregation(plan, aliasName, "Aggregation with fieldExtractPreference", aggClass, fieldType, fieldExtractPreference);
+    }
+
+    private static void assertAggregation(
+        PhysicalPlan plan,
+        String aliasName,
+        String reason,
+        Class<? extends AggregateFunction> aggClass,
+        DataType fieldType,
+        FieldExtractPreference fieldExtractPreference
     ) {
         var aggFunc = assertAggregation(plan, aliasName, aggClass);
         var aggField = as(aggFunc.field(), Attribute.class);
         var spatialAgg = as(aggFunc, SpatialAggregateFunction.class);
-        assertThat(
-            "Expected spatial aggregation to use doc-values",
-            spatialAgg.fieldExtractPreference(),
-            equalTo(useDocValues ? FieldExtractPreference.DOC_VALUES : FieldExtractPreference.NONE)
-        );
-        assertThat("", aggField.dataType(), equalTo(fieldType));
+        assertThat(spatialAgg.fieldExtractPreference(), equalTo(fieldExtractPreference));
+        assertThat(reason, aggField.dataType(), equalTo(fieldType));
     }
 
     private static AggregateFunction assertAggregation(PhysicalPlan plan, String aliasName, Class<? extends AggregateFunction> aggClass) {
@@ -6978,13 +7788,14 @@ private static QueryBuilder findQueryBuilder(BoolQueryBuilder booleanQuery, Stri
     }
 
     private void assertFieldExtractionWithDocValues(FieldExtractExec extract, DataType dataType, String... fieldNames) {
+        var docValuesAttributes = extract.docValuesAttributes();
         extract.attributesToExtract().forEach(attr -> {
             String name = attr.name();
             if (asList(fieldNames).contains(name)) {
-                assertThat("Expected field '" + name + "' to use doc-values", extract.hasDocValuesAttribute(attr), equalTo(true));
+                assertThat("Expected field '" + name + "' to use doc-values", docValuesAttributes.contains(attr), equalTo(true));
                 assertThat("Expected field '" + name + "' to have data type " + dataType, attr.dataType(), equalTo(dataType));
             } else {
-                assertThat("Expected field '" + name + "' to NOT use doc-values", extract.hasDocValuesAttribute(attr), equalTo(false));
+                assertThat("Expected field '" + name + "' to NOT use doc-values", docValuesAttributes.contains(attr), equalTo(false));
             }
         });
     }
@@ -7009,7 +7820,7 @@ private PhysicalPlan optimizedPlan(PhysicalPlan plan, SearchStats searchStats) {
         // individually hence why here the plan is kept as is
 
         var l = p.transformUp(FragmentExec.class, fragment -> {
-            var localPlan = PlannerUtils.localPlan(config, fragment, searchStats);
+            var localPlan = PlannerUtils.localPlan(config, FoldContext.small(), fragment, searchStats);
             return EstimatesRowSize.estimateRowSize(fragment.estimatedRowSize(), localPlan);
         });
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java
index 9fe479dbb8625..e6a7d110f8c09 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.analysis.Analyzer;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 import org.elasticsearch.xpack.esql.plan.physical.EstimatesRowSize;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
@@ -26,7 +27,7 @@ public class TestPlannerOptimizer {
     private final Configuration config;
 
     public TestPlannerOptimizer(Configuration config, Analyzer analyzer) {
-        this(config, analyzer, new LogicalPlanOptimizer(new LogicalOptimizerContext(config)));
+        this(config, analyzer, new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small())));
     }
 
     public TestPlannerOptimizer(Configuration config, Analyzer analyzer, LogicalPlanOptimizer logicalOptimizer) {
@@ -61,8 +62,13 @@ private PhysicalPlan optimizedPlan(PhysicalPlan plan, SearchStats searchStats) {
         // this is of no use in the unit tests, which checks the plan as a whole instead of each
         // individually hence why here the plan is kept as is
 
-        var logicalTestOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(config, searchStats));
-        var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer(new LocalPhysicalOptimizerContext(config, searchStats), true);
+        var logicalTestOptimizer = new LocalLogicalPlanOptimizer(
+            new LocalLogicalOptimizerContext(config, FoldContext.small(), searchStats)
+        );
+        var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer(
+            new LocalPhysicalOptimizerContext(config, FoldContext.small(), searchStats),
+            true
+        );
         var l = PlannerUtils.localPlan(physicalPlan, logicalTestOptimizer, physicalTestOptimizer);
 
         // handle local reduction alignment
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/LogicalOptimizerContextTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/LogicalOptimizerContextTests.java
new file mode 100644
index 0000000000000..5d2fec0fc8181
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/LogicalOptimizerContextTests.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.optimizer.rules;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.xpack.esql.ConfigurationTestUtils;
+import org.elasticsearch.xpack.esql.EsqlTestUtils;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
+import org.elasticsearch.xpack.esql.session.Configuration;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class LogicalOptimizerContextTests extends ESTestCase {
+    public void testToString() {
+        // Random looking numbers for FoldContext are indeed random. Just so we have consistent numbers to assert on in toString.
+        LogicalOptimizerContext ctx = new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG, new FoldContext(102));
+        ctx.foldCtx().trackAllocation(Source.EMPTY, 99);
+        assertThat(
+            ctx.toString(),
+            equalTo("LogicalOptimizerContext[configuration=" + EsqlTestUtils.TEST_CFG + ", foldCtx=FoldContext[3/102]]")
+        );
+    }
+
+    public void testEqualsAndHashCode() {
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(randomLogicalOptimizerContext(), this::copy, this::mutate);
+    }
+
+    private LogicalOptimizerContext randomLogicalOptimizerContext() {
+        return new LogicalOptimizerContext(ConfigurationTestUtils.randomConfiguration(), randomFoldContext());
+    }
+
+    private LogicalOptimizerContext copy(LogicalOptimizerContext c) {
+        return new LogicalOptimizerContext(c.configuration(), c.foldCtx());
+    }
+
+    private LogicalOptimizerContext mutate(LogicalOptimizerContext c) {
+        Configuration configuration = c.configuration();
+        FoldContext foldCtx = c.foldCtx();
+        if (randomBoolean()) {
+            configuration = randomValueOtherThan(configuration, ConfigurationTestUtils::randomConfiguration);
+        } else {
+            foldCtx = randomValueOtherThan(foldCtx, this::randomFoldContext);
+        }
+        return new LogicalOptimizerContext(configuration, foldCtx);
+    }
+
+    private FoldContext randomFoldContext() {
+        FoldContext ctx = new FoldContext(randomNonNegativeLong());
+        if (randomBoolean()) {
+            ctx.trackAllocation(Source.EMPTY, randomLongBetween(0, ctx.initialAllowedBytes()));
+        }
+        return ctx;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsEliminationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsEliminationTests.java
index 08c8612d8097c..c0c145aee5382 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsEliminationTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsEliminationTests.java
@@ -22,25 +22,26 @@
 import static java.util.Arrays.asList;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.notEqualsOf;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.NULL;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
 
 public class BooleanFunctionEqualsEliminationTests extends ESTestCase {
+    private Expression booleanFunctionEqualElimination(BinaryComparison e) {
+        return new BooleanFunctionEqualsElimination().rule(e, unboundLogicalOptimizerContext());
+    }
 
     public void testBoolEqualsSimplificationOnExpressions() {
-        BooleanFunctionEqualsElimination s = new BooleanFunctionEqualsElimination();
         Expression exp = new GreaterThan(EMPTY, getFieldAttribute(), new Literal(EMPTY, 0, DataType.INTEGER), null);
 
-        assertEquals(exp, s.rule(new Equals(EMPTY, exp, TRUE)));
+        assertEquals(exp, booleanFunctionEqualElimination(new Equals(EMPTY, exp, TRUE)));
         // TODO: Replace use of QL Not with ESQL Not
-        assertEquals(new Not(EMPTY, exp), s.rule(new Equals(EMPTY, exp, FALSE)));
+        assertEquals(new Not(EMPTY, exp), booleanFunctionEqualElimination(new Equals(EMPTY, exp, FALSE)));
     }
 
     public void testBoolEqualsSimplificationOnFields() {
-        BooleanFunctionEqualsElimination s = new BooleanFunctionEqualsElimination();
-
         FieldAttribute field = getFieldAttribute();
 
         List<? extends BinaryComparison> comparisons = asList(
@@ -55,7 +56,7 @@ public void testBoolEqualsSimplificationOnFields() {
         );
 
         for (BinaryComparison comparison : comparisons) {
-            assertEquals(comparison, s.rule(comparison));
+            assertEquals(comparison, booleanFunctionEqualElimination(comparison));
         }
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplificationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplificationTests.java
index 3b1f8cfc83af3..5b4bf806518de 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplificationTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplificationTests.java
@@ -9,9 +9,11 @@
 
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
 
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
@@ -20,33 +22,31 @@ public class BooleanSimplificationTests extends ESTestCase {
     private static final Expression DUMMY_EXPRESSION =
         new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 0);
 
-    public void testBoolSimplifyOr() {
-        BooleanSimplification simplification = new BooleanSimplification();
+    private Expression booleanSimplification(ScalarFunction e) {
+        return new BooleanSimplification().rule(e, unboundLogicalOptimizerContext());
+    }
 
-        assertEquals(TRUE, simplification.rule(new Or(EMPTY, TRUE, TRUE)));
-        assertEquals(TRUE, simplification.rule(new Or(EMPTY, TRUE, DUMMY_EXPRESSION)));
-        assertEquals(TRUE, simplification.rule(new Or(EMPTY, DUMMY_EXPRESSION, TRUE)));
+    public void testBoolSimplifyOr() {
+        assertEquals(TRUE, booleanSimplification(new Or(EMPTY, TRUE, TRUE)));
+        assertEquals(TRUE, booleanSimplification(new Or(EMPTY, TRUE, DUMMY_EXPRESSION)));
+        assertEquals(TRUE, booleanSimplification(new Or(EMPTY, DUMMY_EXPRESSION, TRUE)));
 
-        assertEquals(FALSE, simplification.rule(new Or(EMPTY, FALSE, FALSE)));
-        assertEquals(DUMMY_EXPRESSION, simplification.rule(new Or(EMPTY, FALSE, DUMMY_EXPRESSION)));
-        assertEquals(DUMMY_EXPRESSION, simplification.rule(new Or(EMPTY, DUMMY_EXPRESSION, FALSE)));
+        assertEquals(FALSE, booleanSimplification(new Or(EMPTY, FALSE, FALSE)));
+        assertEquals(DUMMY_EXPRESSION, booleanSimplification(new Or(EMPTY, FALSE, DUMMY_EXPRESSION)));
+        assertEquals(DUMMY_EXPRESSION, booleanSimplification(new Or(EMPTY, DUMMY_EXPRESSION, FALSE)));
     }
 
     public void testBoolSimplifyAnd() {
-        BooleanSimplification simplification = new BooleanSimplification();
+        assertEquals(TRUE, booleanSimplification(new And(EMPTY, TRUE, TRUE)));
+        assertEquals(DUMMY_EXPRESSION, booleanSimplification(new And(EMPTY, TRUE, DUMMY_EXPRESSION)));
+        assertEquals(DUMMY_EXPRESSION, booleanSimplification(new And(EMPTY, DUMMY_EXPRESSION, TRUE)));
 
-        assertEquals(TRUE, simplification.rule(new And(EMPTY, TRUE, TRUE)));
-        assertEquals(DUMMY_EXPRESSION, simplification.rule(new And(EMPTY, TRUE, DUMMY_EXPRESSION)));
-        assertEquals(DUMMY_EXPRESSION, simplification.rule(new And(EMPTY, DUMMY_EXPRESSION, TRUE)));
-
-        assertEquals(FALSE, simplification.rule(new And(EMPTY, FALSE, FALSE)));
-        assertEquals(FALSE, simplification.rule(new And(EMPTY, FALSE, DUMMY_EXPRESSION)));
-        assertEquals(FALSE, simplification.rule(new And(EMPTY, DUMMY_EXPRESSION, FALSE)));
+        assertEquals(FALSE, booleanSimplification(new And(EMPTY, FALSE, FALSE)));
+        assertEquals(FALSE, booleanSimplification(new And(EMPTY, FALSE, DUMMY_EXPRESSION)));
+        assertEquals(FALSE, booleanSimplification(new And(EMPTY, DUMMY_EXPRESSION, FALSE)));
     }
 
     public void testBoolCommonFactorExtraction() {
-        BooleanSimplification simplification = new BooleanSimplification();
-
         Expression a1 = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 1);
         Expression a2 = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 1);
         Expression b = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 2);
@@ -55,7 +55,7 @@ public void testBoolCommonFactorExtraction() {
         Or actual = new Or(EMPTY, new And(EMPTY, a1, b), new And(EMPTY, a2, c));
         And expected = new And(EMPTY, a1, new Or(EMPTY, b, c));
 
-        assertEquals(expected, simplification.rule(actual));
+        assertEquals(expected, booleanSimplification(actual));
     }
 
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisonsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisonsTests.java
index d388369e0b167..a0d23731ae82d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisonsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisonsTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual;
@@ -37,6 +38,7 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOf;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOrEqualOf;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.notEqualsOf;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
@@ -45,19 +47,17 @@
 import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
 
 public class CombineBinaryComparisonsTests extends ESTestCase {
-
-    private static final Expression DUMMY_EXPRESSION =
-        new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 0);
+    private Expression combine(BinaryLogic e) {
+        return new CombineBinaryComparisons().rule(e, unboundLogicalOptimizerContext());
+    }
 
     public void testCombineBinaryComparisonsNotComparable() {
         FieldAttribute fa = getFieldAttribute();
         LessThanOrEqual lte = lessThanOrEqualOf(fa, SIX);
         LessThan lt = lessThanOf(fa, FALSE);
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-
         And and = new And(EMPTY, lte, lt);
-        Expression exp = rule.rule(and);
+        Expression exp = combine(and);
         assertEquals(exp, and);
     }
 
@@ -67,9 +67,7 @@ public void testCombineBinaryComparisonsUpper() {
         LessThanOrEqual lte = lessThanOrEqualOf(fa, SIX);
         LessThan lt = lessThanOf(fa, FIVE);
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-
-        Expression exp = rule.rule(new And(EMPTY, lte, lt));
+        Expression exp = combine(new And(EMPTY, lte, lt));
         assertEquals(LessThan.class, exp.getClass());
         LessThan r = (LessThan) exp;
         assertEquals(FIVE, r.right());
@@ -81,9 +79,7 @@ public void testCombineBinaryComparisonsLower() {
         GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, SIX);
         GreaterThan gt = greaterThanOf(fa, FIVE);
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-
-        Expression exp = rule.rule(new And(EMPTY, gte, gt));
+        Expression exp = combine(new And(EMPTY, gte, gt));
         assertEquals(GreaterThanOrEqual.class, exp.getClass());
         GreaterThanOrEqual r = (GreaterThanOrEqual) exp;
         assertEquals(SIX, r.right());
@@ -95,9 +91,7 @@ public void testCombineBinaryComparisonsInclude() {
         GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, FIVE);
         GreaterThan gt = greaterThanOf(fa, FIVE);
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-
-        Expression exp = rule.rule(new And(EMPTY, gte, gt));
+        Expression exp = combine(new And(EMPTY, gte, gt));
         assertEquals(GreaterThan.class, exp.getClass());
         GreaterThan r = (GreaterThan) exp;
         assertEquals(FIVE, r.right());
@@ -111,9 +105,7 @@ public void testCombineMultipleBinaryComparisons() {
         LessThanOrEqual lte = lessThanOrEqualOf(fa, L(7));
         LessThan lt = lessThanOf(fa, SIX);
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-
-        Expression exp = rule.rule(new And(EMPTY, gte, new And(EMPTY, gt, new And(EMPTY, lt, lte))));
+        Expression exp = combine(new And(EMPTY, gte, new And(EMPTY, gt, new And(EMPTY, lt, lte))));
         assertEquals(And.class, exp.getClass());
         And and = (And) exp;
         assertEquals(gt, and.left());
@@ -128,10 +120,8 @@ public void testCombineMixedMultipleBinaryComparisons() {
         LessThanOrEqual lte = lessThanOrEqualOf(fa, L(7));
         Expression ne = notEqualsOf(fa, FIVE);
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-
         // TRUE AND a != 5 AND 4 < a <= 7
-        Expression exp = rule.rule(new And(EMPTY, gte, new And(EMPTY, TRUE, new And(EMPTY, gt, new And(EMPTY, ne, lte)))));
+        Expression exp = combine(new And(EMPTY, gte, new And(EMPTY, TRUE, new And(EMPTY, gt, new And(EMPTY, ne, lte)))));
         assertEquals(And.class, exp.getClass());
         And and = ((And) exp);
         assertEquals(And.class, and.right().getClass());
@@ -150,8 +140,7 @@ public void testCombineComparisonsIntoRange() {
         GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, ONE);
         LessThan lt = lessThanOf(fa, FIVE);
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(new And(EMPTY, gte, lt));
+        Expression exp = combine(new And(EMPTY, gte, lt));
         assertEquals(And.class, exp.getClass());
 
         And and = (And) exp;
@@ -167,8 +156,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndGt3() {
         GreaterThan gt = greaterThanOf(fa, THREE);
         And and = new And(EMPTY, neq, gt);
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
+        Expression exp = combine(and);
         assertEquals(gt, exp);
     }
 
@@ -180,8 +168,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndGte2() {
         GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, TWO);
         And and = new And(EMPTY, neq, gte);
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
+        Expression exp = combine(and);
         assertEquals(GreaterThan.class, exp.getClass());
         GreaterThan gt = (GreaterThan) exp;
         assertEquals(TWO, gt.right());
@@ -195,8 +182,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndGte1() {
         GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, ONE);
         And and = new And(EMPTY, neq, gte);
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
+        Expression exp = combine(and);
         assertEquals(And.class, exp.getClass()); // can't optimize
     }
 
@@ -208,8 +194,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndLte3() {
         LessThanOrEqual lte = lessThanOrEqualOf(fa, THREE);
         And and = new And(EMPTY, neq, lte);
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
+        Expression exp = combine(and);
         assertEquals(and, exp); // can't optimize
     }
 
@@ -221,8 +206,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndLte2() {
         LessThanOrEqual lte = lessThanOrEqualOf(fa, TWO);
         And and = new And(EMPTY, neq, lte);
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
+        Expression exp = combine(and);
         assertEquals(LessThan.class, exp.getClass());
         LessThan lt = (LessThan) exp;
         assertEquals(TWO, lt.right());
@@ -236,8 +220,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndLte1() {
         LessThanOrEqual lte = lessThanOrEqualOf(fa, ONE);
         And and = new And(EMPTY, neq, lte);
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
+        Expression exp = combine(and);
         assertEquals(lte, exp);
     }
 
@@ -251,8 +234,7 @@ public void testCombineBinaryComparisonsDisjunctionNotComparable() {
 
         Or or = new Or(EMPTY, gt1, gt2);
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
+        Expression exp = combine(or);
         assertEquals(exp, or);
     }
 
@@ -266,8 +248,7 @@ public void testCombineBinaryComparisonsDisjunctionLowerBound() {
 
         Or or = new Or(EMPTY, gt1, new Or(EMPTY, gt2, gt3));
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
+        Expression exp = combine(or);
         assertEquals(GreaterThan.class, exp.getClass());
 
         GreaterThan gt = (GreaterThan) exp;
@@ -284,8 +265,7 @@ public void testCombineBinaryComparisonsDisjunctionIncludeLowerBounds() {
 
         Or or = new Or(EMPTY, new Or(EMPTY, gt1, gt2), gte3);
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
+        Expression exp = combine(or);
         assertEquals(GreaterThan.class, exp.getClass());
 
         GreaterThan gt = (GreaterThan) exp;
@@ -302,8 +282,7 @@ public void testCombineBinaryComparisonsDisjunctionUpperBound() {
 
         Or or = new Or(EMPTY, new Or(EMPTY, lt1, lt2), lt3);
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
+        Expression exp = combine(or);
         assertEquals(LessThan.class, exp.getClass());
 
         LessThan lt = (LessThan) exp;
@@ -320,8 +299,7 @@ public void testCombineBinaryComparisonsDisjunctionIncludeUpperBounds() {
 
         Or or = new Or(EMPTY, lt2, new Or(EMPTY, lte2, lt1));
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
+        Expression exp = combine(or);
         assertEquals(LessThanOrEqual.class, exp.getClass());
 
         LessThanOrEqual lte = (LessThanOrEqual) exp;
@@ -340,8 +318,7 @@ public void testCombineBinaryComparisonsDisjunctionOfLowerAndUpperBounds() {
 
         Or or = new Or(EMPTY, new Or(EMPTY, lt2, gt3), new Or(EMPTY, lt1, gt4));
 
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
+        Expression exp = combine(or);
         assertEquals(Or.class, exp.getClass());
 
         Or ro = (Or) exp;
@@ -367,7 +344,7 @@ public void testBooleanSimplificationCommonExpressionSubstraction() {
         And right = new And(EMPTY, a2, common);
         Or or = new Or(EMPTY, left, right);
 
-        Expression exp = new BooleanSimplification().rule(or);
+        Expression exp = new BooleanSimplification().rule(or, unboundLogicalOptimizerContext());
         assertEquals(new And(EMPTY, common, new Or(EMPTY, a1, a2)), exp);
     }
 
@@ -391,8 +368,7 @@ public void testBinaryComparisonAndOutOfRangeNotEqualsDifferentFields() {
         );
 
         for (And and : testCases) {
-            CombineBinaryComparisons rule = new CombineBinaryComparisons();
-            Expression exp = rule.rule(and);
+            Expression exp = combine(and);
             assertEquals("Rule should not have transformed [" + and.nodeString() + "]", and, exp);
         }
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctionsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctionsTests.java
index 043d18dac9fd4..bb5f2fd3505e9 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctionsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctionsTests.java
@@ -38,16 +38,24 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOf;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.relation;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
 import static org.hamcrest.Matchers.contains;
 
 public class CombineDisjunctionsTests extends ESTestCase {
+    private Expression combineDisjunctions(Or e) {
+        return new CombineDisjunctions().rule(e, unboundLogicalOptimizerContext());
+    }
+
+    private LogicalPlan combineDisjunctions(LogicalPlan l) {
+        return new CombineDisjunctions().apply(l, unboundLogicalOptimizerContext());
+    }
 
     public void testTwoEqualsWithOr() {
         FieldAttribute fa = getFieldAttribute();
 
         Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO));
-        Expression e = new CombineDisjunctions().rule(or);
+        Expression e = combineDisjunctions(or);
         assertEquals(In.class, e.getClass());
         In in = (In) e;
         assertEquals(fa, in.value());
@@ -58,7 +66,7 @@ public void testTwoEqualsWithSameValue() {
         FieldAttribute fa = getFieldAttribute();
 
         Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, ONE));
-        Expression e = new CombineDisjunctions().rule(or);
+        Expression e = combineDisjunctions(or);
         assertEquals(Equals.class, e.getClass());
         Equals eq = (Equals) e;
         assertEquals(fa, eq.left());
@@ -69,7 +77,7 @@ public void testOneEqualsOneIn() {
         FieldAttribute fa = getFieldAttribute();
 
         Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, List.of(TWO)));
-        Expression e = new CombineDisjunctions().rule(or);
+        Expression e = combineDisjunctions(or);
         assertEquals(In.class, e.getClass());
         In in = (In) e;
         assertEquals(fa, in.value());
@@ -80,7 +88,7 @@ public void testOneEqualsOneInWithSameValue() {
         FieldAttribute fa = getFieldAttribute();
 
         Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, asList(ONE, TWO)));
-        Expression e = new CombineDisjunctions().rule(or);
+        Expression e = combineDisjunctions(or);
         assertEquals(In.class, e.getClass());
         In in = (In) e;
         assertEquals(fa, in.value());
@@ -92,7 +100,7 @@ public void testSingleValueInToEquals() {
 
         Equals equals = equalsOf(fa, ONE);
         Or or = new Or(EMPTY, equals, new In(EMPTY, fa, List.of(ONE)));
-        Expression e = new CombineDisjunctions().rule(or);
+        Expression e = combineDisjunctions(or);
         assertEquals(equals, e);
     }
 
@@ -101,7 +109,7 @@ public void testEqualsBehindAnd() {
 
         And and = new And(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO));
         Filter dummy = new Filter(EMPTY, relation(), and);
-        LogicalPlan transformed = new CombineDisjunctions().apply(dummy);
+        LogicalPlan transformed = combineDisjunctions(dummy);
         assertSame(dummy, transformed);
         assertEquals(and, ((Filter) transformed).condition());
     }
@@ -111,7 +119,7 @@ public void testTwoEqualsDifferentFields() {
         FieldAttribute fieldTwo = getFieldAttribute("TWO");
 
         Or or = new Or(EMPTY, equalsOf(fieldOne, ONE), equalsOf(fieldTwo, TWO));
-        Expression e = new CombineDisjunctions().rule(or);
+        Expression e = combineDisjunctions(or);
         assertEquals(or, e);
     }
 
@@ -120,7 +128,7 @@ public void testMultipleIn() {
 
         Or firstOr = new Or(EMPTY, new In(EMPTY, fa, List.of(ONE)), new In(EMPTY, fa, List.of(TWO)));
         Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, List.of(THREE)));
-        Expression e = new CombineDisjunctions().rule(secondOr);
+        Expression e = combineDisjunctions(secondOr);
         assertEquals(In.class, e.getClass());
         In in = (In) e;
         assertEquals(fa, in.value());
@@ -132,7 +140,7 @@ public void testOrWithNonCombinableExpressions() {
 
         Or firstOr = new Or(EMPTY, new In(EMPTY, fa, List.of(ONE)), lessThanOf(fa, TWO));
         Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, List.of(THREE)));
-        Expression e = new CombineDisjunctions().rule(secondOr);
+        Expression e = combineDisjunctions(secondOr);
         assertEquals(Or.class, e.getClass());
         Or or = (Or) e;
         assertEquals(or.left(), firstOr.right());
@@ -160,7 +168,7 @@ public void testCombineCIDRMatch() {
         cidrs.add(new CIDRMatch(EMPTY, faa, ipa2));
         cidrs.add(new CIDRMatch(EMPTY, fab, ipb2));
         Or oldOr = (Or) Predicates.combineOr(cidrs);
-        Expression e = new CombineDisjunctions().rule(oldOr);
+        Expression e = combineDisjunctions(oldOr);
         assertEquals(Or.class, e.getClass());
         Or newOr = (Or) e;
         assertEquals(CIDRMatch.class, newOr.left().getClass());
@@ -211,7 +219,7 @@ public void testCombineCIDRMatchEqualsIns() {
 
         Or oldOr = (Or) Predicates.combineOr(all);
 
-        Expression e = new CombineDisjunctions().rule(oldOr);
+        Expression e = combineDisjunctions(oldOr);
         assertEquals(Or.class, e.getClass());
         Or newOr = (Or) e;
         assertEquals(Or.class, newOr.left().getClass());
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java
index c4f4dac67acd3..8a8585b8d0ab5 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java
@@ -11,9 +11,11 @@
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
+import org.elasticsearch.xpack.esql.core.expression.predicate.Range;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
@@ -27,11 +29,16 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mod;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Sub;
+import org.elasticsearch.xpack.esql.plan.logical.Filter;
+import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.FIVE;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.THREE;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.TWO;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.as;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptySource;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.equalsOf;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.fieldAttribute;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.greaterThanOf;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.greaterThanOrEqualOf;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOf;
@@ -39,68 +46,77 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.notEqualsOf;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.of;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.rangeOf;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.NULL;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
 
 public class ConstantFoldingTests extends ESTestCase {
+    private Expression constantFolding(Expression e) {
+        return new ConstantFolding().rule(e, unboundLogicalOptimizerContext());
+    }
 
     public void testConstantFolding() {
         Expression exp = new Add(EMPTY, TWO, THREE);
 
         assertTrue(exp.foldable());
-        Expression result = new ConstantFolding().rule(exp);
-        assertTrue(result instanceof Literal);
-        assertEquals(5, ((Literal) result).value());
+        Expression result = constantFolding(exp);
+        assertEquals(5, as(result, Literal.class).value());
 
         // check now with an alias
-        result = new ConstantFolding().rule(new Alias(EMPTY, "a", exp));
+        result = constantFolding(new Alias(EMPTY, "a", exp));
         assertEquals("a", Expressions.name(result));
         assertEquals(Alias.class, result.getClass());
     }
 
     public void testConstantFoldingBinaryComparison() {
-        assertEquals(FALSE, new ConstantFolding().rule(greaterThanOf(TWO, THREE)).canonical());
-        assertEquals(FALSE, new ConstantFolding().rule(greaterThanOrEqualOf(TWO, THREE)).canonical());
-        assertEquals(FALSE, new ConstantFolding().rule(equalsOf(TWO, THREE)).canonical());
-        assertEquals(TRUE, new ConstantFolding().rule(notEqualsOf(TWO, THREE)).canonical());
-        assertEquals(TRUE, new ConstantFolding().rule(lessThanOrEqualOf(TWO, THREE)).canonical());
-        assertEquals(TRUE, new ConstantFolding().rule(lessThanOf(TWO, THREE)).canonical());
+        assertEquals(FALSE, constantFolding(greaterThanOf(TWO, THREE)).canonical());
+        assertEquals(FALSE, constantFolding(greaterThanOrEqualOf(TWO, THREE)).canonical());
+        assertEquals(FALSE, constantFolding(equalsOf(TWO, THREE)).canonical());
+        assertEquals(TRUE, constantFolding(notEqualsOf(TWO, THREE)).canonical());
+        assertEquals(TRUE, constantFolding(lessThanOrEqualOf(TWO, THREE)).canonical());
+        assertEquals(TRUE, constantFolding(lessThanOf(TWO, THREE)).canonical());
     }
 
     public void testConstantFoldingBinaryLogic() {
-        assertEquals(FALSE, new ConstantFolding().rule(new And(EMPTY, greaterThanOf(TWO, THREE), TRUE)).canonical());
-        assertEquals(TRUE, new ConstantFolding().rule(new Or(EMPTY, greaterThanOrEqualOf(TWO, THREE), TRUE)).canonical());
+        assertEquals(FALSE, constantFolding(new And(EMPTY, greaterThanOf(TWO, THREE), TRUE)).canonical());
+        assertEquals(TRUE, constantFolding(new Or(EMPTY, greaterThanOrEqualOf(TWO, THREE), TRUE)).canonical());
     }
 
     public void testConstantFoldingBinaryLogic_WithNullHandling() {
-        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new And(EMPTY, NULL, TRUE)).canonical().nullable());
-        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new And(EMPTY, TRUE, NULL)).canonical().nullable());
-        assertEquals(FALSE, new ConstantFolding().rule(new And(EMPTY, NULL, FALSE)).canonical());
-        assertEquals(FALSE, new ConstantFolding().rule(new And(EMPTY, FALSE, NULL)).canonical());
-        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new And(EMPTY, NULL, NULL)).canonical().nullable());
-
-        assertEquals(TRUE, new ConstantFolding().rule(new Or(EMPTY, NULL, TRUE)).canonical());
-        assertEquals(TRUE, new ConstantFolding().rule(new Or(EMPTY, TRUE, NULL)).canonical());
-        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new Or(EMPTY, NULL, FALSE)).canonical().nullable());
-        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new Or(EMPTY, FALSE, NULL)).canonical().nullable());
-        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new Or(EMPTY, NULL, NULL)).canonical().nullable());
+        assertEquals(Nullability.TRUE, constantFolding(new And(EMPTY, NULL, TRUE)).canonical().nullable());
+        assertEquals(Nullability.TRUE, constantFolding(new And(EMPTY, TRUE, NULL)).canonical().nullable());
+        assertEquals(FALSE, constantFolding(new And(EMPTY, NULL, FALSE)).canonical());
+        assertEquals(FALSE, constantFolding(new And(EMPTY, FALSE, NULL)).canonical());
+        assertEquals(Nullability.TRUE, constantFolding(new And(EMPTY, NULL, NULL)).canonical().nullable());
+
+        assertEquals(TRUE, constantFolding(new Or(EMPTY, NULL, TRUE)).canonical());
+        assertEquals(TRUE, constantFolding(new Or(EMPTY, TRUE, NULL)).canonical());
+        assertEquals(Nullability.TRUE, constantFolding(new Or(EMPTY, NULL, FALSE)).canonical().nullable());
+        assertEquals(Nullability.TRUE, constantFolding(new Or(EMPTY, FALSE, NULL)).canonical().nullable());
+        assertEquals(Nullability.TRUE, constantFolding(new Or(EMPTY, NULL, NULL)).canonical().nullable());
     }
 
     public void testConstantFoldingRange() {
-        assertEquals(true, new ConstantFolding().rule(rangeOf(FIVE, FIVE, true, new Literal(EMPTY, 10, DataType.INTEGER), false)).fold());
-        assertEquals(false, new ConstantFolding().rule(rangeOf(FIVE, FIVE, false, new Literal(EMPTY, 10, DataType.INTEGER), false)).fold());
+        assertEquals(
+            true,
+            constantFolding(rangeOf(FIVE, FIVE, true, new Literal(EMPTY, 10, DataType.INTEGER), false)).fold(FoldContext.small())
+        );
+        assertEquals(
+            false,
+            constantFolding(rangeOf(FIVE, FIVE, false, new Literal(EMPTY, 10, DataType.INTEGER), false)).fold(FoldContext.small())
+        );
     }
 
     public void testConstantNot() {
-        assertEquals(FALSE, new ConstantFolding().rule(new Not(EMPTY, TRUE)));
-        assertEquals(TRUE, new ConstantFolding().rule(new Not(EMPTY, FALSE)));
+        assertEquals(FALSE, constantFolding(new Not(EMPTY, TRUE)));
+        assertEquals(TRUE, constantFolding(new Not(EMPTY, FALSE)));
     }
 
     public void testConstantFoldingLikes() {
-        assertEquals(TRUE, new ConstantFolding().rule(new WildcardLike(EMPTY, of("test_emp"), new WildcardPattern("test*"))).canonical());
-        assertEquals(TRUE, new ConstantFolding().rule(new RLike(EMPTY, of("test_emp"), new RLikePattern("test.emp"))).canonical());
+        assertEquals(TRUE, constantFolding(new WildcardLike(EMPTY, of("test_emp"), new WildcardPattern("test*"))).canonical());
+        assertEquals(TRUE, constantFolding(new RLike(EMPTY, of("test_emp"), new RLikePattern("test.emp"))).canonical());
     }
 
     public void testArithmeticFolding() {
@@ -111,8 +127,54 @@ public void testArithmeticFolding() {
         assertEquals(1, foldOperator(new Mod(EMPTY, new Literal(EMPTY, 7, DataType.INTEGER), THREE)));
     }
 
-    private static Object foldOperator(BinaryOperator<?, ?, ?, ?> b) {
-        return ((Literal) new ConstantFolding().rule(b)).value();
+    public void testFoldRange() {
+        // 1 + 9 < value AND value < 20-1
+        // with value = 12 and randomly replacing the `<` by `<=`
+        Expression lowerBound = new Add(EMPTY, new Literal(EMPTY, 1, DataType.INTEGER), new Literal(EMPTY, 9, DataType.INTEGER));
+        Expression upperBound = new Sub(EMPTY, new Literal(EMPTY, 20, DataType.INTEGER), new Literal(EMPTY, 1, DataType.INTEGER));
+        Expression value = new Literal(EMPTY, 12, DataType.INTEGER);
+        Range range = new Range(EMPTY, value, lowerBound, randomBoolean(), upperBound, randomBoolean(), randomZone());
+
+        Expression folded = constantFolding(range);
+        assertTrue((Boolean) as(folded, Literal.class).value());
     }
 
+    public void testFoldRangeWithInvalidBoundaries() {
+        // 1 + 9 < value AND value <= 11 - 1
+        // This is always false. We also randomly test versions with `<=`.
+        Expression lowerBound;
+        boolean includeLowerBound = randomBoolean();
+        if (includeLowerBound) {
+            // 1 + 10 <= value
+            lowerBound = new Add(EMPTY, new Literal(EMPTY, 1, DataType.INTEGER), new Literal(EMPTY, 10, DataType.INTEGER));
+        } else {
+            // 1 + 9 < value
+            lowerBound = new Add(EMPTY, new Literal(EMPTY, 1, DataType.INTEGER), new Literal(EMPTY, 9, DataType.INTEGER));
+        }
+
+        boolean includeUpperBound = randomBoolean();
+        // value < 11 - 1
+        // or
+        // value <= 11 - 1
+        Expression upperBound = new Sub(EMPTY, new Literal(EMPTY, 11, DataType.INTEGER), new Literal(EMPTY, 1, DataType.INTEGER));
+
+        Expression value = fieldAttribute();
+
+        Range range = new Range(EMPTY, value, lowerBound, includeLowerBound, upperBound, includeUpperBound, randomZone());
+
+        // We need to test this as part of a logical plan, to correctly simulate how we traverse down the expression tree.
+        // Just applying this to the range directly won't perform a transformDown.
+        LogicalPlan filter = new Filter(EMPTY, emptySource(), range);
+
+        Filter foldedOnce = as(new ConstantFolding().apply(filter, unboundLogicalOptimizerContext()), Filter.class);
+        // We need to run the rule twice, because during the first run only the boundaries can be folded - the range doesn't know it's
+        // foldable, yet.
+        Filter foldedTwice = as(new ConstantFolding().apply(foldedOnce, unboundLogicalOptimizerContext()), Filter.class);
+
+        assertFalse((Boolean) as(foldedTwice.condition(), Literal.class).value());
+    }
+
+    private Object foldOperator(BinaryOperator<?, ?, ?, ?> b) {
+        return ((Literal) constantFolding(b)).value();
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java
index ae31576184938..252b25a214bb8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
@@ -67,9 +68,11 @@
 import java.util.List;
 
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.L;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.as;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.greaterThanOf;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.NULL;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
@@ -85,28 +88,27 @@
 import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
 
 public class FoldNullTests extends ESTestCase {
+    private Expression foldNull(Expression e) {
+        return new FoldNull().rule(e, unboundLogicalOptimizerContext());
+    }
 
     public void testBasicNullFolding() {
-        FoldNull rule = new FoldNull();
-        assertNullLiteral(rule.rule(new Add(EMPTY, L(randomInt()), Literal.NULL)));
-        assertNullLiteral(rule.rule(new Round(EMPTY, Literal.NULL, null)));
-        assertNullLiteral(rule.rule(new Pow(EMPTY, Literal.NULL, Literal.NULL)));
-        assertNullLiteral(rule.rule(new DateFormat(EMPTY, Literal.NULL, Literal.NULL, null)));
-        assertNullLiteral(rule.rule(new DateParse(EMPTY, Literal.NULL, Literal.NULL)));
-        assertNullLiteral(rule.rule(new DateTrunc(EMPTY, Literal.NULL, Literal.NULL)));
-        assertNullLiteral(rule.rule(new Substring(EMPTY, Literal.NULL, Literal.NULL, Literal.NULL)));
+        assertNullLiteral(foldNull(new Add(EMPTY, L(randomInt()), Literal.NULL)));
+        assertNullLiteral(foldNull(new Round(EMPTY, Literal.NULL, null)));
+        assertNullLiteral(foldNull(new Pow(EMPTY, Literal.NULL, Literal.NULL)));
+        assertNullLiteral(foldNull(new DateFormat(EMPTY, Literal.NULL, Literal.NULL, null)));
+        assertNullLiteral(foldNull(new DateParse(EMPTY, Literal.NULL, Literal.NULL)));
+        assertNullLiteral(foldNull(new DateTrunc(EMPTY, Literal.NULL, Literal.NULL)));
+        assertNullLiteral(foldNull(new Substring(EMPTY, Literal.NULL, Literal.NULL, Literal.NULL)));
     }
 
     public void testNullFoldingIsNotNull() {
-        FoldNull foldNull = new FoldNull();
-        assertEquals(true, foldNull.rule(new IsNotNull(EMPTY, TRUE)).fold());
-        assertEquals(false, foldNull.rule(new IsNotNull(EMPTY, NULL)).fold());
+        assertEquals(true, foldNull(new IsNotNull(EMPTY, TRUE)).fold(FoldContext.small()));
+        assertEquals(false, foldNull(new IsNotNull(EMPTY, NULL)).fold(FoldContext.small()));
     }
 
     @SuppressWarnings("unchecked")
     public void testNullFoldingDoesNotApplyOnAbstractMultivalueFunction() throws Exception {
-        FoldNull rule = new FoldNull();
-
         List<Class<? extends AbstractMultivalueFunction>> items = List.of(
             MvDedupe.class,
             MvFirst.class,
@@ -119,119 +121,112 @@ public void testNullFoldingDoesNotApplyOnAbstractMultivalueFunction() throws Exc
         for (Class<? extends AbstractMultivalueFunction> clazz : items) {
             Constructor<? extends AbstractMultivalueFunction> ctor = clazz.getConstructor(Source.class, Expression.class);
             AbstractMultivalueFunction conditionalFunction = ctor.newInstance(EMPTY, getFieldAttribute("a"));
-            assertEquals(conditionalFunction, rule.rule(conditionalFunction));
+            assertEquals(conditionalFunction, foldNull(conditionalFunction));
 
             conditionalFunction = ctor.newInstance(EMPTY, NULL);
-            assertEquals(NULL, rule.rule(conditionalFunction));
+            assertEquals(NULL, foldNull(conditionalFunction));
         }
 
         // avg and count ar different just because they know the return type in advance (all the others infer the type from the input)
         MvAvg avg = new MvAvg(EMPTY, getFieldAttribute("a"));
-        assertEquals(avg, rule.rule(avg));
+        assertEquals(avg, foldNull(avg));
         avg = new MvAvg(EMPTY, NULL);
-        assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(avg));
+        assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(avg));
 
         MvCount count = new MvCount(EMPTY, getFieldAttribute("a"));
-        assertEquals(count, rule.rule(count));
+        assertEquals(count, foldNull(count));
         count = new MvCount(EMPTY, NULL);
-        assertEquals(new Literal(EMPTY, null, INTEGER), rule.rule(count));
+        assertEquals(new Literal(EMPTY, null, INTEGER), foldNull(count));
     }
 
     public void testNullFoldingIsNull() {
-        FoldNull foldNull = new FoldNull();
-        assertEquals(true, foldNull.rule(new IsNull(EMPTY, NULL)).fold());
-        assertEquals(false, foldNull.rule(new IsNull(EMPTY, TRUE)).fold());
+        assertEquals(true, foldNull(new IsNull(EMPTY, NULL)).fold(FoldContext.small()));
+        assertEquals(false, foldNull(new IsNull(EMPTY, TRUE)).fold(FoldContext.small()));
     }
 
     public void testGenericNullableExpression() {
         FoldNull rule = new FoldNull();
         // arithmetic
-        assertNullLiteral(rule.rule(new Add(EMPTY, getFieldAttribute("a"), NULL)));
+        assertNullLiteral(foldNull(new Add(EMPTY, getFieldAttribute("a"), NULL)));
         // comparison
-        assertNullLiteral(rule.rule(greaterThanOf(getFieldAttribute("a"), NULL)));
+        assertNullLiteral(foldNull(greaterThanOf(getFieldAttribute("a"), NULL)));
         // regex
-        assertNullLiteral(rule.rule(new RLike(EMPTY, NULL, new RLikePattern("123"))));
+        assertNullLiteral(foldNull(new RLike(EMPTY, NULL, new RLikePattern("123"))));
         // date functions
-        assertNullLiteral(rule.rule(new DateExtract(EMPTY, NULL, NULL, configuration(""))));
+        assertNullLiteral(foldNull(new DateExtract(EMPTY, NULL, NULL, configuration(""))));
         // math functions
-        assertNullLiteral(rule.rule(new Cos(EMPTY, NULL)));
+        assertNullLiteral(foldNull(new Cos(EMPTY, NULL)));
         // string functions
-        assertNullLiteral(rule.rule(new LTrim(EMPTY, NULL)));
+        assertNullLiteral(foldNull(new LTrim(EMPTY, NULL)));
         // spatial
-        assertNullLiteral(rule.rule(new SpatialCentroid(EMPTY, NULL)));
+        assertNullLiteral(foldNull(new SpatialCentroid(EMPTY, NULL)));
         // ip
-        assertNullLiteral(rule.rule(new CIDRMatch(EMPTY, NULL, List.of(NULL))));
+        assertNullLiteral(foldNull(new CIDRMatch(EMPTY, NULL, List.of(NULL))));
         // conversion
-        assertNullLiteral(rule.rule(new ToString(EMPTY, NULL)));
+        assertNullLiteral(foldNull(new ToString(EMPTY, NULL)));
     }
 
     public void testNullFoldingDoesNotApplyOnLogicalExpressions() {
-        FoldNull rule = new FoldNull();
-
         Or or = new Or(EMPTY, NULL, TRUE);
-        assertEquals(or, rule.rule(or));
+        assertEquals(or, foldNull(or));
         or = new Or(EMPTY, NULL, NULL);
-        assertEquals(or, rule.rule(or));
+        assertEquals(or, foldNull(or));
 
         And and = new And(EMPTY, NULL, TRUE);
-        assertEquals(and, rule.rule(and));
+        assertEquals(and, foldNull(and));
         and = new And(EMPTY, NULL, NULL);
-        assertEquals(and, rule.rule(and));
+        assertEquals(and, foldNull(and));
     }
 
     @SuppressWarnings("unchecked")
     public void testNullFoldingDoesNotApplyOnAggregate() throws Exception {
-        FoldNull rule = new FoldNull();
-
         List<Class<? extends AggregateFunction>> items = List.of(Max.class, Min.class);
         for (Class<? extends AggregateFunction> clazz : items) {
             Constructor<? extends AggregateFunction> ctor = clazz.getConstructor(Source.class, Expression.class);
             AggregateFunction conditionalFunction = ctor.newInstance(EMPTY, getFieldAttribute("a"));
-            assertEquals(conditionalFunction, rule.rule(conditionalFunction));
+            assertEquals(conditionalFunction, foldNull(conditionalFunction));
 
             conditionalFunction = ctor.newInstance(EMPTY, NULL);
-            assertEquals(NULL, rule.rule(conditionalFunction));
+            assertEquals(NULL, foldNull(conditionalFunction));
         }
 
         Avg avg = new Avg(EMPTY, getFieldAttribute("a"));
-        assertEquals(avg, rule.rule(avg));
+        assertEquals(avg, foldNull(avg));
         avg = new Avg(EMPTY, NULL);
-        assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(avg));
+        assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(avg));
 
         Count count = new Count(EMPTY, getFieldAttribute("a"));
-        assertEquals(count, rule.rule(count));
+        assertEquals(count, foldNull(count));
         count = new Count(EMPTY, NULL);
-        assertEquals(count, rule.rule(count));
+        assertEquals(count, foldNull(count));
 
         CountDistinct countd = new CountDistinct(EMPTY, getFieldAttribute("a"), getFieldAttribute("a"));
-        assertEquals(countd, rule.rule(countd));
+        assertEquals(countd, foldNull(countd));
         countd = new CountDistinct(EMPTY, NULL, NULL);
-        assertEquals(new Literal(EMPTY, null, LONG), rule.rule(countd));
+        assertEquals(new Literal(EMPTY, null, LONG), foldNull(countd));
 
         Median median = new Median(EMPTY, getFieldAttribute("a"));
-        assertEquals(median, rule.rule(median));
+        assertEquals(median, foldNull(median));
         median = new Median(EMPTY, NULL);
-        assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(median));
+        assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(median));
 
         MedianAbsoluteDeviation medianad = new MedianAbsoluteDeviation(EMPTY, getFieldAttribute("a"));
-        assertEquals(medianad, rule.rule(medianad));
+        assertEquals(medianad, foldNull(medianad));
         medianad = new MedianAbsoluteDeviation(EMPTY, NULL);
-        assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(medianad));
+        assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(medianad));
 
         Percentile percentile = new Percentile(EMPTY, getFieldAttribute("a"), getFieldAttribute("a"));
-        assertEquals(percentile, rule.rule(percentile));
+        assertEquals(percentile, foldNull(percentile));
         percentile = new Percentile(EMPTY, NULL, NULL);
-        assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(percentile));
+        assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(percentile));
 
         Sum sum = new Sum(EMPTY, getFieldAttribute("a"));
-        assertEquals(sum, rule.rule(sum));
+        assertEquals(sum, foldNull(sum));
         sum = new Sum(EMPTY, NULL);
-        assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(sum));
+        assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(sum));
     }
 
     public void testNullFoldableDoesNotApplyToIsNullAndNotNull() {
-        FoldNull rule = new FoldNull();
-
         DataType numericType = randomFrom(INTEGER, LONG, DOUBLE);
         DataType genericType = randomFrom(INTEGER, LONG, DOUBLE, UNSIGNED_LONG, KEYWORD, TEXT, GEO_POINT, GEO_SHAPE, VERSION, IP);
         List<Expression> items = List.of(
@@ -260,29 +255,26 @@ public void testNullFoldableDoesNotApplyToIsNullAndNotNull() {
         );
         for (Expression item : items) {
             Expression isNull = new IsNull(EMPTY, item);
-            Expression transformed = rule.rule(isNull);
+            Expression transformed = foldNull(isNull);
             assertEquals(isNull, transformed);
 
             IsNotNull isNotNull = new IsNotNull(EMPTY, item);
-            transformed = rule.rule(isNotNull);
+            transformed = foldNull(isNotNull);
             assertEquals(isNotNull, transformed);
         }
     }
 
     public void testNullBucketGetsFolded() {
-        FoldNull foldNull = new FoldNull();
-        assertEquals(NULL, foldNull.rule(new Bucket(EMPTY, NULL, NULL, NULL, NULL)));
+        assertEquals(NULL, foldNull(new Bucket(EMPTY, NULL, NULL, NULL, NULL)));
     }
 
     public void testNullCategorizeGroupingNotFolded() {
-        FoldNull foldNull = new FoldNull();
         Categorize categorize = new Categorize(EMPTY, NULL);
-        assertEquals(categorize, foldNull.rule(categorize));
+        assertEquals(categorize, foldNull(categorize));
     }
 
     private void assertNullLiteral(Expression expression) {
-        assertEquals(Literal.class, expression.getClass());
-        assertNull(expression.fold());
+        assertNull(as(expression, Literal.class).value());
     }
 
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRightTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRightTests.java
index 17e69e81444c5..1664e9f4653bb 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRightTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRightTests.java
@@ -15,6 +15,7 @@
 
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.FIVE;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.equalsOf;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
 import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
 
@@ -22,7 +23,7 @@ public class LiteralsOnTheRightTests extends ESTestCase {
 
     public void testLiteralsOnTheRight() {
         Alias a = new Alias(EMPTY, "a", new Literal(EMPTY, 10, INTEGER));
-        Expression result = new LiteralsOnTheRight().rule(equalsOf(FIVE, a));
+        Expression result = new LiteralsOnTheRight().rule(equalsOf(FIVE, a), unboundLogicalOptimizerContext());
         assertTrue(result instanceof Equals);
         Equals eq = (Equals) result;
         assertEquals(a, eq.left());
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEqualsTests.java
index 55091653e75d4..a6c0d838b2c21 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEqualsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEqualsTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Range;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
@@ -37,11 +38,15 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOrEqualOf;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.notEqualsOf;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.rangeOf;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
 
 public class PropagateEqualsTests extends ESTestCase {
+    private Expression propagateEquals(BinaryLogic e) {
+        return new PropagateEquals().rule(e, unboundLogicalOptimizerContext());
+    }
 
     // a == 1 AND a == 2 -> FALSE
     public void testDualEqualsConjunction() {
@@ -49,8 +54,7 @@ public void testDualEqualsConjunction() {
         Equals eq1 = equalsOf(fa, ONE);
         Equals eq2 = equalsOf(fa, TWO);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq1, eq2));
+        Expression exp = propagateEquals(new And(EMPTY, eq1, eq2));
         assertEquals(FALSE, exp);
     }
 
@@ -60,8 +64,7 @@ public void testEliminateRangeByEqualsOutsideInterval() {
         Equals eq1 = equalsOf(fa, new Literal(EMPTY, 10, DataType.INTEGER));
         Range r = rangeOf(fa, ONE, false, new Literal(EMPTY, 10, DataType.INTEGER), false);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq1, r));
+        Expression exp = propagateEquals(new And(EMPTY, eq1, r));
         assertEquals(FALSE, exp);
     }
 
@@ -71,8 +74,7 @@ public void testPropagateEquals_VarNeq3AndVarEq3() {
         NotEquals neq = notEqualsOf(fa, THREE);
         Equals eq = equalsOf(fa, THREE);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, neq, eq));
+        Expression exp = propagateEquals(new And(EMPTY, neq, eq));
         assertEquals(FALSE, exp);
     }
 
@@ -82,8 +84,7 @@ public void testPropagateEquals_VarNeq4AndVarEq3() {
         NotEquals neq = notEqualsOf(fa, FOUR);
         Equals eq = equalsOf(fa, THREE);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, neq, eq));
+        Expression exp = propagateEquals(new And(EMPTY, neq, eq));
         assertEquals(Equals.class, exp.getClass());
         assertEquals(eq, exp);
     }
@@ -94,8 +95,7 @@ public void testPropagateEquals_VarEq2AndVarLt2() {
         Equals eq = equalsOf(fa, TWO);
         LessThan lt = lessThanOf(fa, TWO);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq, lt));
+        Expression exp = propagateEquals(new And(EMPTY, eq, lt));
         assertEquals(FALSE, exp);
     }
 
@@ -105,8 +105,7 @@ public void testPropagateEquals_VarEq2AndVarLte2() {
         Equals eq = equalsOf(fa, TWO);
         LessThanOrEqual lt = lessThanOrEqualOf(fa, TWO);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq, lt));
+        Expression exp = propagateEquals(new And(EMPTY, eq, lt));
         assertEquals(eq, exp);
     }
 
@@ -116,8 +115,7 @@ public void testPropagateEquals_VarEq2AndVarLte1() {
         Equals eq = equalsOf(fa, TWO);
         LessThanOrEqual lt = lessThanOrEqualOf(fa, ONE);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq, lt));
+        Expression exp = propagateEquals(new And(EMPTY, eq, lt));
         assertEquals(FALSE, exp);
     }
 
@@ -127,8 +125,7 @@ public void testPropagateEquals_VarEq2AndVarGt2() {
         Equals eq = equalsOf(fa, TWO);
         GreaterThan gt = greaterThanOf(fa, TWO);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq, gt));
+        Expression exp = propagateEquals(new And(EMPTY, eq, gt));
         assertEquals(FALSE, exp);
     }
 
@@ -138,8 +135,7 @@ public void testPropagateEquals_VarEq2AndVarGte2() {
         Equals eq = equalsOf(fa, TWO);
         GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, TWO);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq, gte));
+        Expression exp = propagateEquals(new And(EMPTY, eq, gte));
         assertEquals(eq, exp);
     }
 
@@ -149,8 +145,7 @@ public void testPropagateEquals_VarEq2AndVarLt3() {
         Equals eq = equalsOf(fa, TWO);
         GreaterThan gt = greaterThanOf(fa, THREE);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq, gt));
+        Expression exp = propagateEquals(new And(EMPTY, eq, gt));
         assertEquals(FALSE, exp);
     }
 
@@ -162,9 +157,8 @@ public void testPropagateEquals_VarEq2AndVarLt3AndVarGt1AndVarNeq4() {
         GreaterThan gt = greaterThanOf(fa, ONE);
         NotEquals neq = notEqualsOf(fa, FOUR);
 
-        PropagateEquals rule = new PropagateEquals();
         Expression and = Predicates.combineAnd(asList(eq, lt, gt, neq));
-        Expression exp = rule.rule((And) and);
+        Expression exp = propagateEquals((And) and);
         assertEquals(eq, exp);
     }
 
@@ -176,9 +170,8 @@ public void testPropagateEquals_VarEq2AndVarRangeGt1Lt3AndVarGt0AndVarNeq4() {
         GreaterThan gt = greaterThanOf(fa, new Literal(EMPTY, 0, DataType.INTEGER));
         NotEquals neq = notEqualsOf(fa, FOUR);
 
-        PropagateEquals rule = new PropagateEquals();
         Expression and = Predicates.combineAnd(asList(eq, range, gt, neq));
-        Expression exp = rule.rule((And) and);
+        Expression exp = propagateEquals((And) and);
         assertEquals(eq, exp);
     }
 
@@ -188,8 +181,7 @@ public void testPropagateEquals_VarEq2OrVarGt1() {
         Equals eq = equalsOf(fa, TWO);
         GreaterThan gt = greaterThanOf(fa, ONE);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, gt));
+        Expression exp = propagateEquals(new Or(EMPTY, eq, gt));
         assertEquals(gt, exp);
     }
 
@@ -199,8 +191,7 @@ public void testPropagateEquals_VarEq2OrVarGte2() {
         Equals eq = equalsOf(fa, TWO);
         GreaterThan gt = greaterThanOf(fa, TWO);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, gt));
+        Expression exp = propagateEquals(new Or(EMPTY, eq, gt));
         assertEquals(GreaterThanOrEqual.class, exp.getClass());
         GreaterThanOrEqual gte = (GreaterThanOrEqual) exp;
         assertEquals(TWO, gte.right());
@@ -212,8 +203,7 @@ public void testPropagateEquals_VarEq2OrVarLt3() {
         Equals eq = equalsOf(fa, TWO);
         LessThan lt = lessThanOf(fa, THREE);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, lt));
+        Expression exp = propagateEquals(new Or(EMPTY, eq, lt));
         assertEquals(lt, exp);
     }
 
@@ -223,8 +213,7 @@ public void testPropagateEquals_VarEq3OrVarLt3() {
         Equals eq = equalsOf(fa, THREE);
         LessThan lt = lessThanOf(fa, THREE);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, lt));
+        Expression exp = propagateEquals(new Or(EMPTY, eq, lt));
         assertEquals(LessThanOrEqual.class, exp.getClass());
         LessThanOrEqual lte = (LessThanOrEqual) exp;
         assertEquals(THREE, lte.right());
@@ -236,8 +225,7 @@ public void testPropagateEquals_VarEq2OrVarRangeGt1Lt3() {
         Equals eq = equalsOf(fa, TWO);
         Range range = rangeOf(fa, ONE, false, THREE, false);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, range));
+        Expression exp = propagateEquals(new Or(EMPTY, eq, range));
         assertEquals(range, exp);
     }
 
@@ -247,8 +235,7 @@ public void testPropagateEquals_VarEq2OrVarRangeGt2Lt3() {
         Equals eq = equalsOf(fa, TWO);
         Range range = rangeOf(fa, TWO, false, THREE, false);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, range));
+        Expression exp = propagateEquals(new Or(EMPTY, eq, range));
         assertEquals(Range.class, exp.getClass());
         Range r = (Range) exp;
         assertEquals(TWO, r.lower());
@@ -263,8 +250,7 @@ public void testPropagateEquals_VarEq3OrVarRangeGt2Lt3() {
         Equals eq = equalsOf(fa, THREE);
         Range range = rangeOf(fa, TWO, false, THREE, false);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, range));
+        Expression exp = propagateEquals(new Or(EMPTY, eq, range));
         assertEquals(Range.class, exp.getClass());
         Range r = (Range) exp;
         assertEquals(TWO, r.lower());
@@ -279,8 +265,7 @@ public void testPropagateEquals_VarEq2OrVarNeq2() {
         Equals eq = equalsOf(fa, TWO);
         NotEquals neq = notEqualsOf(fa, TWO);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, neq));
+        Expression exp = propagateEquals(new Or(EMPTY, eq, neq));
         assertEquals(TRUE, exp);
     }
 
@@ -290,8 +275,7 @@ public void testPropagateEquals_VarEq2OrVarNeq5() {
         Equals eq = equalsOf(fa, TWO);
         NotEquals neq = notEqualsOf(fa, FIVE);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, neq));
+        Expression exp = propagateEquals(new Or(EMPTY, eq, neq));
         assertEquals(NotEquals.class, exp.getClass());
         NotEquals ne = (NotEquals) exp;
         assertEquals(FIVE, ne.right());
@@ -305,8 +289,7 @@ public void testPropagateEquals_VarEq2OrVarRangeGt3Lt4OrVarGt2OrVarNe2() {
         GreaterThan gt = greaterThanOf(fa, TWO);
         NotEquals neq = notEqualsOf(fa, TWO);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule((Or) Predicates.combineOr(asList(eq, range, neq, gt)));
+        Expression exp = propagateEquals((Or) Predicates.combineOr(asList(eq, range, neq, gt)));
         assertEquals(TRUE, exp);
     }
 
@@ -317,8 +300,7 @@ public void testPropagateEquals_ignoreDateTimeFields() {
         Equals eq2 = equalsOf(fa, TWO);
         And and = new And(EMPTY, eq1, eq2);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(and);
+        Expression exp = propagateEquals(and);
         assertEquals(and, exp);
     }
 
@@ -328,8 +310,7 @@ public void testEliminateRangeByEqualsInInterval() {
         Equals eq1 = equalsOf(fa, ONE);
         Range r = rangeOf(fa, ONE, true, new Literal(EMPTY, 10, DataType.INTEGER), false);
 
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq1, r));
+        Expression exp = propagateEquals(new And(EMPTY, eq1, r));
         assertEquals(eq1, exp);
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullableTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullableTests.java
index d1d6a7fbaa208..d35890e5b56bb 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullableTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullableTests.java
@@ -31,11 +31,20 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.greaterThanOf;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOf;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.relation;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
 import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
 
 public class PropagateNullableTests extends ESTestCase {
+    private Expression propagateNullable(And e) {
+        return new PropagateNullable().rule(e, unboundLogicalOptimizerContext());
+    }
+
+    private LogicalPlan propagateNullable(LogicalPlan p) {
+        return new PropagateNullable().apply(p, unboundLogicalOptimizerContext());
+    }
+
     private Literal nullOf(DataType dataType) {
         return new Literal(Source.EMPTY, null, dataType);
     }
@@ -45,7 +54,7 @@ public void testIsNullAndNotNull() {
         FieldAttribute fa = getFieldAttribute();
 
         And and = new And(EMPTY, new IsNull(EMPTY, fa), new IsNotNull(EMPTY, fa));
-        assertEquals(FALSE, new PropagateNullable().rule(and));
+        assertEquals(FALSE, propagateNullable(and));
     }
 
     // a IS NULL AND b IS NOT NULL AND c IS NULL AND d IS NOT NULL AND e IS NULL AND a IS NOT NULL => false
@@ -58,7 +67,7 @@ public void testIsNullAndNotNullMultiField() {
 
         And and = new And(EMPTY, andOne, new And(EMPTY, andThree, andTwo));
 
-        assertEquals(FALSE, new PropagateNullable().rule(and));
+        assertEquals(FALSE, propagateNullable(and));
     }
 
     // a IS NULL AND a > 1 => a IS NULL AND false
@@ -67,7 +76,7 @@ public void testIsNullAndComparison() {
         IsNull isNull = new IsNull(EMPTY, fa);
 
         And and = new And(EMPTY, isNull, greaterThanOf(fa, ONE));
-        assertEquals(new And(EMPTY, isNull, nullOf(BOOLEAN)), new PropagateNullable().rule(and));
+        assertEquals(new And(EMPTY, isNull, nullOf(BOOLEAN)), propagateNullable(and));
     }
 
     // a IS NULL AND b < 1 AND c < 1 AND a < 1 => a IS NULL AND b < 1 AND c < 1 => a IS NULL AND b < 1 AND c < 1
@@ -79,7 +88,7 @@ public void testIsNullAndMultipleComparison() {
         And and = new And(EMPTY, isNull, nestedAnd);
         And top = new And(EMPTY, and, lessThanOf(fa, ONE));
 
-        Expression optimized = new PropagateNullable().rule(top);
+        Expression optimized = propagateNullable(top);
         Expression expected = new And(EMPTY, and, nullOf(BOOLEAN));
         assertEquals(Predicates.splitAnd(expected), Predicates.splitAnd(optimized));
     }
@@ -97,7 +106,7 @@ public void testIsNullAndDeeplyNestedExpression() {
         Expression kept = new And(EMPTY, isNull, lessThanOf(getFieldAttribute("b"), THREE));
         And and = new And(EMPTY, nullified, kept);
 
-        Expression optimized = new PropagateNullable().rule(and);
+        Expression optimized = propagateNullable(and);
         Expression expected = new And(EMPTY, new And(EMPTY, nullOf(BOOLEAN), nullOf(BOOLEAN)), kept);
 
         assertEquals(Predicates.splitAnd(expected), Predicates.splitAnd(optimized));
@@ -110,13 +119,13 @@ public void testIsNullInDisjunction() {
 
         Or or = new Or(EMPTY, new IsNull(EMPTY, fa), new IsNotNull(EMPTY, fa));
         Filter dummy = new Filter(EMPTY, relation(), or);
-        LogicalPlan transformed = new PropagateNullable().apply(dummy);
+        LogicalPlan transformed = propagateNullable(dummy);
         assertSame(dummy, transformed);
         assertEquals(or, ((Filter) transformed).condition());
 
         or = new Or(EMPTY, new IsNull(EMPTY, fa), greaterThanOf(fa, ONE));
         dummy = new Filter(EMPTY, relation(), or);
-        transformed = new PropagateNullable().apply(dummy);
+        transformed = propagateNullable(dummy);
         assertSame(dummy, transformed);
         assertEquals(or, ((Filter) transformed).condition());
     }
@@ -129,7 +138,7 @@ public void testIsNullDisjunction() {
         Or or = new Or(EMPTY, isNull, greaterThanOf(fa, THREE));
         And and = new And(EMPTY, new Add(EMPTY, fa, ONE), or);
 
-        assertEquals(and, new PropagateNullable().rule(and));
+        assertEquals(and, propagateNullable(and));
     }
 
     public void testDoNotOptimizeIsNullAndMultipleComparisonWithConstants() {
@@ -141,7 +150,7 @@ public void testDoNotOptimizeIsNullAndMultipleComparisonWithConstants() {
         And aIsNull_AND_bLT1_AND_cLT1 = new And(EMPTY, aIsNull, bLT1_AND_cLT1);
         And aIsNull_AND_bLT1_AND_cLT1_AND_aLT1 = new And(EMPTY, aIsNull_AND_bLT1_AND_cLT1, lessThanOf(a, ONE));
 
-        Expression optimized = new PropagateNullable().rule(aIsNull_AND_bLT1_AND_cLT1_AND_aLT1);
+        Expression optimized = propagateNullable(aIsNull_AND_bLT1_AND_cLT1_AND_aLT1);
         Literal nullLiteral = new Literal(EMPTY, null, BOOLEAN);
         assertEquals(asList(aIsNull, nullLiteral, nullLiteral, nullLiteral), Predicates.splitAnd(optimized));
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java
index c7206c6971bde..b9ffc39e5e130 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java
@@ -10,8 +10,10 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
+import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
@@ -20,16 +22,20 @@
 
 import static java.util.Arrays.asList;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
 
 public class ReplaceRegexMatchTests extends ESTestCase {
+    private Expression replaceRegexMatch(RegexMatch<?> e) {
+        return new ReplaceRegexMatch().rule(e, unboundLogicalOptimizerContext());
+    }
 
     public void testMatchAllWildcardLikeToExist() {
         for (String s : asList("*", "**", "***")) {
             WildcardPattern pattern = new WildcardPattern(s);
             FieldAttribute fa = getFieldAttribute();
             WildcardLike l = new WildcardLike(EMPTY, fa, pattern);
-            Expression e = new ReplaceRegexMatch().rule(l);
+            Expression e = replaceRegexMatch(l);
             assertEquals(IsNotNull.class, e.getClass());
             IsNotNull inn = (IsNotNull) e;
             assertEquals(fa, inn.field());
@@ -40,7 +46,7 @@ public void testMatchAllRLikeToExist() {
         RLikePattern pattern = new RLikePattern(".*");
         FieldAttribute fa = getFieldAttribute();
         RLike l = new RLike(EMPTY, fa, pattern);
-        Expression e = new ReplaceRegexMatch().rule(l);
+        Expression e = replaceRegexMatch(l);
         assertEquals(IsNotNull.class, e.getClass());
         IsNotNull inn = (IsNotNull) e;
         assertEquals(fa, inn.field());
@@ -51,11 +57,11 @@ public void testExactMatchWildcardLike() {
             WildcardPattern pattern = new WildcardPattern(s);
             FieldAttribute fa = getFieldAttribute();
             WildcardLike l = new WildcardLike(EMPTY, fa, pattern);
-            Expression e = new ReplaceRegexMatch().rule(l);
+            Expression e = replaceRegexMatch(l);
             assertEquals(Equals.class, e.getClass());
             Equals eq = (Equals) e;
             assertEquals(fa, eq.left());
-            assertEquals(s.replace("\\", StringUtils.EMPTY), eq.right().fold());
+            assertEquals(s.replace("\\", StringUtils.EMPTY), eq.right().fold(FoldContext.small()));
         }
     }
 
@@ -63,11 +69,11 @@ public void testExactMatchRLike() {
         RLikePattern pattern = new RLikePattern("abc");
         FieldAttribute fa = getFieldAttribute();
         RLike l = new RLike(EMPTY, fa, pattern);
-        Expression e = new ReplaceRegexMatch().rule(l);
+        Expression e = replaceRegexMatch(l);
         assertEquals(Equals.class, e.getClass());
         Equals eq = (Equals) e;
         assertEquals(fa, eq.left());
-        assertEquals("abc", eq.right().fold());
+        assertEquals("abc", eq.right().fold(FoldContext.small()));
     }
 
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java
index 2429bcb1a1b04..90c8ae1032325 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
@@ -417,7 +418,7 @@ private static void assertNoPushdownSort(TestPhysicalPlanBuilder builder, String
 
     private static PhysicalPlan pushTopNToSource(TopNExec topNExec) {
         var configuration = EsqlTestUtils.configuration("from test");
-        var ctx = new LocalPhysicalOptimizerContext(configuration, SearchStats.EMPTY);
+        var ctx = new LocalPhysicalOptimizerContext(configuration, FoldContext.small(), SearchStats.EMPTY);
         var pushTopNToSource = new PushTopNToSource();
         return pushTopNToSource.rule(topNExec, ctx);
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java
index e6fef186721a0..99a04b6ed8f10 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java
@@ -151,8 +151,12 @@ void expectInvalidIndexNameErrorWithLineNumber(String query, String indexString,
         expectInvalidIndexNameErrorWithLineNumber(query, "\"" + indexString + "\"", lineNumber, indexString);
     }
 
-    void expectInvalidIndexNameErrorWithLineNumber(String query, String indexString, String lineNumber, String error) {
-        expectError(LoggerMessageFormat.format(null, query, indexString), lineNumber + "Invalid index name [" + error);
+    void expectInvalidIndexNameErrorWithLineNumber(String query, String indexString, String lineNumber, String name) {
+        expectError(LoggerMessageFormat.format(null, query, indexString), lineNumber + "Invalid index name [" + name);
+    }
+
+    void expectInvalidIndexNameErrorWithLineNumber(String query, String indexString, String lineNumber, String name, String error) {
+        expectError(LoggerMessageFormat.format(null, query, indexString), lineNumber + "Invalid index name [" + name + "], " + error);
     }
 
     void expectDateMathErrorWithLineNumber(String query, String arg, String lineNumber, String error) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java
index 710637c05a900..85d4017b166fa 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedStar;
@@ -617,16 +618,14 @@ public void testSimplifyInWithSingleElementList() {
         Equals eq = (Equals) e;
         assertThat(eq.left(), instanceOf(UnresolvedAttribute.class));
         assertThat(((UnresolvedAttribute) eq.left()).name(), equalTo("a"));
-        assertThat(eq.right(), instanceOf(Literal.class));
-        assertThat(eq.right().fold(), equalTo(1));
+        assertThat(as(eq.right(), Literal.class).value(), equalTo(1));
 
         e = whereExpression("1 IN (a)");
         assertThat(e, instanceOf(Equals.class));
         eq = (Equals) e;
         assertThat(eq.right(), instanceOf(UnresolvedAttribute.class));
         assertThat(((UnresolvedAttribute) eq.right()).name(), equalTo("a"));
-        assertThat(eq.left(), instanceOf(Literal.class));
-        assertThat(eq.left().fold(), equalTo(1));
+        assertThat(eq.left().fold(FoldContext.small()), equalTo(1));
 
         e = whereExpression("1 NOT IN (a)");
         assertThat(e, instanceOf(Not.class));
@@ -635,9 +634,7 @@ public void testSimplifyInWithSingleElementList() {
         eq = (Equals) e;
         assertThat(eq.right(), instanceOf(UnresolvedAttribute.class));
         assertThat(((UnresolvedAttribute) eq.right()).name(), equalTo("a"));
-        assertThat(eq.left(), instanceOf(Literal.class));
-        assertThat(eq.left().fold(), equalTo(1));
-
+        assertThat(eq.left().fold(FoldContext.small()), equalTo(1));
     }
 
     private Expression whereExpression(String e) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
index b83892ea47049..a6243d25ba579 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
@@ -577,6 +578,9 @@ public void testInvalidCharacterInIndexPattern() {
             expectInvalidIndexNameErrorWithLineNumber(command, "indexpattern, --indexpattern", lineNumber, "-indexpattern");
             expectInvalidIndexNameErrorWithLineNumber(command, "indexpattern, \"--indexpattern\"", lineNumber, "-indexpattern");
             expectInvalidIndexNameErrorWithLineNumber(command, "\"indexpattern, --indexpattern\"", commands.get(command), "-indexpattern");
+            expectInvalidIndexNameErrorWithLineNumber(command, "\"- , -\"", commands.get(command), "", "must not be empty");
+            expectInvalidIndexNameErrorWithLineNumber(command, "\"indexpattern,-\"", commands.get(command), "", "must not be empty");
+            clustersAndIndices(command, "indexpattern", "*-");
             clustersAndIndices(command, "indexpattern", "-indexpattern");
         }
 
@@ -621,11 +625,8 @@ public void testInvalidQuotingAsFromIndexPattern() {
         expectError("FROM \"foo\"bar\"", ": token recognition error at: '\"'");
         expectError("FROM \"foo\"\"bar\"", ": extraneous input '\"bar\"' expecting <EOF>");
 
-        expectError("FROM \"\"\"foo\"\"\"bar\"\"\"", ": mismatched input 'bar' expecting {<EOF>, '|', ',', OPENING_BRACKET, 'metadata'}");
-        expectError(
-            "FROM \"\"\"foo\"\"\"\"\"\"bar\"\"\"",
-            ": mismatched input '\"bar\"' expecting {<EOF>, '|', ',', OPENING_BRACKET, 'metadata'}"
-        );
+        expectError("FROM \"\"\"foo\"\"\"bar\"\"\"", ": mismatched input 'bar' expecting {<EOF>, '|', ',', 'metadata'}");
+        expectError("FROM \"\"\"foo\"\"\"\"\"\"bar\"\"\"", ": mismatched input '\"bar\"' expecting {<EOF>, '|', ',', 'metadata'}");
     }
 
     public void testInvalidQuotingAsMetricsIndexPattern() {
@@ -905,10 +906,7 @@ public void testDeprecatedIsNullFunction() {
     public void testMetadataFieldOnOtherSources() {
         expectError("row a = 1 metadata _index", "line 1:20: extraneous input '_index' expecting <EOF>");
         expectError("show info metadata _index", "line 1:11: token recognition error at: 'm'");
-        expectError(
-            "explain [from foo] metadata _index",
-            "line 1:20: mismatched input 'metadata' expecting {'|', ',', OPENING_BRACKET, ']', 'metadata'}"
-        );
+        expectError("explain [from foo] metadata _index", "line 1:20: mismatched input 'metadata' expecting {'|', ',', ']', 'metadata'}");
     }
 
     public void testMetadataFieldMultipleDeclarations() {
@@ -1129,51 +1127,51 @@ public void testInputParams() {
         assertThat(field.name(), is("x"));
         assertThat(field, instanceOf(Alias.class));
         Alias alias = (Alias) field;
-        assertThat(alias.child().fold(), is(1));
+        assertThat(alias.child().fold(FoldContext.small()), is(1));
 
         field = row.fields().get(1);
         assertThat(field.name(), is("y"));
         assertThat(field, instanceOf(Alias.class));
         alias = (Alias) field;
-        assertThat(alias.child().fold(), is("2"));
+        assertThat(alias.child().fold(FoldContext.small()), is("2"));
 
         field = row.fields().get(2);
         assertThat(field.name(), is("a"));
         assertThat(field, instanceOf(Alias.class));
         alias = (Alias) field;
-        assertThat(alias.child().fold(), is("2 days"));
+        assertThat(alias.child().fold(FoldContext.small()), is("2 days"));
 
         field = row.fields().get(3);
         assertThat(field.name(), is("b"));
         assertThat(field, instanceOf(Alias.class));
         alias = (Alias) field;
-        assertThat(alias.child().fold(), is("4 hours"));
+        assertThat(alias.child().fold(FoldContext.small()), is("4 hours"));
 
         field = row.fields().get(4);
         assertThat(field.name(), is("c"));
         assertThat(field, instanceOf(Alias.class));
         alias = (Alias) field;
-        assertThat(alias.child().fold().getClass(), is(String.class));
-        assertThat(alias.child().fold().toString(), is("1.2.3"));
+        assertThat(alias.child().fold(FoldContext.small()).getClass(), is(String.class));
+        assertThat(alias.child().fold(FoldContext.small()).toString(), is("1.2.3"));
 
         field = row.fields().get(5);
         assertThat(field.name(), is("d"));
         assertThat(field, instanceOf(Alias.class));
         alias = (Alias) field;
-        assertThat(alias.child().fold().getClass(), is(String.class));
-        assertThat(alias.child().fold().toString(), is("127.0.0.1"));
+        assertThat(alias.child().fold(FoldContext.small()).getClass(), is(String.class));
+        assertThat(alias.child().fold(FoldContext.small()).toString(), is("127.0.0.1"));
 
         field = row.fields().get(6);
         assertThat(field.name(), is("e"));
         assertThat(field, instanceOf(Alias.class));
         alias = (Alias) field;
-        assertThat(alias.child().fold(), is(9));
+        assertThat(alias.child().fold(FoldContext.small()), is(9));
 
         field = row.fields().get(7);
         assertThat(field.name(), is("f"));
         assertThat(field, instanceOf(Alias.class));
         alias = (Alias) field;
-        assertThat(alias.child().fold(), is(11));
+        assertThat(alias.child().fold(FoldContext.small()), is(11));
     }
 
     public void testMissingInputParams() {
@@ -1190,13 +1188,13 @@ public void testNamedParams() {
         assertThat(field.name(), is("x"));
         assertThat(field, instanceOf(Alias.class));
         Alias alias = (Alias) field;
-        assertThat(alias.child().fold(), is(1));
+        assertThat(alias.child().fold(FoldContext.small()), is(1));
 
         field = row.fields().get(1);
         assertThat(field.name(), is("y"));
         assertThat(field, instanceOf(Alias.class));
         alias = (Alias) field;
-        assertThat(alias.child().fold(), is(1));
+        assertThat(alias.child().fold(FoldContext.small()), is(1));
     }
 
     public void testInvalidNamedParams() {
@@ -1237,13 +1235,13 @@ public void testPositionalParams() {
         assertThat(field.name(), is("x"));
         assertThat(field, instanceOf(Alias.class));
         Alias alias = (Alias) field;
-        assertThat(alias.child().fold(), is(1));
+        assertThat(alias.child().fold(FoldContext.small()), is(1));
 
         field = row.fields().get(1);
         assertThat(field.name(), is("y"));
         assertThat(field, instanceOf(Alias.class));
         alias = (Alias) field;
-        assertThat(alias.child().fold(), is(1));
+        assertThat(alias.child().fold(FoldContext.small()), is(1));
     }
 
     public void testInvalidPositionalParams() {
@@ -2057,7 +2055,7 @@ private void assertStringAsLookupIndexPattern(String string, String statement) {
         var plan = statement(statement);
         var lookup = as(plan, Lookup.class);
         var tableName = as(lookup.tableName(), Literal.class);
-        assertThat(tableName.fold(), equalTo(string));
+        assertThat(tableName.fold(FoldContext.small()), equalTo(string));
     }
 
     public void testIdPatternUnquoted() throws Exception {
@@ -2125,7 +2123,7 @@ public void testLookup() {
         var plan = statement(query);
         var lookup = as(plan, Lookup.class);
         var tableName = as(lookup.tableName(), Literal.class);
-        assertThat(tableName.fold(), equalTo("t"));
+        assertThat(tableName.fold(FoldContext.small()), equalTo("t"));
         assertThat(lookup.matchFields(), hasSize(1));
         var matchField = as(lookup.matchFields().get(0), UnresolvedAttribute.class);
         assertThat(matchField.name(), equalTo("j"));
@@ -2306,7 +2304,7 @@ public void testMatchOperatorConstantQueryString() {
         var match = (Match) filter.condition();
         var matchField = (UnresolvedAttribute) match.field();
         assertThat(matchField.name(), equalTo("field"));
-        assertThat(match.query().fold(), equalTo("value"));
+        assertThat(match.query().fold(FoldContext.small()), equalTo("value"));
     }
 
     public void testInvalidMatchOperator() {
@@ -2341,7 +2339,7 @@ public void testMatchFunctionFieldCasting() {
         var toInteger = (ToInteger) function.children().get(0);
         var matchField = (UnresolvedAttribute) toInteger.field();
         assertThat(matchField.name(), equalTo("field"));
-        assertThat(function.children().get(1).fold(), equalTo("value"));
+        assertThat(function.children().get(1).fold(FoldContext.small()), equalTo("value"));
     }
 
     public void testMatchOperatorFieldCasting() {
@@ -2351,6 +2349,13 @@ public void testMatchOperatorFieldCasting() {
         var toInteger = (ToInteger) match.field();
         var matchField = (UnresolvedAttribute) toInteger.field();
         assertThat(matchField.name(), equalTo("field"));
-        assertThat(match.query().fold(), equalTo("value"));
+        assertThat(match.query().fold(FoldContext.small()), equalTo("value"));
+    }
+
+    public void testFailingMetadataWithSquareBrackets() {
+        expectError(
+            "FROM test [METADATA _index] | STATS count(*)",
+            "line 1:11: mismatched input '[' expecting {<EOF>, '|', ',', 'metadata'}"
+        );
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java
index a254207865ad5..2f47a672a68d0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
@@ -42,7 +43,7 @@ public void testTransformWithExpressionTopLevel() throws Exception {
 
         assertEquals(Limit.class, transformed.getClass());
         Limit l = (Limit) transformed;
-        assertEquals(24, l.limit().fold());
+        assertEquals(24, l.limit().fold(FoldContext.small()));
     }
 
     public void testTransformWithExpressionTree() throws Exception {
@@ -53,7 +54,7 @@ public void testTransformWithExpressionTree() throws Exception {
         assertEquals(OrderBy.class, transformed.getClass());
         OrderBy order = (OrderBy) transformed;
         assertEquals(Limit.class, order.child().getClass());
-        assertEquals(24, ((Limit) order.child()).limit().fold());
+        assertEquals(24, ((Limit) order.child()).limit().fold(FoldContext.small()));
     }
 
     public void testTransformWithExpressionTopLevelInCollection() throws Exception {
@@ -83,12 +84,12 @@ public void testForEachWithExpressionTopLevel() throws Exception {
 
         List<Object> list = new ArrayList<>();
         project.forEachExpression(Literal.class, l -> {
-            if (l.fold().equals(42)) {
-                list.add(l.fold());
+            if (l.value().equals(42)) {
+                list.add(l.value());
             }
         });
 
-        assertEquals(singletonList(one.child().fold()), list);
+        assertEquals(singletonList(one.child().fold(FoldContext.small())), list);
     }
 
     public void testForEachWithExpressionTree() throws Exception {
@@ -97,12 +98,12 @@ public void testForEachWithExpressionTree() throws Exception {
 
         List<Object> list = new ArrayList<>();
         o.forEachExpressionDown(Literal.class, l -> {
-            if (l.fold().equals(42)) {
-                list.add(l.fold());
+            if (l.value().equals(42)) {
+                list.add(l.value());
             }
         });
 
-        assertEquals(singletonList(limit.limit().fold()), list);
+        assertEquals(singletonList(limit.limit().fold(FoldContext.small())), list);
     }
 
     public void testForEachWithExpressionTopLevelInCollection() throws Exception {
@@ -129,12 +130,12 @@ public void testForEachWithExpressionTreeInCollection() throws Exception {
 
         List<Object> list = new ArrayList<>();
         project.forEachExpression(Literal.class, l -> {
-            if (l.fold().equals(42)) {
-                list.add(l.fold());
+            if (l.value().equals(42)) {
+                list.add(l.value());
             }
         });
 
-        assertEquals(singletonList(one.child().fold()), list);
+        assertEquals(singletonList(one.child().fold(FoldContext.small())), list);
     }
 
     public void testPlanExpressions() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java
index 5a7547d011c0f..e2eb05b0c14d3 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.xpack.esql.TestBlockFactory;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
@@ -145,7 +146,7 @@ public void testEvaluatorSuppliers() {
         lb.append(LONG);
         Layout layout = lb.build();
 
-        var supplier = EvalMapper.toEvaluator(expression, layout);
+        var supplier = EvalMapper.toEvaluator(FoldContext.small(), expression, layout);
         EvalOperator.ExpressionEvaluator evaluator1 = supplier.get(driverContext());
         EvalOperator.ExpressionEvaluator evaluator2 = supplier.get(driverContext());
         assertNotNull(evaluator1);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java
index 55f32d07fc2cb..4191f42f08237 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
-import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
 import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerContext;
 import org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizer;
@@ -52,6 +51,7 @@
 import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning;
 import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization;
 import static org.elasticsearch.xpack.esql.core.util.Queries.Clause.FILTER;
@@ -78,7 +78,7 @@ public static void init() {
         Map<String, EsField> mapping = loadMapping("mapping-basic.json");
         EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD));
         IndexResolution getIndexResult = IndexResolution.valid(test);
-        logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG));
+        logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext());
         physicalPlanOptimizer = new PhysicalPlanOptimizer(new PhysicalOptimizerContext(EsqlTestUtils.TEST_CFG));
         mapper = new Mapper();
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java
index 5d8da21c6faad..a1648c67d9bd4 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.xpack.esql.TestBlockFactory;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -83,6 +84,7 @@ public void closeIndex() throws IOException {
     public void testLuceneSourceOperatorHugeRowSize() throws IOException {
         int estimatedRowSize = randomEstimatedRowSize(estimatedRowSizeIsHuge);
         LocalExecutionPlanner.LocalExecutionPlan plan = planner().plan(
+            FoldContext.small(),
             new EsQueryExec(Source.EMPTY, index(), IndexMode.STANDARD, List.of(), null, null, null, estimatedRowSize)
         );
         assertThat(plan.driverFactories.size(), lessThanOrEqualTo(pragmas.taskConcurrency()));
@@ -98,6 +100,7 @@ public void testLuceneTopNSourceOperator() throws IOException {
         EsQueryExec.FieldSort sort = new EsQueryExec.FieldSort(sortField, Order.OrderDirection.ASC, Order.NullsPosition.LAST);
         Literal limit = new Literal(Source.EMPTY, 10, DataType.INTEGER);
         LocalExecutionPlanner.LocalExecutionPlan plan = planner().plan(
+            FoldContext.small(),
             new EsQueryExec(Source.EMPTY, index(), IndexMode.STANDARD, List.of(), null, limit, List.of(sort), estimatedRowSize)
         );
         assertThat(plan.driverFactories.size(), lessThanOrEqualTo(pragmas.taskConcurrency()));
@@ -113,6 +116,7 @@ public void testLuceneTopNSourceOperatorDistanceSort() throws IOException {
         EsQueryExec.GeoDistanceSort sort = new EsQueryExec.GeoDistanceSort(sortField, Order.OrderDirection.ASC, 1, -1);
         Literal limit = new Literal(Source.EMPTY, 10, DataType.INTEGER);
         LocalExecutionPlanner.LocalExecutionPlan plan = planner().plan(
+            FoldContext.small(),
             new EsQueryExec(Source.EMPTY, index(), IndexMode.STANDARD, List.of(), null, limit, List.of(sort), estimatedRowSize)
         );
         assertThat(plan.driverFactories.size(), lessThanOrEqualTo(pragmas.taskConcurrency()));
@@ -187,7 +191,7 @@ private EsPhysicalOperationProviders esPhysicalOperationProviders() throws IOExc
             );
         }
         releasables.add(searcher);
-        return new EsPhysicalOperationProviders(shardContexts, null);
+        return new EsPhysicalOperationProviders(FoldContext.small(), shardContexts, null);
     }
 
     private IndexReader reader() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java
index e91fc6e49312d..628737aa36c6c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java
@@ -30,16 +30,23 @@
 import org.elasticsearch.compute.operator.OrdinalsGroupingOperator;
 import org.elasticsearch.compute.operator.SourceOperator;
 import org.elasticsearch.compute.operator.SourceOperator.SourceOperatorFactory;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
-import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference;
 import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.plugins.scanners.StablePluginsRegistry;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.TestBlockFactory;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField;
 import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes;
+import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction;
 import org.elasticsearch.xpack.esql.plan.physical.AggregateExec;
 import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec;
 import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec;
@@ -48,8 +55,12 @@
 import org.elasticsearch.xpack.ml.MachineLearning;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.OptionalInt;
 import java.util.Random;
+import java.util.function.BiFunction;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.IntStream;
@@ -57,26 +68,33 @@
 import static com.carrotsearch.randomizedtesting.generators.RandomNumbers.randomIntBetween;
 import static java.util.stream.Collectors.joining;
 import static org.apache.lucene.tests.util.LuceneTestCase.createTempDir;
-import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.DOC_VALUES;
-import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.NONE;
 
 public class TestPhysicalOperationProviders extends AbstractPhysicalOperationProviders {
+    private final List<IndexPage> indexPages;
 
-    private final Page testData;
-    private final List<String> columnNames;
+    private TestPhysicalOperationProviders(FoldContext foldContext, List<IndexPage> indexPages, AnalysisRegistry analysisRegistry) {
+        super(foldContext, analysisRegistry);
+        this.indexPages = indexPages;
+    }
 
-    public TestPhysicalOperationProviders(Page testData, List<String> columnNames) throws IOException {
-        super(
-            new AnalysisModule(
-                TestEnvironment.newEnvironment(
-                    Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build()
-                ),
-                List.of(new MachineLearning(Settings.EMPTY), new CommonAnalysisPlugin()),
-                new StablePluginsRegistry()
-            ).getAnalysisRegistry()
-        );
-        this.testData = testData;
-        this.columnNames = columnNames;
+    public static TestPhysicalOperationProviders create(FoldContext foldContext, List<IndexPage> indexPages) throws IOException {
+        return new TestPhysicalOperationProviders(foldContext, indexPages, createAnalysisRegistry());
+    }
+
+    public record IndexPage(String index, Page page, List<String> columnNames) {
+        OptionalInt columnIndex(String columnName) {
+            return IntStream.range(0, columnNames.size()).filter(i -> columnNames.get(i).equals(columnName)).findFirst();
+        }
+    }
+
+    private static AnalysisRegistry createAnalysisRegistry() throws IOException {
+        return new AnalysisModule(
+            TestEnvironment.newEnvironment(
+                Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build()
+            ),
+            List.of(new MachineLearning(Settings.EMPTY), new CommonAnalysisPlugin()),
+            new StablePluginsRegistry()
+        ).getAnalysisRegistry();
     }
 
     @Override
@@ -86,7 +104,10 @@ public PhysicalOperation fieldExtractPhysicalOperation(FieldExtractExec fieldExt
         for (Attribute attr : fieldExtractExec.attributesToExtract()) {
             layout.append(attr);
             op = op.with(
-                new TestFieldExtractOperatorFactory(attr, PlannerUtils.extractPreference(fieldExtractExec.hasDocValuesAttribute(attr))),
+                new TestFieldExtractOperatorFactory(
+                    attr,
+                    PlannerUtils.extractPreference(fieldExtractExec.docValuesAttributes().contains(attr))
+                ),
                 layout.build()
             );
         }
@@ -115,13 +136,12 @@ public Operator.OperatorFactory ordinalGroupingOperatorFactory(
             aggregatorFactories,
             groupElementType,
             context.bigArrays(),
-            attrSource.name()
+            attrSource
         );
     }
 
     private class TestSourceOperator extends SourceOperator {
-
-        boolean finished = false;
+        private int index = 0;
         private final DriverContext driverContext;
 
         TestSourceOperator(DriverContext driverContext) {
@@ -130,28 +150,29 @@ private class TestSourceOperator extends SourceOperator {
 
         @Override
         public Page getOutput() {
-            if (finished == false) {
-                finish();
-            }
-
+            var pageIndex = indexPages.get(index);
+            var page = pageIndex.page;
             BlockFactory blockFactory = driverContext.blockFactory();
             DocVector docVector = new DocVector(
-                blockFactory.newConstantIntVector(0, testData.getPositionCount()),
-                blockFactory.newConstantIntVector(0, testData.getPositionCount()),
-                blockFactory.newIntArrayVector(IntStream.range(0, testData.getPositionCount()).toArray(), testData.getPositionCount()),
+                // The shard ID is used to encode the index ID.
+                blockFactory.newConstantIntVector(index, page.getPositionCount()),
+                blockFactory.newConstantIntVector(0, page.getPositionCount()),
+                blockFactory.newIntArrayVector(IntStream.range(0, page.getPositionCount()).toArray(), page.getPositionCount()),
                 true
             );
-            return new Page(docVector.asBlock());
+            var block = docVector.asBlock();
+            index++;
+            return new Page(block);
         }
 
         @Override
         public boolean isFinished() {
-            return finished;
+            return index == indexPages.size();
         }
 
         @Override
         public void finish() {
-            finished = true;
+            index = indexPages.size();
         }
 
         @Override
@@ -174,24 +195,19 @@ public String describe() {
     }
 
     private class TestFieldExtractOperator implements Operator {
-
+        private final Attribute attribute;
         private Page lastPage;
         boolean finished;
-        String columnName;
-        private final DataType dataType;
-        private final MappedFieldType.FieldExtractPreference extractPreference;
-
-        TestFieldExtractOperator(String columnName, DataType dataType, MappedFieldType.FieldExtractPreference extractPreference) {
-            assert columnNames.contains(columnName);
-            this.columnName = columnName;
-            this.dataType = dataType;
+        private final FieldExtractPreference extractPreference;
+
+        TestFieldExtractOperator(Attribute attr, FieldExtractPreference extractPreference) {
+            this.attribute = attr;
             this.extractPreference = extractPreference;
         }
 
         @Override
         public void addInput(Page page) {
-            Block block = extractBlockForColumn(page, columnName, dataType, extractPreference);
-            lastPage = page.appendBlock(block);
+            lastPage = page.appendBlock(getBlock(page.getBlock(0), attribute, extractPreference));
         }
 
         @Override
@@ -223,12 +239,12 @@ public void close() {
     }
 
     private class TestFieldExtractOperatorFactory implements Operator.OperatorFactory {
-        final Operator op;
-        private String columnName;
+        private final Operator op;
+        private final Attribute attribute;
 
-        TestFieldExtractOperatorFactory(Attribute attr, MappedFieldType.FieldExtractPreference extractPreference) {
-            this.op = new TestFieldExtractOperator(attr.name(), attr.dataType(), extractPreference);
-            this.columnName = attr.name();
+        TestFieldExtractOperatorFactory(Attribute attr, FieldExtractPreference extractPreference) {
+            this.op = new TestFieldExtractOperator(attr, extractPreference);
+            this.attribute = attr;
         }
 
         @Override
@@ -238,27 +254,88 @@ public Operator get(DriverContext driverContext) {
 
         @Override
         public String describe() {
-            return "TestFieldExtractOperator(" + columnName + ")";
+            return "TestFieldExtractOperator(" + attribute.name() + ")";
+        }
+    }
+
+    private Block getBlock(DocBlock docBlock, Attribute attribute, FieldExtractPreference extractPreference) {
+        if (attribute instanceof UnsupportedAttribute) {
+            return docBlock.blockFactory().newConstantNullBlock(docBlock.getPositionCount());
+        }
+        return extractBlockForColumn(
+            docBlock,
+            attribute.dataType(),
+            extractPreference,
+            attribute instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField multiTypeEsField
+                ? (indexDoc, blockCopier) -> getBlockForMultiType(indexDoc, multiTypeEsField, blockCopier)
+                : (indexDoc, blockCopier) -> extractBlockForSingleDoc(indexDoc, attribute.name(), blockCopier)
+        );
+    }
+
+    private Block getBlockForMultiType(DocBlock indexDoc, MultiTypeEsField multiTypeEsField, TestBlockCopier blockCopier) {
+        var indexId = indexDoc.asVector().shards().getInt(0);
+        var indexPage = indexPages.get(indexId);
+        var conversion = (AbstractConvertFunction) multiTypeEsField.getConversionExpressionForIndex(indexPage.index);
+        Supplier<Block> nulls = () -> indexDoc.blockFactory().newConstantNullBlock(indexDoc.getPositionCount());
+        if (conversion == null) {
+            return nulls.get();
+        }
+        var field = (FieldAttribute) conversion.field();
+        return indexPage.columnIndex(field.fieldName()).isEmpty()
+            ? nulls.get()
+            : TypeConverter.fromConvertFunction(conversion).convert(extractBlockForSingleDoc(indexDoc, field.fieldName(), blockCopier));
+    }
+
+    private Block extractBlockForSingleDoc(DocBlock docBlock, String columnName, TestBlockCopier blockCopier) {
+        var indexId = docBlock.asVector().shards().getInt(0);
+        var indexPage = indexPages.get(indexId);
+        int columnIndex = indexPage.columnIndex(columnName)
+            .orElseThrow(() -> new EsqlIllegalArgumentException("Cannot find column named [{}] in {}", columnName, indexPage.columnNames));
+        var originalData = indexPage.page.getBlock(columnIndex);
+        return blockCopier.copyBlock(originalData);
+    }
+
+    private static void foreachIndexDoc(DocBlock docBlock, Consumer<DocBlock> indexDocConsumer) {
+        var currentIndex = -1;
+        List<Integer> currentList = null;
+        DocVector vector = docBlock.asVector();
+        for (int i = 0; i < docBlock.getPositionCount(); i++) {
+            int indexId = vector.shards().getInt(i);
+            if (indexId != currentIndex) {
+                consumeIndexDoc(indexDocConsumer, vector, currentList);
+                currentList = new ArrayList<>();
+                currentIndex = indexId;
+            }
+            currentList.add(i);
+        }
+        consumeIndexDoc(indexDocConsumer, vector, currentList);
+    }
+
+    private static void consumeIndexDoc(Consumer<DocBlock> indexDocConsumer, DocVector vector, @Nullable List<Integer> currentList) {
+        if (currentList != null) {
+            try (DocVector indexDocVector = vector.filter(currentList.stream().mapToInt(Integer::intValue).toArray())) {
+                indexDocConsumer.accept(indexDocVector.asBlock());
+            }
         }
     }
 
     private class TestHashAggregationOperator extends HashAggregationOperator {
 
-        private final String columnName;
+        private final Attribute attribute;
 
         TestHashAggregationOperator(
             List<GroupingAggregator.Factory> aggregators,
             Supplier<BlockHash> blockHash,
-            String columnName,
+            Attribute attribute,
             DriverContext driverContext
         ) {
             super(aggregators, blockHash, driverContext);
-            this.columnName = columnName;
+            this.attribute = attribute;
         }
 
         @Override
         protected Page wrapPage(Page page) {
-            return page.appendBlock(extractBlockForColumn(page, columnName, null, NONE));
+            return page.appendBlock(getBlock(page.getBlock(0), attribute, FieldExtractPreference.NONE));
         }
     }
 
@@ -267,24 +344,24 @@ protected Page wrapPage(Page page) {
      * {@link HashAggregationOperator}.
      */
     private class TestOrdinalsGroupingAggregationOperatorFactory implements Operator.OperatorFactory {
-        private int groupByChannel;
-        private List<GroupingAggregator.Factory> aggregators;
-        private ElementType groupElementType;
-        private BigArrays bigArrays;
-        private String columnName;
+        private final int groupByChannel;
+        private final List<GroupingAggregator.Factory> aggregators;
+        private final ElementType groupElementType;
+        private final BigArrays bigArrays;
+        private final Attribute attribute;
 
         TestOrdinalsGroupingAggregationOperatorFactory(
             int channelIndex,
             List<GroupingAggregator.Factory> aggregatorFactories,
             ElementType groupElementType,
             BigArrays bigArrays,
-            String name
+            Attribute attribute
         ) {
             this.groupByChannel = channelIndex;
             this.aggregators = aggregatorFactories;
             this.groupElementType = groupElementType;
             this.bigArrays = bigArrays;
-            this.columnName = name;
+            this.attribute = attribute;
         }
 
         @Override
@@ -299,7 +376,7 @@ public Operator get(DriverContext driverContext) {
                     pageSize,
                     false
                 ),
-                columnName,
+                attribute,
                 driverContext
             );
         }
@@ -315,32 +392,34 @@ public String describe() {
     }
 
     private Block extractBlockForColumn(
-        Page page,
-        String columnName,
+        DocBlock docBlock,
         DataType dataType,
-        MappedFieldType.FieldExtractPreference extractPreference
+        FieldExtractPreference extractPreference,
+        BiFunction<DocBlock, TestBlockCopier, Block> extractBlock
     ) {
-        var columnIndex = -1;
-        // locate the block index corresponding to "columnName"
-        for (int i = 0, size = columnNames.size(); i < size && columnIndex < 0; i++) {
-            if (columnNames.get(i).equals(columnName)) {
-                columnIndex = i;
-            }
-        }
-        if (columnIndex < 0) {
-            throw new EsqlIllegalArgumentException("Cannot find column named [{}] in {}", columnName, columnNames);
+        BlockFactory blockFactory = docBlock.blockFactory();
+        boolean mapToDocValues = shouldMapToDocValues(dataType, extractPreference);
+        try (
+            Block.Builder blockBuilder = mapToDocValues
+                ? blockFactory.newLongBlockBuilder(docBlock.getPositionCount())
+                : blockBuilder(dataType, docBlock.getPositionCount(), TestBlockFactory.getNonBreakingInstance())
+        ) {
+            foreachIndexDoc(docBlock, indexDoc -> {
+                TestBlockCopier blockCopier = mapToDocValues
+                    ? TestSpatialPointStatsBlockCopier.create(indexDoc.asVector().docs(), dataType)
+                    : new TestBlockCopier(indexDoc.asVector().docs());
+                Block blockForIndex = extractBlock.apply(indexDoc, blockCopier);
+                blockBuilder.copyFrom(blockForIndex, 0, blockForIndex.getPositionCount());
+            });
+            var result = blockBuilder.build();
+            assert result.getPositionCount() == docBlock.getPositionCount()
+                : "Expected " + docBlock.getPositionCount() + " rows, got " + result.getPositionCount();
+            return result;
         }
-        DocBlock docBlock = page.getBlock(0);
-        IntVector docIndices = docBlock.asVector().docs();
-        Block originalData = testData.getBlock(columnIndex);
-        var blockCopier = shouldMapToDocValues(dataType, extractPreference)
-            ? TestSpatialPointStatsBlockCopier.create(docIndices, dataType)
-            : new TestBlockCopier(docIndices);
-        return blockCopier.copyBlock(originalData);
     }
 
-    private boolean shouldMapToDocValues(DataType dataType, MappedFieldType.FieldExtractPreference extractPreference) {
-        return extractPreference == DOC_VALUES && DataType.isSpatialPoint(dataType);
+    private boolean shouldMapToDocValues(DataType dataType, FieldExtractPreference extractPreference) {
+        return extractPreference == FieldExtractPreference.DOC_VALUES && DataType.isSpatialPoint(dataType);
     }
 
     private static class TestBlockCopier {
@@ -406,9 +485,9 @@ protected Block copyBlock(Block originalData) {
         }
 
         private static TestSpatialPointStatsBlockCopier create(IntVector docIndices, DataType dataType) {
-            Function<BytesRef, Long> encoder = switch (dataType.esType()) {
-                case "geo_point" -> SpatialCoordinateTypes.GEO::wkbAsLong;
-                case "cartesian_point" -> SpatialCoordinateTypes.CARTESIAN::wkbAsLong;
+            Function<BytesRef, Long> encoder = switch (dataType) {
+                case GEO_POINT -> SpatialCoordinateTypes.GEO::wkbAsLong;
+                case CARTESIAN_POINT -> SpatialCoordinateTypes.CARTESIAN::wkbAsLong;
                 default -> throw new IllegalArgumentException("Unsupported spatial data type: " + dataType);
             };
             return new TestSpatialPointStatsBlockCopier(docIndices) {
@@ -419,4 +498,13 @@ protected long encode(BytesRef wkb) {
             };
         }
     }
+
+    private static Block.Builder blockBuilder(DataType dataType, int estimatedSize, BlockFactory blockFactory) {
+        ElementType elementType = switch (dataType) {
+            case SHORT -> ElementType.INT;
+            case FLOAT, HALF_FLOAT, SCALED_FLOAT -> ElementType.DOUBLE;
+            default -> PlannerUtils.toElementType(dataType);
+        };
+        return elementType.newBlockBuilder(estimatedSize, blockFactory);
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java
index f2a619f0dbd89..f3b1d84e507a5 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
 import org.elasticsearch.xpack.esql.index.EsIndex;
 import org.elasticsearch.xpack.esql.index.IndexResolution;
-import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
@@ -39,10 +38,10 @@
 
 import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration;
 import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomTables;
-import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_CFG;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptyPolicyResolution;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -187,7 +186,7 @@ static LogicalPlan parse(String query) {
         Map<String, EsField> mapping = loadMapping("mapping-basic.json");
         EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD));
         IndexResolution getIndexResult = IndexResolution.valid(test);
-        var logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(TEST_CFG));
+        var logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext());
         var analyzer = new Analyzer(
             new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), getIndexResult, emptyPolicyResolution()),
             TEST_VERIFIER
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java
index b606f99df437c..ae0db127491f7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java
@@ -460,8 +460,7 @@ public void testCancelOnFailure() throws Exception {
                 }), TimeValue.timeValueNanos(between(0, 100)), threadPool.generic());
             }
         }
-        assertBusy(rootListener::isDone);
-        ExecutionException failure = expectThrows(ExecutionException.class, () -> rootListener.get(1, TimeUnit.SECONDS));
+        ExecutionException failure = expectThrows(ExecutionException.class, () -> rootListener.get(10, TimeUnit.SECONDS));
         Throwable cause = failure.getCause();
         assertNotNull(failure);
         assertThat(cause, instanceOf(CircuitBreakingException.class));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java
index d1ce064c35d81..fac3495697da8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.analysis.Analyzer;
 import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
 import org.elasticsearch.xpack.esql.index.EsIndex;
@@ -92,7 +93,8 @@ protected DataNodeRequest createTestInstance() {
             aliasFilters,
             physicalPlan,
             generateRandomStringArray(10, 10, false, false),
-            IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())
+            IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()),
+            randomBoolean()
         );
         request.setParentTask(randomAlphaOfLength(10), randomNonNegativeLong());
         return request;
@@ -100,7 +102,7 @@ protected DataNodeRequest createTestInstance() {
 
     @Override
     protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException {
-        return switch (between(0, 8)) {
+        return switch (between(0, 9)) {
             case 0 -> {
                 var request = new DataNodeRequest(
                     randomAlphaOfLength(20),
@@ -110,7 +112,8 @@ protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException
                     in.aliasFilters(),
                     in.plan(),
                     in.indices(),
-                    in.indicesOptions()
+                    in.indicesOptions(),
+                    in.runNodeLevelReduction()
                 );
                 request.setParentTask(in.getParentTask());
                 yield request;
@@ -124,7 +127,8 @@ protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException
                     in.aliasFilters(),
                     in.plan(),
                     in.indices(),
-                    in.indicesOptions()
+                    in.indicesOptions(),
+                    in.runNodeLevelReduction()
                 );
                 request.setParentTask(in.getParentTask());
                 yield request;
@@ -139,7 +143,8 @@ protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException
                     in.aliasFilters(),
                     in.plan(),
                     in.indices(),
-                    in.indicesOptions()
+                    in.indicesOptions(),
+                    in.runNodeLevelReduction()
                 );
                 request.setParentTask(in.getParentTask());
                 yield request;
@@ -166,7 +171,8 @@ protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException
                     in.aliasFilters(),
                     mapAndMaybeOptimize(parse(newQuery)),
                     in.indices(),
-                    in.indicesOptions()
+                    in.indicesOptions(),
+                    in.runNodeLevelReduction()
                 );
                 request.setParentTask(in.getParentTask());
                 yield request;
@@ -186,7 +192,8 @@ protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException
                     aliasFilters,
                     in.plan(),
                     in.indices(),
-                    in.indicesOptions()
+                    in.indicesOptions(),
+                    in.runNodeLevelReduction()
                 );
                 request.setParentTask(request.getParentTask());
                 yield request;
@@ -200,7 +207,8 @@ protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException
                     in.aliasFilters(),
                     in.plan(),
                     in.indices(),
-                    in.indicesOptions()
+                    in.indicesOptions(),
+                    in.runNodeLevelReduction()
                 );
                 request.setParentTask(
                     randomValueOtherThan(request.getParentTask().getNodeId(), () -> randomAlphaOfLength(10)),
@@ -218,7 +226,8 @@ protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException
                     in.aliasFilters(),
                     in.plan(),
                     in.indices(),
-                    in.indicesOptions()
+                    in.indicesOptions(),
+                    in.runNodeLevelReduction()
                 );
                 request.setParentTask(request.getParentTask());
                 yield request;
@@ -233,7 +242,8 @@ protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException
                     in.aliasFilters(),
                     in.plan(),
                     indices,
-                    in.indicesOptions()
+                    in.indicesOptions(),
+                    in.runNodeLevelReduction()
                 );
                 request.setParentTask(request.getParentTask());
                 yield request;
@@ -251,7 +261,23 @@ protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException
                     in.aliasFilters(),
                     in.plan(),
                     in.indices(),
-                    indicesOptions
+                    indicesOptions,
+                    in.runNodeLevelReduction()
+                );
+                request.setParentTask(request.getParentTask());
+                yield request;
+            }
+            case 9 -> {
+                var request = new DataNodeRequest(
+                    in.sessionId(),
+                    in.configuration(),
+                    in.clusterAlias(),
+                    in.shardIds(),
+                    in.aliasFilters(),
+                    in.plan(),
+                    in.indices(),
+                    in.indicesOptions(),
+                    in.runNodeLevelReduction() == false
                 );
                 request.setParentTask(request.getParentTask());
                 yield request;
@@ -264,7 +290,7 @@ static LogicalPlan parse(String query) {
         Map<String, EsField> mapping = loadMapping("mapping-basic.json");
         EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD));
         IndexResolution getIndexResult = IndexResolution.valid(test);
-        var logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(TEST_CFG));
+        var logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(TEST_CFG, FoldContext.small()));
         var analyzer = new Analyzer(
             new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), getIndexResult, emptyPolicyResolution()),
             TEST_VERIFIER
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java
index d0c5ddd0dc927..fcb99c92af25d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java
@@ -39,7 +39,8 @@ public void testNoIndexPlaceholder() {
             Collections.emptyMap(),
             null,
             generateRandomStringArray(10, 10, false, false),
-            IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())
+            IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()),
+            randomBoolean()
         );
 
         assertThat(request.shardIds(), equalTo(shardIds));
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQueryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/BoolQueryTests.java
similarity index 92%
rename from x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQueryTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/BoolQueryTests.java
index 1c9d6bc54aebf..1aa5d47ed07ea 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQueryTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/BoolQueryTests.java
@@ -4,9 +4,13 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.core.querydsl.query;
+package org.elasticsearch.xpack.esql.querydsl.query;
 
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.core.querydsl.query.BoolQuery;
+import org.elasticsearch.xpack.esql.core.querydsl.query.ExistsQuery;
+import org.elasticsearch.xpack.esql.core.querydsl.query.NotQuery;
+import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.tree.SourceTests;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQueryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQueryTests.java
similarity index 96%
rename from x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQueryTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQueryTests.java
index 4316bd21ffe94..49d1a9ad19d09 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQueryTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQueryTests.java
@@ -4,17 +4,17 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.core.querydsl.query;
+package org.elasticsearch.xpack.esql.querydsl.query;
 
 import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
-import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.tree.SourceTests;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
+import org.elasticsearch.xpack.esql.expression.predicate.fulltext.MatchQueryPredicate;
 
 import java.util.Arrays;
 import java.util.List;
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/MultiMatchQueryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQueryTests.java
similarity index 94%
rename from x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/MultiMatchQueryTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQueryTests.java
index 9ca9765ed0542..93c285f5e3ab0 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/MultiMatchQueryTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQueryTests.java
@@ -4,14 +4,14 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.core.querydsl.query;
+package org.elasticsearch.xpack.esql.querydsl.query;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MultiMatchQueryPredicate;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
+import org.elasticsearch.xpack.esql.expression.predicate.fulltext.MultiMatchQueryPredicate;
 
 import java.util.HashMap;
 import java.util.Map;
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/QueryStringQueryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/QueryStringQueryTests.java
similarity index 94%
rename from x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/QueryStringQueryTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/QueryStringQueryTests.java
index 22e7b93e84ce1..3114b852aac70 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/QueryStringQueryTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/QueryStringQueryTests.java
@@ -4,12 +4,13 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.core.querydsl.query;
+package org.elasticsearch.xpack.esql.querydsl.query;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.index.query.QueryStringQueryBuilder;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtilsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtilsTests.java
index 1000c05282fdb..6b01010ffa5f4 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtilsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtilsTests.java
@@ -644,6 +644,7 @@ public void testMissingIndicesIsFatal() {
 
     public void testCheckForCcsLicense() {
         final TestIndicesExpressionGrouper indicesGrouper = new TestIndicesExpressionGrouper();
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
 
         // this seems to be used only for tracking usage of features, not for checking if a license is expired
         final LongSupplier currTime = () -> System.currentTimeMillis();
@@ -671,22 +672,22 @@ public void testCheckForCcsLicense() {
             List<TableInfo> indices = new ArrayList<>();
             indices.add(new TableInfo(new TableIdentifier(EMPTY, null, randomFrom("idx", "idx1,idx2*"))));
 
-            checkForCcsLicense(indices, indicesGrouper, enterpriseLicenseValid);
-            checkForCcsLicense(indices, indicesGrouper, platinumLicenseValid);
-            checkForCcsLicense(indices, indicesGrouper, goldLicenseValid);
-            checkForCcsLicense(indices, indicesGrouper, trialLicenseValid);
-            checkForCcsLicense(indices, indicesGrouper, basicLicenseValid);
-            checkForCcsLicense(indices, indicesGrouper, standardLicenseValid);
-            checkForCcsLicense(indices, indicesGrouper, missingLicense);
-            checkForCcsLicense(indices, indicesGrouper, nullLicense);
-
-            checkForCcsLicense(indices, indicesGrouper, enterpriseLicenseInactive);
-            checkForCcsLicense(indices, indicesGrouper, platinumLicenseInactive);
-            checkForCcsLicense(indices, indicesGrouper, goldLicenseInactive);
-            checkForCcsLicense(indices, indicesGrouper, trialLicenseInactive);
-            checkForCcsLicense(indices, indicesGrouper, basicLicenseInactive);
-            checkForCcsLicense(indices, indicesGrouper, standardLicenseInactive);
-            checkForCcsLicense(indices, indicesGrouper, missingLicenseInactive);
+            checkForCcsLicense(executionInfo, indices, indicesGrouper, enterpriseLicenseValid);
+            checkForCcsLicense(executionInfo, indices, indicesGrouper, platinumLicenseValid);
+            checkForCcsLicense(executionInfo, indices, indicesGrouper, goldLicenseValid);
+            checkForCcsLicense(executionInfo, indices, indicesGrouper, trialLicenseValid);
+            checkForCcsLicense(executionInfo, indices, indicesGrouper, basicLicenseValid);
+            checkForCcsLicense(executionInfo, indices, indicesGrouper, standardLicenseValid);
+            checkForCcsLicense(executionInfo, indices, indicesGrouper, missingLicense);
+            checkForCcsLicense(executionInfo, indices, indicesGrouper, nullLicense);
+
+            checkForCcsLicense(executionInfo, indices, indicesGrouper, enterpriseLicenseInactive);
+            checkForCcsLicense(executionInfo, indices, indicesGrouper, platinumLicenseInactive);
+            checkForCcsLicense(executionInfo, indices, indicesGrouper, goldLicenseInactive);
+            checkForCcsLicense(executionInfo, indices, indicesGrouper, trialLicenseInactive);
+            checkForCcsLicense(executionInfo, indices, indicesGrouper, basicLicenseInactive);
+            checkForCcsLicense(executionInfo, indices, indicesGrouper, standardLicenseInactive);
+            checkForCcsLicense(executionInfo, indices, indicesGrouper, missingLicenseInactive);
         }
 
         // cross-cluster search requires a valid (active, non-expired) enterprise license OR a valid trial license
@@ -701,8 +702,8 @@ public void testCheckForCcsLicense() {
             }
 
             // licenses that work
-            checkForCcsLicense(indices, indicesGrouper, enterpriseLicenseValid);
-            checkForCcsLicense(indices, indicesGrouper, trialLicenseValid);
+            checkForCcsLicense(executionInfo, indices, indicesGrouper, enterpriseLicenseValid);
+            checkForCcsLicense(executionInfo, indices, indicesGrouper, trialLicenseValid);
 
             // all others fail ---
 
@@ -739,9 +740,10 @@ private void assertLicenseCheckFails(
         XPackLicenseState licenseState,
         String expectedErrorMessageSuffix
     ) {
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
         ElasticsearchStatusException e = expectThrows(
             ElasticsearchStatusException.class,
-            () -> checkForCcsLicense(indices, indicesGrouper, licenseState)
+            () -> checkForCcsLicense(executionInfo, indices, indicesGrouper, licenseState)
         );
         assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
         assertThat(
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java
index 0fe89b24dfc6a..4db4f7925d4ff 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.Build;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 import org.elasticsearch.xpack.esql.parser.ParsingException;
 
@@ -1316,25 +1317,25 @@ public void testCountStar() {
     }
 
     public void testEnrichOnDefaultFieldWithKeep() {
-        Set<String> fieldNames = EsqlSession.fieldNames(parser.createStatement("""
+        Set<String> fieldNames = fieldNames("""
             from employees
             | enrich languages_policy
-            | keep emp_no"""), Set.of("language_name"));
+            | keep emp_no""", Set.of("language_name"));
         assertThat(fieldNames, equalTo(Set.of("emp_no", "emp_no.*", "language_name", "language_name.*")));
     }
 
     public void testDissectOverwriteName() {
-        Set<String> fieldNames = EsqlSession.fieldNames(parser.createStatement("""
+        Set<String> fieldNames = fieldNames("""
             from employees
             | dissect first_name "%{first_name} %{more}"
-            | keep emp_no, first_name, more"""), Set.of());
+            | keep emp_no, first_name, more""", Set.of());
         assertThat(fieldNames, equalTo(Set.of("emp_no", "emp_no.*", "first_name", "first_name.*")));
     }
 
     public void testEnrichOnDefaultField() {
-        Set<String> fieldNames = EsqlSession.fieldNames(parser.createStatement("""
+        Set<String> fieldNames = fieldNames("""
             from employees
-            | enrich languages_policy"""), Set.of("language_name"));
+            | enrich languages_policy""", Set.of("language_name"));
         assertThat(fieldNames, equalTo(ALL_FIELDS));
     }
 
@@ -1345,7 +1346,7 @@ public void testMetrics() {
             assertThat(e.getMessage(), containsString("line 1:1: mismatched input 'METRICS' expecting {"));
             return;
         }
-        Set<String> fieldNames = EsqlSession.fieldNames(parser.createStatement(query), Set.of());
+        Set<String> fieldNames = fieldNames(query, Set.of());
         assertThat(
             fieldNames,
             equalTo(
@@ -1363,8 +1364,230 @@ public void testMetrics() {
         );
     }
 
+    public void testLookupJoin() {
+        assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+        assertFieldNames(
+            "FROM employees | KEEP languages | RENAME languages AS language_code | LOOKUP JOIN languages_lookup ON language_code",
+            Set.of("languages", "languages.*", "language_code", "language_code.*"),
+            Set.of("languages_lookup") // Since we have KEEP before the LOOKUP JOIN we need to wildcard the lookup index
+        );
+    }
+
+    public void testLookupJoinKeep() {
+        assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+        assertFieldNames(
+            """
+                FROM employees
+                | KEEP languages
+                | RENAME languages AS language_code
+                | LOOKUP JOIN languages_lookup ON language_code
+                | KEEP languages, language_code, language_name""",
+            Set.of("languages", "languages.*", "language_code", "language_code.*", "language_name", "language_name.*"),
+            Set.of()  // Since we have KEEP after the LOOKUP, we can use the global field names instead of wildcarding the lookup index
+        );
+    }
+
+    public void testLookupJoinKeepWildcard() {
+        assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+        assertFieldNames(
+            """
+                FROM employees
+                | KEEP languages
+                | RENAME languages AS language_code
+                | LOOKUP JOIN languages_lookup ON language_code
+                | KEEP language*""",
+            Set.of("language*", "languages", "languages.*", "language_code", "language_code.*"),
+            Set.of()  // Since we have KEEP after the LOOKUP, we can use the global field names instead of wildcarding the lookup index
+        );
+    }
+
+    public void testMultiLookupJoin() {
+        assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+        assertFieldNames(
+            """
+                FROM sample_data
+                | EVAL client_ip = client_ip::keyword
+                | LOOKUP JOIN clientips_lookup ON client_ip
+                | LOOKUP JOIN message_types_lookup ON message""",
+            Set.of("*"), // With no KEEP we should keep all fields
+            Set.of() // since global field names are wildcarded, we don't need to wildcard any indices
+        );
+    }
+
+    public void testMultiLookupJoinKeepBefore() {
+        assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+        assertFieldNames(
+            """
+                FROM sample_data
+                | EVAL client_ip = client_ip::keyword
+                | KEEP @timestamp, client_ip, event_duration, message
+                | LOOKUP JOIN clientips_lookup ON client_ip
+                | LOOKUP JOIN message_types_lookup ON message""",
+            Set.of("@timestamp", "@timestamp.*", "client_ip", "client_ip.*", "event_duration", "event_duration.*", "message", "message.*"),
+            Set.of("clientips_lookup", "message_types_lookup") // Since the KEEP is before both JOINS we need to wildcard both indices
+        );
+    }
+
+    public void testMultiLookupJoinKeepBetween() {
+        assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+        assertFieldNames(
+            """
+                FROM sample_data
+                | EVAL client_ip = client_ip::keyword
+                | LOOKUP JOIN clientips_lookup ON client_ip
+                | KEEP @timestamp, client_ip, event_duration, message, env
+                | LOOKUP JOIN message_types_lookup ON message""",
+            Set.of(
+                "@timestamp",
+                "@timestamp.*",
+                "client_ip",
+                "client_ip.*",
+                "event_duration",
+                "event_duration.*",
+                "message",
+                "message.*",
+                "env",
+                "env.*"
+            ),
+            Set.of("message_types_lookup")  // Since the KEEP is before the second JOIN, we need to wildcard the second index
+        );
+    }
+
+    public void testMultiLookupJoinKeepAfter() {
+        assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+        assertFieldNames(
+            """
+                FROM sample_data
+                | EVAL client_ip = client_ip::keyword
+                | LOOKUP JOIN clientips_lookup ON client_ip
+                | LOOKUP JOIN message_types_lookup ON message
+                | KEEP @timestamp, client_ip, event_duration, message, env, type""",
+            Set.of(
+                "@timestamp",
+                "@timestamp.*",
+                "client_ip",
+                "client_ip.*",
+                "event_duration",
+                "event_duration.*",
+                "message",
+                "message.*",
+                "env",
+                "env.*",
+                "type",
+                "type.*"
+            ),
+            Set.of()  // Since the KEEP is after both JOINs, we can use the global field names
+        );
+    }
+
+    public void testMultiLookupJoinKeepAfterWildcard() {
+        assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+        assertFieldNames(
+            """
+                FROM sample_data
+                | EVAL client_ip = client_ip::keyword
+                | LOOKUP JOIN clientips_lookup ON client_ip
+                | LOOKUP JOIN message_types_lookup ON message
+                | KEEP *env*, *type*""",
+            Set.of("*env*", "*type*", "client_ip", "client_ip.*", "message", "message.*"),
+            Set.of()  // Since the KEEP is after both JOINs, we can use the global field names
+        );
+    }
+
+    public void testMultiLookupJoinSameIndex() {
+        assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+        assertFieldNames(
+            """
+                FROM sample_data
+                | EVAL client_ip = client_ip::keyword
+                | LOOKUP JOIN clientips_lookup ON client_ip
+                | EVAL client_ip = message
+                | LOOKUP JOIN clientips_lookup ON client_ip""",
+            Set.of("*"), // With no KEEP we should keep all fields
+            Set.of() // since global field names are wildcarded, we don't need to wildcard any indices
+        );
+    }
+
+    public void testMultiLookupJoinSameIndexKeepBefore() {
+        assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+        assertFieldNames(
+            """
+                FROM sample_data
+                | EVAL client_ip = client_ip::keyword
+                | KEEP @timestamp, client_ip, event_duration, message
+                | LOOKUP JOIN clientips_lookup ON client_ip
+                | EVAL client_ip = message
+                | LOOKUP JOIN clientips_lookup ON client_ip""",
+            Set.of("@timestamp", "@timestamp.*", "client_ip", "client_ip.*", "event_duration", "event_duration.*", "message", "message.*"),
+            Set.of("clientips_lookup") // Since there is no KEEP after the last JOIN, we need to wildcard the index
+        );
+    }
+
+    public void testMultiLookupJoinSameIndexKeepBetween() {
+        assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+        assertFieldNames(
+            """
+                FROM sample_data
+                | EVAL client_ip = client_ip::keyword
+                | LOOKUP JOIN clientips_lookup ON client_ip
+                | KEEP @timestamp, client_ip, event_duration, message, env
+                | EVAL client_ip = message
+                | LOOKUP JOIN clientips_lookup ON client_ip""",
+            Set.of(
+                "@timestamp",
+                "@timestamp.*",
+                "client_ip",
+                "client_ip.*",
+                "event_duration",
+                "event_duration.*",
+                "message",
+                "message.*",
+                "env",
+                "env.*"
+            ),
+            Set.of("clientips_lookup") // Since there is no KEEP after the last JOIN, we need to wildcard the index
+        );
+    }
+
+    public void testMultiLookupJoinSameIndexKeepAfter() {
+        assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled());
+        assertFieldNames(
+            """
+                FROM sample_data
+                | EVAL client_ip = client_ip::keyword
+                | LOOKUP JOIN clientips_lookup ON client_ip
+                | EVAL client_ip = message
+                | LOOKUP JOIN clientips_lookup ON client_ip
+                | KEEP @timestamp, client_ip, event_duration, message, env""",
+            Set.of(
+                "@timestamp",
+                "@timestamp.*",
+                "client_ip",
+                "client_ip.*",
+                "event_duration",
+                "event_duration.*",
+                "message",
+                "message.*",
+                "env",
+                "env.*"
+            ),
+            Set.of()  // Since the KEEP is after both JOINs, we can use the global field names
+        );
+    }
+
+    private Set<String> fieldNames(String query, Set<String> enrichPolicyMatchFields) {
+        var preAnalysisResult = new EsqlSession.PreAnalysisResult(null);
+        return EsqlSession.fieldNames(parser.createStatement(query), enrichPolicyMatchFields, preAnalysisResult).fieldNames();
+    }
+
     private void assertFieldNames(String query, Set<String> expected) {
-        Set<String> fieldNames = EsqlSession.fieldNames(parser.createStatement(query), Collections.emptySet());
+        Set<String> fieldNames = fieldNames(query, Collections.emptySet());
         assertThat(fieldNames, equalTo(expected));
     }
+
+    private void assertFieldNames(String query, Set<String> expected, Set<String> wildCardIndices) {
+        var preAnalysisResult = EsqlSession.fieldNames(parser.createStatement(query), Set.of(), new EsqlSession.PreAnalysisResult(null));
+        assertThat("Query-wide field names", preAnalysisResult.fieldNames(), equalTo(expected));
+        assertThat("Lookup Indices that expect wildcard lookups", preAnalysisResult.wildcardJoinIndices(), equalTo(wildCardIndices));
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java
index b323efad2b4c3..a3c5cd9168b4f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.xpack.esql.action.EsqlQueryRequest;
 import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction;
 import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolver;
 import org.elasticsearch.xpack.esql.execution.PlanExecutor;
 import org.elasticsearch.xpack.esql.session.EsqlSession;
@@ -119,10 +120,12 @@ public void testFailedMetric() {
             request,
             randomAlphaOfLength(10),
             EsqlTestUtils.TEST_CFG,
+            FoldContext.small(),
             enrichResolver,
             new EsqlExecutionInfo(randomBoolean()),
             groupIndicesByCluster,
             runPhase,
+            EsqlTestUtils.MOCK_QUERY_BUILDER_RESOLVER,
             new ActionListener<>() {
                 @Override
                 public void onResponse(Result result) {
@@ -148,10 +151,12 @@ public void onFailure(Exception e) {
             request,
             randomAlphaOfLength(10),
             EsqlTestUtils.TEST_CFG,
+            FoldContext.small(),
             enrichResolver,
             new EsqlExecutionInfo(randomBoolean()),
             groupIndicesByCluster,
             runPhase,
+            EsqlTestUtils.MOCK_QUERY_BUILDER_RESOLVER,
             new ActionListener<>() {
                 @Override
                 public void onResponse(Result result) {}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
index c1d94933537f0..c3f7080ebdd92 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttributeTests;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedNamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.function.Function;
-import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.FullTextPredicate;
 import org.elasticsearch.xpack.esql.core.tree.AbstractNodeTestCase;
 import org.elasticsearch.xpack.esql.core.tree.Node;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
@@ -40,6 +39,8 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Pow;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat;
+import org.elasticsearch.xpack.esql.expression.predicate.fulltext.FullTextPredicate;
+import org.elasticsearch.xpack.esql.index.EsIndex;
 import org.elasticsearch.xpack.esql.plan.logical.Dissect;
 import org.elasticsearch.xpack.esql.plan.logical.Grok;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
@@ -87,6 +88,7 @@
 import static java.util.Collections.emptyList;
 import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration;
 import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
+import static org.elasticsearch.xpack.esql.index.EsIndexSerializationTests.randomEsIndex;
 import static org.mockito.Mockito.mock;
 
 /**
@@ -491,6 +493,9 @@ public void accept(Page page) {
         if (argClass == Configuration.class) {
             return randomConfiguration();
         }
+        if (argClass == EsIndex.class) {
+            return randomEsIndex();
+        }
         if (argClass == JoinConfig.class) {
             return new JoinConfig(
                 JoinTypes.LEFT,
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java
index 8a57dfa968ccd..9a30c2281d742 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java
@@ -7,9 +7,11 @@
 
 package org.elasticsearch.xpack.esql.type;
 
+import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.List;
 
@@ -50,11 +52,19 @@
 public class EsqlDataTypeConverterTests extends ESTestCase {
 
     public void testNanoTimeToString() {
-        long expected = randomLong();
+        long expected = randomNonNegativeLong();
         long actual = EsqlDataTypeConverter.dateNanosToLong(EsqlDataTypeConverter.nanoTimeToString(expected));
         assertEquals(expected, actual);
     }
 
+    public void testStringToDateNanos() {
+        assertEquals(
+            DateUtils.toLong(Instant.parse("2023-01-01T00:00:00.000Z")),
+            EsqlDataTypeConverter.convert("2023-01-01T00:00:00.000000000", DATE_NANOS)
+        );
+        assertEquals(DateUtils.toLong(Instant.parse("2023-01-01T00:00:00.000Z")), EsqlDataTypeConverter.convert("2023-01-01", DATE_NANOS));
+    }
+
     public void testCommonTypeNull() {
         for (DataType dataType : DataType.values()) {
             assertEqualsCommonType(dataType, NULL, dataType);
diff --git a/x-pack/plugin/fleet/build.gradle b/x-pack/plugin/fleet/build.gradle
index 9fef7a928bcf8..c00a2af430f4a 100644
--- a/x-pack/plugin/fleet/build.gradle
+++ b/x-pack/plugin/fleet/build.gradle
@@ -10,9 +10,9 @@ apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 esplugin {
-  name 'x-pack-fleet'
-  description 'Elasticsearch Expanded Pack Plugin - Plugin exposing APIs for Fleet system indices'
-  classname 'org.elasticsearch.xpack.fleet.Fleet'
+  name = 'x-pack-fleet'
+  description = 'Elasticsearch Expanded Pack Plugin - Plugin exposing APIs for Fleet system indices'
+  classname ='org.elasticsearch.xpack.fleet.Fleet'
   extendedPlugins = ['x-pack-core']
 }
 
diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java
index 138a07833e584..b5d1b2ead1680 100644
--- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java
+++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java
@@ -79,11 +79,11 @@ public class Fleet extends Plugin implements SystemIndexPlugin {
     private static final int CURRENT_INDEX_VERSION = 7;
     private static final String MAPPING_VERSION_VARIABLE = "fleet.version";
     private static final List<String> ALLOWED_PRODUCTS = List.of("kibana", "fleet");
-    private static final int FLEET_ACTIONS_MAPPINGS_VERSION = 1;
-    private static final int FLEET_AGENTS_MAPPINGS_VERSION = 1;
-    private static final int FLEET_ENROLLMENT_API_KEYS_MAPPINGS_VERSION = 1;
+    private static final int FLEET_ACTIONS_MAPPINGS_VERSION = 2;
+    private static final int FLEET_AGENTS_MAPPINGS_VERSION = 2;
+    private static final int FLEET_ENROLLMENT_API_KEYS_MAPPINGS_VERSION = 2;
     private static final int FLEET_SECRETS_MAPPINGS_VERSION = 1;
-    private static final int FLEET_POLICIES_MAPPINGS_VERSION = 1;
+    private static final int FLEET_POLICIES_MAPPINGS_VERSION = 2;
     private static final int FLEET_POLICIES_LEADER_MAPPINGS_VERSION = 1;
     private static final int FLEET_SERVERS_MAPPINGS_VERSION = 1;
     private static final int FLEET_ARTIFACTS_MAPPINGS_VERSION = 1;
diff --git a/x-pack/plugin/frozen-indices/build.gradle b/x-pack/plugin/frozen-indices/build.gradle
index d4423f9df2741..8b1aa7f0ba8ed 100644
--- a/x-pack/plugin/frozen-indices/build.gradle
+++ b/x-pack/plugin/frozen-indices/build.gradle
@@ -1,9 +1,9 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 esplugin {
-  name 'frozen-indices'
-  description 'A plugin for the frozen indices functionality'
-  classname 'org.elasticsearch.xpack.frozen.FrozenIndices'
+  name = 'frozen-indices'
+  description = 'A plugin for the frozen indices functionality'
+  classname = 'org.elasticsearch.xpack.frozen.FrozenIndices'
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java
index 05b75fe6b01ca..0b106f6e304da 100644
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java
+++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java
@@ -8,36 +8,22 @@
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsFilter;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.EngineFactory;
 import org.elasticsearch.index.engine.frozen.FrozenEngine;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.EnginePlugin;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction;
 import org.elasticsearch.xpack.frozen.action.TransportFreezeIndexAction;
-import org.elasticsearch.xpack.frozen.rest.action.RestFreezeIndexAction;
 
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
-import java.util.function.Predicate;
-import java.util.function.Supplier;
 
 public class FrozenIndices extends Plugin implements ActionPlugin, EnginePlugin {
 
@@ -63,19 +49,4 @@ public List<Setting<?>> getSettings() {
         actions.add(new ActionHandler<>(FreezeIndexAction.INSTANCE, TransportFreezeIndexAction.class));
         return actions;
     }
-
-    @Override
-    public List<RestHandler> getRestHandlers(
-        Settings settings,
-        NamedWriteableRegistry namedWriteableRegistry,
-        RestController restController,
-        ClusterSettings clusterSettings,
-        IndexScopedSettings indexScopedSettings,
-        SettingsFilter settingsFilter,
-        IndexNameExpressionResolver indexNameExpressionResolver,
-        Supplier<DiscoveryNodes> nodesInCluster,
-        Predicate<NodeFeature> clusterSupportsFeature
-    ) {
-        return Collections.singletonList(new RestFreezeIndexAction());
-    }
 }
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java
deleted file mode 100644
index f07d17fee87f5..0000000000000
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.frozen.rest.action;
-
-import org.elasticsearch.action.support.ActiveShardCount;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.UpdateForV9;
-import org.elasticsearch.protocol.xpack.frozen.FreezeRequest;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction;
-
-import java.util.List;
-
-import static org.elasticsearch.rest.RestRequest.Method.POST;
-import static org.elasticsearch.rest.RestUtils.getAckTimeout;
-import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
-
-public final class RestFreezeIndexAction extends BaseRestHandler {
-
-    private static final String FREEZE_REMOVED = "It is no longer possible to freeze indices, but existing frozen indices can still be "
-        + "unfrozen";
-
-    private static final String UNFREEZE_DEPRECATED = "Frozen indices are deprecated because they provide no benefit given improvements "
-        + "in heap memory utilization. They will be removed in a future release.";
-
-    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_INDEXING)
-    // these routes were ".deprecated" in RestApiVersion.V_8 which will require use of REST API compatibility headers to access
-    // this API in v9. It is unclear if this was intentional for v9, and the code has been updated to ".deprecateAndKeep" which will
-    // continue to emit deprecations warnings but will not require any special headers to access the API in v9.
-    // Please review and update the code and tests as needed. The original code remains commented out below for reference.
-    @Override
-    public List<Route> routes() {
-        return List.of(
-            // Route.builder(POST, "/{index}/_unfreeze").deprecated(UNFREEZE_DEPRECATED, RestApiVersion.V_8).build()
-            Route.builder(POST, "/{index}/_unfreeze").deprecateAndKeep(UNFREEZE_DEPRECATED).build()
-        );
-    }
-
-    @Override
-    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
-        final var freezeRequest = new FreezeRequest(
-            getMasterNodeTimeout(request),
-            getAckTimeout(request),
-            Strings.splitStringByCommaToArray(request.param("index"))
-        );
-        freezeRequest.indicesOptions(IndicesOptions.fromRequest(request, freezeRequest.indicesOptions()));
-        String waitForActiveShards = request.param("wait_for_active_shards");
-        if (waitForActiveShards != null) {
-            freezeRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards));
-        }
-        freezeRequest.setFreeze(false);
-        return channel -> client.execute(FreezeIndexAction.INSTANCE, freezeRequest, new RestToXContentListener<>(channel));
-    }
-
-    @Override
-    public String getName() {
-        return "freeze_index";
-    }
-}
diff --git a/x-pack/plugin/geoip-enterprise-downloader/build.gradle b/x-pack/plugin/geoip-enterprise-downloader/build.gradle
index ab16609ac7aad..7ab9159eed4c9 100644
--- a/x-pack/plugin/geoip-enterprise-downloader/build.gradle
+++ b/x-pack/plugin/geoip-enterprise-downloader/build.gradle
@@ -2,9 +2,9 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 esplugin {
-  name 'x-pack-geoip-enterprise-downloader'
-  description 'Elasticsearch Expanded Pack Plugin - Geoip Enterprise Downloader'
-  classname 'org.elasticsearch.xpack.geoip.EnterpriseDownloaderPlugin'
+  name = 'x-pack-geoip-enterprise-downloader'
+  description = 'Elasticsearch Expanded Pack Plugin - Geoip Enterprise Downloader'
+  classname ='org.elasticsearch.xpack.geoip.EnterpriseDownloaderPlugin'
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/graph/build.gradle b/x-pack/plugin/graph/build.gradle
index 7f622b2283435..b944a3b5b8a7b 100644
--- a/x-pack/plugin/graph/build.gradle
+++ b/x-pack/plugin/graph/build.gradle
@@ -1,9 +1,9 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 esplugin {
-  name 'x-pack-graph'
-  description 'Elasticsearch Expanded Pack Plugin - Graph'
-  classname 'org.elasticsearch.xpack.graph.Graph'
+  name = 'x-pack-graph'
+  description = 'Elasticsearch Expanded Pack Plugin - Graph'
+  classname ='org.elasticsearch.xpack.graph.Graph'
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java
index 973151e73d8dc..78870cbb9530b 100644
--- a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java
+++ b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.protocol.xpack.graph.GraphExploreRequest;
 import org.elasticsearch.protocol.xpack.graph.GraphExploreRequest.TermBoost;
@@ -42,10 +41,6 @@
 @ServerlessScope(Scope.PUBLIC)
 public class RestGraphAction extends BaseRestHandler {
 
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGraphAction.class);
-    public static final String TYPES_DEPRECATION_MESSAGE = "[types removal]" + " Specifying types in graph requests is deprecated.";
-    private static final String URI_BASE = "/_xpack";
-
     public static final ParseField TIMEOUT_FIELD = new ParseField("timeout");
     public static final ParseField SIGNIFICANCE_FIELD = new ParseField("use_significance");
     public static final ParseField RETURN_DETAILED_INFO = new ParseField("return_detailed_stats");
diff --git a/x-pack/plugin/identity-provider/build.gradle b/x-pack/plugin/identity-provider/build.gradle
index f9c121da0f550..2d0b3e09db2ca 100644
--- a/x-pack/plugin/identity-provider/build.gradle
+++ b/x-pack/plugin/identity-provider/build.gradle
@@ -9,9 +9,9 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.publish'
 apply plugin: 'elasticsearch.internal-cluster-test'
 esplugin {
-  name 'x-pack-identity-provider'
-  description 'Elasticsearch Expanded Pack Plugin - Identity Provider'
-  classname 'org.elasticsearch.xpack.idp.IdentityProviderPlugin'
+  name = 'x-pack-identity-provider'
+  description = 'Elasticsearch Expanded Pack Plugin - Identity Provider'
+  classname = 'org.elasticsearch.xpack.idp.IdentityProviderPlugin'
   extendedPlugins = ['x-pack-core']
 }
 
diff --git a/x-pack/plugin/identity-provider/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/identity-provider/src/main/plugin-metadata/entitlement-policy.yaml
new file mode 100644
index 0000000000000..d826de8ca8725
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/plugin-metadata/entitlement-policy.yaml
@@ -0,0 +1,2 @@
+ALL-UNNAMED:
+  - set_https_connection_properties # potentially required by apache.httpcomponents
diff --git a/x-pack/plugin/ilm/build.gradle b/x-pack/plugin/ilm/build.gradle
index e0dbd0b8db00e..9ef2ad5b05924 100644
--- a/x-pack/plugin/ilm/build.gradle
+++ b/x-pack/plugin/ilm/build.gradle
@@ -2,12 +2,12 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  name 'x-pack-ilm'
-  description 'Elasticsearch Expanded Pack Plugin - Index Lifecycle Management'
-  classname 'org.elasticsearch.xpack.ilm.IndexLifecycle'
+  name = 'x-pack-ilm'
+  description = 'Elasticsearch Expanded Pack Plugin - Index Lifecycle Management'
+  classname ='org.elasticsearch.xpack.ilm.IndexLifecycle'
   extendedPlugins = ['x-pack-core']
-  hasNativeController false
-  requiresKeystore true
+  hasNativeController =false
+  requiresKeystore =true
 }
 base {
   archivesName = 'x-pack-ilm'
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java
index 28f97adec8814..91afa4da560b5 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.WarningFailureException;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Template;
@@ -216,29 +217,27 @@ public void testReadOnlyAction() throws Exception {
     }
 
     public void testFreezeAction() throws Exception {
-        createNewSingletonPolicy(client(), policyName, "cold", FreezeAction.INSTANCE);
+        try {
+            createNewSingletonPolicy(client(), policyName, "cold", FreezeAction.INSTANCE);
+        } catch (WarningFailureException e) {
+            assertThat(e.getMessage(), containsString("The freeze action in ILM is deprecated and will be removed in a future version"));
+        }
         createComposableTemplate(client(), template, dataStream + "*", getTemplate(policyName));
         indexDocument(client(), dataStream, true);
 
+        // The freeze action is a noop action with only noop steps and should pass through to complete the phase asap.
         String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1);
-        assertBusy(
-            () -> assertThat(
-                "index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore",
-                explainIndex(client(), backingIndexName).get("step"),
-                is(CheckNotDataStreamWriteIndexStep.NAME)
-            ),
-            30,
-            TimeUnit.SECONDS
-        );
-
-        // Manual rollover the original index such that it's not the write index in the data stream anymore
-        rolloverMaxOneDocCondition(client(), dataStream);
-
-        assertBusy(
-            () -> assertThat(explainIndex(client(), backingIndexName).get("step"), is(PhaseCompleteStep.NAME)),
-            30,
-            TimeUnit.SECONDS
-        );
+        assertBusy(() -> {
+            try {
+                assertThat(explainIndex(client(), backingIndexName).get("step"), is(PhaseCompleteStep.NAME));
+                fail("expected a deprecation warning");
+            } catch (WarningFailureException e) {
+                assertThat(
+                    e.getMessage(),
+                    containsString("The freeze action in ILM is deprecated and will be removed in a future version")
+                );
+            }
+        }, 30, TimeUnit.SECONDS);
 
         Map<String, Object> settings = getOnlyIndexSettings(client(), backingIndexName);
         assertNull(settings.get("index.frozen"));
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
index 4c53d711ffdef..23a8dbd220118 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.WarningFailureException;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
@@ -190,7 +191,11 @@ public void testUpdatePolicyToNotContainFailedStep() throws Exception {
     }
 
     public void testFreezeNoop() throws Exception {
-        createNewSingletonPolicy(client(), policy, "cold", FreezeAction.INSTANCE);
+        try {
+            createNewSingletonPolicy(client(), policy, "cold", FreezeAction.INSTANCE);
+        } catch (WarningFailureException e) {
+            assertThat(e.getMessage(), containsString("The freeze action in ILM is deprecated and will be removed in a future version"));
+        }
 
         createIndexWithSettings(
             client(),
@@ -202,11 +207,16 @@ public void testFreezeNoop() throws Exception {
                 .put("index.lifecycle.name", policy)
         );
 
-        assertBusy(
-            () -> assertThat(getStepKeyForIndex(client(), index), equalTo(PhaseCompleteStep.finalStep("cold").getKey())),
-            30,
-            TimeUnit.SECONDS
-        );
+        assertBusy(() -> {
+            try {
+                assertThat(getStepKeyForIndex(client(), index), equalTo(PhaseCompleteStep.finalStep("cold").getKey()));
+            } catch (WarningFailureException e) {
+                assertThat(
+                    e.getMessage(),
+                    containsString("The freeze action in ILM is deprecated and will be removed in a future version")
+                );
+            }
+        }, 30, TimeUnit.SECONDS);
         assertFalse(getOnlyIndexSettings(client(), index).containsKey("index.frozen"));
     }
 
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java
index 15e59a1593337..bae3064971a5e 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.WarningFailureException;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Template;
@@ -361,9 +362,10 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex
             null
         );
 
+        String template = randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT);
         createComposableTemplate(
             client(),
-            randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT),
+            template,
             dataStream,
             new Template(
                 Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 5).put(LifecycleSettings.LIFECYCLE_NAME, policy).build(),
@@ -407,19 +409,24 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex
         // snapshot)
         assertOK(client().performRequest(new Request("DELETE", "/_data_stream/" + dataStream)));
 
-        createPolicy(
-            client(),
-            policy,
-            new Phase("hot", TimeValue.ZERO, Map.of()),
-            new Phase(
-                "warm",
-                TimeValue.ZERO,
-                Map.of(ShrinkAction.NAME, new ShrinkAction(1, null, false), ForceMergeAction.NAME, new ForceMergeAction(1, null))
-            ),
-            new Phase("cold", TimeValue.ZERO, Map.of(FreezeAction.NAME, FreezeAction.INSTANCE)),
-            null,
-            null
-        );
+        try {
+            createPolicy(
+                client(),
+                policy,
+                new Phase("hot", TimeValue.ZERO, Map.of()),
+                new Phase(
+                    "warm",
+                    TimeValue.ZERO,
+                    Map.of(ShrinkAction.NAME, new ShrinkAction(1, null, false), ForceMergeAction.NAME, new ForceMergeAction(1, null))
+                ),
+                new Phase("cold", TimeValue.ZERO, Map.of(FreezeAction.NAME, FreezeAction.INSTANCE)),
+                null,
+                null
+            );
+            fail("Expected a deprecation warning.");
+        } catch (WarningFailureException e) {
+            assertThat(e.getMessage(), containsString("The freeze action in ILM is deprecated and will be removed in a future version"));
+        }
 
         // restore the datastream
         Request restoreSnapshot = new Request("POST", "/_snapshot/" + snapshotRepo + "/" + dsSnapshotName + "/_restore");
@@ -432,7 +439,16 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex
 
         // the restored index is now managed by the now updated ILM policy and needs to go through the warm and cold phase
         assertBusy(() -> {
-            Step.StepKey stepKeyForIndex = getStepKeyForIndex(client(), searchableSnapMountedIndexName);
+            Step.StepKey stepKeyForIndex;
+            try {
+                stepKeyForIndex = getStepKeyForIndex(client(), searchableSnapMountedIndexName);
+            } catch (WarningFailureException e) {
+                assertThat(
+                    e.getMessage(),
+                    containsString("The freeze action in ILM is deprecated and will be removed in a future version")
+                );
+                stepKeyForIndex = getKeyForIndex(e.getResponse(), searchableSnapMountedIndexName);
+            }
             assertThat(stepKeyForIndex.phase(), is("cold"));
             assertThat(stepKeyForIndex.name(), is(PhaseCompleteStep.NAME));
         }, 30, TimeUnit.SECONDS);
@@ -984,4 +1000,18 @@ private void triggerStateChange() throws IOException {
         Request rerouteRequest = new Request("POST", "/_cluster/reroute");
         client().performRequest(rerouteRequest);
     }
+
+    private Step.StepKey getKeyForIndex(Response response, String indexName) throws IOException {
+        Map<String, Object> responseMap;
+        try (InputStream is = response.getEntity().getContent()) {
+            responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true);
+        }
+
+        @SuppressWarnings("unchecked")
+        Map<String, Object> indexResponse = ((Map<String, Map<String, Object>>) responseMap.get("indices")).get(indexName);
+        String phase = (String) indexResponse.get("phase");
+        String action = (String) indexResponse.get("action");
+        String step = (String) indexResponse.get("step");
+        return new Step.StepKey(phase, action, step);
+    }
 }
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java
index f25028824b56e..5d4b837dc4d1d 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java
@@ -108,7 +108,7 @@ public void testWaitInShrunkShardsAllocatedExceedsThreshold() throws Exception {
 
         String[] firstAttemptShrinkIndexName = new String[1];
         assertBusy(() -> {
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex);
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex);
             ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
 
             IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex);
@@ -118,7 +118,7 @@ public void testWaitInShrunkShardsAllocatedExceedsThreshold() throws Exception {
 
         // let's check ILM for the managed index is waiting in the `shrunk-shards-allocated` step
         assertBusy(() -> {
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex);
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex);
             ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
 
             IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex);
@@ -160,7 +160,7 @@ public void onFailure(Exception e) {
 
         String[] secondCycleShrinkIndexName = new String[1];
         assertBusy(() -> {
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex);
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex);
             ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
 
             IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex);
@@ -180,7 +180,9 @@ public void onFailure(Exception e) {
         // situation and allow for shrink to complete by reducing the number of shards for the shrunk index to 0
         setReplicaCount(0, secondCycleShrinkIndexName[0]);
         assertBusy(() -> {
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(secondCycleShrinkIndexName[0]);
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(
+                secondCycleShrinkIndexName[0]
+            );
             ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
             IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses()
                 .get(secondCycleShrinkIndexName[0]);
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java
index 21924634ff6ab..d9cc4de344a39 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java
@@ -149,7 +149,10 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception {
             List<String> backingIndices = getBackingIndices(dataStreamName);
             String firstGenerationIndex = backingIndices.get(0);
             String secondGenerationIndex = backingIndices.get(1);
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(firstGenerationIndex, secondGenerationIndex);
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(
+                firstGenerationIndex,
+                secondGenerationIndex
+            );
             ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
 
             IndexLifecycleExplainResponse firstGenerationExplain = explainResponse.getIndexResponses().get(firstGenerationIndex);
@@ -193,7 +196,7 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception {
             String firstGenerationIndex = backingIndices.get(0);
             String secondGenerationIndex = backingIndices.get(1);
             String writeIndex = backingIndices.get(2);
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(
                 firstGenerationIndex,
                 secondGenerationIndex,
                 writeIndex
@@ -270,7 +273,7 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception {
             String secondGenerationIndex = backingIndices.get(1);
             String thirdGenerationIndex = backingIndices.get(2);
             String writeIndex = backingIndices.get(3);
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(
                 firstGenerationIndex,
                 secondGenerationIndex,
                 thirdGenerationIndex,
@@ -348,7 +351,10 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro
             List<String> backingIndices = getBackingIndices(dataStreamName);
             String firstGenerationIndex = backingIndices.get(0);
             String secondGenerationIndex = backingIndices.get(1);
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(firstGenerationIndex, secondGenerationIndex);
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(
+                firstGenerationIndex,
+                secondGenerationIndex
+            );
             ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
 
             IndexLifecycleExplainResponse firstGenerationExplain = explainResponse.getIndexResponses().get(firstGenerationIndex);
@@ -401,7 +407,7 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro
             String firstGenerationIndex = backingIndices.get(0);
             String secondGenerationIndex = backingIndices.get(1);
             String thirdGenerationIndex = backingIndices.get(2);
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(
                 firstGenerationIndex,
                 secondGenerationIndex,
                 thirdGenerationIndex
@@ -465,7 +471,7 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro
             String secondGenerationIndex = backingIndices.get(1);
             String thirdGenerationIndex = backingIndices.get(2);
             String writeIndex = backingIndices.get(3);
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(
                 firstGenerationIndex,
                 secondGenerationIndex,
                 thirdGenerationIndex,
@@ -543,7 +549,10 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce
             List<String> backingIndices = getBackingIndices(dataStreamName);
             String firstGenerationIndex = backingIndices.get(0);
             String secondGenerationIndex = backingIndices.get(1);
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(firstGenerationIndex, secondGenerationIndex);
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(
+                firstGenerationIndex,
+                secondGenerationIndex
+            );
             ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
 
             IndexLifecycleExplainResponse firstGenerationExplain = explainResponse.getIndexResponses().get(firstGenerationIndex);
@@ -591,7 +600,7 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce
             String firstGenerationIndex = backingIndices.get(0);
             String secondGenerationIndex = backingIndices.get(1);
             String writeIndex = backingIndices.get(2);
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(
                 firstGenerationIndex,
                 secondGenerationIndex,
                 writeIndex
@@ -663,7 +672,7 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce
             String secondGenerationIndex = backingIndices.get(1);
             String thirdGenerationIndex = backingIndices.get(2);
             String writeIndex = backingIndices.get(3);
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(
                 firstGenerationIndex,
                 secondGenerationIndex,
                 thirdGenerationIndex,
@@ -720,7 +729,7 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce
             String secondGenerationIndex = backingIndices.get(1);
             String thirdGenerationIndex = backingIndices.get(2);
             String writeIndex = backingIndices.get(3);
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(
                 firstGenerationIndex,
                 secondGenerationIndex,
                 thirdGenerationIndex,
@@ -846,7 +855,7 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw
                 assertThat(index.isManagedByLifecycle(), is(true));
             }
 
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(writeIndex);
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(writeIndex);
             ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
 
             IndexLifecycleExplainResponse writeIndexExplain = explainResponse.getIndexResponses().get(writeIndex);
@@ -875,7 +884,10 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw
             String thirdGenerationIndex = backingIndices.get(2);
             String writeIndex = backingIndices.get(3);
 
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(thirdGenerationIndex, writeIndex);
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(
+                thirdGenerationIndex,
+                writeIndex
+            );
             ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
 
             IndexLifecycleExplainResponse thirdGenerationExplain = explainResponse.getIndexResponses().get(thirdGenerationIndex);
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java
index 6d409bf474cfc..d483401931830 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java
@@ -116,7 +116,7 @@ public void testIndexDataTierMigration() throws Exception {
         assertTrue(res.isAcknowledged());
 
         assertBusy(() -> {
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex);
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex);
             ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
 
             IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex);
@@ -127,7 +127,7 @@ public void testIndexDataTierMigration() throws Exception {
         logger.info("starting a warm data node");
         internalCluster().startNode(warmNode(Settings.EMPTY));
         assertBusy(() -> {
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex);
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex);
             ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
 
             IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex);
@@ -140,7 +140,7 @@ public void testIndexDataTierMigration() throws Exception {
 
         // wait for lifecycle to complete in the cold phase after the index has been migrated to the cold node
         assertBusy(() -> {
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex);
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex);
             ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
 
             IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex);
@@ -177,7 +177,7 @@ public void testUserOptsOutOfTierMigration() throws Exception {
         assertTrue(res.isAcknowledged());
 
         assertBusy(() -> {
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex);
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex);
             ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
 
             IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex);
@@ -197,7 +197,7 @@ public void testUserOptsOutOfTierMigration() throws Exception {
         // 1. start another cold node so both the primary and replica can relocate to the cold nodes
         // 2. remove the tier routing setting from the index again (we're doing this below)
         assertBusy(() -> {
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex);
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex);
             ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
 
             IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex);
@@ -210,7 +210,7 @@ public void testUserOptsOutOfTierMigration() throws Exception {
 
         // wait for lifecycle to complete in the cold phase
         assertBusy(() -> {
-            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex);
+            ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex);
             ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
 
             IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex);
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java
index 2c4c1c9e20bb6..3f92abc7e3da2 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java
@@ -99,8 +99,10 @@ public void testShrinkOnTiers() throws Exception {
         prepareIndex(index).setCreate(true).setId("1").setSource("@timestamp", "2020-09-09").get();
 
         assertBusy(() -> {
-            ExplainLifecycleResponse explain = client().execute(ExplainLifecycleAction.INSTANCE, new ExplainLifecycleRequest().indices("*"))
-                .get();
+            ExplainLifecycleResponse explain = client().execute(
+                ExplainLifecycleAction.INSTANCE,
+                new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices("*")
+            ).get();
             logger.info("--> explain: {}", Strings.toString(explain));
 
             String backingIndexName = DataStream.getDefaultBackingIndexName(index, 1);
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java
index b91a309a23ae5..c695dc8f87b68 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java
@@ -99,8 +99,10 @@ public void testShrinkOnTiers() throws Exception {
         prepareIndex(index).setCreate(true).setId("1").setSource("@timestamp", "2020-09-09").get();
 
         assertBusy(() -> {
-            ExplainLifecycleResponse explain = client().execute(ExplainLifecycleAction.INSTANCE, new ExplainLifecycleRequest().indices("*"))
-                .get();
+            ExplainLifecycleResponse explain = client().execute(
+                ExplainLifecycleAction.INSTANCE,
+                new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices("*")
+            ).get();
             logger.info("--> explain: {}", Strings.toString(explain));
 
             String backingIndexName = DataStream.getDefaultBackingIndexName(index, 1);
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
index 644f88dc533b9..8999adc08f97b 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
@@ -388,7 +388,7 @@ public void testExplainParseOriginationDate() throws Exception {
 
     private IndexLifecycleExplainResponse executeExplainRequestAndGetTestIndexResponse(String indexName) throws ExecutionException,
         InterruptedException {
-        ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest();
+        ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT);
         ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
         assertThat(explainResponse.getIndexResponses().size(), equalTo(1));
         return explainResponse.getIndexResponses().get(indexName);
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java
index 195f989eab055..3950edf648e18 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java
@@ -36,12 +36,11 @@ public String getName() {
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
         String[] indexes = Strings.splitStringByCommaToArray(restRequest.param("index"));
-        ExplainLifecycleRequest explainLifecycleRequest = new ExplainLifecycleRequest();
+        ExplainLifecycleRequest explainLifecycleRequest = new ExplainLifecycleRequest(getMasterNodeTimeout(restRequest));
         explainLifecycleRequest.indices(indexes);
         explainLifecycleRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, IndicesOptions.strictExpandOpen()));
         explainLifecycleRequest.onlyManaged(restRequest.paramAsBoolean("only_managed", false));
         explainLifecycleRequest.onlyErrors(restRequest.paramAsBoolean("only_errors", false));
-        explainLifecycleRequest.masterNodeTimeout(getMasterNodeTimeout(restRequest));
         return channel -> client.execute(ExplainLifecycleAction.INSTANCE, explainLifecycleRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java
index c50ea682ca9a2..85acc3fa8da6f 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java
@@ -153,6 +153,7 @@ static IndexLifecycleExplainResponse getIndexLifecycleExplainResponse(
                 // Try to add default rollover conditions to the response.
                 var phase = phaseExecutionInfo.getPhase();
                 if (phase != null) {
+                    phase.maybeAddDeprecationWarningForFreezeAction(policyName);
                     var rolloverAction = (RolloverAction) phase.getActions().get(RolloverAction.NAME);
                     if (rolloverAction != null) {
                         var conditions = applyDefaultConditions(rolloverAction.getConditions(), rolloverOnlyIfHasDocuments);
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java
index 92d182ea6d44a..7bdb95ff14673 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java
@@ -109,7 +109,7 @@ protected void masterOperation(
         Map<String, String> filteredHeaders = ClientHelper.getPersistableSafeSecurityHeaders(threadPool.getThreadContext(), state);
 
         LifecyclePolicy.validatePolicyName(request.getPolicy().getName());
-
+        request.getPolicy().maybeAddDeprecationWarningForFreezeAction(request.getPolicy().getName());
         {
             IndexLifecycleMetadata lifecycleMetadata = state.metadata().custom(IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata.EMPTY);
             LifecyclePolicyMetadata existingPolicy = lifecycleMetadata.getPolicyMetadatas().get(request.getPolicy().getName());
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleServiceTests.java
index b77e643bc2853..e70b1be1d108d 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleServiceTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleServiceTests.java
@@ -601,6 +601,7 @@ public void testIndicesOnShuttingDownNodesInDangerousStep() {
                                     "shutdown_node",
                                     SingleNodeShutdownMetadata.builder()
                                         .setNodeId("shutdown_node")
+                                        .setNodeEphemeralId("shutdown_node")
                                         .setReason("shut down for test")
                                         .setStartedAtMillis(randomNonNegativeLong())
                                         .setType(SingleNodeShutdownMetadata.Type.RESTART)
@@ -632,6 +633,7 @@ public void testIndicesOnShuttingDownNodesInDangerousStep() {
                                     "shutdown_node",
                                     SingleNodeShutdownMetadata.builder()
                                         .setNodeId("shutdown_node")
+                                        .setNodeEphemeralId("shutdown_node")
                                         .setReason("shut down for test")
                                         .setStartedAtMillis(randomNonNegativeLong())
                                         .setType(type)
diff --git a/x-pack/plugin/inference/build.gradle b/x-pack/plugin/inference/build.gradle
index 1d0236a5834e5..cb3ccbd171304 100644
--- a/x-pack/plugin/inference/build.gradle
+++ b/x-pack/plugin/inference/build.gradle
@@ -16,9 +16,9 @@ restResources {
 }
 
 esplugin {
-  name 'x-pack-inference'
-  description 'Configuration and evaluation of inference models'
-  classname 'org.elasticsearch.xpack.inference.InferencePlugin'
+  name = 'x-pack-inference'
+  description = 'Configuration and evaluation of inference models'
+  classname ='org.elasticsearch.xpack.inference.InferencePlugin'
   extendedPlugins = ['x-pack-core']
 }
 
@@ -34,6 +34,7 @@ dependencies {
   testImplementation(testArtifact(project(':server')))
   testImplementation(project(':x-pack:plugin:inference:qa:test-service-plugin'))
   testImplementation project(':modules:reindex')
+  testImplementation project(':modules:mapper-extras')
   clusterPlugins project(':x-pack:plugin:inference:qa:test-service-plugin')
 
   api "com.ibm.icu:icu4j:${versions.icu4j}"
@@ -66,8 +67,8 @@ dependencies {
   api "software.amazon.awssdk:checksums-spi:${versions.awsv2sdk}"
   api "software.amazon.awssdk:checksums:${versions.awsv2sdk}"
   api "software.amazon.awssdk:sdk-core:${versions.awsv2sdk}"
-  api "org.reactivestreams:reactive-streams:1.0.4"
-  api "org.reactivestreams:reactive-streams-tck:1.0.4"
+  api "org.reactivestreams:reactive-streams:${versions.reactive_streams}"
+  api "org.reactivestreams:reactive-streams-tck:${versions.reactive_streams}"
   api "software.amazon.awssdk:profiles:${versions.awsv2sdk}"
   api "software.amazon.awssdk:retries:${versions.awsv2sdk}"
   api "software.amazon.awssdk:auth:${versions.awsv2sdk}"
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java
index 5e6c4d53f4c58..cdc6d9b2dff5f 100644
--- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java
@@ -34,6 +34,7 @@
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
 
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
@@ -341,31 +342,44 @@ protected Map<String, Object> infer(String modelId, List<String> input) throws I
         return inferInternal(endpoint, input, null, Map.of());
     }
 
-    protected Deque<ServerSentEvent> streamInferOnMockService(String modelId, TaskType taskType, List<String> input) throws Exception {
+    protected Deque<ServerSentEvent> streamInferOnMockService(
+        String modelId,
+        TaskType taskType,
+        List<String> input,
+        @Nullable Consumer<Response> responseConsumerCallback
+    ) throws Exception {
         var endpoint = Strings.format("_inference/%s/%s/_stream", taskType, modelId);
-        return callAsync(endpoint, input);
+        return callAsync(endpoint, input, responseConsumerCallback);
     }
 
-    protected Deque<ServerSentEvent> unifiedCompletionInferOnMockService(String modelId, TaskType taskType, List<String> input)
-        throws Exception {
+    protected Deque<ServerSentEvent> unifiedCompletionInferOnMockService(
+        String modelId,
+        TaskType taskType,
+        List<String> input,
+        @Nullable Consumer<Response> responseConsumerCallback
+    ) throws Exception {
         var endpoint = Strings.format("_inference/%s/%s/_unified", taskType, modelId);
-        return callAsyncUnified(endpoint, input, "user");
+        return callAsyncUnified(endpoint, input, "user", responseConsumerCallback);
     }
 
-    private Deque<ServerSentEvent> callAsync(String endpoint, List<String> input) throws Exception {
+    private Deque<ServerSentEvent> callAsync(String endpoint, List<String> input, @Nullable Consumer<Response> responseConsumerCallback)
+        throws Exception {
         var request = new Request("POST", endpoint);
         request.setJsonEntity(jsonBody(input, null));
 
-        return execAsyncCall(request);
+        return execAsyncCall(request, responseConsumerCallback);
     }
 
-    private Deque<ServerSentEvent> execAsyncCall(Request request) throws Exception {
+    private Deque<ServerSentEvent> execAsyncCall(Request request, @Nullable Consumer<Response> responseConsumerCallback) throws Exception {
         var responseConsumer = new AsyncInferenceResponseConsumer();
         request.setOptions(RequestOptions.DEFAULT.toBuilder().setHttpAsyncResponseConsumerFactory(() -> responseConsumer).build());
         var latch = new CountDownLatch(1);
         client().performRequestAsync(request, new ResponseListener() {
             @Override
             public void onSuccess(Response response) {
+                if (responseConsumerCallback != null) {
+                    responseConsumerCallback.accept(response);
+                }
                 latch.countDown();
             }
 
@@ -378,11 +392,16 @@ public void onFailure(Exception exception) {
         return responseConsumer.events();
     }
 
-    private Deque<ServerSentEvent> callAsyncUnified(String endpoint, List<String> input, String role) throws Exception {
+    private Deque<ServerSentEvent> callAsyncUnified(
+        String endpoint,
+        List<String> input,
+        String role,
+        @Nullable Consumer<Response> responseConsumerCallback
+    ) throws Exception {
         var request = new Request("POST", endpoint);
 
         request.setJsonEntity(createUnifiedJsonBody(input, role));
-        return execAsyncCall(request);
+        return execAsyncCall(request, responseConsumerCallback);
     }
 
     private String createUnifiedJsonBody(List<String> input, String role) throws IOException {
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBasicLicenseIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBasicLicenseIT.java
new file mode 100644
index 0000000000000..4400ad8bbb538
--- /dev/null
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBasicLicenseIT.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.junit.ClassRule;
+
+import static org.elasticsearch.xpack.inference.InferenceBaseRestTest.mockSparseServiceModelConfig;
+
+public class InferenceBasicLicenseIT extends InferenceLicenseBaseRestTest {
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .setting("xpack.license.self_generated.type", "basic")
+        .setting("xpack.security.enabled", "true")
+        .user("x_pack_rest_user", "x-pack-test-password")
+        .plugin("inference-service-test")
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("x_pack_rest_user", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+    public void testPutModel_RestrictedWithBasicLicense() throws Exception {
+        var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id");
+        var modelConfig = mockSparseServiceModelConfig(null, true);
+        sendRestrictedRequest("PUT", endpoint, modelConfig);
+    }
+
+    public void testUpdateModel_RestrictedWithBasicLicense() throws Exception {
+        var endpoint = Strings.format("_inference/%s/%s/_update?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id");
+        var requestBody = """
+            {
+              "task_settings": {
+                "num_threads": 2
+              }
+            }
+            """;
+        sendRestrictedRequest("PUT", endpoint, requestBody);
+    }
+
+    public void testPerformInference_RestrictedWithBasicLicense() throws Exception {
+        var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id");
+        var requestBody = """
+            {
+              "input": ["washing", "machine"]
+            }
+            """;
+        sendRestrictedRequest("POST", endpoint, requestBody);
+    }
+
+    public void testGetServices_NonRestrictedWithBasicLicense() throws Exception {
+        var endpoint = "_inference/_services";
+        sendNonRestrictedRequest("GET", endpoint, null, 200, false);
+    }
+
+    public void testGetModels_NonRestrictedWithBasicLicense() throws Exception {
+        var endpoint = "_inference/_all";
+        sendNonRestrictedRequest("GET", endpoint, null, 200, false);
+    }
+
+    public void testDeleteModel_NonRestrictedWithBasicLicense() throws Exception {
+        var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id");
+        sendNonRestrictedRequest("DELETE", endpoint, null, 404, true);
+    }
+}
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java
index fc593a6a8b0fa..50f83dc7b02eb 100644
--- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java
@@ -10,6 +10,7 @@
 package org.elasticsearch.xpack.inference;
 
 import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
@@ -28,6 +29,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
@@ -37,9 +39,15 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.equalToIgnoringCase;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
 
 public class InferenceCrudIT extends InferenceBaseRestTest {
 
+    private static final Consumer<Response> VALIDATE_ELASTIC_PRODUCT_HEADER_CONSUMER = (r) -> assertThat(
+        r.getHeader("X-elastic-product"),
+        is("Elasticsearch")
+    );
+
     @SuppressWarnings("unchecked")
     public void testCRUD() throws IOException {
         for (int i = 0; i < 5; i++) {
@@ -142,19 +150,17 @@ public void testGetServicesWithoutTaskType() throws IOException {
         List<Object> services = getAllServices();
         if ((ElasticInferenceServiceFeature.DEPRECATED_ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled()
             || ElasticInferenceServiceFeature.ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled())) {
-            assertThat(services.size(), equalTo(18));
+            assertThat(services.size(), equalTo(19));
         } else {
-            assertThat(services.size(), equalTo(17));
+            assertThat(services.size(), equalTo(18));
         }
 
         String[] providers = new String[services.size()];
         for (int i = 0; i < services.size(); i++) {
             Map<String, Object> serviceConfig = (Map<String, Object>) services.get(i);
-            providers[i] = (String) serviceConfig.get("provider");
+            providers[i] = (String) serviceConfig.get("service");
         }
 
-        Arrays.sort(providers);
-
         var providerList = new ArrayList<>(
             Arrays.asList(
                 "alibabacloud-ai-search",
@@ -167,6 +173,7 @@ public void testGetServicesWithoutTaskType() throws IOException {
                 "googleaistudio",
                 "googlevertexai",
                 "hugging_face",
+                "jinaai",
                 "mistral",
                 "openai",
                 "streaming_completion_test_service",
@@ -180,23 +187,21 @@ public void testGetServicesWithoutTaskType() throws IOException {
             || ElasticInferenceServiceFeature.ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled())) {
             providerList.add(6, "elastic");
         }
-        assertArrayEquals(providers, providerList.toArray());
+        assertArrayEquals(providerList.toArray(), providers);
     }
 
     @SuppressWarnings("unchecked")
     public void testGetServicesWithTextEmbeddingTaskType() throws IOException {
         List<Object> services = getServices(TaskType.TEXT_EMBEDDING);
-        assertThat(services.size(), equalTo(13));
+        assertThat(services.size(), equalTo(14));
 
         String[] providers = new String[services.size()];
         for (int i = 0; i < services.size(); i++) {
             Map<String, Object> serviceConfig = (Map<String, Object>) services.get(i);
-            providers[i] = (String) serviceConfig.get("provider");
+            providers[i] = (String) serviceConfig.get("service");
         }
 
-        Arrays.sort(providers);
         assertArrayEquals(
-            providers,
             List.of(
                 "alibabacloud-ai-search",
                 "amazonbedrock",
@@ -207,46 +212,50 @@ public void testGetServicesWithTextEmbeddingTaskType() throws IOException {
                 "googleaistudio",
                 "googlevertexai",
                 "hugging_face",
+                "jinaai",
                 "mistral",
                 "openai",
                 "text_embedding_test_service",
                 "watsonxai"
-            ).toArray()
+            ).toArray(),
+            providers
         );
     }
 
     @SuppressWarnings("unchecked")
     public void testGetServicesWithRerankTaskType() throws IOException {
         List<Object> services = getServices(TaskType.RERANK);
-        assertThat(services.size(), equalTo(5));
+        assertThat(services.size(), equalTo(6));
 
         String[] providers = new String[services.size()];
         for (int i = 0; i < services.size(); i++) {
             Map<String, Object> serviceConfig = (Map<String, Object>) services.get(i);
-            providers[i] = (String) serviceConfig.get("provider");
+            providers[i] = (String) serviceConfig.get("service");
         }
 
-        Arrays.sort(providers);
         assertArrayEquals(
-            providers,
-            List.of("alibabacloud-ai-search", "cohere", "elasticsearch", "googlevertexai", "test_reranking_service").toArray()
+            List.of("alibabacloud-ai-search", "cohere", "elasticsearch", "googlevertexai", "jinaai", "test_reranking_service").toArray(),
+            providers
         );
     }
 
     @SuppressWarnings("unchecked")
     public void testGetServicesWithCompletionTaskType() throws IOException {
         List<Object> services = getServices(TaskType.COMPLETION);
-        assertThat(services.size(), equalTo(9));
+        if ((ElasticInferenceServiceFeature.DEPRECATED_ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled()
+            || ElasticInferenceServiceFeature.ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled())) {
+            assertThat(services.size(), equalTo(10));
+        } else {
+            assertThat(services.size(), equalTo(9));
+        }
 
         String[] providers = new String[services.size()];
         for (int i = 0; i < services.size(); i++) {
             Map<String, Object> serviceConfig = (Map<String, Object>) services.get(i);
-            providers[i] = (String) serviceConfig.get("provider");
+            providers[i] = (String) serviceConfig.get("service");
         }
 
-        Arrays.sort(providers);
-        assertArrayEquals(
-            providers,
+        var providerList = new ArrayList<>(
             List.of(
                 "alibabacloud-ai-search",
                 "amazonbedrock",
@@ -257,8 +266,15 @@ public void testGetServicesWithCompletionTaskType() throws IOException {
                 "googleaistudio",
                 "openai",
                 "streaming_completion_test_service"
-            ).toArray()
+            )
         );
+
+        if ((ElasticInferenceServiceFeature.DEPRECATED_ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled()
+            || ElasticInferenceServiceFeature.ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled())) {
+            providerList.add(6, "elastic");
+        }
+
+        assertArrayEquals(providers, providerList.toArray());
     }
 
     @SuppressWarnings("unchecked")
@@ -275,11 +291,9 @@ public void testGetServicesWithSparseEmbeddingTaskType() throws IOException {
         String[] providers = new String[services.size()];
         for (int i = 0; i < services.size(); i++) {
             Map<String, Object> serviceConfig = (Map<String, Object>) services.get(i);
-            providers[i] = (String) serviceConfig.get("provider");
+            providers[i] = (String) serviceConfig.get("service");
         }
 
-        Arrays.sort(providers);
-
         var providerList = new ArrayList<>(Arrays.asList("alibabacloud-ai-search", "elasticsearch", "hugging_face", "test_service"));
         if ((ElasticInferenceServiceFeature.DEPRECATED_ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled()
             || ElasticInferenceServiceFeature.ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled())) {
@@ -442,7 +456,7 @@ public void testUnsupportedStream() throws Exception {
         assertEquals(TaskType.SPARSE_EMBEDDING.toString(), singleModel.get("task_type"));
 
         try {
-            var events = streamInferOnMockService(modelId, TaskType.SPARSE_EMBEDDING, List.of(randomUUID()));
+            var events = streamInferOnMockService(modelId, TaskType.SPARSE_EMBEDDING, List.of(randomUUID()), null);
             assertThat(events.size(), equalTo(2));
             events.forEach(event -> {
                 switch (event.name()) {
@@ -469,7 +483,7 @@ public void testSupportedStream() throws Exception {
 
         var input = IntStream.range(1, 2 + randomInt(8)).mapToObj(i -> randomAlphanumericOfLength(5)).toList();
         try {
-            var events = streamInferOnMockService(modelId, TaskType.COMPLETION, input);
+            var events = streamInferOnMockService(modelId, TaskType.COMPLETION, input, VALIDATE_ELASTIC_PRODUCT_HEADER_CONSUMER);
 
             var expectedResponses = Stream.concat(
                 input.stream().map(s -> s.toUpperCase(Locale.ROOT)).map(str -> "{\"completion\":[{\"delta\":\"" + str + "\"}]}"),
@@ -496,7 +510,7 @@ public void testUnifiedCompletionInference() throws Exception {
 
         var input = IntStream.range(1, 2 + randomInt(8)).mapToObj(i -> randomAlphanumericOfLength(5)).toList();
         try {
-            var events = unifiedCompletionInferOnMockService(modelId, TaskType.COMPLETION, input);
+            var events = unifiedCompletionInferOnMockService(modelId, TaskType.COMPLETION, input, VALIDATE_ELASTIC_PRODUCT_HEADER_CONSUMER);
             var expectedResponses = expectedResultsIterator(input);
             assertThat(events.size(), equalTo((input.size() + 1) * 2));
             events.forEach(event -> {
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceLicenseBaseRestTest.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceLicenseBaseRestTest.java
new file mode 100644
index 0000000000000..43183bae73252
--- /dev/null
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceLicenseBaseRestTest.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class InferenceLicenseBaseRestTest extends ESRestTestCase {
+    protected void sendRestrictedRequest(String method, String endpoint, String body) throws IOException {
+        var request = new Request(method, endpoint);
+        request.setJsonEntity(body);
+
+        var exception = assertThrows(ResponseException.class, () -> client().performRequest(request));
+        assertEquals(403, exception.getResponse().getStatusLine().getStatusCode());
+        assertThat(exception.getMessage(), containsString("current license is non-compliant for [inference]"));
+    }
+
+    protected void sendNonRestrictedRequest(String method, String endpoint, String body, int expectedStatusCode, boolean exceptionExpected)
+        throws IOException {
+        var request = new Request(method, endpoint);
+        request.setJsonEntity(body);
+
+        int actualStatusCode;
+        if (exceptionExpected) {
+            var exception = assertThrows(ResponseException.class, () -> client().performRequest(request));
+            actualStatusCode = exception.getResponse().getStatusLine().getStatusCode();
+        } else {
+            var response = client().performRequest(request);
+            actualStatusCode = response.getStatusLine().getStatusCode();
+        }
+        assertEquals(expectedStatusCode, actualStatusCode);
+    }
+}
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceTrialLicenseIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceTrialLicenseIT.java
new file mode 100644
index 0000000000000..c7066a827fa7e
--- /dev/null
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceTrialLicenseIT.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.junit.ClassRule;
+
+import static org.elasticsearch.xpack.inference.InferenceBaseRestTest.mockSparseServiceModelConfig;
+
+public class InferenceTrialLicenseIT extends InferenceLicenseBaseRestTest {
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .setting("xpack.license.self_generated.type", "trial")
+        .setting("xpack.security.enabled", "true")
+        .user("x_pack_rest_user", "x-pack-test-password")
+        .plugin("inference-service-test")
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("x_pack_rest_user", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+    public void testPutModel_NonRestrictedWithTrialLicense() throws Exception {
+        var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id");
+        var modelConfig = mockSparseServiceModelConfig(null, true);
+        sendNonRestrictedRequest("PUT", endpoint, modelConfig, 200, false);
+    }
+
+    public void testUpdateModel_NonRestrictedWithTrialLicense() throws Exception {
+        var endpoint = Strings.format("_inference/%s/%s/_update?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id");
+        var requestBody = """
+            {
+              "task_settings": {
+                "num_threads": 2
+              }
+            }
+            """;
+        sendNonRestrictedRequest("PUT", endpoint, requestBody, 404, true);
+    }
+
+    public void testPerformInference_NonRestrictedWithTrialLicense() throws Exception {
+        var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id");
+        var requestBody = """
+            {
+              "input": ["washing", "machine"]
+            }
+            """;
+        sendNonRestrictedRequest("POST", endpoint, requestBody, 404, true);
+    }
+
+    public void testGetServices_NonRestrictedWithBasicLicense() throws Exception {
+        var endpoint = "_inference/_services";
+        sendNonRestrictedRequest("GET", endpoint, null, 200, false);
+    }
+
+    public void testGetModels_NonRestrictedWithBasicLicense() throws Exception {
+        var endpoint = "_inference/_all";
+        sendNonRestrictedRequest("GET", endpoint, null, 200, false);
+    }
+
+    public void testDeleteModel_NonRestrictedWithBasicLicense() throws Exception {
+        var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id");
+        sendNonRestrictedRequest("DELETE", endpoint, null, 404, true);
+    }
+}
diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java
index d38503a884092..880557c59f11c 100644
--- a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java
+++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java
@@ -12,8 +12,11 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.http.MockWebServer;
-import org.elasticsearch.upgrades.AbstractRollingUpgradeTestCase;
+import org.elasticsearch.upgrades.ParameterizedRollingUpgradeTestCase;
+import org.junit.ClassRule;
 
 import java.io.IOException;
 import java.util.LinkedList;
@@ -22,7 +25,7 @@
 
 import static org.elasticsearch.core.Strings.format;
 
-public class InferenceUpgradeTestCase extends AbstractRollingUpgradeTestCase {
+public class InferenceUpgradeTestCase extends ParameterizedRollingUpgradeTestCase {
 
     static final String MODELS_RENAMED_TO_ENDPOINTS = "8.15.0";
 
@@ -30,6 +33,20 @@ public InferenceUpgradeTestCase(@Name("upgradedNodes") int upgradedNodes) {
         super(upgradedNodes);
     }
 
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .version(getOldClusterTestVersion())
+        .nodes(NODE_NUM)
+        .setting("xpack.security.enabled", "false")
+        .setting("xpack.license.self_generated.type", "trial")
+        .build();
+
+    @Override
+    protected ElasticsearchCluster getUpgradeCluster() {
+        return cluster;
+    }
+
     protected static String getUrl(MockWebServer webServer) {
         return format("http://%s:%s", webServer.getHostName(), webServer.getPort());
     }
diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java
index 7b43a5b86e619..b6dcfcb84a77f 100644
--- a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java
+++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java
@@ -16,6 +16,8 @@
 import org.junit.BeforeClass;
 
 import java.io.IOException;
+import java.util.Collection;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 
@@ -83,8 +85,9 @@ public void testOpenAiEmbeddings() throws IOException {
             assertEquals("openai", configs.get(0).get("service"));
             var serviceSettings = (Map<String, Object>) configs.get(0).get("service_settings");
             var taskSettings = (Map<String, Object>) configs.get(0).get("task_settings");
-            var modelIdFound = serviceSettings.containsKey("model_id") || taskSettings.containsKey("model_id");
-            assertTrue("model_id not found in config: " + configs.toString(), modelIdFound);
+            var modelIdFound = serviceSettings.containsKey("model_id")
+                || (taskSettings.containsKey("model") && getOldClusterTestVersion().onOrBefore(OPEN_AI_EMBEDDINGS_MODEL_SETTING_MOVED));
+            assertTrue("model_id not found in config: " + configs, modelIdFound);
 
             assertEmbeddingInference(oldClusterId);
         } else if (isUpgradedCluster()) {
@@ -139,9 +142,11 @@ public void testOpenAiCompletions() throws IOException {
 
             assertCompletionInference(oldClusterId);
         } else if (isMixedCluster()) {
-            var configs = (List<Map<String, Object>>) get(testTaskType, oldClusterId).get("endpoints");
+            List<Map<String, Object>> configs = new LinkedList<>();
+            var request = get(testTaskType, oldClusterId);
+            configs.addAll((Collection<? extends Map<String, Object>>) request.getOrDefault("endpoints", List.of()));
             if (oldClusterHasFeature("gte_v" + MODELS_RENAMED_TO_ENDPOINTS) == false) {
-                configs.addAll((List<Map<String, Object>>) get(testTaskType, oldClusterId).get(old_cluster_endpoint_identifier));
+                configs.addAll((List<Map<String, Object>>) request.getOrDefault(old_cluster_endpoint_identifier, List.of()));
                 // in version 8.15, there was a breaking change where "models" was renamed to "endpoints"
             }
             assertEquals("openai", configs.get(0).get("service"));
diff --git a/x-pack/plugin/inference/qa/test-service-plugin/build.gradle b/x-pack/plugin/inference/qa/test-service-plugin/build.gradle
index 031c7519154b1..145869bf6c34d 100644
--- a/x-pack/plugin/inference/qa/test-service-plugin/build.gradle
+++ b/x-pack/plugin/inference/qa/test-service-plugin/build.gradle
@@ -3,9 +3,9 @@ apply plugin: 'elasticsearch.base-internal-es-plugin'
 apply plugin: 'elasticsearch.internal-java-rest-test'
 
 esplugin {
-  name 'inference-service-test'
-  description 'A mock inference service'
-  classname 'org.elasticsearch.xpack.inference.mock.TestInferenceServicePlugin'
+  name = 'inference-service-test'
+  description = 'A mock inference service'
+  classname = 'org.elasticsearch.xpack.inference.mock.TestInferenceServicePlugin'
   extendedPlugins = ['x-pack-inference']
 }
 
diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java
index a6888f28159f4..89c79dd148598 100644
--- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java
+++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.inference.ChunkedInference;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceExtension;
 import org.elasticsearch.inference.InferenceServiceResults;
@@ -29,10 +28,8 @@
 import org.elasticsearch.inference.ServiceSettings;
 import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.SimilarityMeasure;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.inference.UnifiedCompletionRequest;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -260,23 +257,18 @@ public static InferenceServiceConfiguration get() {
 
                     configurationMap.put(
                         "model",
-                        new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                        new SettingsConfiguration.Builder().setDescription("")
                             .setLabel("Model")
-                            .setOrder(1)
                             .setRequired(true)
                             .setSensitive(true)
-                            .setTooltip("")
                             .setType(SettingsConfigurationFieldType.STRING)
                             .build()
                     );
 
-                    return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                        Map<String, SettingsConfiguration> taskSettingsConfig;
-                        switch (t) {
-                            default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                        }
-                        return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                    }).toList()).setConfiguration(configurationMap).build();
+                    return new InferenceServiceConfiguration.Builder().setService(NAME)
+                        .setTaskTypes(supportedTaskTypes)
+                        .setConfigurations(configurationMap)
+                        .build();
                 }
             );
         }
diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java
index bbb773aa5129a..77c762a38baaf 100644
--- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java
+++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceExtension;
 import org.elasticsearch.inference.InferenceServiceResults;
@@ -27,10 +26,8 @@
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.ServiceSettings;
 import org.elasticsearch.inference.SettingsConfiguration;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.inference.UnifiedCompletionRequest;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -174,23 +171,18 @@ public static InferenceServiceConfiguration get() {
 
                     configurationMap.put(
                         "model",
-                        new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                        new SettingsConfiguration.Builder().setDescription("")
                             .setLabel("Model")
-                            .setOrder(1)
                             .setRequired(true)
                             .setSensitive(true)
-                            .setTooltip("")
                             .setType(SettingsConfigurationFieldType.STRING)
                             .build()
                     );
 
-                    return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                        Map<String, SettingsConfiguration> taskSettingsConfig;
-                        switch (t) {
-                            default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                        }
-                        return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                    }).toList()).setConfiguration(configurationMap).build();
+                    return new InferenceServiceConfiguration.Builder().setService(NAME)
+                        .setTaskTypes(supportedTaskTypes)
+                        .setConfigurations(configurationMap)
+                        .build();
                 }
             );
         }
diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java
index eea64304f503a..bef0b1812beda 100644
--- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java
+++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceExtension;
 import org.elasticsearch.inference.InferenceServiceResults;
@@ -27,10 +26,8 @@
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.ServiceSettings;
 import org.elasticsearch.inference.SettingsConfiguration;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.inference.UnifiedCompletionRequest;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -208,35 +205,28 @@ public static InferenceServiceConfiguration get() {
 
                     configurationMap.put(
                         "model",
-                        new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                        new SettingsConfiguration.Builder().setDescription("")
                             .setLabel("Model")
-                            .setOrder(1)
                             .setRequired(true)
                             .setSensitive(false)
-                            .setTooltip("")
                             .setType(SettingsConfigurationFieldType.STRING)
                             .build()
                     );
 
                     configurationMap.put(
                         "hidden_field",
-                        new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                        new SettingsConfiguration.Builder().setDescription("")
                             .setLabel("Hidden Field")
-                            .setOrder(2)
                             .setRequired(true)
                             .setSensitive(false)
-                            .setTooltip("")
                             .setType(SettingsConfigurationFieldType.STRING)
                             .build()
                     );
 
-                    return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                        Map<String, SettingsConfiguration> taskSettingsConfig;
-                        switch (t) {
-                            default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                        }
-                        return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                    }).toList()).setConfiguration(configurationMap).build();
+                    return new InferenceServiceConfiguration.Builder().setService(NAME)
+                        .setTaskTypes(supportedTaskTypes)
+                        .setConfigurations(configurationMap)
+                        .build();
                 }
             );
         }
diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java
index a39be00d9e7fa..e071b704c233e 100644
--- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java
+++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceExtension;
 import org.elasticsearch.inference.InferenceServiceResults;
@@ -28,10 +27,8 @@
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ServiceSettings;
 import org.elasticsearch.inference.SettingsConfiguration;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.inference.UnifiedCompletionRequest;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -260,23 +257,18 @@ public static InferenceServiceConfiguration get() {
 
                     configurationMap.put(
                         "model_id",
-                        new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                        new SettingsConfiguration.Builder().setDescription("")
                             .setLabel("Model ID")
-                            .setOrder(1)
                             .setRequired(true)
                             .setSensitive(true)
-                            .setTooltip("")
                             .setType(SettingsConfigurationFieldType.STRING)
                             .build()
                     );
 
-                    return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                        Map<String, SettingsConfiguration> taskSettingsConfig;
-                        switch (t) {
-                            default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                        }
-                        return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                    }).toList()).setConfiguration(configurationMap).build();
+                    return new InferenceServiceConfiguration.Builder().setService(NAME)
+                        .setTaskTypes(supportedTaskTypes)
+                        .setConfigurations(configurationMap)
+                        .build();
                 }
             );
         }
diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java
index c7b3a9d42f579..9da6b52555498 100644
--- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java
+++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java
@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.inference.action.filter;
 
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
@@ -17,6 +19,9 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.update.UpdateRequestBuilder;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.inference.SimilarityMeasure;
 import org.elasticsearch.plugins.Plugin;
@@ -30,9 +35,9 @@
 
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
@@ -41,9 +46,19 @@
 import static org.hamcrest.Matchers.equalTo;
 
 public class ShardBulkInferenceActionFilterIT extends ESIntegTestCase {
-
     public static final String INDEX_NAME = "test-index";
 
+    private final boolean useLegacyFormat;
+
+    public ShardBulkInferenceActionFilterIT(boolean useLegacyFormat) {
+        this.useLegacyFormat = useLegacyFormat;
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return List.of(new Object[] { true }, new Object[] { false });
+    }
+
     @Before
     public void setup() throws Exception {
         Utils.storeSparseModel(client());
@@ -62,8 +77,16 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(LocalStateInferencePlugin.class);
     }
 
+    @Override
+    public Settings indexSettings() {
+        return Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 10))
+            .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat)
+            .build();
+    }
+
     public void testBulkOperations() throws Exception {
-        Map<String, Integer> shardsSettings = Collections.singletonMap(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 10));
         indicesAdmin().prepareCreate(INDEX_NAME)
             .setMapping(
                 String.format(
@@ -86,7 +109,6 @@ public void testBulkOperations() throws Exception {
                     TestDenseInferenceServiceExtension.TestInferenceService.NAME
                 )
             )
-            .setSettings(shardsSettings)
             .get();
 
         int totalBulkReqs = randomIntBetween(2, 100);
@@ -152,5 +174,4 @@ public void testBulkOperations() throws Exception {
             searchResponse.decRef();
         }
     }
-
 }
diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListenerTests.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListenerTests.java
index ab3f466f3c11f..5fc4448c80940 100644
--- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListenerTests.java
+++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListenerTests.java
@@ -17,6 +17,7 @@
 import org.apache.http.nio.util.SimpleInputBuffer;
 import org.apache.http.protocol.HttpContext;
 import org.apache.http.util.EntityUtils;
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
@@ -25,6 +26,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -33,6 +35,7 @@
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
@@ -43,6 +46,7 @@
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEvent;
@@ -52,6 +56,7 @@
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Deque;
 import java.util.Iterator;
 import java.util.List;
@@ -96,6 +101,14 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     public static class StreamingPlugin extends Plugin implements ActionPlugin {
+        private final SetOnce<ThreadPool> threadPool = new SetOnce<>();
+
+        @Override
+        public Collection<?> createComponents(PluginServices services) {
+            threadPool.set(services.threadPool());
+            return Collections.emptyList();
+        }
+
         @Override
         public Collection<RestHandler> getRestHandlers(
             Settings settings,
@@ -122,7 +135,7 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c
                     var publisher = new RandomPublisher(requestCount, withError);
                     var inferenceServiceResults = new StreamingInferenceServiceResults(publisher);
                     var inferenceResponse = new InferenceAction.Response(inferenceServiceResults, inferenceServiceResults.publisher());
-                    new ServerSentEventsRestActionListener(channel).onResponse(inferenceResponse);
+                    new ServerSentEventsRestActionListener(channel, threadPool).onResponse(inferenceResponse);
                 }
             }, new RestHandler() {
                 @Override
@@ -132,7 +145,7 @@ public List<Route> routes() {
 
                 @Override
                 public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) {
-                    new ServerSentEventsRestActionListener(channel).onFailure(expectedException);
+                    new ServerSentEventsRestActionListener(channel, threadPool).onFailure(expectedException);
                 }
             }, new RestHandler() {
                 @Override
@@ -143,7 +156,7 @@ public List<Route> routes() {
                 @Override
                 public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) {
                     var inferenceResponse = new InferenceAction.Response(new SingleInferenceServiceResults());
-                    new ServerSentEventsRestActionListener(channel).onResponse(inferenceResponse);
+                    new ServerSentEventsRestActionListener(channel, threadPool).onResponse(inferenceResponse);
                 }
             });
         }
@@ -231,7 +244,11 @@ private static class RandomString implements ChunkedToXContent {
         @Override
         public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
             var randomString = randomUnicodeOfLengthBetween(2, 20);
-            return ChunkedToXContent.builder(params).object(b -> b.field("delta", randomString));
+            return Iterators.concat(
+                ChunkedToXContentHelper.startObject(),
+                ChunkedToXContentHelper.field("delta", randomString),
+                ChunkedToXContentHelper.endObject()
+            );
         }
     }
 
@@ -264,7 +281,7 @@ public void writeTo(StreamOutput out) {
 
         @Override
         public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-            return ChunkedToXContent.builder(params).field("result", randomUnicodeOfLengthBetween(2, 20));
+            return ChunkedToXContentHelper.field("result", randomUnicodeOfLengthBetween(2, 20));
         }
     }
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java
index 3b7613b8b0e1f..7d6069572ba21 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java
@@ -9,31 +9,21 @@
 
 import org.elasticsearch.features.FeatureSpecification;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsMapper;
 import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper;
-import org.elasticsearch.xpack.inference.queries.SemanticMatchQueryRewriteInterceptor;
-import org.elasticsearch.xpack.inference.queries.SemanticQueryBuilder;
-import org.elasticsearch.xpack.inference.rank.random.RandomRankRetrieverBuilder;
 import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankRetrieverBuilder;
 
 import java.util.Set;
 
+import static org.elasticsearch.xpack.inference.queries.SemanticKnnVectorQueryRewriteInterceptor.SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED;
+import static org.elasticsearch.xpack.inference.queries.SemanticMatchQueryRewriteInterceptor.SEMANTIC_MATCH_QUERY_REWRITE_INTERCEPTION_SUPPORTED;
+import static org.elasticsearch.xpack.inference.queries.SemanticSparseVectorQueryRewriteInterceptor.SEMANTIC_SPARSE_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED;
+
 /**
  * Provides inference features.
  */
 public class InferenceFeatures implements FeatureSpecification {
 
-    @Override
-    public Set<NodeFeature> getFeatures() {
-        return Set.of(
-            TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED,
-            RandomRankRetrieverBuilder.RANDOM_RERANKER_RETRIEVER_SUPPORTED,
-            SemanticTextFieldMapper.SEMANTIC_TEXT_SEARCH_INFERENCE_ID,
-            SemanticQueryBuilder.SEMANTIC_TEXT_INNER_HITS,
-            SemanticTextFieldMapper.SEMANTIC_TEXT_DEFAULT_ELSER_2,
-            TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_COMPOSITION_SUPPORTED
-        );
-    }
-
     private static final NodeFeature SEMANTIC_TEXT_HIGHLIGHTER = new NodeFeature("semantic_text.highlighter");
 
     @Override
@@ -45,7 +35,11 @@ public Set<NodeFeature> getTestFeatures() {
             SemanticTextFieldMapper.SEMANTIC_TEXT_ZERO_SIZE_FIX,
             SemanticTextFieldMapper.SEMANTIC_TEXT_ALWAYS_EMIT_INFERENCE_ID_FIX,
             SEMANTIC_TEXT_HIGHLIGHTER,
-            SemanticMatchQueryRewriteInterceptor.SEMANTIC_MATCH_QUERY_REWRITE_INTERCEPTION_SUPPORTED
+            SEMANTIC_MATCH_QUERY_REWRITE_INTERCEPTION_SUPPORTED,
+            SEMANTIC_SPARSE_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED,
+            SemanticInferenceMetadataFieldsMapper.EXPLICIT_NULL_FIXES,
+            SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED,
+            TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_ALIAS_HANDLING_FIX
         );
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
index 71fbcf6d8ef49..6fc9870034018 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
@@ -75,6 +75,11 @@
 import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceServiceSettings;
 import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserServiceSettings;
 import org.elasticsearch.xpack.inference.services.ibmwatsonx.embeddings.IbmWatsonxEmbeddingsServiceSettings;
+import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettings;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsServiceSettings;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettings;
+import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankServiceSettings;
+import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankTaskSettings;
 import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsServiceSettings;
 import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionServiceSettings;
 import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionTaskSettings;
@@ -132,6 +137,7 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         addAmazonBedrockNamedWriteables(namedWriteables);
         addEisNamedWriteables(namedWriteables);
         addAlibabaCloudSearchNamedWriteables(namedWriteables);
+        addJinaAINamedWriteables(namedWriteables);
 
         addUnifiedNamedWriteables(namedWriteables);
 
@@ -569,6 +575,28 @@ private static void addAlibabaCloudSearchNamedWriteables(List<NamedWriteableRegi
 
     }
 
+    private static void addJinaAINamedWriteables(List<NamedWriteableRegistry.Entry> namedWriteables) {
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(ServiceSettings.class, JinaAIServiceSettings.NAME, JinaAIServiceSettings::new)
+        );
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(
+                ServiceSettings.class,
+                JinaAIEmbeddingsServiceSettings.NAME,
+                JinaAIEmbeddingsServiceSettings::new
+            )
+        );
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(TaskSettings.class, JinaAIEmbeddingsTaskSettings.NAME, JinaAIEmbeddingsTaskSettings::new)
+        );
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(ServiceSettings.class, JinaAIRerankServiceSettings.NAME, JinaAIRerankServiceSettings::new)
+        );
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(TaskSettings.class, JinaAIRerankTaskSettings.NAME, JinaAIRerankTaskSettings::new)
+        );
+    }
+
     private static void addEisNamedWriteables(List<NamedWriteableRegistry.Entry> namedWriteables) {
         namedWriteables.add(
             new NamedWriteableRegistry.Entry(
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
index 93743a5485c2c..6ba529cb66eaa 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
@@ -25,9 +25,12 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.inference.InferenceServiceExtension;
 import org.elasticsearch.inference.InferenceServiceRegistry;
+import org.elasticsearch.license.License;
+import org.elasticsearch.license.LicensedFeature;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.node.PluginComponentBinding;
 import org.elasticsearch.plugins.ActionPlugin;
@@ -44,6 +47,7 @@
 import org.elasticsearch.search.rank.RankDoc;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ScalingExecutorBuilder;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.XPackPlugin;
@@ -77,9 +81,12 @@
 import org.elasticsearch.xpack.inference.highlight.SemanticTextHighlighter;
 import org.elasticsearch.xpack.inference.logging.ThrottlerManager;
 import org.elasticsearch.xpack.inference.mapper.OffsetSourceFieldMapper;
+import org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsMapper;
 import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper;
+import org.elasticsearch.xpack.inference.queries.SemanticKnnVectorQueryRewriteInterceptor;
 import org.elasticsearch.xpack.inference.queries.SemanticMatchQueryRewriteInterceptor;
 import org.elasticsearch.xpack.inference.queries.SemanticQueryBuilder;
+import org.elasticsearch.xpack.inference.queries.SemanticSparseVectorQueryRewriteInterceptor;
 import org.elasticsearch.xpack.inference.rank.random.RandomRankBuilder;
 import org.elasticsearch.xpack.inference.rank.random.RandomRankRetrieverBuilder;
 import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankBuilder;
@@ -111,6 +118,7 @@
 import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceService;
 import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserService;
 import org.elasticsearch.xpack.inference.services.ibmwatsonx.IbmWatsonxService;
+import org.elasticsearch.xpack.inference.services.jinaai.JinaAIService;
 import org.elasticsearch.xpack.inference.services.mistral.MistralService;
 import org.elasticsearch.xpack.inference.services.openai.OpenAiService;
 import org.elasticsearch.xpack.inference.telemetry.InferenceStats;
@@ -146,6 +154,12 @@ public class InferencePlugin extends Plugin implements ActionPlugin, ExtensibleP
         Setting.Property.Dynamic
     );
 
+    public static final LicensedFeature.Momentary INFERENCE_API_FEATURE = LicensedFeature.momentary(
+        "inference",
+        "api",
+        License.OperationMode.ENTERPRISE
+    );
+
     public static final String NAME = "inference";
     public static final String UTILITY_THREAD_POOL_NAME = "inference_utility";
 
@@ -156,6 +170,9 @@ public class InferencePlugin extends Plugin implements ActionPlugin, ExtensibleP
     private final SetOnce<AmazonBedrockRequestSender.Factory> amazonBedrockFactory = new SetOnce<>();
     private final SetOnce<HttpRequestSender.Factory> elasicInferenceServiceFactory = new SetOnce<>();
     private final SetOnce<ServiceComponents> serviceComponents = new SetOnce<>();
+    // This is mainly so that the rest handlers can access the ThreadPool in a way that avoids potential null pointers from it
+    // not being initialized yet
+    private final SetOnce<ThreadPool> threadPoolSetOnce = new SetOnce<>();
     private final SetOnce<ElasticInferenceServiceComponents> elasticInferenceServiceComponents = new SetOnce<>();
     private final SetOnce<InferenceServiceRegistry> inferenceServiceRegistry = new SetOnce<>();
     private final SetOnce<ShardBulkInferenceActionFilter> shardBulkInferenceActionFilter = new SetOnce<>();
@@ -201,7 +218,7 @@ public List<RestHandler> getRestHandlers(
     ) {
         var availableRestActions = List.of(
             new RestInferenceAction(),
-            new RestStreamInferenceAction(),
+            new RestStreamInferenceAction(threadPoolSetOnce),
             new RestGetInferenceModelAction(),
             new RestPutInferenceModelAction(),
             new RestUpdateInferenceModelAction(),
@@ -210,7 +227,7 @@ public List<RestHandler> getRestHandlers(
             new RestGetInferenceServicesAction()
         );
         List<RestHandler> conditionalRestActions = UnifiedCompletionFeature.UNIFIED_COMPLETION_FEATURE_FLAG.isEnabled()
-            ? List.of(new RestUnifiedCompletionInferenceAction())
+            ? List.of(new RestUnifiedCompletionInferenceAction(threadPoolSetOnce))
             : List.of();
 
         return Stream.concat(availableRestActions.stream(), conditionalRestActions.stream()).toList();
@@ -221,6 +238,7 @@ public Collection<?> createComponents(PluginServices services) {
         var throttlerManager = new ThrottlerManager(settings, services.threadPool(), services.clusterService());
         var truncator = new Truncator(settings, services.clusterService());
         serviceComponents.set(new ServiceComponents(services.threadPool(), throttlerManager, settings, truncator));
+        threadPoolSetOnce.set(services.threadPool());
 
         var httpClientManager = HttpClientManager.create(settings, services.threadPool(), services.clusterService(), throttlerManager);
         var httpRequestSenderFactory = new HttpRequestSender.Factory(serviceComponents.get(), httpClientManager, services.clusterService());
@@ -285,7 +303,7 @@ public Collection<?> createComponents(PluginServices services) {
         }
         inferenceServiceRegistry.set(registry);
 
-        var actionFilter = new ShardBulkInferenceActionFilter(registry, modelRegistry);
+        var actionFilter = new ShardBulkInferenceActionFilter(services.clusterService(), registry, modelRegistry);
         shardBulkInferenceActionFilter.set(actionFilter);
 
         var meterRegistry = services.telemetryProvider().getMeterRegistry();
@@ -314,6 +332,7 @@ public List<InferenceServiceExtension.Factory> getInferenceServiceFactories() {
             context -> new AmazonBedrockService(httpFactory.get(), amazonBedrockFactory.get(), serviceComponents.get()),
             context -> new AlibabaCloudSearchService(httpFactory.get(), serviceComponents.get()),
             context -> new IbmWatsonxService(httpFactory.get(), serviceComponents.get()),
+            context -> new JinaAIService(httpFactory.get(), serviceComponents.get()),
             ElasticsearchInternalService::new
         );
     }
@@ -419,6 +438,11 @@ public void close() {
         IOUtils.closeWhileHandlingException(inferenceServiceRegistry.get(), throttlerToClose);
     }
 
+    @Override
+    public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
+        return Map.of(SemanticInferenceMetadataFieldsMapper.NAME, SemanticInferenceMetadataFieldsMapper.PARSER);
+    }
+
     @Override
     public Map<String, Mapper.TypeParser> getMappers() {
         return Map.of(
@@ -440,7 +464,11 @@ public List<QuerySpec<?>> getQueries() {
 
     @Override
     public List<QueryRewriteInterceptor> getQueryRewriteInterceptors() {
-        return List.of(new SemanticMatchQueryRewriteInterceptor());
+        return List.of(
+            new SemanticKnnVectorQueryRewriteInterceptor(),
+            new SemanticMatchQueryRewriteInterceptor(),
+            new SemanticSparseVectorQueryRewriteInterceptor()
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java
index 2a0e8e1775279..b6c7d26b36f9a 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java
@@ -23,9 +23,12 @@
 import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.inference.UnparsedModel;
+import org.elasticsearch.license.LicenseUtils;
+import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.inference.action.BaseInferenceActionRequest;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager;
@@ -38,6 +41,7 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.core.Strings.format;
+import static org.elasticsearch.xpack.inference.InferencePlugin.INFERENCE_API_FEATURE;
 import static org.elasticsearch.xpack.inference.telemetry.InferenceStats.modelAttributes;
 import static org.elasticsearch.xpack.inference.telemetry.InferenceStats.responseAttributes;
 
@@ -48,6 +52,7 @@ public abstract class BaseTransportInferenceAction<Request extends BaseInference
     private static final Logger log = LogManager.getLogger(BaseTransportInferenceAction.class);
     private static final String STREAMING_INFERENCE_TASK_TYPE = "streaming_inference";
     private static final String STREAMING_TASK_ACTION = "xpack/inference/streaming_inference[n]";
+    private final XPackLicenseState licenseState;
     private final ModelRegistry modelRegistry;
     private final InferenceServiceRegistry serviceRegistry;
     private final InferenceStats inferenceStats;
@@ -57,6 +62,7 @@ public BaseTransportInferenceAction(
         String inferenceActionName,
         TransportService transportService,
         ActionFilters actionFilters,
+        XPackLicenseState licenseState,
         ModelRegistry modelRegistry,
         InferenceServiceRegistry serviceRegistry,
         InferenceStats inferenceStats,
@@ -64,6 +70,7 @@ public BaseTransportInferenceAction(
         Writeable.Reader<Request> requestReader
     ) {
         super(inferenceActionName, transportService, actionFilters, requestReader, EsExecutors.DIRECT_EXECUTOR_SERVICE);
+        this.licenseState = licenseState;
         this.modelRegistry = modelRegistry;
         this.serviceRegistry = serviceRegistry;
         this.inferenceStats = inferenceStats;
@@ -72,6 +79,11 @@ public BaseTransportInferenceAction(
 
     @Override
     protected void doExecute(Task task, Request request, ActionListener<InferenceAction.Response> listener) {
+        if (INFERENCE_API_FEATURE.check(licenseState) == false) {
+            listener.onFailure(LicenseUtils.newComplianceException(XPackField.INFERENCE));
+            return;
+        }
+
         var timer = InferenceTimer.start();
 
         var getModelListener = ActionListener.wrap((UnparsedModel unparsedModel) -> {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java
index c1dbd8cfec9d5..bdbd190a5120e 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java
@@ -205,16 +205,11 @@ private static String buildErrorString(String inferenceEndpointId, Set<String> p
     }
 
     private static Set<String> endpointIsReferencedInIndex(final ClusterState state, final String inferenceEndpointId) {
-        Set<String> indexes = extractIndexesReferencingInferenceEndpoints(state.getMetadata(), Set.of(inferenceEndpointId));
-        return indexes;
+        return extractIndexesReferencingInferenceEndpoints(state.getMetadata(), Set.of(inferenceEndpointId));
     }
 
     private static Set<String> endpointIsReferencedInPipelines(final ClusterState state, final String inferenceEndpointId) {
-        Set<String> modelIdsReferencedByPipelines = InferenceProcessorInfoExtractor.pipelineIdsForResource(
-            state,
-            Set.of(inferenceEndpointId)
-        );
-        return modelIdsReferencedByPipelines;
+        return InferenceProcessorInfoExtractor.pipelineIdsForResource(state, Set.of(inferenceEndpointId));
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceServicesAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceServicesAction.java
index 002b2b0fe93b0..0d8d7e81019a6 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceServicesAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceServicesAction.java
@@ -21,9 +21,9 @@
 import org.elasticsearch.xpack.core.inference.action.GetInferenceServicesAction;
 
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.stream.Collectors;
 
 public class TransportGetInferenceServicesAction extends HandledTransportAction<
@@ -72,7 +72,8 @@ private void getServiceConfigurationsForTaskType(
                 service -> service.getValue().hideFromConfigurationApi() == false
                     && service.getValue().supportedTaskTypes().contains(requestedTaskType)
             )
-            .collect(Collectors.toSet());
+            .sorted(Comparator.comparing(service -> service.getValue().name()))
+            .collect(Collectors.toCollection(ArrayList::new));
 
         getServiceConfigurationsForServices(filteredServices, listener.delegateFailureAndWrap((delegate, configurations) -> {
             delegate.onResponse(new GetInferenceServicesAction.Response(configurations));
@@ -84,14 +85,15 @@ private void getAllServiceConfigurations(ActionListener<GetInferenceServicesActi
             .entrySet()
             .stream()
             .filter(service -> service.getValue().hideFromConfigurationApi() == false)
-            .collect(Collectors.toSet());
+            .sorted(Comparator.comparing(service -> service.getValue().name()))
+            .collect(Collectors.toCollection(ArrayList::new));
         getServiceConfigurationsForServices(availableServices, listener.delegateFailureAndWrap((delegate, configurations) -> {
             delegate.onResponse(new GetInferenceServicesAction.Response(configurations));
         }));
     }
 
     private void getServiceConfigurationsForServices(
-        Set<Map.Entry<String, InferenceService>> services,
+        ArrayList<Map.Entry<String, InferenceService>> services,
         ActionListener<List<InferenceServiceConfiguration>> listener
     ) {
         try {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java
index 08e6d869a553d..24ef0d7d610d0 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.UnparsedModel;
 import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager;
@@ -28,6 +29,7 @@ public class TransportInferenceAction extends BaseTransportInferenceAction<Infer
     public TransportInferenceAction(
         TransportService transportService,
         ActionFilters actionFilters,
+        XPackLicenseState licenseState,
         ModelRegistry modelRegistry,
         InferenceServiceRegistry serviceRegistry,
         InferenceStats inferenceStats,
@@ -37,6 +39,7 @@ public TransportInferenceAction(
             InferenceAction.NAME,
             transportService,
             actionFilters,
+            licenseState,
             modelRegistry,
             serviceRegistry,
             inferenceStats,
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java
index 2baee7f8afd66..a5d987d198c38 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
@@ -30,12 +29,15 @@
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.license.LicenseUtils;
+import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.inference.action.PutInferenceModelAction;
 import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignmentUtils;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
@@ -50,6 +52,7 @@
 import java.util.Map;
 
 import static org.elasticsearch.core.Strings.format;
+import static org.elasticsearch.xpack.inference.InferencePlugin.INFERENCE_API_FEATURE;
 import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.OLD_ELSER_SERVICE_NAME;
 
 public class TransportPutInferenceModelAction extends TransportMasterNodeAction<
@@ -58,9 +61,9 @@ public class TransportPutInferenceModelAction extends TransportMasterNodeAction<
 
     private static final Logger logger = LogManager.getLogger(TransportPutInferenceModelAction.class);
 
+    private final XPackLicenseState licenseState;
     private final ModelRegistry modelRegistry;
     private final InferenceServiceRegistry serviceRegistry;
-    private final Client client;
     private volatile boolean skipValidationAndStart;
 
     @Inject
@@ -70,9 +73,9 @@ public TransportPutInferenceModelAction(
         ThreadPool threadPool,
         ActionFilters actionFilters,
         IndexNameExpressionResolver indexNameExpressionResolver,
+        XPackLicenseState licenseState,
         ModelRegistry modelRegistry,
         InferenceServiceRegistry serviceRegistry,
-        Client client,
         Settings settings
     ) {
         super(
@@ -86,9 +89,9 @@ public TransportPutInferenceModelAction(
             PutInferenceModelAction.Response::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
+        this.licenseState = licenseState;
         this.modelRegistry = modelRegistry;
         this.serviceRegistry = serviceRegistry;
-        this.client = client;
         this.skipValidationAndStart = InferencePlugin.SKIP_VALIDATE_AND_START.get(settings);
         clusterService.getClusterSettings()
             .addSettingsUpdateConsumer(InferencePlugin.SKIP_VALIDATE_AND_START, this::setSkipValidationAndStart);
@@ -101,6 +104,11 @@ protected void masterOperation(
         ClusterState state,
         ActionListener<PutInferenceModelAction.Response> listener
     ) throws Exception {
+        if (INFERENCE_API_FEATURE.check(licenseState) == false) {
+            listener.onFailure(LicenseUtils.newComplianceException(XPackField.INFERENCE));
+            return;
+        }
+
         var requestAsMap = requestToMap(request);
         var resolvedTaskType = ServiceUtils.resolveTaskType(request.getTaskType(), (String) requestAsMap.remove(TaskType.NAME));
 
@@ -153,7 +161,7 @@ protected void masterOperation(
         if ((assignments == null || assignments.isEmpty()) == false) {
             listener.onFailure(
                 ExceptionsHelper.badRequestException(
-                    Messages.MODEL_ID_MATCHES_EXISTING_MODEL_IDS_BUT_MUST_NOT,
+                    Messages.INFERENCE_ID_MATCHES_EXISTING_MODEL_IDS_BUT_MUST_NOT,
                     request.getInferenceEntityId()
                 )
             );
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java
index f0906231d8f42..1478130f6a6c8 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.inference.UnparsedModel;
 import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction;
@@ -30,6 +31,7 @@ public class TransportUnifiedCompletionInferenceAction extends BaseTransportInfe
     public TransportUnifiedCompletionInferenceAction(
         TransportService transportService,
         ActionFilters actionFilters,
+        XPackLicenseState licenseState,
         ModelRegistry modelRegistry,
         InferenceServiceRegistry serviceRegistry,
         InferenceStats inferenceStats,
@@ -39,6 +41,7 @@ public TransportUnifiedCompletionInferenceAction(
             UnifiedCompletionAction.NAME,
             transportService,
             actionFilters,
+            licenseState,
             modelRegistry,
             serviceRegistry,
             inferenceStats,
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java
index 03a88e5228fa8..4a90f73722438 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.inference.InferenceService;
@@ -35,12 +34,15 @@
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.inference.UnparsedModel;
 import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.license.LicenseUtils;
+import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.inference.action.UpdateInferenceModelAction;
 import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction;
 import org.elasticsearch.xpack.core.ml.action.UpdateTrainedModelDeploymentAction;
@@ -58,6 +60,7 @@
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicReference;
 
+import static org.elasticsearch.xpack.inference.InferencePlugin.INFERENCE_API_FEATURE;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.resolveTaskType;
 import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS;
 
@@ -67,6 +70,7 @@ public class TransportUpdateInferenceModelAction extends TransportMasterNodeActi
 
     private static final Logger logger = LogManager.getLogger(TransportUpdateInferenceModelAction.class);
 
+    private final XPackLicenseState licenseState;
     private final ModelRegistry modelRegistry;
     private final InferenceServiceRegistry serviceRegistry;
     private final Client client;
@@ -78,10 +82,10 @@ public TransportUpdateInferenceModelAction(
         ThreadPool threadPool,
         ActionFilters actionFilters,
         IndexNameExpressionResolver indexNameExpressionResolver,
+        XPackLicenseState licenseState,
         ModelRegistry modelRegistry,
         InferenceServiceRegistry serviceRegistry,
-        Client client,
-        Settings settings
+        Client client
     ) {
         super(
             UpdateInferenceModelAction.NAME,
@@ -94,6 +98,7 @@ public TransportUpdateInferenceModelAction(
             UpdateInferenceModelAction.Response::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
+        this.licenseState = licenseState;
         this.modelRegistry = modelRegistry;
         this.serviceRegistry = serviceRegistry;
         this.client = client;
@@ -106,6 +111,11 @@ protected void masterOperation(
         ClusterState state,
         ActionListener<UpdateInferenceModelAction.Response> masterListener
     ) {
+        if (INFERENCE_API_FEATURE.check(licenseState) == false) {
+            masterListener.onFailure(LicenseUtils.newComplianceException(XPackField.INFERENCE));
+            return;
+        }
+
         var bodyTaskType = request.getContentAsSettings().taskType();
         var resolvedTaskType = resolveTaskType(request.getTaskType(), bodyTaskType != null ? bodyTaskType.toString() : null);
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java
index a9195ea24af3a..f4aa49bad1648 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java
@@ -24,11 +24,13 @@
 import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.inference.ChunkedInference;
 import org.elasticsearch.inference.InferenceService;
 import org.elasticsearch.inference.InferenceServiceRegistry;
@@ -37,24 +39,24 @@
 import org.elasticsearch.inference.UnparsedModel;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
+import org.elasticsearch.xcontent.XContent;
 import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceError;
 import org.elasticsearch.xpack.inference.mapper.SemanticTextField;
 import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper;
+import org.elasticsearch.xpack.inference.mapper.SemanticTextUtils;
 import org.elasticsearch.xpack.inference.registry.ModelRegistry;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.toSemanticTextFieldChunks;
-
 /**
  * A {@link MappedActionFilter} that intercepts {@link BulkShardRequest} to apply inference on fields specified
  * as {@link SemanticTextFieldMapper} in the index mapping. For each semantic text field referencing fields in
@@ -67,16 +69,29 @@
  */
 public class ShardBulkInferenceActionFilter implements MappedActionFilter {
     protected static final int DEFAULT_BATCH_SIZE = 512;
+    private static final Object EXPLICIT_NULL = new Object();
+    private static final ChunkedInference EMPTY_CHUNKED_INFERENCE = new EmptyChunkedInference();
 
+    private final ClusterService clusterService;
     private final InferenceServiceRegistry inferenceServiceRegistry;
     private final ModelRegistry modelRegistry;
     private final int batchSize;
 
-    public ShardBulkInferenceActionFilter(InferenceServiceRegistry inferenceServiceRegistry, ModelRegistry modelRegistry) {
-        this(inferenceServiceRegistry, modelRegistry, DEFAULT_BATCH_SIZE);
+    public ShardBulkInferenceActionFilter(
+        ClusterService clusterService,
+        InferenceServiceRegistry inferenceServiceRegistry,
+        ModelRegistry modelRegistry
+    ) {
+        this(clusterService, inferenceServiceRegistry, modelRegistry, DEFAULT_BATCH_SIZE);
     }
 
-    public ShardBulkInferenceActionFilter(InferenceServiceRegistry inferenceServiceRegistry, ModelRegistry modelRegistry, int batchSize) {
+    public ShardBulkInferenceActionFilter(
+        ClusterService clusterService,
+        InferenceServiceRegistry inferenceServiceRegistry,
+        ModelRegistry modelRegistry,
+        int batchSize
+    ) {
+        this.clusterService = clusterService;
         this.inferenceServiceRegistry = inferenceServiceRegistry;
         this.modelRegistry = modelRegistry;
         this.batchSize = batchSize;
@@ -112,7 +127,9 @@ private void processBulkShardRequest(
         BulkShardRequest bulkShardRequest,
         Runnable onCompletion
     ) {
-        new AsyncBulkShardInferenceAction(fieldInferenceMap, bulkShardRequest, onCompletion).run();
+        var index = clusterService.state().getMetadata().index(bulkShardRequest.index());
+        boolean useLegacyFormat = InferenceMetadataFieldsMapper.isEnabled(index.getSettings()) == false;
+        new AsyncBulkShardInferenceAction(useLegacyFormat, fieldInferenceMap, bulkShardRequest, onCompletion).run();
     }
 
     private record InferenceProvider(InferenceService service, Model model) {}
@@ -121,26 +138,29 @@ private record InferenceProvider(InferenceService service, Model model) {}
      * A field inference request on a single input.
      * @param index The index of the request in the original bulk request.
      * @param field The target field.
+     * @param sourceField The source field.
      * @param input The input to run inference on.
      * @param inputOrder The original order of the input.
-     * @param isOriginalFieldInput Whether the input is part of the original values of the field.
+     * @param offsetAdjustment The adjustment to apply to the chunk text offsets.
      */
-    private record FieldInferenceRequest(int index, String field, String input, int inputOrder, boolean isOriginalFieldInput) {}
+    private record FieldInferenceRequest(int index, String field, String sourceField, String input, int inputOrder, int offsetAdjustment) {}
 
     /**
      * The field inference response.
      * @param field The target field.
+     * @param sourceField The input that was used to run inference.
      * @param input The input that was used to run inference.
      * @param inputOrder The original order of the input.
-     * @param isOriginalFieldInput Whether the input is part of the original values of the field.
+     * @param offsetAdjustment The adjustment to apply to the chunk text offsets.
      * @param model The model used to run inference.
      * @param chunkedResults The actual results.
      */
     private record FieldInferenceResponse(
         String field,
+        String sourceField,
         String input,
         int inputOrder,
-        boolean isOriginalFieldInput,
+        int offsetAdjustment,
         Model model,
         ChunkedInference chunkedResults
     ) {}
@@ -165,16 +185,19 @@ void addFailure(Exception exc) {
     }
 
     private class AsyncBulkShardInferenceAction implements Runnable {
+        private final boolean useLegacyFormat;
         private final Map<String, InferenceFieldMetadata> fieldInferenceMap;
         private final BulkShardRequest bulkShardRequest;
         private final Runnable onCompletion;
         private final AtomicArray<FieldInferenceResponseAccumulator> inferenceResults;
 
         private AsyncBulkShardInferenceAction(
+            boolean useLegacyFormat,
             Map<String, InferenceFieldMetadata> fieldInferenceMap,
             BulkShardRequest bulkShardRequest,
             Runnable onCompletion
         ) {
+            this.useLegacyFormat = useLegacyFormat;
             this.fieldInferenceMap = fieldInferenceMap;
             this.bulkShardRequest = bulkShardRequest;
             this.inferenceResults = new AtomicArray<>(bulkShardRequest.items().length);
@@ -295,9 +318,10 @@ public void onResponse(List<ChunkedInference> results) {
                                 acc.addOrUpdateResponse(
                                     new FieldInferenceResponse(
                                         request.field(),
+                                        request.sourceField(),
                                         request.input(),
                                         request.inputOrder(),
-                                        request.isOriginalFieldInput(),
+                                        request.offsetAdjustment(),
                                         inferenceProvider.model,
                                         result
                                     )
@@ -357,8 +381,7 @@ private void addInferenceResponseFailure(int id, Exception failure) {
         /**
          * Applies the {@link FieldInferenceResponseAccumulator} to the provided {@link BulkItemRequest}.
          * If the response contains failures, the bulk item request is marked as failed for the downstream action.
-         * Otherwise, the source of the request is augmented with the field inference results under the
-         * {@link SemanticTextField#INFERENCE_FIELD} field.
+         * Otherwise, the source of the request is augmented with the field inference results.
          */
         private void applyInferenceResponses(BulkItemRequest item, FieldInferenceResponseAccumulator response) throws IOException {
             if (response.failures().isEmpty() == false) {
@@ -370,25 +393,65 @@ private void applyInferenceResponses(BulkItemRequest item, FieldInferenceRespons
 
             final IndexRequest indexRequest = getIndexRequestOrNull(item.request());
             var newDocMap = indexRequest.sourceAsMap();
+            Map<String, Object> inferenceFieldsMap = new HashMap<>();
             for (var entry : response.responses.entrySet()) {
                 var fieldName = entry.getKey();
                 var responses = entry.getValue();
-                var model = responses.get(0).model();
+                Model model = null;
+
+                InferenceFieldMetadata inferenceFieldMetadata = fieldInferenceMap.get(fieldName);
+                if (inferenceFieldMetadata == null) {
+                    throw new IllegalStateException("No inference field metadata for field [" + fieldName + "]");
+                }
+
                 // ensure that the order in the original field is consistent in case of multiple inputs
                 Collections.sort(responses, Comparator.comparingInt(FieldInferenceResponse::inputOrder));
-                List<String> inputs = responses.stream().filter(r -> r.isOriginalFieldInput).map(r -> r.input).collect(Collectors.toList());
-                List<ChunkedInference> results = responses.stream().map(r -> r.chunkedResults).collect(Collectors.toList());
+                Map<String, List<SemanticTextField.Chunk>> chunkMap = new LinkedHashMap<>();
+                for (var resp : responses) {
+                    // Get the first non-null model from the response list
+                    if (model == null) {
+                        model = resp.model;
+                    }
+
+                    var lst = chunkMap.computeIfAbsent(resp.sourceField, k -> new ArrayList<>());
+                    lst.addAll(
+                        SemanticTextField.toSemanticTextFieldChunks(
+                            resp.input,
+                            resp.offsetAdjustment,
+                            resp.chunkedResults,
+                            indexRequest.getContentType(),
+                            useLegacyFormat
+                        )
+                    );
+                }
+
+                List<String> inputs = responses.stream()
+                    .filter(r -> r.sourceField().equals(fieldName))
+                    .map(r -> r.input)
+                    .collect(Collectors.toList());
+
+                // The model can be null if we are only processing update requests that clear inference results. This is ok because we will
+                // merge in the field's existing model settings on the data node.
                 var result = new SemanticTextField(
+                    useLegacyFormat,
                     fieldName,
-                    inputs,
+                    useLegacyFormat ? inputs : null,
                     new SemanticTextField.InferenceResult(
-                        model.getInferenceEntityId(),
-                        new SemanticTextField.ModelSettings(model),
-                        toSemanticTextFieldChunks(results, indexRequest.getContentType())
+                        inferenceFieldMetadata.getInferenceId(),
+                        model != null ? new SemanticTextField.ModelSettings(model) : null,
+                        chunkMap
                     ),
                     indexRequest.getContentType()
                 );
-                SemanticTextFieldMapper.insertValue(fieldName, newDocMap, result);
+
+                if (useLegacyFormat) {
+                    SemanticTextUtils.insertValue(fieldName, newDocMap, result);
+                } else {
+                    inferenceFieldsMap.put(fieldName, result);
+                }
+            }
+            if (useLegacyFormat == false) {
+                newDocMap.put(InferenceMetadataFieldsMapper.NAME, inferenceFieldsMap);
             }
             indexRequest.source(newDocMap, indexRequest.getContentType());
         }
@@ -437,17 +500,44 @@ private Map<String, List<FieldInferenceRequest>> createFieldInferenceRequests(Bu
                 for (var entry : fieldInferenceMap.values()) {
                     String field = entry.getName();
                     String inferenceId = entry.getInferenceId();
-                    var originalFieldValue = XContentMapValues.extractValue(field, docMap);
-                    if (originalFieldValue instanceof Map || (originalFieldValue == null && entry.getSourceFields().length == 1)) {
-                        // Inference has already been computed, or there is no inference required.
-                        continue;
+
+                    if (useLegacyFormat) {
+                        var originalFieldValue = XContentMapValues.extractValue(field, docMap);
+                        if (originalFieldValue instanceof Map || (originalFieldValue == null && entry.getSourceFields().length == 1)) {
+                            // Inference has already been computed, or there is no inference required.
+                            continue;
+                        }
+                    } else {
+                        var inferenceMetadataFieldsValue = XContentMapValues.extractValue(
+                            InferenceMetadataFieldsMapper.NAME + "." + field,
+                            docMap,
+                            EXPLICIT_NULL
+                        );
+                        if (inferenceMetadataFieldsValue != null) {
+                            // Inference has already been computed
+                            continue;
+                        }
                     }
+
                     int order = 0;
                     for (var sourceField : entry.getSourceFields()) {
-                        boolean isOriginalFieldInput = sourceField.equals(field);
-                        var valueObj = XContentMapValues.extractValue(sourceField, docMap);
-                        if (valueObj == null) {
-                            if (isUpdateRequest) {
+                        var valueObj = XContentMapValues.extractValue(sourceField, docMap, EXPLICIT_NULL);
+                        if (useLegacyFormat == false && isUpdateRequest && valueObj == EXPLICIT_NULL) {
+                            /**
+                             * It's an update request, and the source field is explicitly set to null,
+                             * so we need to propagate this information to the inference fields metadata
+                             * to overwrite any inference previously computed on the field.
+                             * This ensures that the field is treated as intentionally cleared,
+                             * preventing any unintended carryover of prior inference results.
+                             */
+                            var slot = ensureResponseAccumulatorSlot(itemIndex);
+                            slot.addOrUpdateResponse(
+                                new FieldInferenceResponse(field, sourceField, null, order++, 0, null, EMPTY_CHUNKED_INFERENCE)
+                            );
+                            continue;
+                        }
+                        if (valueObj == null || valueObj == EXPLICIT_NULL) {
+                            if (isUpdateRequest && useLegacyFormat) {
                                 addInferenceResponseFailure(
                                     item.id(),
                                     new ElasticsearchStatusException(
@@ -464,14 +554,21 @@ private Map<String, List<FieldInferenceRequest>> createFieldInferenceRequests(Bu
                         ensureResponseAccumulatorSlot(itemIndex);
                         final List<String> values;
                         try {
-                            values = nodeStringValues(field, valueObj);
+                            values = SemanticTextUtils.nodeStringValues(field, valueObj);
                         } catch (Exception exc) {
                             addInferenceResponseFailure(item.id(), exc);
                             break;
                         }
+
                         List<FieldInferenceRequest> fieldRequests = fieldRequestsMap.computeIfAbsent(inferenceId, k -> new ArrayList<>());
-                        for (var v : values) {
-                            fieldRequests.add(new FieldInferenceRequest(itemIndex, field, v, order++, isOriginalFieldInput));
+                        int offsetAdjustment = 0;
+                        for (String v : values) {
+                            fieldRequests.add(new FieldInferenceRequest(itemIndex, field, sourceField, v, order++, offsetAdjustment));
+
+                            // When using the inference metadata fields format, all the input values are concatenated so that the
+                            // chunk text offsets are expressed in the context of a single string. Calculate the offset adjustment
+                            // to apply to account for this.
+                            offsetAdjustment += v.length() + 1; // Add one for separator char length
                         }
                     }
                 }
@@ -480,41 +577,6 @@ private Map<String, List<FieldInferenceRequest>> createFieldInferenceRequests(Bu
         }
     }
 
-    /**
-     * This method converts the given {@code valueObj} into a list of strings.
-     * If {@code valueObj} is not a string or a collection of strings, it throws an ElasticsearchStatusException.
-     */
-    private static List<String> nodeStringValues(String field, Object valueObj) {
-        if (valueObj instanceof Number || valueObj instanceof Boolean) {
-            return List.of(valueObj.toString());
-        } else if (valueObj instanceof String value) {
-            return List.of(value);
-        } else if (valueObj instanceof Collection<?> values) {
-            List<String> valuesString = new ArrayList<>();
-            for (var v : values) {
-                if (v instanceof Number || v instanceof Boolean) {
-                    valuesString.add(v.toString());
-                } else if (v instanceof String value) {
-                    valuesString.add(value);
-                } else {
-                    throw new ElasticsearchStatusException(
-                        "Invalid format for field [{}], expected [String] got [{}]",
-                        RestStatus.BAD_REQUEST,
-                        field,
-                        valueObj.getClass().getSimpleName()
-                    );
-                }
-            }
-            return valuesString;
-        }
-        throw new ElasticsearchStatusException(
-            "Invalid format for field [{}], expected [String] got [{}]",
-            RestStatus.BAD_REQUEST,
-            field,
-            valueObj.getClass().getSimpleName()
-        );
-    }
-
     static IndexRequest getIndexRequestOrNull(DocWriteRequest<?> docWriteRequest) {
         if (docWriteRequest instanceof IndexRequest indexRequest) {
             return indexRequest;
@@ -524,4 +586,11 @@ static IndexRequest getIndexRequestOrNull(DocWriteRequest<?> docWriteRequest) {
             return null;
         }
     }
+
+    private static class EmptyChunkedInference implements ChunkedInference {
+        @Override
+        public Iterator<Chunk> chunksAsMatchedTextAndByteReference(XContent xcontent) {
+            return Collections.emptyIterator();
+        }
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionCreator.java
new file mode 100644
index 0000000000000..4d5827a3bf266
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionCreator.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.action.jinaai;
+
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction;
+import org.elasticsearch.xpack.inference.external.http.sender.JinaAIEmbeddingsRequestManager;
+import org.elasticsearch.xpack.inference.external.http.sender.JinaAIRerankRequestManager;
+import org.elasticsearch.xpack.inference.external.http.sender.Sender;
+import org.elasticsearch.xpack.inference.services.ServiceComponents;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModel;
+import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankModel;
+
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage;
+
+/**
+ * Provides a way to construct an {@link ExecutableAction} using the visitor pattern based on the jinaai model type.
+ */
+public class JinaAIActionCreator implements JinaAIActionVisitor {
+    private final Sender sender;
+    private final ServiceComponents serviceComponents;
+
+    public JinaAIActionCreator(Sender sender, ServiceComponents serviceComponents) {
+        this.sender = Objects.requireNonNull(sender);
+        this.serviceComponents = Objects.requireNonNull(serviceComponents);
+    }
+
+    @Override
+    public ExecutableAction create(JinaAIEmbeddingsModel model, Map<String, Object> taskSettings, InputType inputType) {
+        var overriddenModel = JinaAIEmbeddingsModel.of(model, taskSettings, inputType);
+        var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(
+            overriddenModel.getServiceSettings().getCommonSettings().uri(),
+            "JinaAI embeddings"
+        );
+        var requestCreator = JinaAIEmbeddingsRequestManager.of(overriddenModel, serviceComponents.threadPool());
+        return new SenderExecutableAction(sender, requestCreator, failedToSendRequestErrorMessage);
+    }
+
+    @Override
+    public ExecutableAction create(JinaAIRerankModel model, Map<String, Object> taskSettings) {
+        var overriddenModel = JinaAIRerankModel.of(model, taskSettings);
+        var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(
+            overriddenModel.getServiceSettings().getCommonSettings().uri(),
+            "JinaAI rerank"
+        );
+        var requestCreator = JinaAIRerankRequestManager.of(overriddenModel, serviceComponents.threadPool());
+        return new SenderExecutableAction(sender, requestCreator, failedToSendRequestErrorMessage);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionVisitor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionVisitor.java
new file mode 100644
index 0000000000000..c585e68e3d731
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionVisitor.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.action.jinaai;
+
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModel;
+import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankModel;
+
+import java.util.Map;
+
+public interface JinaAIActionVisitor {
+    ExecutableAction create(JinaAIEmbeddingsModel model, Map<String, Object> taskSettings, InputType inputType);
+
+    ExecutableAction create(JinaAIRerankModel model, Map<String, Object> taskSettings);
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceUnifiedChatCompletionResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceUnifiedChatCompletionResponseHandler.java
new file mode 100644
index 0000000000000..c0bccb9b2cd49
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceUnifiedChatCompletionResponseHandler.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.elastic;
+
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.xpack.core.inference.results.StreamingUnifiedChatCompletionResults;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser;
+import org.elasticsearch.xpack.inference.external.openai.OpenAiUnifiedStreamingProcessor;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventParser;
+import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventProcessor;
+
+import java.util.concurrent.Flow;
+
+public class ElasticInferenceServiceUnifiedChatCompletionResponseHandler extends ElasticInferenceServiceResponseHandler {
+    public ElasticInferenceServiceUnifiedChatCompletionResponseHandler(String requestType, ResponseParser parseFunction) {
+        super(requestType, parseFunction);
+    }
+
+    @Override
+    public boolean canHandleStreamingResponses() {
+        return true;
+    }
+
+    @Override
+    public InferenceServiceResults parseResult(Request request, Flow.Publisher<HttpResult> flow) {
+        var serverSentEventProcessor = new ServerSentEventProcessor(new ServerSentEventParser());
+        var openAiProcessor = new OpenAiUnifiedStreamingProcessor(); // EIS uses the unified API spec
+
+        flow.subscribe(serverSentEventProcessor);
+        serverSentEventProcessor.subscribe(openAiProcessor);
+        return new StreamingUnifiedChatCompletionResults(openAiProcessor);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceUnifiedCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceUnifiedCompletionRequestManager.java
new file mode 100644
index 0000000000000..66314db1e05bd
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceUnifiedCompletionRequestManager.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.http.sender;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.inference.external.elastic.ElasticInferenceServiceUnifiedChatCompletionResponseHandler;
+import org.elasticsearch.xpack.inference.external.http.retry.RequestSender;
+import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler;
+import org.elasticsearch.xpack.inference.external.request.elastic.ElasticInferenceServiceUnifiedChatCompletionRequest;
+import org.elasticsearch.xpack.inference.external.response.openai.OpenAiChatCompletionResponseEntity;
+import org.elasticsearch.xpack.inference.services.elastic.completion.ElasticInferenceServiceCompletionModel;
+import org.elasticsearch.xpack.inference.telemetry.TraceContext;
+
+import java.util.Objects;
+import java.util.function.Supplier;
+
+public class ElasticInferenceServiceUnifiedCompletionRequestManager extends ElasticInferenceServiceRequestManager {
+
+    private static final Logger logger = LogManager.getLogger(ElasticInferenceServiceUnifiedCompletionRequestManager.class);
+
+    private static final ResponseHandler HANDLER = createCompletionHandler();
+
+    public static ElasticInferenceServiceUnifiedCompletionRequestManager of(
+        ElasticInferenceServiceCompletionModel model,
+        ThreadPool threadPool,
+        TraceContext traceContext
+    ) {
+        return new ElasticInferenceServiceUnifiedCompletionRequestManager(
+            Objects.requireNonNull(model),
+            Objects.requireNonNull(threadPool),
+            Objects.requireNonNull(traceContext)
+        );
+    }
+
+    private final ElasticInferenceServiceCompletionModel model;
+    private final TraceContext traceContext;
+
+    private ElasticInferenceServiceUnifiedCompletionRequestManager(
+        ElasticInferenceServiceCompletionModel model,
+        ThreadPool threadPool,
+        TraceContext traceContext
+    ) {
+        super(threadPool, model);
+        this.model = model;
+        this.traceContext = traceContext;
+    }
+
+    @Override
+    public void execute(
+        InferenceInputs inferenceInputs,
+        RequestSender requestSender,
+        Supplier<Boolean> hasRequestCompletedFunction,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+
+        ElasticInferenceServiceUnifiedChatCompletionRequest request = new ElasticInferenceServiceUnifiedChatCompletionRequest(
+            inferenceInputs.castTo(UnifiedChatInput.class),
+            model,
+            traceContext
+        );
+
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
+    }
+
+    private static ResponseHandler createCompletionHandler() {
+        return new ElasticInferenceServiceUnifiedChatCompletionResponseHandler(
+            "elastic inference service completion",
+            // We use OpenAiChatCompletionResponseEntity here as the ElasticInferenceServiceResponseEntity fields are a subset of the OpenAI
+            // one.
+            OpenAiChatCompletionResponseEntity::fromResponse
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIEmbeddingsRequestManager.java
new file mode 100644
index 0000000000000..c0828224cd1a9
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIEmbeddingsRequestManager.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.http.sender;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.inference.external.http.retry.RequestSender;
+import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler;
+import org.elasticsearch.xpack.inference.external.jinaai.JinaAIResponseHandler;
+import org.elasticsearch.xpack.inference.external.request.jinaai.JinaAIEmbeddingsRequest;
+import org.elasticsearch.xpack.inference.external.response.jinaai.JinaAIEmbeddingsResponseEntity;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModel;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+public class JinaAIEmbeddingsRequestManager extends JinaAIRequestManager {
+    private static final Logger logger = LogManager.getLogger(JinaAIEmbeddingsRequestManager.class);
+    private static final ResponseHandler HANDLER = createEmbeddingsHandler();
+
+    private static ResponseHandler createEmbeddingsHandler() {
+        return new JinaAIResponseHandler("jinaai text embedding", JinaAIEmbeddingsResponseEntity::fromResponse);
+    }
+
+    public static JinaAIEmbeddingsRequestManager of(JinaAIEmbeddingsModel model, ThreadPool threadPool) {
+        return new JinaAIEmbeddingsRequestManager(Objects.requireNonNull(model), Objects.requireNonNull(threadPool));
+    }
+
+    private final JinaAIEmbeddingsModel model;
+
+    private JinaAIEmbeddingsRequestManager(JinaAIEmbeddingsModel model, ThreadPool threadPool) {
+        super(threadPool, model);
+        this.model = Objects.requireNonNull(model);
+    }
+
+    @Override
+    public void execute(
+        InferenceInputs inferenceInputs,
+        RequestSender requestSender,
+        Supplier<Boolean> hasRequestCompletedFunction,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+        List<String> docsInput = DocumentsOnlyInput.of(inferenceInputs).getInputs();
+        JinaAIEmbeddingsRequest request = new JinaAIEmbeddingsRequest(docsInput, model);
+
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIRequestManager.java
new file mode 100644
index 0000000000000..3a0d6e4e17f5b
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIRequestManager.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.http.sender;
+
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.inference.services.jinaai.JinaAIModel;
+
+import java.util.Objects;
+
+abstract class JinaAIRequestManager extends BaseRequestManager {
+
+    protected JinaAIRequestManager(ThreadPool threadPool, JinaAIModel model) {
+        super(threadPool, model.getInferenceEntityId(), RateLimitGrouping.of(model), model.rateLimitServiceSettings().rateLimitSettings());
+    }
+
+    record RateLimitGrouping(int apiKeyHash) {
+        public static RateLimitGrouping of(JinaAIModel model) {
+            Objects.requireNonNull(model);
+
+            return new RateLimitGrouping(model.apiKey().hashCode());
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIRerankRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIRerankRequestManager.java
new file mode 100644
index 0000000000000..26f134873bca0
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIRerankRequestManager.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.http.sender;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.inference.external.http.retry.RequestSender;
+import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler;
+import org.elasticsearch.xpack.inference.external.jinaai.JinaAIResponseHandler;
+import org.elasticsearch.xpack.inference.external.request.jinaai.JinaAIRerankRequest;
+import org.elasticsearch.xpack.inference.external.response.jinaai.JinaAIRerankResponseEntity;
+import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankModel;
+
+import java.util.Objects;
+import java.util.function.Supplier;
+
+public class JinaAIRerankRequestManager extends JinaAIRequestManager {
+    private static final Logger logger = LogManager.getLogger(JinaAIRerankRequestManager.class);
+    private static final ResponseHandler HANDLER = createJinaAIResponseHandler();
+
+    private static ResponseHandler createJinaAIResponseHandler() {
+        return new JinaAIResponseHandler("jinaai rerank", (request, response) -> JinaAIRerankResponseEntity.fromResponse(response));
+    }
+
+    public static JinaAIRerankRequestManager of(JinaAIRerankModel model, ThreadPool threadPool) {
+        return new JinaAIRerankRequestManager(Objects.requireNonNull(model), Objects.requireNonNull(threadPool));
+    }
+
+    private final JinaAIRerankModel model;
+
+    private JinaAIRerankRequestManager(JinaAIRerankModel model, ThreadPool threadPool) {
+        super(threadPool, model);
+        this.model = model;
+    }
+
+    @Override
+    public void execute(
+        InferenceInputs inferenceInputs,
+        RequestSender requestSender,
+        Supplier<Boolean> hasRequestCompletedFunction,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+        var rerankInput = QueryAndDocsInputs.of(inferenceInputs);
+        JinaAIRerankRequest request = new JinaAIRerankRequest(rerankInput.getQuery(), rerankInput.getChunks(), model);
+
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInput.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInput.java
index f89fa1ee37a6f..fceec7c431182 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInput.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInput.java
@@ -40,15 +40,7 @@ public UnifiedChatInput(List<String> inputs, String roleValue, boolean stream) {
 
     private static List<UnifiedCompletionRequest.Message> convertToMessages(List<String> inputs, String roleValue) {
         return inputs.stream()
-            .map(
-                value -> new UnifiedCompletionRequest.Message(
-                    new UnifiedCompletionRequest.ContentString(value),
-                    roleValue,
-                    null,
-                    null,
-                    null
-                )
-            )
+            .map(value -> new UnifiedCompletionRequest.Message(new UnifiedCompletionRequest.ContentString(value), roleValue, null, null))
             .toList();
     }
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIAccount.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIAccount.java
new file mode 100644
index 0000000000000..722a785db4795
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIAccount.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.jinaai;
+
+import org.elasticsearch.common.CheckedSupplier;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.xpack.inference.services.jinaai.JinaAIModel;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.external.request.RequestUtils.buildUri;
+
+public record JinaAIAccount(URI uri, SecureString apiKey) {
+
+    public static JinaAIAccount of(JinaAIModel model, CheckedSupplier<URI, URISyntaxException> uriBuilder) {
+        var uri = buildUri(model.uri(), "JinaAI", uriBuilder);
+
+        return new JinaAIAccount(uri, model.apiKey());
+    }
+
+    public JinaAIAccount {
+        Objects.requireNonNull(uri);
+        Objects.requireNonNull(apiKey);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIResponseHandler.java
new file mode 100644
index 0000000000000..66dc85b3bdb6a
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIResponseHandler.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.jinaai;
+
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler;
+import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser;
+import org.elasticsearch.xpack.inference.external.http.retry.RetryException;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.external.response.jinaai.JinaAIErrorResponseEntity;
+
+/**
+ * Defines how to handle various errors returned from the JinaAI integration.
+ *
+ */
+public class JinaAIResponseHandler extends BaseResponseHandler {
+    static final String VALIDATION_ERROR_MESSAGE = "Received an input validation error response";
+    static final String PAYMENT_ERROR_MESSAGE = "Payment required";
+
+    public JinaAIResponseHandler(String requestType, ResponseParser parseFunction) {
+        super(requestType, parseFunction, JinaAIErrorResponseEntity::fromResponse);
+    }
+
+    /**
+     * Validates the status code throws an RetryException if not in the range [200, 300).
+     *
+     * @param request The http request
+     * @param result  The http response and body
+     * @throws RetryException Throws if status code is {@code >= 300 or < 200 }
+     */
+    @Override
+    protected void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException {
+        if (result.isSuccessfulResponse()) {
+            return;
+        }
+
+        // handle error codes
+        int statusCode = result.response().getStatusLine().getStatusCode();
+        if (statusCode == 500) {
+            throw new RetryException(true, buildError(SERVER_ERROR, request, result));
+        } else if (statusCode > 500) {
+            throw new RetryException(false, buildError(SERVER_ERROR, request, result));
+        } else if (statusCode == 429) {
+            throw new RetryException(true, buildError(RATE_LIMIT, request, result));
+        } else if (statusCode == 400 || statusCode == 422) {
+            throw new RetryException(false, buildError(VALIDATION_ERROR_MESSAGE, request, result));
+        } else if (statusCode == 401) {
+            throw new RetryException(false, buildError(AUTHENTICATION, request, result));
+        } else if (statusCode == 402) {
+            throw new RetryException(false, buildError(PAYMENT_ERROR_MESSAGE, request, result));
+        } else if (statusCode >= 300 && statusCode < 400) {
+            throw new RetryException(false, buildError(REDIRECTION, request, result));
+        } else {
+            throw new RetryException(false, buildError(UNSUCCESSFUL, request, result));
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java
index 1fc61d3331d20..0804cb332b15d 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java
@@ -39,7 +39,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.field(TEXTS_FIELD, input);
 
-        String inputType = covertToString(taskSettings.getInputType());
+        String inputType = convertToString(taskSettings.getInputType());
         if (inputType != null) {
             builder.field(INPUT_TYPE_FIELD, inputType);
         }
@@ -49,7 +49,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     }
 
     // default for testing
-    static String covertToString(InputType inputType) {
+    static String convertToString(InputType inputType) {
         if (inputType == null) {
             return null;
         }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java
index 8fae9408b860d..22676c11dc548 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java
@@ -34,7 +34,7 @@ public record AlibabaCloudSearchSparseRequestEntity(List<String> input, AlibabaC
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(TEXTS_FIELD, input);
-        String inputType = AlibabaCloudSearchEmbeddingsRequestEntity.covertToString(taskSettings.getInputType());
+        String inputType = AlibabaCloudSearchEmbeddingsRequestEntity.convertToString(taskSettings.getInputType());
         if (inputType != null) {
             builder.field(INPUT_TYPE_FIELD, inputType);
         }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntity.java
index 63cc5c3cb7261..a972cbbac959d 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntity.java
@@ -51,7 +51,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
 
         if (taskSettings.getInputType() != null) {
-            builder.field(INPUT_TYPE_FIELD, covertToString(taskSettings.getInputType()));
+            builder.field(INPUT_TYPE_FIELD, convertToString(taskSettings.getInputType()));
         }
 
         if (embeddingType != null) {
@@ -67,7 +67,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     }
 
     // default for testing
-    static String covertToString(InputType inputType) {
+    static String convertToString(InputType inputType) {
         return switch (inputType) {
             case INGEST -> SEARCH_DOCUMENT;
             case SEARCH -> SEARCH_QUERY;
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java
index d445a779f8230..d1aaa6d5f984f 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java
@@ -12,13 +12,13 @@
 import org.apache.http.entity.ByteArrayEntity;
 import org.apache.http.message.BasicHeader;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.tasks.Task;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.inference.common.Truncator;
 import org.elasticsearch.xpack.inference.external.request.HttpRequest;
 import org.elasticsearch.xpack.inference.external.request.Request;
 import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModel;
 import org.elasticsearch.xpack.inference.telemetry.TraceContext;
+import org.elasticsearch.xpack.inference.telemetry.TraceContextHandler;
 
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
@@ -27,13 +27,10 @@
 public class ElasticInferenceServiceSparseEmbeddingsRequest implements ElasticInferenceServiceRequest {
 
     private final URI uri;
-
     private final ElasticInferenceServiceSparseEmbeddingsModel model;
-
     private final Truncator.TruncationResult truncationResult;
     private final Truncator truncator;
-
-    private final TraceContext traceContext;
+    private final TraceContextHandler traceContextHandler;
 
     public ElasticInferenceServiceSparseEmbeddingsRequest(
         Truncator truncator,
@@ -45,7 +42,7 @@ public ElasticInferenceServiceSparseEmbeddingsRequest(
         this.truncationResult = truncationResult;
         this.model = Objects.requireNonNull(model);
         this.uri = model.uri();
-        this.traceContext = traceContext;
+        this.traceContextHandler = new TraceContextHandler(traceContext);
     }
 
     @Override
@@ -56,15 +53,16 @@ public HttpRequest createHttpRequest() {
         ByteArrayEntity byteEntity = new ByteArrayEntity(requestEntity.getBytes(StandardCharsets.UTF_8));
         httpPost.setEntity(byteEntity);
 
-        if (traceContext != null) {
-            propagateTraceContext(httpPost);
-        }
-
+        traceContextHandler.propagateTraceContext(httpPost);
         httpPost.setHeader(new BasicHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType()));
 
         return new HttpRequest(httpPost, getInferenceEntityId());
     }
 
+    public TraceContext getTraceContext() {
+        return traceContextHandler.traceContext();
+    }
+
     @Override
     public String getInferenceEntityId() {
         return model.getInferenceEntityId();
@@ -75,15 +73,10 @@ public URI getURI() {
         return this.uri;
     }
 
-    public TraceContext getTraceContext() {
-        return traceContext;
-    }
-
     @Override
     public Request truncate() {
         var truncatedInput = truncator.truncate(truncationResult.input());
-
-        return new ElasticInferenceServiceSparseEmbeddingsRequest(truncator, truncatedInput, model, traceContext);
+        return new ElasticInferenceServiceSparseEmbeddingsRequest(truncator, truncatedInput, model, traceContextHandler.traceContext());
     }
 
     @Override
@@ -91,16 +84,4 @@ public boolean[] getTruncationInfo() {
         return truncationResult.truncated().clone();
     }
 
-    private void propagateTraceContext(HttpPost httpPost) {
-        var traceParent = traceContext.traceParent();
-        var traceState = traceContext.traceState();
-
-        if (traceParent != null) {
-            httpPost.setHeader(Task.TRACE_PARENT_HTTP_HEADER, traceParent);
-        }
-
-        if (traceState != null) {
-            httpPost.setHeader(Task.TRACE_STATE, traceState);
-        }
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceUnifiedChatCompletionRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceUnifiedChatCompletionRequest.java
new file mode 100644
index 0000000000000..112ead7057933
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceUnifiedChatCompletionRequest.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.elastic;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.message.BasicHeader;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput;
+import org.elasticsearch.xpack.inference.external.request.HttpRequest;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.services.elastic.completion.ElasticInferenceServiceCompletionModel;
+import org.elasticsearch.xpack.inference.telemetry.TraceContext;
+import org.elasticsearch.xpack.inference.telemetry.TraceContextHandler;
+
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+
+public class ElasticInferenceServiceUnifiedChatCompletionRequest implements Request {
+
+    private final ElasticInferenceServiceCompletionModel model;
+    private final UnifiedChatInput unifiedChatInput;
+    private final TraceContextHandler traceContextHandler;
+
+    public ElasticInferenceServiceUnifiedChatCompletionRequest(
+        UnifiedChatInput unifiedChatInput,
+        ElasticInferenceServiceCompletionModel model,
+        TraceContext traceContext
+    ) {
+        this.unifiedChatInput = Objects.requireNonNull(unifiedChatInput);
+        this.model = Objects.requireNonNull(model);
+        this.traceContextHandler = new TraceContextHandler(traceContext);
+    }
+
+    @Override
+    public HttpRequest createHttpRequest() {
+        var httpPost = new HttpPost(model.uri());
+        var requestEntity = Strings.toString(
+            new ElasticInferenceServiceUnifiedChatCompletionRequestEntity(unifiedChatInput, model.getServiceSettings().modelId())
+        );
+
+        ByteArrayEntity byteEntity = new ByteArrayEntity(requestEntity.getBytes(StandardCharsets.UTF_8));
+        httpPost.setEntity(byteEntity);
+
+        traceContextHandler.propagateTraceContext(httpPost);
+        httpPost.setHeader(new BasicHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType()));
+
+        return new HttpRequest(httpPost, getInferenceEntityId());
+    }
+
+    @Override
+    public URI getURI() {
+        return model.uri();
+    }
+
+    @Override
+    public Request truncate() {
+        // No truncation
+        return this;
+    }
+
+    @Override
+    public boolean[] getTruncationInfo() {
+        // No truncation
+        return null;
+    }
+
+    @Override
+    public String getInferenceEntityId() {
+        return model.getInferenceEntityId();
+    }
+
+    @Override
+    public boolean isStreaming() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceUnifiedChatCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceUnifiedChatCompletionRequestEntity.java
new file mode 100644
index 0000000000000..ded8a074478cf
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceUnifiedChatCompletionRequestEntity.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.elastic;
+
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput;
+import org.elasticsearch.xpack.inference.external.unified.UnifiedChatCompletionRequestEntity;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class ElasticInferenceServiceUnifiedChatCompletionRequestEntity implements ToXContentObject {
+    private static final String MODEL_FIELD = "model";
+
+    private final UnifiedChatCompletionRequestEntity unifiedRequestEntity;
+    private final String modelId;
+
+    public ElasticInferenceServiceUnifiedChatCompletionRequestEntity(UnifiedChatInput unifiedChatInput, String modelId) {
+        this.unifiedRequestEntity = new UnifiedChatCompletionRequestEntity(Objects.requireNonNull(unifiedChatInput));
+        this.modelId = Objects.requireNonNull(modelId);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        unifiedRequestEntity.toXContent(builder, params);
+        builder.field(MODEL_FIELD, modelId);
+        builder.endObject();
+
+        return builder;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequest.java
new file mode 100644
index 0000000000000..d99f15a7703ae
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequest.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.jinaai;
+
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.entity.ByteArrayEntity;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xpack.inference.external.jinaai.JinaAIAccount;
+import org.elasticsearch.xpack.inference.external.request.HttpRequest;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModel;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettings;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Objects;
+
+public class JinaAIEmbeddingsRequest extends JinaAIRequest {
+
+    private final JinaAIAccount account;
+    private final List<String> input;
+    private final JinaAIEmbeddingsTaskSettings taskSettings;
+    private final String model;
+    private final String inferenceEntityId;
+
+    public JinaAIEmbeddingsRequest(List<String> input, JinaAIEmbeddingsModel embeddingsModel) {
+        Objects.requireNonNull(embeddingsModel);
+
+        account = JinaAIAccount.of(embeddingsModel, JinaAIEmbeddingsRequest::buildDefaultUri);
+        this.input = Objects.requireNonNull(input);
+        taskSettings = embeddingsModel.getTaskSettings();
+        model = embeddingsModel.getServiceSettings().getCommonSettings().modelId();
+        inferenceEntityId = embeddingsModel.getInferenceEntityId();
+    }
+
+    @Override
+    public HttpRequest createHttpRequest() {
+        HttpPost httpPost = new HttpPost(account.uri());
+
+        ByteArrayEntity byteEntity = new ByteArrayEntity(
+            Strings.toString(new JinaAIEmbeddingsRequestEntity(input, taskSettings, model)).getBytes(StandardCharsets.UTF_8)
+        );
+        httpPost.setEntity(byteEntity);
+
+        decorateWithAuthHeader(httpPost, account);
+
+        return new HttpRequest(httpPost, getInferenceEntityId());
+    }
+
+    @Override
+    public String getInferenceEntityId() {
+        return inferenceEntityId;
+    }
+
+    @Override
+    public URI getURI() {
+        return account.uri();
+    }
+
+    @Override
+    public Request truncate() {
+        return this;
+    }
+
+    @Override
+    public boolean[] getTruncationInfo() {
+        return null;
+    }
+
+    public static URI buildDefaultUri() throws URISyntaxException {
+        return new URIBuilder().setScheme("https")
+            .setHost(JinaAIUtils.HOST)
+            .setPathSegments(JinaAIUtils.VERSION_1, JinaAIUtils.EMBEDDINGS_PATH)
+            .build();
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestEntity.java
new file mode 100644
index 0000000000000..d4f98f1eb52ca
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestEntity.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.jinaai;
+
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettings;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettings.invalidInputTypeMessage;
+
+public record JinaAIEmbeddingsRequestEntity(List<String> input, JinaAIEmbeddingsTaskSettings taskSettings, @Nullable String model)
+    implements
+        ToXContentObject {
+
+    private static final String SEARCH_DOCUMENT = "retrieval.passage";
+    private static final String SEARCH_QUERY = "retrieval.query";
+    private static final String CLUSTERING = "separation";
+    private static final String CLASSIFICATION = "classification";
+    private static final String INPUT_FIELD = "input";
+    private static final String MODEL_FIELD = "model";
+    public static final String TASK_TYPE_FIELD = "task";
+
+    public JinaAIEmbeddingsRequestEntity {
+        Objects.requireNonNull(input);
+        Objects.requireNonNull(taskSettings);
+        Objects.requireNonNull(model);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(INPUT_FIELD, input);
+        builder.field(MODEL_FIELD, model);
+
+        if (taskSettings.getInputType() != null) {
+            builder.field(TASK_TYPE_FIELD, convertToString(taskSettings.getInputType()));
+        }
+
+        builder.endObject();
+        return builder;
+    }
+
+    // default for testing
+    static String convertToString(InputType inputType) {
+        return switch (inputType) {
+            case INGEST -> SEARCH_DOCUMENT;
+            case SEARCH -> SEARCH_QUERY;
+            case CLASSIFICATION -> CLASSIFICATION;
+            case CLUSTERING -> CLUSTERING;
+            default -> {
+                assert false : invalidInputTypeMessage(inputType);
+                yield null;
+            }
+        };
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRequest.java
new file mode 100644
index 0000000000000..8b1e26a36238b
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRequest.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.jinaai;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.HttpPost;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.external.jinaai.JinaAIAccount;
+import org.elasticsearch.xpack.inference.external.request.Request;
+
+import static org.elasticsearch.xpack.inference.external.request.RequestUtils.createAuthBearerHeader;
+
+public abstract class JinaAIRequest implements Request {
+
+    public static void decorateWithAuthHeader(HttpPost request, JinaAIAccount account) {
+        request.setHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType());
+        request.setHeader(createAuthBearerHeader(account.apiKey()));
+        request.setHeader(JinaAIUtils.createRequestSourceHeader());
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequest.java
new file mode 100644
index 0000000000000..93d4ab830c604
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequest.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.jinaai;
+
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.entity.ByteArrayEntity;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xpack.inference.external.jinaai.JinaAIAccount;
+import org.elasticsearch.xpack.inference.external.request.HttpRequest;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankModel;
+import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankTaskSettings;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Objects;
+
+public class JinaAIRerankRequest extends JinaAIRequest {
+
+    private final JinaAIAccount account;
+    private final String query;
+    private final List<String> input;
+    private final JinaAIRerankTaskSettings taskSettings;
+    private final String model;
+    private final String inferenceEntityId;
+
+    public JinaAIRerankRequest(String query, List<String> input, JinaAIRerankModel model) {
+        Objects.requireNonNull(model);
+
+        this.account = JinaAIAccount.of(model, JinaAIRerankRequest::buildDefaultUri);
+        this.input = Objects.requireNonNull(input);
+        this.query = Objects.requireNonNull(query);
+        taskSettings = model.getTaskSettings();
+        this.model = model.getServiceSettings().modelId();
+        inferenceEntityId = model.getInferenceEntityId();
+    }
+
+    @Override
+    public HttpRequest createHttpRequest() {
+        HttpPost httpPost = new HttpPost(account.uri());
+
+        ByteArrayEntity byteEntity = new ByteArrayEntity(
+            Strings.toString(new JinaAIRerankRequestEntity(query, input, taskSettings, model)).getBytes(StandardCharsets.UTF_8)
+        );
+        httpPost.setEntity(byteEntity);
+
+        decorateWithAuthHeader(httpPost, account);
+
+        return new HttpRequest(httpPost, getInferenceEntityId());
+    }
+
+    @Override
+    public String getInferenceEntityId() {
+        return inferenceEntityId;
+    }
+
+    @Override
+    public URI getURI() {
+        return account.uri();
+    }
+
+    @Override
+    public Request truncate() {
+        return this;
+    }
+
+    @Override
+    public boolean[] getTruncationInfo() {
+        return null;
+    }
+
+    public static URI buildDefaultUri() throws URISyntaxException {
+        return new URIBuilder().setScheme("https")
+            .setHost(JinaAIUtils.HOST)
+            .setPathSegments(JinaAIUtils.VERSION_1, JinaAIUtils.RERANK_PATH)
+            .build();
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestEntity.java
new file mode 100644
index 0000000000000..7f470d5fa91f5
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestEntity.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.jinaai;
+
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankTaskSettings;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+public record JinaAIRerankRequestEntity(String model, String query, List<String> documents, JinaAIRerankTaskSettings taskSettings)
+    implements
+        ToXContentObject {
+
+    private static final String DOCUMENTS_FIELD = "documents";
+    private static final String QUERY_FIELD = "query";
+    private static final String MODEL_FIELD = "model";
+
+    public JinaAIRerankRequestEntity {
+        Objects.requireNonNull(query);
+        Objects.requireNonNull(documents);
+        Objects.requireNonNull(model);
+        Objects.requireNonNull(taskSettings);
+    }
+
+    public JinaAIRerankRequestEntity(String query, List<String> input, JinaAIRerankTaskSettings taskSettings, String model) {
+        this(model, query, input, taskSettings != null ? taskSettings : JinaAIRerankTaskSettings.EMPTY_SETTINGS);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        builder.field(MODEL_FIELD, model);
+        builder.field(QUERY_FIELD, query);
+        builder.field(DOCUMENTS_FIELD, documents);
+
+        if (taskSettings.getTopNDocumentsOnly() != null) {
+            builder.field(JinaAIRerankTaskSettings.TOP_N_DOCS_ONLY, taskSettings.getTopNDocumentsOnly());
+        }
+
+        var return_documents = taskSettings.getDoesReturnDocuments();
+        if (return_documents != null) {
+            builder.field(JinaAIRerankTaskSettings.RETURN_DOCUMENTS, return_documents);
+        }
+
+        builder.endObject();
+        return builder;
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIUtils.java
new file mode 100644
index 0000000000000..fccbd1e230556
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIUtils.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.jinaai;
+
+import org.apache.http.Header;
+import org.apache.http.message.BasicHeader;
+
+public class JinaAIUtils {
+    public static final String HOST = "api.jina.ai";
+    public static final String VERSION_1 = "v1";
+    public static final String EMBEDDINGS_PATH = "embeddings";
+    public static final String RERANK_PATH = "rerank";
+    public static final String REQUEST_SOURCE_HEADER = "Request-Source";
+    public static final String ELASTIC_REQUEST_SOURCE = "unspecified:elasticsearch";
+
+    public static Header createRequestSourceHeader() {
+        return new BasicHeader(REQUEST_SOURCE_HEADER, ELASTIC_REQUEST_SOURCE);
+    }
+
+    private JinaAIUtils() {}
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiEmbeddingsRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiEmbeddingsRequest.java
index f82e7ff3f5260..7f8626dacc684 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiEmbeddingsRequest.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiEmbeddingsRequest.java
@@ -27,7 +27,7 @@
 import static org.elasticsearch.xpack.inference.external.request.RequestUtils.createAuthBearerHeader;
 import static org.elasticsearch.xpack.inference.external.request.openai.OpenAiUtils.createOrgHeader;
 
-public class OpenAiEmbeddingsRequest implements OpenAiRequest {
+public class OpenAiEmbeddingsRequest implements Request {
 
     private final Truncator truncator;
     private final OpenAiAccount account;
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequest.java
index 2e6bdb748fd33..e5b85633a499b 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequest.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequest.java
@@ -27,7 +27,7 @@
 import static org.elasticsearch.xpack.inference.external.request.RequestUtils.createAuthBearerHeader;
 import static org.elasticsearch.xpack.inference.external.request.openai.OpenAiUtils.createOrgHeader;
 
-public class OpenAiUnifiedChatCompletionRequest implements OpenAiRequest {
+public class OpenAiUnifiedChatCompletionRequest implements Request {
 
     private final OpenAiAccount account;
     private final OpenAiChatCompletionModel model;
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestEntity.java
index 50339bf851f7d..b80100c9e2f79 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestEntity.java
@@ -8,10 +8,10 @@
 package org.elasticsearch.xpack.inference.external.request.openai;
 
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.inference.UnifiedCompletionRequest;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput;
+import org.elasticsearch.xpack.inference.external.unified.UnifiedChatCompletionRequestEntity;
 import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModel;
 
 import java.io.IOException;
@@ -19,165 +19,28 @@
 
 public class OpenAiUnifiedChatCompletionRequestEntity implements ToXContentObject {
 
-    public static final String NAME_FIELD = "name";
-    public static final String TOOL_CALL_ID_FIELD = "tool_call_id";
-    public static final String TOOL_CALLS_FIELD = "tool_calls";
-    public static final String ID_FIELD = "id";
-    public static final String FUNCTION_FIELD = "function";
-    public static final String ARGUMENTS_FIELD = "arguments";
-    public static final String DESCRIPTION_FIELD = "description";
-    public static final String PARAMETERS_FIELD = "parameters";
-    public static final String STRICT_FIELD = "strict";
-    public static final String TOP_P_FIELD = "top_p";
     public static final String USER_FIELD = "user";
-    public static final String STREAM_FIELD = "stream";
-    private static final String NUMBER_OF_RETURNED_CHOICES_FIELD = "n";
     private static final String MODEL_FIELD = "model";
-    public static final String MESSAGES_FIELD = "messages";
-    private static final String ROLE_FIELD = "role";
-    private static final String CONTENT_FIELD = "content";
-    private static final String MAX_COMPLETION_TOKENS_FIELD = "max_completion_tokens";
-    private static final String STOP_FIELD = "stop";
-    private static final String TEMPERATURE_FIELD = "temperature";
-    private static final String TOOL_CHOICE_FIELD = "tool_choice";
-    private static final String TOOL_FIELD = "tools";
-    private static final String TEXT_FIELD = "text";
-    private static final String TYPE_FIELD = "type";
-    private static final String STREAM_OPTIONS_FIELD = "stream_options";
-    private static final String INCLUDE_USAGE_FIELD = "include_usage";
 
-    private final UnifiedCompletionRequest unifiedRequest;
-    private final boolean stream;
     private final OpenAiChatCompletionModel model;
+    private final UnifiedChatCompletionRequestEntity unifiedRequestEntity;
 
     public OpenAiUnifiedChatCompletionRequestEntity(UnifiedChatInput unifiedChatInput, OpenAiChatCompletionModel model) {
-        Objects.requireNonNull(unifiedChatInput);
-
-        this.unifiedRequest = unifiedChatInput.getRequest();
-        this.stream = unifiedChatInput.stream();
+        this.unifiedRequestEntity = new UnifiedChatCompletionRequestEntity(Objects.requireNonNull(unifiedChatInput));
         this.model = Objects.requireNonNull(model);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.startArray(MESSAGES_FIELD);
-        {
-            for (UnifiedCompletionRequest.Message message : unifiedRequest.messages()) {
-                builder.startObject();
-                {
-                    switch (message.content()) {
-                        case UnifiedCompletionRequest.ContentString contentString -> builder.field(CONTENT_FIELD, contentString.content());
-                        case UnifiedCompletionRequest.ContentObjects contentObjects -> {
-                            builder.startArray(CONTENT_FIELD);
-                            for (UnifiedCompletionRequest.ContentObject contentObject : contentObjects.contentObjects()) {
-                                builder.startObject();
-                                builder.field(TEXT_FIELD, contentObject.text());
-                                builder.field(TYPE_FIELD, contentObject.type());
-                                builder.endObject();
-                            }
-                            builder.endArray();
-                        }
-                    }
-
-                    builder.field(ROLE_FIELD, message.role());
-                    if (message.name() != null) {
-                        builder.field(NAME_FIELD, message.name());
-                    }
-                    if (message.toolCallId() != null) {
-                        builder.field(TOOL_CALL_ID_FIELD, message.toolCallId());
-                    }
-                    if (message.toolCalls() != null) {
-                        builder.startArray(TOOL_CALLS_FIELD);
-                        for (UnifiedCompletionRequest.ToolCall toolCall : message.toolCalls()) {
-                            builder.startObject();
-                            {
-                                builder.field(ID_FIELD, toolCall.id());
-                                builder.startObject(FUNCTION_FIELD);
-                                {
-                                    builder.field(ARGUMENTS_FIELD, toolCall.function().arguments());
-                                    builder.field(NAME_FIELD, toolCall.function().name());
-                                }
-                                builder.endObject();
-                                builder.field(TYPE_FIELD, toolCall.type());
-                            }
-                            builder.endObject();
-                        }
-                        builder.endArray();
-                    }
-                }
-                builder.endObject();
-            }
-        }
-        builder.endArray();
+        unifiedRequestEntity.toXContent(builder, params);
 
         builder.field(MODEL_FIELD, model.getServiceSettings().modelId());
-        if (unifiedRequest.maxCompletionTokens() != null) {
-            builder.field(MAX_COMPLETION_TOKENS_FIELD, unifiedRequest.maxCompletionTokens());
-        }
-
-        builder.field(NUMBER_OF_RETURNED_CHOICES_FIELD, 1);
-
-        if (unifiedRequest.stop() != null && unifiedRequest.stop().isEmpty() == false) {
-            builder.field(STOP_FIELD, unifiedRequest.stop());
-        }
-        if (unifiedRequest.temperature() != null) {
-            builder.field(TEMPERATURE_FIELD, unifiedRequest.temperature());
-        }
-        if (unifiedRequest.toolChoice() != null) {
-            if (unifiedRequest.toolChoice() instanceof UnifiedCompletionRequest.ToolChoiceString) {
-                builder.field(TOOL_CHOICE_FIELD, ((UnifiedCompletionRequest.ToolChoiceString) unifiedRequest.toolChoice()).value());
-            } else if (unifiedRequest.toolChoice() instanceof UnifiedCompletionRequest.ToolChoiceObject) {
-                builder.startObject(TOOL_CHOICE_FIELD);
-                {
-                    builder.field(TYPE_FIELD, ((UnifiedCompletionRequest.ToolChoiceObject) unifiedRequest.toolChoice()).type());
-                    builder.startObject(FUNCTION_FIELD);
-                    {
-                        builder.field(
-                            NAME_FIELD,
-                            ((UnifiedCompletionRequest.ToolChoiceObject) unifiedRequest.toolChoice()).function().name()
-                        );
-                    }
-                    builder.endObject();
-                }
-                builder.endObject();
-            }
-        }
-        if (unifiedRequest.tools() != null && unifiedRequest.tools().isEmpty() == false) {
-            builder.startArray(TOOL_FIELD);
-            for (UnifiedCompletionRequest.Tool t : unifiedRequest.tools()) {
-                builder.startObject();
-                {
-                    builder.field(TYPE_FIELD, t.type());
-                    builder.startObject(FUNCTION_FIELD);
-                    {
-                        builder.field(DESCRIPTION_FIELD, t.function().description());
-                        builder.field(NAME_FIELD, t.function().name());
-                        builder.field(PARAMETERS_FIELD, t.function().parameters());
-                        if (t.function().strict() != null) {
-                            builder.field(STRICT_FIELD, t.function().strict());
-                        }
-                    }
-                    builder.endObject();
-                }
-                builder.endObject();
-            }
-            builder.endArray();
-        }
-        if (unifiedRequest.topP() != null) {
-            builder.field(TOP_P_FIELD, unifiedRequest.topP());
-        }
 
         if (Strings.isNullOrEmpty(model.getTaskSettings().user()) == false) {
             builder.field(USER_FIELD, model.getTaskSettings().user());
         }
 
-        builder.field(STREAM_FIELD, stream);
-        if (stream) {
-            builder.startObject(STREAM_OPTIONS_FIELD);
-            builder.field(INCLUDE_USAGE_FIELD, true);
-            builder.endObject();
-        }
         builder.endObject();
 
         return builder;
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntity.java
new file mode 100644
index 0000000000000..26bde5f5f48ad
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntity.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ *
+ * this file was contributed to by a generative AI
+ */
+
+package org.elasticsearch.xpack.inference.external.response.jinaai;
+
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.external.response.XContentUtils;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.parseList;
+import static org.elasticsearch.xpack.inference.external.response.XContentUtils.consumeUntilObjectEnd;
+import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken;
+import static org.elasticsearch.xpack.inference.external.response.XContentUtils.positionParserAtTokenAfterField;
+
+public class JinaAIEmbeddingsResponseEntity {
+    private static final String FAILED_TO_FIND_FIELD_TEMPLATE = "Failed to find required field [%s] in JinaAI embeddings response";
+
+    /**
+     * Parses the JinaAI json response.
+     * For a request like:
+     *
+     * <pre>
+     *     <code>
+     *        {
+     *            "inputs": ["hello this is my name", "I wish I was there!"]
+     *        }
+     *     </code>
+     * </pre>
+     *
+     * The response would look like:
+     *
+     * <pre>
+     * <code>
+     * {
+     *  "object": "list",
+     *  "data": [
+     *      {
+     *          "object": "embedding",
+     *          "embedding": [
+     *              -0.009327292,
+     *              -0.0028842222,
+     *          ],
+     *          "index": 0
+     *      },
+     *      {
+     *          "object": "embedding",
+     *          "embedding": [ ... ],
+     *          "index": 1
+     *      }
+     *  ],
+     *  "model": "jina-embeddings-v3",
+     *  "usage": {
+     *      "prompt_tokens": 8,
+     *      "total_tokens": 8
+     *  }
+     * }
+     * </code>
+     * </pre>
+     */
+    public static InferenceTextEmbeddingFloatResults fromResponse(Request request, HttpResult response) throws IOException {
+        var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE);
+
+        try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) {
+            moveToFirstToken(jsonParser);
+
+            XContentParser.Token token = jsonParser.currentToken();
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
+
+            positionParserAtTokenAfterField(jsonParser, "data", FAILED_TO_FIND_FIELD_TEMPLATE);
+
+            List<InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding> embeddingList = parseList(
+                jsonParser,
+                JinaAIEmbeddingsResponseEntity::parseEmbeddingObject
+            );
+
+            return new InferenceTextEmbeddingFloatResults(embeddingList);
+        }
+    }
+
+    private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseEmbeddingObject(XContentParser parser)
+        throws IOException {
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
+
+        positionParserAtTokenAfterField(parser, "embedding", FAILED_TO_FIND_FIELD_TEMPLATE);
+
+        List<Float> embeddingValuesList = parseList(parser, XContentUtils::parseFloat);
+        // parse and discard the rest of the object
+        consumeUntilObjectEnd(parser);
+
+        return InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(embeddingValuesList);
+    }
+
+    private JinaAIEmbeddingsResponseEntity() {}
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIErrorResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIErrorResponseEntity.java
new file mode 100644
index 0000000000000..99d29b26e4a04
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIErrorResponseEntity.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.jinaai;
+
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse;
+
+public class JinaAIErrorResponseEntity extends ErrorResponse {
+
+    private JinaAIErrorResponseEntity(String errorMessage) {
+        super(errorMessage);
+    }
+
+    /**
+     * Parse an HTTP response into a JinaAIErrorResponseEntity
+     *
+     * @param response The error response
+     * @return An error entity if the response is JSON with a `detail` field containing the error message
+     * or null if the response does not contain the message field
+     */
+    public static ErrorResponse fromResponse(HttpResult response) {
+        try (
+            XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON)
+                .createParser(XContentParserConfiguration.EMPTY, response.body())
+        ) {
+            var responseMap = jsonParser.map();
+            var message = (String) responseMap.get("detail");
+            if (message != null) {
+                return new JinaAIErrorResponseEntity(message);
+            }
+        } catch (Exception e) {
+            // swallow the error
+        }
+
+        return ErrorResponse.UNDEFINED_ERROR;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIRerankResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIRerankResponseEntity.java
new file mode 100644
index 0000000000000..d22bc875041e0
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIRerankResponseEntity.java
@@ -0,0 +1,158 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ *
+ * this file was contributed to by a generative AI
+ */
+
+package org.elasticsearch.xpack.inference.external.response.jinaai;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.inference.results.RankedDocsResults;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.parseList;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownToken;
+import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken;
+import static org.elasticsearch.xpack.inference.external.response.XContentUtils.positionParserAtTokenAfterField;
+
+public class JinaAIRerankResponseEntity {
+
+    private static final Logger logger = LogManager.getLogger(JinaAIRerankResponseEntity.class);
+
+    /**
+     * Parses the JinaAI ranked response.
+     *
+     * For a request like:
+     *     "model": "jina-reranker-v2-base-multilingual",
+     *     "query": "What is the capital of the United States?",
+     *     "top_n": 3,
+     *     "documents": ["Carson City is the capital city of the American state of Nevada.",
+     *                   "The Commonwealth of the Northern Mariana ... Its capital is Saipan.",
+     *                   "Washington, D.C. (also known as simply Washington or D.C., ... It is a federal district.",
+     *                   "Capital punishment (the death penalty) ... As of 2017, capital punishment is legal in 30 of the 50 states."]
+     * <p>
+     *  The response will look like (without whitespace):
+     *     {
+     *     "id": "1983d114-a6e8-4940-b121-eb4ac3f6f703",
+     *     "results": [
+     *         {
+     *             "document": {
+     *                 "text": "Washington, D.C.  is the capital of the United States. It is a federal district."
+     *             },
+     *             "index": 2,
+     *             "relevance_score": 0.98005307
+     *         },
+     *         {
+     *             "document": {
+     *                 "text": "Capital punishment (the death penalty) As of 2017, capital punishment is legal in 30 of the 50 states."
+     *             },
+     *             "index": 3,
+     *             "relevance_score": 0.27904198
+     *         },
+     *         {
+     *             "document": {
+     *                 "text": "Carson City is the capital city of the American state of Nevada."
+     *             },
+     *             "index": 0,
+     *             "relevance_score": 0.10194652
+     *         }
+     *     ],
+     *     "usage": {"total_tokens": 15}
+     *     }
+     *
+     * @param response the http response from JinaAI
+     * @return the parsed response
+     * @throws IOException if there is an error parsing the response
+     */
+    public static InferenceServiceResults fromResponse(HttpResult response) throws IOException {
+        var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE);
+
+        try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) {
+            moveToFirstToken(jsonParser);
+
+            XContentParser.Token token = jsonParser.currentToken();
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
+
+            positionParserAtTokenAfterField(jsonParser, "results", FAILED_TO_FIND_FIELD_TEMPLATE);
+
+            token = jsonParser.currentToken();
+            if (token == XContentParser.Token.START_ARRAY) {
+                return new RankedDocsResults(parseList(jsonParser, JinaAIRerankResponseEntity::parseRankedDocObject));
+            } else {
+                throwUnknownToken(token, jsonParser);
+            }
+
+            // This should never be reached. The above code should either return successfully or hit the throwUnknownToken
+            // or throw a parsing exception
+            throw new IllegalStateException("Reached an invalid state while parsing the JinaAI response");
+        }
+    }
+
+    private static RankedDocsResults.RankedDoc parseRankedDocObject(XContentParser parser) throws IOException {
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
+        int index = -1;
+        float relevanceScore = -1;
+        String documentText = null;
+        parser.nextToken();
+        while (parser.currentToken() != XContentParser.Token.END_OBJECT) {
+            if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
+                switch (parser.currentName()) {
+                    case "index":
+                        parser.nextToken(); // move to VALUE_NUMBER
+                        index = parser.intValue();
+                        parser.nextToken(); // move to next FIELD_NAME or END_OBJECT
+                        break;
+                    case "relevance_score":
+                        parser.nextToken(); // move to VALUE_NUMBER
+                        relevanceScore = parser.floatValue();
+                        parser.nextToken(); // move to next FIELD_NAME or END_OBJECT
+                        break;
+                    case "document":
+                        parser.nextToken(); // move to START_OBJECT; document text is wrapped in an object
+                        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
+                        do {
+                            if (parser.currentToken() == XContentParser.Token.FIELD_NAME && parser.currentName().equals("text")) {
+                                parser.nextToken(); // move to VALUE_STRING
+                                documentText = parser.text();
+                            }
+                        } while (parser.nextToken() != XContentParser.Token.END_OBJECT);
+                        parser.nextToken();// move past END_OBJECT
+                        // parser should now be at the next FIELD_NAME or END_OBJECT
+                        break;
+                    default:
+                        throwUnknownField(parser.currentName(), parser);
+                }
+            } else {
+                parser.nextToken();
+            }
+        }
+
+        if (index == -1) {
+            logger.warn("Failed to find required field [index] in JinaAI rerank response");
+        }
+        if (relevanceScore == -1) {
+            logger.warn("Failed to find required field [relevance_score] in JinaAI rerank response");
+        }
+        // documentText may or may not be present depending on the request parameter
+
+        return new RankedDocsResults.RankedDoc(index, relevanceScore, documentText);
+    }
+
+    private JinaAIRerankResponseEntity() {}
+
+    static String FAILED_TO_FIND_FIELD_TEMPLATE = "Failed to find required field [%s] in JinaAI rerank response";
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/unified/UnifiedChatCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/unified/UnifiedChatCompletionRequestEntity.java
new file mode 100644
index 0000000000000..5e6d09cde2b9f
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/unified/UnifiedChatCompletionRequestEntity.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.unified;
+
+import org.elasticsearch.inference.UnifiedCompletionRequest;
+import org.elasticsearch.xcontent.ToXContentFragment;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class UnifiedChatCompletionRequestEntity implements ToXContentFragment {
+
+    public static final String NAME_FIELD = "name";
+    public static final String TOOL_CALL_ID_FIELD = "tool_call_id";
+    public static final String TOOL_CALLS_FIELD = "tool_calls";
+    public static final String ID_FIELD = "id";
+    public static final String FUNCTION_FIELD = "function";
+    public static final String ARGUMENTS_FIELD = "arguments";
+    public static final String DESCRIPTION_FIELD = "description";
+    public static final String PARAMETERS_FIELD = "parameters";
+    public static final String STRICT_FIELD = "strict";
+    public static final String TOP_P_FIELD = "top_p";
+    public static final String STREAM_FIELD = "stream";
+    private static final String NUMBER_OF_RETURNED_CHOICES_FIELD = "n";
+    public static final String MESSAGES_FIELD = "messages";
+    private static final String ROLE_FIELD = "role";
+    private static final String CONTENT_FIELD = "content";
+    private static final String MAX_COMPLETION_TOKENS_FIELD = "max_completion_tokens";
+    private static final String STOP_FIELD = "stop";
+    private static final String TEMPERATURE_FIELD = "temperature";
+    private static final String TOOL_CHOICE_FIELD = "tool_choice";
+    private static final String TOOL_FIELD = "tools";
+    private static final String TEXT_FIELD = "text";
+    private static final String TYPE_FIELD = "type";
+    private static final String STREAM_OPTIONS_FIELD = "stream_options";
+    private static final String INCLUDE_USAGE_FIELD = "include_usage";
+
+    private final UnifiedCompletionRequest unifiedRequest;
+    private final boolean stream;
+
+    public UnifiedChatCompletionRequestEntity(UnifiedChatInput unifiedChatInput) {
+        Objects.requireNonNull(unifiedChatInput);
+
+        this.unifiedRequest = unifiedChatInput.getRequest();
+        this.stream = unifiedChatInput.stream();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startArray(MESSAGES_FIELD);
+        {
+            for (UnifiedCompletionRequest.Message message : unifiedRequest.messages()) {
+                builder.startObject();
+                {
+                    switch (message.content()) {
+                        case UnifiedCompletionRequest.ContentString contentString -> builder.field(CONTENT_FIELD, contentString.content());
+                        case UnifiedCompletionRequest.ContentObjects contentObjects -> {
+                            builder.startArray(CONTENT_FIELD);
+                            for (UnifiedCompletionRequest.ContentObject contentObject : contentObjects.contentObjects()) {
+                                builder.startObject();
+                                builder.field(TEXT_FIELD, contentObject.text());
+                                builder.field(TYPE_FIELD, contentObject.type());
+                                builder.endObject();
+                            }
+                            builder.endArray();
+                        }
+                        case null -> {
+                            // do nothing because content is optional
+                        }
+                    }
+
+                    builder.field(ROLE_FIELD, message.role());
+                    if (message.toolCallId() != null) {
+                        builder.field(TOOL_CALL_ID_FIELD, message.toolCallId());
+                    }
+                    if (message.toolCalls() != null) {
+                        builder.startArray(TOOL_CALLS_FIELD);
+                        for (UnifiedCompletionRequest.ToolCall toolCall : message.toolCalls()) {
+                            builder.startObject();
+                            {
+                                builder.field(ID_FIELD, toolCall.id());
+                                builder.startObject(FUNCTION_FIELD);
+                                {
+                                    builder.field(ARGUMENTS_FIELD, toolCall.function().arguments());
+                                    builder.field(NAME_FIELD, toolCall.function().name());
+                                }
+                                builder.endObject();
+                                builder.field(TYPE_FIELD, toolCall.type());
+                            }
+                            builder.endObject();
+                        }
+                        builder.endArray();
+                    }
+                }
+                builder.endObject();
+            }
+        }
+        builder.endArray();
+
+        if (unifiedRequest.maxCompletionTokens() != null) {
+            builder.field(MAX_COMPLETION_TOKENS_FIELD, unifiedRequest.maxCompletionTokens());
+        }
+
+        // Underlying providers expect OpenAI to only return 1 possible choice.
+        builder.field(NUMBER_OF_RETURNED_CHOICES_FIELD, 1);
+
+        if (unifiedRequest.stop() != null && unifiedRequest.stop().isEmpty() == false) {
+            builder.field(STOP_FIELD, unifiedRequest.stop());
+        }
+        if (unifiedRequest.temperature() != null) {
+            builder.field(TEMPERATURE_FIELD, unifiedRequest.temperature());
+        }
+        if (unifiedRequest.toolChoice() != null) {
+            if (unifiedRequest.toolChoice() instanceof UnifiedCompletionRequest.ToolChoiceString) {
+                builder.field(TOOL_CHOICE_FIELD, ((UnifiedCompletionRequest.ToolChoiceString) unifiedRequest.toolChoice()).value());
+            } else if (unifiedRequest.toolChoice() instanceof UnifiedCompletionRequest.ToolChoiceObject) {
+                builder.startObject(TOOL_CHOICE_FIELD);
+                {
+                    builder.field(TYPE_FIELD, ((UnifiedCompletionRequest.ToolChoiceObject) unifiedRequest.toolChoice()).type());
+                    builder.startObject(FUNCTION_FIELD);
+                    {
+                        builder.field(
+                            NAME_FIELD,
+                            ((UnifiedCompletionRequest.ToolChoiceObject) unifiedRequest.toolChoice()).function().name()
+                        );
+                    }
+                    builder.endObject();
+                }
+                builder.endObject();
+            }
+        }
+        boolean usesTools = unifiedRequest.tools() != null && unifiedRequest.tools().isEmpty() == false;
+
+        if (usesTools) {
+            builder.startArray(TOOL_FIELD);
+            for (UnifiedCompletionRequest.Tool tool : unifiedRequest.tools()) {
+                builder.startObject();
+                {
+                    builder.field(TYPE_FIELD, tool.type());
+                    builder.startObject(FUNCTION_FIELD);
+                    {
+                        builder.field(DESCRIPTION_FIELD, tool.function().description());
+                        builder.field(NAME_FIELD, tool.function().name());
+                        builder.field(PARAMETERS_FIELD, tool.function().parameters());
+                        if (tool.function().strict() != null) {
+                            builder.field(STRICT_FIELD, tool.function().strict());
+                        }
+                    }
+                    builder.endObject();
+                }
+                builder.endObject();
+            }
+            builder.endArray();
+        }
+        if (unifiedRequest.topP() != null) {
+            builder.field(TOP_P_FIELD, unifiedRequest.topP());
+        }
+
+        builder.field(STREAM_FIELD, stream);
+        if (stream) {
+            builder.startObject(STREAM_OPTIONS_FIELD);
+            builder.field(INCLUDE_USAGE_FIELD, true);
+            builder.endObject();
+        }
+
+        return builder;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighter.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighter.java
index f2bfa72ec617a..ae8997dd8c1f0 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighter.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighter.java
@@ -26,20 +26,31 @@
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.DenseVectorFieldType;
 import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper.SparseVectorFieldType;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.search.fetch.FetchSubPhase;
+import org.elasticsearch.search.fetch.subphase.highlight.DefaultHighlighter;
 import org.elasticsearch.search.fetch.subphase.highlight.FieldHighlightContext;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightUtils;
 import org.elasticsearch.search.fetch.subphase.highlight.Highlighter;
 import org.elasticsearch.search.vectors.VectorData;
 import org.elasticsearch.xpack.core.ml.search.SparseVectorQueryWrapper;
+import org.elasticsearch.xpack.inference.mapper.OffsetSourceField;
+import org.elasticsearch.xpack.inference.mapper.OffsetSourceFieldMapper;
 import org.elasticsearch.xpack.inference.mapper.SemanticTextField;
 import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper;
+import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SemanticTextFieldType;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.function.Function;
+
+import static org.elasticsearch.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR;
 
 /**
  * A {@link Highlighter} designed for the {@link SemanticTextFieldMapper}.
@@ -49,19 +60,19 @@
 public class SemanticTextHighlighter implements Highlighter {
     public static final String NAME = "semantic";
 
-    private record OffsetAndScore(int offset, float score) {}
+    private record OffsetAndScore(int index, OffsetSourceFieldMapper.OffsetSource offset, float score) {}
 
     @Override
     public boolean canHighlight(MappedFieldType fieldType) {
-        if (fieldType instanceof SemanticTextFieldMapper.SemanticTextFieldType) {
-            return true;
-        }
-        return false;
+        return fieldType instanceof SemanticTextFieldType;
     }
 
     @Override
     public HighlightField highlight(FieldHighlightContext fieldContext) throws IOException {
-        SemanticTextFieldMapper.SemanticTextFieldType fieldType = (SemanticTextFieldMapper.SemanticTextFieldType) fieldContext.fieldType;
+        if (canHighlight(fieldContext.fieldType) == false) {
+            return null;
+        }
+        SemanticTextFieldType fieldType = (SemanticTextFieldType) fieldContext.fieldType;
         if (fieldType.getEmbeddingsField() == null) {
             // nothing indexed yet
             return null;
@@ -104,28 +115,36 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc
         int size = Math.min(chunks.size(), numberOfFragments);
         if (fieldContext.field.fieldOptions().scoreOrdered() == false) {
             chunks = chunks.subList(0, size);
-            chunks.sort(Comparator.comparingInt(c -> c.offset));
+            chunks.sort(Comparator.comparingInt(c -> c.index));
         }
         Text[] snippets = new Text[size];
-        List<Map<?, ?>> nestedSources = XContentMapValues.extractNestedSources(
-            fieldType.getChunksField().fullPath(),
-            fieldContext.hitContext.source().source()
-        );
+        final Function<OffsetAndScore, String> offsetToContent;
+        if (fieldType.useLegacyFormat()) {
+            List<Map<?, ?>> nestedSources = XContentMapValues.extractNestedSources(
+                fieldType.getChunksField().fullPath(),
+                fieldContext.hitContext.source().source()
+            );
+            offsetToContent = entry -> getContentFromLegacyNestedSources(fieldType.name(), entry, nestedSources);
+        } else {
+            Map<String, String> fieldToContent = new HashMap<>();
+            offsetToContent = entry -> {
+                String content = fieldToContent.computeIfAbsent(entry.offset().field(), key -> {
+                    try {
+                        return extractFieldContent(
+                            fieldContext.context.getSearchExecutionContext(),
+                            fieldContext.hitContext,
+                            entry.offset.field()
+                        );
+                    } catch (IOException e) {
+                        throw new UncheckedIOException("Error extracting field content from field " + entry.offset.field(), e);
+                    }
+                });
+                return content.substring(entry.offset().start(), entry.offset().end());
+            };
+        }
         for (int i = 0; i < size; i++) {
             var chunk = chunks.get(i);
-            if (nestedSources.size() <= chunk.offset) {
-                throw new IllegalStateException(
-                    String.format(
-                        Locale.ROOT,
-                        "Invalid content detected for field [%s]: the chunks size is [%d], "
-                            + "but a reference to offset [%d] was found in the result.",
-                        fieldType.name(),
-                        nestedSources.size(),
-                        chunk.offset
-                    )
-                );
-            }
-            String content = (String) nestedSources.get(chunk.offset).get(SemanticTextField.CHUNKED_TEXT_FIELD);
+            String content = offsetToContent.apply(chunk);
             if (content == null) {
                 throw new IllegalStateException(
                     String.format(
@@ -142,10 +161,43 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc
         return new HighlightField(fieldContext.fieldName, snippets);
     }
 
+    private String extractFieldContent(SearchExecutionContext searchContext, FetchSubPhase.HitContext hitContext, String sourceField)
+        throws IOException {
+        var sourceFieldType = searchContext.getMappingLookup().getFieldType(sourceField);
+        if (sourceFieldType == null) {
+            return null;
+        }
+
+        var values = HighlightUtils.loadFieldValues(sourceFieldType, searchContext, hitContext)
+            .stream()
+            .<Object>map((s) -> DefaultHighlighter.convertFieldValue(sourceFieldType, s))
+            .toList();
+        if (values.size() == 0) {
+            return null;
+        }
+        return DefaultHighlighter.mergeFieldValues(values, MULTIVAL_SEP_CHAR);
+    }
+
+    private String getContentFromLegacyNestedSources(String fieldName, OffsetAndScore cand, List<Map<?, ?>> nestedSources) {
+        if (nestedSources.size() <= cand.index) {
+            throw new IllegalStateException(
+                String.format(
+                    Locale.ROOT,
+                    "Invalid content detected for field [%s]: the chunks size is [%d], "
+                        + "but a reference to offset [%d] was found in the result.",
+                    fieldName,
+                    nestedSources.size(),
+                    cand.index
+                )
+            );
+        }
+        return (String) nestedSources.get(cand.index).get(SemanticTextField.CHUNKED_TEXT_FIELD);
+    }
+
     private List<OffsetAndScore> extractOffsetAndScores(
         SearchExecutionContext context,
         LeafReader reader,
-        SemanticTextFieldMapper.SemanticTextFieldType fieldType,
+        SemanticTextFieldType fieldType,
         int docId,
         List<Query> leafQueries
     ) throws IOException {
@@ -163,10 +215,31 @@ private List<OffsetAndScore> extractOffsetAndScores(
         } else if (scorer.iterator().nextDoc() == DocIdSetIterator.NO_MORE_DOCS) {
             return List.of();
         }
+
+        OffsetSourceField.OffsetSourceLoader offsetReader = null;
+        if (fieldType.useLegacyFormat() == false) {
+            var terms = reader.terms(fieldType.getOffsetsField().fullPath());
+            if (terms == null) {
+                // The field is empty
+                return List.of();
+            }
+            offsetReader = OffsetSourceField.loader(terms);
+        }
+
         List<OffsetAndScore> results = new ArrayList<>();
-        int offset = 0;
+        int index = 0;
         while (scorer.docID() < docId) {
-            results.add(new OffsetAndScore(offset++, scorer.score()));
+            if (offsetReader != null) {
+                var offset = offsetReader.advanceTo(scorer.docID());
+                if (offset == null) {
+                    throw new IllegalStateException(
+                        "Cannot highlight field [" + fieldType.name() + "], missing offsets for doc [" + docId + "]"
+                    );
+                }
+                results.add(new OffsetAndScore(index++, offset, scorer.score()));
+            } else {
+                results.add(new OffsetAndScore(index++, null, scorer.score()));
+            }
             if (scorer.iterator().nextDoc() == DocIdSetIterator.NO_MORE_DOCS) {
                 break;
             }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapper.java
new file mode 100644
index 0000000000000..3f49973d6e35f
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapper.java
@@ -0,0 +1,180 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.mapper;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.BitSetProducer;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.index.mapper.ContentPath;
+import org.elasticsearch.index.mapper.DocumentParserContext;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MappingLookup;
+import org.elasticsearch.index.mapper.ValueFetcher;
+import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.search.fetch.StoredFieldsSpec;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.xcontent.XContentLocation;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * An {@link InferenceMetadataFieldsMapper} that delegates parsing of underlying fields
+ * to the corresponding {@link SemanticTextFieldMapper}.
+ */
+public class SemanticInferenceMetadataFieldsMapper extends InferenceMetadataFieldsMapper {
+    private static final SemanticInferenceMetadataFieldsMapper INSTANCE = new SemanticInferenceMetadataFieldsMapper();
+
+    public static final NodeFeature EXPLICIT_NULL_FIXES = new NodeFeature("semantic_text.inference_metadata_fields.explicit_null_fixes");
+
+    public static final TypeParser PARSER = new FixedTypeParser(
+        c -> InferenceMetadataFieldsMapper.isEnabled(c.getSettings()) ? INSTANCE : null
+    );
+
+    static class FieldType extends InferenceMetadataFieldType {
+        private static final FieldType INSTANCE = new FieldType();
+
+        FieldType() {
+            super();
+        }
+
+        @Override
+        public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
+            return valueFetcher(context.getMappingLookup(), context::bitsetFilter, context.searcher());
+        }
+
+        @Override
+        public ValueFetcher valueFetcher(MappingLookup mappingLookup, Function<Query, BitSetProducer> bitSetCache, IndexSearcher searcher) {
+            Map<String, ValueFetcher> fieldFetchers = new HashMap<>();
+            for (var inferenceField : mappingLookup.inferenceFields().keySet()) {
+                MappedFieldType ft = mappingLookup.getFieldType(inferenceField);
+                if (ft instanceof SemanticTextFieldMapper.SemanticTextFieldType semanticTextFieldType) {
+                    fieldFetchers.put(inferenceField, semanticTextFieldType.valueFetcherWithInferenceResults(bitSetCache, searcher));
+                } else {
+                    throw new IllegalArgumentException(
+                        "Invalid inference field [" + ft.name() + "]. Expected field type [semantic_text] but got [" + ft.typeName() + "]"
+                    );
+                }
+            }
+            if (fieldFetchers.isEmpty()) {
+                return ValueFetcher.EMPTY;
+            }
+            return new ValueFetcher() {
+                @Override
+                public void setNextReader(LeafReaderContext context) {
+                    fieldFetchers.values().forEach(f -> f.setNextReader(context));
+                }
+
+                @Override
+                public List<Object> fetchValues(Source source, int doc, List<Object> ignoredValues) throws IOException {
+                    Map<String, Object> result = new HashMap<>();
+                    for (var entry : fieldFetchers.entrySet()) {
+                        var values = entry.getValue().fetchValues(source, doc, ignoredValues);
+                        if (values.size() > 0) {
+                            assert values.size() == 1;
+                            result.put(entry.getKey(), values.get(0));
+                        }
+                    }
+                    return result.isEmpty() ? List.of() : List.of(result);
+                }
+
+                @Override
+                public StoredFieldsSpec storedFieldsSpec() {
+                    return StoredFieldsSpec.NO_REQUIREMENTS;
+                }
+            };
+        }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        @Override
+        public Query termQuery(Object value, SearchExecutionContext context) {
+            throw new QueryShardException(
+                context,
+                "[" + name() + "] field which is of type [" + typeName() + "], does not support term queries"
+            );
+        }
+    }
+
+    private SemanticInferenceMetadataFieldsMapper() {
+        super(FieldType.INSTANCE);
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+
+    @Override
+    protected boolean supportsParsingObject() {
+        return true;
+    }
+
+    @Override
+    protected void parseCreateField(DocumentParserContext context) throws IOException {
+        final boolean isWithinLeaf = context.path().isWithinLeafObject();
+        try {
+            // make sure that we don't expand dots in field names while parsing
+            context.path().setWithinLeafObject(true);
+            XContentParser parser = context.parser();
+            XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
+            while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser);
+                String fieldName = parser.currentName();
+
+                // Set the path to that of semantic text field so the parser acts as if we are parsing the semantic text field value
+                // directly. We can safely split on all "." chars because semantic text fields cannot be used when subobjects == false.
+                String[] fieldNameParts = fieldName.split("\\.");
+                setPath(context.path(), fieldNameParts);
+
+                var parent = context.parent().findParentMapper(fieldName);
+                if (parent == null) {
+                    throw new IllegalArgumentException("Field [" + fieldName + "] does not have a parent mapper");
+                }
+                String suffix = parent != context.parent() ? fieldName.substring(parent.fullPath().length() + 1) : fieldName;
+                var mapper = parent.getMapper(suffix);
+                if (mapper instanceof SemanticTextFieldMapper fieldMapper) {
+                    XContentLocation xContentLocation = context.parser().getTokenLocation();
+                    var input = fieldMapper.parseSemanticTextField(context);
+                    if (input != null) {
+                        fieldMapper.parseCreateFieldFromContext(context, input, xContentLocation);
+                    }
+                } else {
+                    throw new IllegalArgumentException(
+                        "Field [" + fieldName + "] is not a [" + SemanticTextFieldMapper.CONTENT_TYPE + "] field"
+                    );
+                }
+            }
+        } finally {
+            context.path().setWithinLeafObject(isWithinLeaf);
+            setPath(context.path(), new String[] { InferenceMetadataFieldsMapper.NAME });
+        }
+    }
+
+    private static void setPath(ContentPath contentPath, String[] newPath) {
+        while (contentPath.length() > 0) {
+            contentPath.remove();
+        }
+
+        for (String pathPart : newPath) {
+            contentPath.add(pathPart);
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java
index d651729dee259..fddff17dab4cf 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java
@@ -10,8 +10,10 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.core.Tuple;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.inference.ChunkedInference;
 import org.elasticsearch.inference.Model;
@@ -31,6 +33,9 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -46,13 +51,18 @@
  * the inference results under the {@link SemanticTextField#INFERENCE_FIELD}.
  *
  * @param fieldName The original field name.
- * @param originalValues The original values associated with the field name.
+ * @param originalValues The original values associated with the field name for indices created before
+ *                       {@link IndexVersions#INFERENCE_METADATA_FIELDS}, null otherwise.
  * @param inference The inference result.
  * @param contentType The {@link XContentType} used to store the embeddings chunks.
  */
-public record SemanticTextField(String fieldName, List<String> originalValues, InferenceResult inference, XContentType contentType)
-    implements
-        ToXContentObject {
+public record SemanticTextField(
+    boolean useLegacyFormat,
+    String fieldName,
+    @Nullable List<String> originalValues,
+    InferenceResult inference,
+    XContentType contentType
+) implements ToXContentObject {
 
     static final String TEXT_FIELD = "text";
     static final String INFERENCE_FIELD = "inference";
@@ -61,15 +71,20 @@ public record SemanticTextField(String fieldName, List<String> originalValues, I
     static final String CHUNKS_FIELD = "chunks";
     static final String CHUNKED_EMBEDDINGS_FIELD = "embeddings";
     public static final String CHUNKED_TEXT_FIELD = "text";
+    static final String CHUNKED_OFFSET_FIELD = "offset";
+    static final String CHUNKED_START_OFFSET_FIELD = "start_offset";
+    static final String CHUNKED_END_OFFSET_FIELD = "end_offset";
     static final String MODEL_SETTINGS_FIELD = "model_settings";
     static final String TASK_TYPE_FIELD = "task_type";
     static final String DIMENSIONS_FIELD = "dimensions";
     static final String SIMILARITY_FIELD = "similarity";
     static final String ELEMENT_TYPE_FIELD = "element_type";
 
-    public record InferenceResult(String inferenceId, ModelSettings modelSettings, List<Chunk> chunks) {}
+    public record InferenceResult(String inferenceId, ModelSettings modelSettings, Map<String, List<Chunk>> chunks) {}
 
-    record Chunk(String text, BytesReference rawEmbeddings) {}
+    public record Chunk(@Nullable String text, int startOffset, int endOffset, BytesReference rawEmbeddings) {}
+
+    public record Offset(String sourceFieldName, int startOffset, int endOffset) {}
 
     public record ModelSettings(
         TaskType taskType,
@@ -187,12 +202,14 @@ public static String getEmbeddingsFieldName(String fieldName) {
         return getChunksFieldName(fieldName) + "." + CHUNKED_EMBEDDINGS_FIELD;
     }
 
-    static SemanticTextField parse(XContentParser parser, Tuple<String, XContentType> context) throws IOException {
-        return SEMANTIC_TEXT_FIELD_PARSER.parse(parser, context);
+    public static String getOffsetsFieldName(String fieldName) {
+        return getChunksFieldName(fieldName) + "." + CHUNKED_OFFSET_FIELD;
     }
 
-    static ModelSettings parseModelSettings(XContentParser parser) throws IOException {
-        return MODEL_SETTINGS_PARSER.parse(parser, null);
+    record ParserContext(boolean useLegacyFormat, String fieldName, XContentType xContentType) {}
+
+    static SemanticTextField parse(XContentParser parser, ParserContext context) throws IOException {
+        return SEMANTIC_TEXT_FIELD_PARSER.parse(parser, context);
     }
 
     static ModelSettings parseModelSettingsFromMap(Object node) {
@@ -207,63 +224,102 @@ static ModelSettings parseModelSettingsFromMap(Object node) {
                 map,
                 XContentType.JSON
             );
-            return parseModelSettings(parser);
+            return MODEL_SETTINGS_PARSER.parse(parser, null);
         } catch (Exception exc) {
             throw new ElasticsearchException(exc);
         }
     }
 
+    @Override
+    public List<String> originalValues() {
+        return originalValues != null ? originalValues : Collections.emptyList();
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        if (originalValues.isEmpty() == false) {
+        List<String> originalValues = originalValues();
+        if (useLegacyFormat && originalValues.isEmpty() == false) {
             builder.field(TEXT_FIELD, originalValues.size() == 1 ? originalValues.get(0) : originalValues);
         }
         builder.startObject(INFERENCE_FIELD);
         builder.field(INFERENCE_ID_FIELD, inference.inferenceId);
         builder.field(MODEL_SETTINGS_FIELD, inference.modelSettings);
-        builder.startArray(CHUNKS_FIELD);
-        for (var chunk : inference.chunks) {
-            builder.startObject();
-            builder.field(CHUNKED_TEXT_FIELD, chunk.text);
-            XContentParser parser = XContentHelper.createParserNotCompressed(
-                XContentParserConfiguration.EMPTY,
-                chunk.rawEmbeddings,
-                contentType
-            );
-            builder.field(CHUNKED_EMBEDDINGS_FIELD).copyCurrentStructure(parser);
+        if (useLegacyFormat) {
+            builder.startArray(CHUNKS_FIELD);
+        } else {
+            builder.startObject(CHUNKS_FIELD);
+        }
+        for (var entry : inference.chunks.entrySet()) {
+            if (useLegacyFormat == false) {
+                builder.startArray(entry.getKey());
+            }
+            for (var chunk : entry.getValue()) {
+                builder.startObject();
+                if (useLegacyFormat) {
+                    builder.field(TEXT_FIELD, chunk.text);
+                } else {
+                    builder.field(CHUNKED_START_OFFSET_FIELD, chunk.startOffset);
+                    builder.field(CHUNKED_END_OFFSET_FIELD, chunk.endOffset);
+                }
+                XContentParser parser = XContentHelper.createParserNotCompressed(
+                    XContentParserConfiguration.EMPTY,
+                    chunk.rawEmbeddings,
+                    contentType
+                );
+                builder.field(CHUNKED_EMBEDDINGS_FIELD).copyCurrentStructure(parser);
+                builder.endObject();
+            }
+            if (useLegacyFormat == false) {
+                builder.endArray();
+            }
+        }
+        if (useLegacyFormat) {
+            builder.endArray();
+        } else {
             builder.endObject();
         }
-        builder.endArray();
         builder.endObject();
         builder.endObject();
         return builder;
     }
 
     @SuppressWarnings("unchecked")
-    private static final ConstructingObjectParser<SemanticTextField, Tuple<String, XContentType>> SEMANTIC_TEXT_FIELD_PARSER =
-        new ConstructingObjectParser<>(
-            SemanticTextFieldMapper.CONTENT_TYPE,
-            true,
-            (args, context) -> new SemanticTextField(
-                context.v1(),
-                (List<String>) (args[0] == null ? List.of() : args[0]),
+    private static final ConstructingObjectParser<SemanticTextField, ParserContext> SEMANTIC_TEXT_FIELD_PARSER =
+        new ConstructingObjectParser<>(SemanticTextFieldMapper.CONTENT_TYPE, true, (args, context) -> {
+            List<String> originalValues = (List<String>) args[0];
+            if (context.useLegacyFormat() == false) {
+                if (originalValues != null && originalValues.isEmpty() == false) {
+                    throw new IllegalArgumentException("Unknown field [" + TEXT_FIELD + "]");
+                }
+                originalValues = null;
+            }
+            return new SemanticTextField(
+                context.useLegacyFormat(),
+                context.fieldName(),
+                originalValues,
                 (InferenceResult) args[1],
-                context.v2()
-            )
-        );
+                context.xContentType()
+            );
+        });
 
     @SuppressWarnings("unchecked")
-    private static final ConstructingObjectParser<InferenceResult, Void> INFERENCE_RESULT_PARSER = new ConstructingObjectParser<>(
+    private static final ConstructingObjectParser<InferenceResult, ParserContext> INFERENCE_RESULT_PARSER = new ConstructingObjectParser<>(
         INFERENCE_FIELD,
         true,
-        args -> new InferenceResult((String) args[0], (ModelSettings) args[1], (List<Chunk>) args[2])
+        args -> new InferenceResult((String) args[0], (ModelSettings) args[1], (Map<String, List<Chunk>>) args[2])
     );
 
-    private static final ConstructingObjectParser<Chunk, Void> CHUNKS_PARSER = new ConstructingObjectParser<>(
+    private static final ConstructingObjectParser<Chunk, ParserContext> CHUNKS_PARSER = new ConstructingObjectParser<>(
         CHUNKS_FIELD,
         true,
-        args -> new Chunk((String) args[0], (BytesReference) args[1])
+        (args, context) -> {
+            String text = (String) args[0];
+            if (context.useLegacyFormat() && text == null) {
+                throw new IllegalArgumentException("Missing chunk text");
+            }
+            return new Chunk(text, args[1] != null ? (int) args[1] : -1, args[2] != null ? (int) args[2] : -1, (BytesReference) args[3]);
+        }
     );
 
     private static final ConstructingObjectParser<ModelSettings, Void> MODEL_SETTINGS_PARSER = new ConstructingObjectParser<>(
@@ -282,17 +338,25 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     static {
         SEMANTIC_TEXT_FIELD_PARSER.declareStringArray(optionalConstructorArg(), new ParseField(TEXT_FIELD));
-        SEMANTIC_TEXT_FIELD_PARSER.declareObject(
-            constructorArg(),
-            (p, c) -> INFERENCE_RESULT_PARSER.parse(p, null),
-            new ParseField(INFERENCE_FIELD)
-        );
+        SEMANTIC_TEXT_FIELD_PARSER.declareObject(constructorArg(), INFERENCE_RESULT_PARSER, new ParseField(INFERENCE_FIELD));
 
         INFERENCE_RESULT_PARSER.declareString(constructorArg(), new ParseField(INFERENCE_ID_FIELD));
-        INFERENCE_RESULT_PARSER.declareObject(constructorArg(), MODEL_SETTINGS_PARSER, new ParseField(MODEL_SETTINGS_FIELD));
-        INFERENCE_RESULT_PARSER.declareObjectArray(constructorArg(), CHUNKS_PARSER, new ParseField(CHUNKS_FIELD));
+        INFERENCE_RESULT_PARSER.declareObjectOrNull(
+            constructorArg(),
+            (p, c) -> MODEL_SETTINGS_PARSER.parse(p, null),
+            null,
+            new ParseField(MODEL_SETTINGS_FIELD)
+        );
+        INFERENCE_RESULT_PARSER.declareField(constructorArg(), (p, c) -> {
+            if (c.useLegacyFormat()) {
+                return Map.of(c.fieldName, parseChunksArrayLegacy(p, c));
+            }
+            return parseChunksMap(p, c);
+        }, new ParseField(CHUNKS_FIELD), ObjectParser.ValueType.OBJECT_ARRAY);
 
-        CHUNKS_PARSER.declareString(constructorArg(), new ParseField(CHUNKED_TEXT_FIELD));
+        CHUNKS_PARSER.declareString(optionalConstructorArg(), new ParseField(TEXT_FIELD));
+        CHUNKS_PARSER.declareInt(optionalConstructorArg(), new ParseField(CHUNKED_START_OFFSET_FIELD));
+        CHUNKS_PARSER.declareInt(optionalConstructorArg(), new ParseField(CHUNKED_END_OFFSET_FIELD));
         CHUNKS_PARSER.declareField(constructorArg(), (p, c) -> {
             XContentBuilder b = XContentBuilder.builder(p.contentType().xContent());
             b.copyCurrentStructure(p);
@@ -305,18 +369,64 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         MODEL_SETTINGS_PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField(ELEMENT_TYPE_FIELD));
     }
 
+    private static Map<String, List<Chunk>> parseChunksMap(XContentParser parser, ParserContext context) throws IOException {
+        Map<String, List<Chunk>> resultMap = new LinkedHashMap<>();
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
+        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+            XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser);
+            String fieldName = parser.currentName();
+            XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser);
+            var chunks = resultMap.computeIfAbsent(fieldName, k -> new ArrayList<>());
+            while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                chunks.add(CHUNKS_PARSER.parse(parser, context));
+            }
+        }
+        return resultMap;
+    }
+
+    private static List<Chunk> parseChunksArrayLegacy(XContentParser parser, ParserContext context) throws IOException {
+        List<Chunk> results = new ArrayList<>();
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
+        while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+            results.add(CHUNKS_PARSER.parse(parser, context));
+        }
+        return results;
+    }
+
     /**
      * Converts the provided {@link ChunkedInference} into a list of {@link Chunk}.
      */
-    public static List<Chunk> toSemanticTextFieldChunks(List<ChunkedInference> results, XContentType contentType) throws IOException {
+    public static List<Chunk> toSemanticTextFieldChunks(
+        String input,
+        int offsetAdjustment,
+        ChunkedInference results,
+        XContentType contentType,
+        boolean useLegacyFormat
+    ) throws IOException {
         List<Chunk> chunks = new ArrayList<>();
-        for (var result : results) {
-            for (var it = result.chunksAsMatchedTextAndByteReference(contentType.xContent()); it.hasNext();) {
-                var chunkAsByteReference = it.next();
-                chunks.add(new Chunk(chunkAsByteReference.matchedText(), chunkAsByteReference.bytesReference()));
-            }
+        Iterator<ChunkedInference.Chunk> it = results.chunksAsMatchedTextAndByteReference(contentType.xContent());
+        while (it.hasNext()) {
+            chunks.add(toSemanticTextFieldChunk(input, offsetAdjustment, it.next(), useLegacyFormat));
         }
         return chunks;
     }
 
+    public static Chunk toSemanticTextFieldChunk(
+        String input,
+        int offsetAdjustment,
+        ChunkedInference.Chunk chunk,
+        boolean useLegacyFormat
+    ) {
+        String text = null;
+        int startOffset = -1;
+        int endOffset = -1;
+        if (useLegacyFormat) {
+            text = input.substring(chunk.textOffset().start(), chunk.textOffset().end());
+        } else {
+            startOffset = chunk.textOffset().start() + offsetAdjustment;
+            endOffset = chunk.textOffset().end() + offsetAdjustment;
+        }
+
+        return new Chunk(text, startOffset, endOffset, chunk.bytesReference());
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java
index cda77233bdfd4..115d0d7947de2 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java
@@ -8,16 +8,23 @@
 package org.elasticsearch.xpack.inference.mapper;
 
 import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Weight;
 import org.apache.lucene.search.join.BitSetProducer;
 import org.apache.lucene.search.join.ScoreMode;
+import org.apache.lucene.util.BitSet;
 import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
@@ -29,6 +36,7 @@
 import org.elasticsearch.index.mapper.DocumentParsingException;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.InferenceFieldMapper;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
@@ -38,7 +46,9 @@
 import org.elasticsearch.index.mapper.NestedObjectMapper;
 import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.index.mapper.SimpleMappedFieldType;
+import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.SourceValueFetcher;
+import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.mapper.TextSearchInfo;
 import org.elasticsearch.index.mapper.ValueFetcher;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
@@ -49,19 +59,25 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.search.fetch.StoredFieldsSpec;
+import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.search.vectors.KnnVectorQueryBuilder;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentLocation;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults;
 import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults;
 import org.elasticsearch.xpack.core.ml.search.SparseVectorQueryBuilder;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -71,7 +87,7 @@
 
 import static org.elasticsearch.search.SearchService.DEFAULT_SIZE;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKED_EMBEDDINGS_FIELD;
-import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKED_TEXT_FIELD;
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKED_OFFSET_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKS_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.INFERENCE_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.INFERENCE_ID_FIELD;
@@ -80,6 +96,7 @@
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.TEXT_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getChunksFieldName;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getEmbeddingsFieldName;
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getOffsetsFieldName;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getOriginalTextFieldName;
 import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.DEFAULT_ELSER_ID;
 
@@ -87,8 +104,6 @@
  * A {@link FieldMapper} for semantic text fields.
  */
 public class SemanticTextFieldMapper extends FieldMapper implements InferenceFieldMapper {
-    public static final NodeFeature SEMANTIC_TEXT_SEARCH_INFERENCE_ID = new NodeFeature("semantic_text.search_inference_id");
-    public static final NodeFeature SEMANTIC_TEXT_DEFAULT_ELSER_2 = new NodeFeature("semantic_text.default_elser_2");
     public static final NodeFeature SEMANTIC_TEXT_IN_OBJECT_FIELD_FIX = new NodeFeature("semantic_text.in_object_field_fix");
     public static final NodeFeature SEMANTIC_TEXT_SINGLE_FIELD_UPDATE_FIX = new NodeFeature("semantic_text.single_field_update_fix");
     public static final NodeFeature SEMANTIC_TEXT_DELETE_FIX = new NodeFeature("semantic_text.delete_fix");
@@ -100,16 +115,14 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
     public static final String CONTENT_TYPE = "semantic_text";
     public static final String DEFAULT_ELSER_2_INFERENCE_ID = DEFAULT_ELSER_ID;
 
-    private final IndexSettings indexSettings;
-
     public static final TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.indexVersionCreated(), c::bitSetProducer, c.getIndexSettings()),
+        (n, c) -> new Builder(n, c::bitSetProducer, c.getIndexSettings()),
         List.of(notInMultiFields(CONTENT_TYPE), notFromDynamicTemplates(CONTENT_TYPE))
     );
 
     public static class Builder extends FieldMapper.Builder {
+        private final boolean useLegacyFormat;
         private final IndexVersion indexVersionCreated;
-        private final IndexSettings indexSettings;
 
         private final Parameter<String> inferenceId = Parameter.stringParam(
             INFERENCE_ID_FIELD,
@@ -154,26 +167,21 @@ public static class Builder extends FieldMapper.Builder {
         public static Builder from(SemanticTextFieldMapper mapper) {
             Builder builder = new Builder(
                 mapper.leafName(),
-                mapper.fieldType().indexVersionCreated,
                 mapper.fieldType().getChunksField().bitsetProducer(),
-                mapper.indexSettings
+                mapper.fieldType().getChunksField().indexSettings()
             );
             builder.init(mapper);
             return builder;
         }
 
-        public Builder(
-            String name,
-            IndexVersion indexVersionCreated,
-            Function<Query, BitSetProducer> bitSetProducer,
-            IndexSettings indexSettings
-        ) {
+        public Builder(String name, Function<Query, BitSetProducer> bitSetProducer, IndexSettings indexSettings) {
             super(name);
-            this.indexVersionCreated = indexVersionCreated;
-            this.indexSettings = indexSettings;
+            this.indexVersionCreated = indexSettings.getIndexVersionCreated();
+            this.useLegacyFormat = InferenceMetadataFieldsMapper.isEnabled(indexSettings.getSettings()) == false;
             this.inferenceFieldBuilder = c -> createInferenceField(
                 c,
-                indexVersionCreated,
+                indexSettings.getIndexVersionCreated(),
+                useLegacyFormat,
                 modelSettings.get(),
                 bitSetProducer,
                 indexSettings
@@ -238,10 +246,10 @@ public SemanticTextFieldMapper build(MapperBuilderContext context) {
                     modelSettings.getValue(),
                     inferenceField,
                     indexVersionCreated,
+                    useLegacyFormat,
                     meta.getValue()
                 ),
-                builderParams(this, context),
-                indexSettings
+                builderParams(this, context)
             );
         }
 
@@ -265,14 +273,8 @@ private SemanticTextFieldMapper copySettings(SemanticTextFieldMapper mapper, Map
         }
     }
 
-    private SemanticTextFieldMapper(
-        String simpleName,
-        MappedFieldType mappedFieldType,
-        BuilderParams builderParams,
-        IndexSettings indexSettings
-    ) {
+    private SemanticTextFieldMapper(String simpleName, MappedFieldType mappedFieldType, BuilderParams builderParams) {
         super(simpleName, mappedFieldType, builderParams);
-        this.indexSettings = indexSettings;
     }
 
     @Override
@@ -289,21 +291,48 @@ public FieldMapper.Builder getMergeBuilder() {
 
     @Override
     protected void parseCreateField(DocumentParserContext context) throws IOException {
-        XContentParser parser = context.parser();
-        if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
+        final XContentParser parser = context.parser();
+        final XContentLocation xContentLocation = parser.getTokenLocation();
+
+        if (fieldType().useLegacyFormat == false) {
+            // Detect if field value is an object, which we don't support parsing
+            if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+                throw new DocumentParsingException(
+                    xContentLocation,
+                    "[" + CONTENT_TYPE + "] field [" + fullPath() + "] does not support object values"
+                );
+            }
+
+            // ignore the rest of the field value
+            parser.skipChildren();
             return;
         }
 
-        XContentLocation xContentLocation = parser.getTokenLocation();
-        final SemanticTextField field;
+        final SemanticTextField field = parseSemanticTextField(context);
+        if (field != null) {
+            parseCreateFieldFromContext(context, field, xContentLocation);
+        }
+    }
+
+    SemanticTextField parseSemanticTextField(DocumentParserContext context) throws IOException {
+        XContentParser parser = context.parser();
+        if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
+            return null;
+        }
         boolean isWithinLeaf = context.path().isWithinLeafObject();
         try {
             context.path().setWithinLeafObject(true);
-            field = SemanticTextField.parse(parser, new Tuple<>(fullPath(), context.parser().contentType()));
+            return SemanticTextField.parse(
+                context.parser(),
+                new SemanticTextField.ParserContext(fieldType().useLegacyFormat, fullPath(), context.parser().contentType())
+            );
         } finally {
             context.path().setWithinLeafObject(isWithinLeaf);
         }
+    }
 
+    void parseCreateFieldFromContext(DocumentParserContext context, SemanticTextField field, XContentLocation xContentLocation)
+        throws IOException {
         final String fullFieldName = fieldType().name();
         if (field.inference().inferenceId().equals(fieldType().getInferenceId()) == false) {
             throw new DocumentParsingException(
@@ -324,9 +353,8 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
             context.path().remove();
             Builder builder = (Builder) new Builder(
                 leafName(),
-                fieldType().indexVersionCreated,
                 fieldType().getChunksField().bitsetProducer(),
-                indexSettings
+                fieldType().getChunksField().indexSettings()
             ).init(this);
             try {
                 mapper = builder.setModelSettings(field.inference().modelSettings())
@@ -355,19 +383,57 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
             mapper = this;
         }
 
+        if (mapper.fieldType().getModelSettings() == null) {
+            for (var chunkList : field.inference().chunks().values()) {
+                if (chunkList.isEmpty() == false) {
+                    throw new DocumentParsingException(
+                        xContentLocation,
+                        "[" + MODEL_SETTINGS_FIELD + "] must be set for field [" + fullFieldName + "] when chunks are provided"
+                    );
+                }
+            }
+        }
+
         var chunksField = mapper.fieldType().getChunksField();
         var embeddingsField = mapper.fieldType().getEmbeddingsField();
-        for (var chunk : field.inference().chunks()) {
-            try (
-                XContentParser subParser = XContentHelper.createParserNotCompressed(
-                    XContentParserConfiguration.EMPTY,
-                    chunk.rawEmbeddings(),
-                    context.parser().contentType()
-                )
-            ) {
-                DocumentParserContext subContext = context.createNestedContext(chunksField).switchParser(subParser);
-                subParser.nextToken();
-                embeddingsField.parse(subContext);
+        var offsetsField = mapper.fieldType().getOffsetsField();
+        for (var entry : field.inference().chunks().entrySet()) {
+            for (var chunk : entry.getValue()) {
+                var nestedContext = context.createNestedContext(chunksField);
+                try (
+                    XContentParser subParser = XContentHelper.createParserNotCompressed(
+                        XContentParserConfiguration.EMPTY,
+                        chunk.rawEmbeddings(),
+                        context.parser().contentType()
+                    )
+                ) {
+                    DocumentParserContext subContext = nestedContext.switchParser(subParser);
+                    subParser.nextToken();
+                    embeddingsField.parse(subContext);
+                }
+
+                if (fieldType().useLegacyFormat) {
+                    continue;
+                }
+
+                try (XContentBuilder builder = XContentFactory.contentBuilder(context.parser().contentType())) {
+                    builder.startObject();
+                    builder.field("field", entry.getKey());
+                    builder.field("start", chunk.startOffset());
+                    builder.field("end", chunk.endOffset());
+                    builder.endObject();
+                    try (
+                        XContentParser subParser = XContentHelper.createParserNotCompressed(
+                            XContentParserConfiguration.EMPTY,
+                            BytesReference.bytes(builder),
+                            context.parser().contentType()
+                        )
+                    ) {
+                        DocumentParserContext subContext = nestedContext.switchParser(subParser);
+                        subParser.nextToken();
+                        offsetsField.parse(subContext);
+                    }
+                }
             }
         }
     }
@@ -390,20 +456,6 @@ public InferenceFieldMetadata getMetadata(Set<String> sourcePaths) {
         return new InferenceFieldMetadata(fullPath(), fieldType().getInferenceId(), fieldType().getSearchInferenceId(), copyFields);
     }
 
-    @Override
-    public Object getOriginalValue(Map<String, Object> sourceAsMap) {
-        Object fieldValue = sourceAsMap.get(fullPath());
-        if (fieldValue == null) {
-            return null;
-        } else if (fieldValue instanceof Map<?, ?> == false) {
-            // Don't try to further validate the non-map value, that will be handled when the source is fully parsed
-            return fieldValue;
-        }
-
-        Map<String, Object> fieldValueMap = XContentMapValues.nodeMapValue(fieldValue, "Field [" + fullPath() + "]");
-        return XContentMapValues.extractValue(TEXT_FIELD, fieldValueMap);
-    }
-
     @Override
     protected void doValidate(MappingLookup mappers) {
         int parentPathIndex = fullPath().lastIndexOf(leafName());
@@ -429,6 +481,7 @@ public static class SemanticTextFieldType extends SimpleMappedFieldType {
         private final SemanticTextField.ModelSettings modelSettings;
         private final ObjectMapper inferenceField;
         private final IndexVersion indexVersionCreated;
+        private final boolean useLegacyFormat;
 
         public SemanticTextFieldType(
             String name,
@@ -437,6 +490,7 @@ public SemanticTextFieldType(
             SemanticTextField.ModelSettings modelSettings,
             ObjectMapper inferenceField,
             IndexVersion indexVersionCreated,
+            boolean useLegacyFormat,
             Map<String, String> meta
         ) {
             super(name, true, false, false, TextSearchInfo.NONE, meta);
@@ -445,6 +499,11 @@ public SemanticTextFieldType(
             this.modelSettings = modelSettings;
             this.inferenceField = inferenceField;
             this.indexVersionCreated = indexVersionCreated;
+            this.useLegacyFormat = useLegacyFormat;
+        }
+
+        public boolean useLegacyFormat() {
+            return useLegacyFormat;
         }
 
         @Override
@@ -452,6 +511,11 @@ public String typeName() {
             return CONTENT_TYPE;
         }
 
+        @Override
+        public String familyTypeName() {
+            return TextFieldMapper.CONTENT_TYPE;
+        }
+
         public String getInferenceId() {
             return inferenceId;
         }
@@ -476,6 +540,10 @@ public FieldMapper getEmbeddingsField() {
             return (FieldMapper) getChunksField().getMapper(CHUNKED_EMBEDDINGS_FIELD);
         }
 
+        public FieldMapper getOffsetsField() {
+            return (FieldMapper) getChunksField().getMapper(CHUNKED_OFFSET_FIELD);
+        }
+
         @Override
         public Query termQuery(Object value, SearchExecutionContext context) {
             throw new IllegalArgumentException(CONTENT_TYPE + " fields do not support term query");
@@ -498,8 +566,30 @@ public Query existsQuery(SearchExecutionContext context) {
 
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
-            // Redirect the fetcher to load the original values of the field
-            return SourceValueFetcher.toString(getOriginalTextFieldName(name()), context, format);
+            if (useLegacyFormat) {
+                // Redirect the fetcher to load the original values of the field
+                return SourceValueFetcher.toString(getOriginalTextFieldName(name()), context, format);
+            }
+            return SourceValueFetcher.toString(name(), context, null);
+        }
+
+        ValueFetcher valueFetcherWithInferenceResults(Function<Query, BitSetProducer> bitSetCache, IndexSearcher searcher) {
+            var embeddingsField = getEmbeddingsField();
+            if (embeddingsField == null) {
+                return ValueFetcher.EMPTY;
+            }
+            try {
+                var embeddingsLoader = embeddingsField.syntheticFieldLoader();
+                var bitSetFilter = bitSetCache.apply(getChunksField().parentTypeFilter());
+                var childWeight = searcher.createWeight(
+                    getChunksField().nestedTypeFilter(),
+                    org.apache.lucene.search.ScoreMode.COMPLETE_NO_SCORES,
+                    1
+                );
+                return new SemanticTextFieldValueFetcher(bitSetFilter, childWeight, embeddingsLoader);
+            } catch (IOException exc) {
+                throw new UncheckedIOException(exc);
+            }
         }
 
         @Override
@@ -622,158 +712,174 @@ private String generateInvalidQueryInferenceResultsMessage(StringBuilder baseMes
 
         @Override
         public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) {
-            SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name().concat(".text")));
+            String name = useLegacyFormat ? name().concat(".text") : name();
+            SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name));
             return new BlockSourceReader.BytesRefsBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll());
         }
-    }
 
-    /**
-     * <p>
-     * Insert or replace the path's value in the map with the provided new value. The map will be modified in-place.
-     * If the complete path does not exist in the map, it will be added to the deepest (sub-)map possible.
-     * </p>
-     * <p>
-     * For example, given the map:
-     * </p>
-     * <pre>
-     * {
-     *   "path1": {
-     *     "path2": {
-     *       "key1": "value1"
-     *     }
-     *   }
-     * }
-     * </pre>
-     * <p>
-     * And the caller wanted to insert {@code "path1.path2.path3.key2": "value2"}, the method would emit the modified map:
-     * </p>
-     * <pre>
-     * {
-     *   "path1": {
-     *     "path2": {
-     *       "key1": "value1",
-     *       "path3.key2": "value2"
-     *     }
-     *   }
-     * }
-     * </pre>
-     *
-     * @param path the value's path in the map.
-     * @param map the map to search and modify in-place.
-     * @param newValue the new value to assign to the path.
-     *
-     * @throws IllegalArgumentException If either the path cannot be fully traversed or there is ambiguity about where to insert the new
-     *                                  value.
-     */
-    public static void insertValue(String path, Map<?, ?> map, Object newValue) {
-        String[] pathElements = path.split("\\.");
-        if (pathElements.length == 0) {
-            return;
-        }
+        private class SemanticTextFieldValueFetcher implements ValueFetcher {
+            private final BitSetProducer parentBitSetProducer;
+            private final Weight childWeight;
+            private final SourceLoader.SyntheticFieldLoader fieldLoader;
 
-        List<SuffixMap> suffixMaps = extractSuffixMaps(pathElements, 0, map);
-        if (suffixMaps.isEmpty()) {
-            // This should never happen. Throw in case it does for some reason.
-            throw new IllegalStateException("extractSuffixMaps returned an empty suffix map list");
-        } else if (suffixMaps.size() == 1) {
-            SuffixMap suffixMap = suffixMaps.getFirst();
-            suffixMap.map().put(suffixMap.suffix(), newValue);
-        } else {
-            throw new IllegalArgumentException(
-                "Path [" + path + "] could be inserted in " + suffixMaps.size() + " distinct ways, it is ambiguous which one to use"
-            );
-        }
-    }
+            private BitSet bitSet;
+            private Scorer childScorer;
+            private SourceLoader.SyntheticFieldLoader.DocValuesLoader dvLoader;
+            private OffsetSourceField.OffsetSourceLoader offsetsLoader;
 
-    private record SuffixMap(String suffix, Map<String, Object> map) {}
-
-    private static List<SuffixMap> extractSuffixMaps(String[] pathElements, int index, Object currentValue) {
-        if (currentValue instanceof List<?> valueList) {
-            List<SuffixMap> suffixMaps = new ArrayList<>(valueList.size());
-            for (Object o : valueList) {
-                suffixMaps.addAll(extractSuffixMaps(pathElements, index, o));
+            private SemanticTextFieldValueFetcher(
+                BitSetProducer bitSetProducer,
+                Weight childWeight,
+                SourceLoader.SyntheticFieldLoader fieldLoader
+            ) {
+                this.parentBitSetProducer = bitSetProducer;
+                this.childWeight = childWeight;
+                this.fieldLoader = fieldLoader;
             }
 
-            return suffixMaps;
-        } else if (currentValue instanceof Map<?, ?>) {
-            @SuppressWarnings("unchecked")
-            Map<String, Object> map = (Map<String, Object>) currentValue;
-            List<SuffixMap> suffixMaps = new ArrayList<>(map.size());
-
-            String key = pathElements[index];
-            while (index < pathElements.length) {
-                if (map.containsKey(key)) {
-                    if (index + 1 == pathElements.length) {
-                        // We found the complete path
-                        suffixMaps.add(new SuffixMap(key, map));
-                    } else {
-                        // We've matched that path partially, keep traversing to try to match it fully
-                        suffixMaps.addAll(extractSuffixMaps(pathElements, index + 1, map.get(key)));
+            @Override
+            public void setNextReader(LeafReaderContext context) {
+                try {
+                    bitSet = parentBitSetProducer.getBitSet(context);
+                    childScorer = childWeight.scorer(context);
+                    if (childScorer != null) {
+                        childScorer.iterator().nextDoc();
                     }
+                    dvLoader = fieldLoader.docValuesLoader(context.reader(), null);
+                    var terms = context.reader().terms(getOffsetsFieldName(name()));
+                    offsetsLoader = terms != null ? OffsetSourceField.loader(terms) : null;
+                } catch (IOException exc) {
+                    throw new UncheckedIOException(exc);
                 }
+            }
 
-                if (++index < pathElements.length) {
-                    key += "." + pathElements[index];
+            @Override
+            public List<Object> fetchValues(Source source, int doc, List<Object> ignoredValues) throws IOException {
+                if (childScorer == null || offsetsLoader == null || doc == 0) {
+                    return List.of();
+                }
+                int previousParent = bitSet.prevSetBit(doc - 1);
+                var it = childScorer.iterator();
+                if (it.docID() < previousParent) {
+                    it.advance(previousParent);
+                }
+                Map<String, List<SemanticTextField.Chunk>> chunkMap = new LinkedHashMap<>();
+                while (it.docID() < doc) {
+                    if (dvLoader == null || dvLoader.advanceToDoc(it.docID()) == false) {
+                        throw new IllegalStateException(
+                            "Cannot fetch values for field [" + name() + "], missing embeddings for doc [" + doc + "]"
+                        );
+                    }
+                    var offset = offsetsLoader.advanceTo(it.docID());
+                    if (offset == null) {
+                        throw new IllegalStateException(
+                            "Cannot fetch values for field [" + name() + "], missing offsets for doc [" + doc + "]"
+                        );
+                    }
+                    var chunks = chunkMap.computeIfAbsent(offset.field(), k -> new ArrayList<>());
+                    chunks.add(
+                        new SemanticTextField.Chunk(
+                            null,
+                            offset.start(),
+                            offset.end(),
+                            rawEmbeddings(fieldLoader::write, source.sourceContentType())
+                        )
+                    );
+                    if (it.nextDoc() == DocIdSetIterator.NO_MORE_DOCS) {
+                        break;
+                    }
+                }
+                if (chunkMap.isEmpty()) {
+                    return List.of();
                 }
+                return List.of(
+                    new SemanticTextField(
+                        useLegacyFormat,
+                        name(),
+                        null,
+                        new SemanticTextField.InferenceResult(inferenceId, modelSettings, chunkMap),
+                        source.sourceContentType()
+                    )
+                );
             }
 
-            if (suffixMaps.isEmpty()) {
-                // We checked for all remaining elements in the path, and they do not exist. This means we found a leaf map that we should
-                // add the value to.
-                suffixMaps.add(new SuffixMap(key, map));
+            private BytesReference rawEmbeddings(CheckedConsumer<XContentBuilder, IOException> writer, XContentType xContentType)
+                throws IOException {
+                try (var result = XContentFactory.contentBuilder(xContentType)) {
+                    try (var builder = XContentFactory.contentBuilder(xContentType)) {
+                        builder.startObject();
+                        writer.accept(builder);
+                        builder.endObject();
+                        try (
+                            XContentParser parser = XContentHelper.createParserNotCompressed(
+                                XContentParserConfiguration.EMPTY,
+                                BytesReference.bytes(builder),
+                                xContentType
+                            )
+                        ) {
+                            XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
+                            XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser);
+                            parser.nextToken();
+                            result.copyCurrentStructure(parser);
+                        }
+                        return BytesReference.bytes(result);
+                    }
+                }
             }
 
-            return suffixMaps;
-        } else {
-            throw new IllegalArgumentException(
-                "Path ["
-                    + String.join(".", Arrays.copyOfRange(pathElements, 0, index))
-                    + "] has value ["
-                    + currentValue
-                    + "] of type ["
-                    + currentValue.getClass().getSimpleName()
-                    + "], which cannot be traversed into further"
-            );
+            @Override
+            public StoredFieldsSpec storedFieldsSpec() {
+                return StoredFieldsSpec.NO_REQUIREMENTS;
+            }
         }
     }
 
     private static ObjectMapper createInferenceField(
         MapperBuilderContext context,
         IndexVersion indexVersionCreated,
+        boolean useLegacyFormat,
         @Nullable SemanticTextField.ModelSettings modelSettings,
         Function<Query, BitSetProducer> bitSetProducer,
         IndexSettings indexSettings
     ) {
         return new ObjectMapper.Builder(INFERENCE_FIELD, Optional.of(ObjectMapper.Subobjects.ENABLED)).dynamic(ObjectMapper.Dynamic.FALSE)
-            .add(createChunksField(indexVersionCreated, modelSettings, bitSetProducer, indexSettings))
+            .add(createChunksField(indexVersionCreated, useLegacyFormat, modelSettings, bitSetProducer, indexSettings))
             .build(context);
     }
 
     private static NestedObjectMapper.Builder createChunksField(
         IndexVersion indexVersionCreated,
+        boolean useLegacyFormat,
         @Nullable SemanticTextField.ModelSettings modelSettings,
         Function<Query, BitSetProducer> bitSetProducer,
         IndexSettings indexSettings
     ) {
         NestedObjectMapper.Builder chunksField = new NestedObjectMapper.Builder(
-            CHUNKS_FIELD,
-            indexVersionCreated,
+            SemanticTextField.CHUNKS_FIELD,
+            indexSettings.getIndexVersionCreated(),
             bitSetProducer,
             indexSettings
         );
         chunksField.dynamic(ObjectMapper.Dynamic.FALSE);
-        KeywordFieldMapper.Builder chunkTextField = new KeywordFieldMapper.Builder(CHUNKED_TEXT_FIELD, indexVersionCreated).indexed(false)
-            .docValues(false);
         if (modelSettings != null) {
-            chunksField.add(createEmbeddingsField(indexVersionCreated, modelSettings));
+            chunksField.add(createEmbeddingsField(indexSettings.getIndexVersionCreated(), modelSettings, useLegacyFormat));
+        }
+        if (useLegacyFormat) {
+            var chunkTextField = new KeywordFieldMapper.Builder(TEXT_FIELD, indexVersionCreated).indexed(false).docValues(false);
+            chunksField.add(chunkTextField);
+        } else {
+            chunksField.add(new OffsetSourceFieldMapper.Builder(CHUNKED_OFFSET_FIELD));
         }
-        chunksField.add(chunkTextField);
         return chunksField;
     }
 
-    private static Mapper.Builder createEmbeddingsField(IndexVersion indexVersionCreated, SemanticTextField.ModelSettings modelSettings) {
+    private static Mapper.Builder createEmbeddingsField(
+        IndexVersion indexVersionCreated,
+        SemanticTextField.ModelSettings modelSettings,
+        boolean useLegacyFormat
+    ) {
         return switch (modelSettings.taskType()) {
-            case SPARSE_EMBEDDING -> new SparseVectorFieldMapper.Builder(CHUNKED_EMBEDDINGS_FIELD);
+            case SPARSE_EMBEDDING -> new SparseVectorFieldMapper.Builder(CHUNKED_EMBEDDINGS_FIELD).setStored(useLegacyFormat == false);
             case TEXT_EMBEDDING -> {
                 DenseVectorFieldMapper.Builder denseVectorMapperBuilder = new DenseVectorFieldMapper.Builder(
                     CHUNKED_EMBEDDINGS_FIELD,
@@ -808,7 +914,7 @@ private static boolean canMergeModelSettings(
         if (Objects.equals(previous, current)) {
             return true;
         }
-        if (previous == null) {
+        if (previous == null || current == null) {
             return true;
         }
         conflicts.addConflict("model_settings", "");
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextUtils.java
new file mode 100644
index 0000000000000..4d3c2e8752367
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextUtils.java
@@ -0,0 +1,163 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.mapper;
+
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.rest.RestStatus;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+public interface SemanticTextUtils {
+    /**
+     * This method converts the given {@code valueObj} into a list of strings.
+     */
+    static List<String> nodeStringValues(String field, Object valueObj) {
+        if (valueObj instanceof Number || valueObj instanceof Boolean) {
+            return List.of(valueObj.toString());
+        } else if (valueObj instanceof String value) {
+            return List.of(value);
+        } else if (valueObj instanceof Collection<?> values) {
+            List<String> valuesString = new ArrayList<>();
+            for (var v : values) {
+                if (v instanceof Number || v instanceof Boolean) {
+                    valuesString.add(v.toString());
+                } else if (v instanceof String value) {
+                    valuesString.add(value);
+                } else {
+                    throw new ElasticsearchStatusException(
+                        "Invalid format for field [{}], expected [String|Number|Boolean] got [{}]",
+                        RestStatus.BAD_REQUEST,
+                        field,
+                        valueObj.getClass().getSimpleName()
+                    );
+                }
+            }
+            return valuesString;
+        }
+        throw new ElasticsearchStatusException(
+            "Invalid format for field [{}], expected [String|Number|Boolean] got [{}]",
+            RestStatus.BAD_REQUEST,
+            field,
+            valueObj.getClass().getSimpleName()
+        );
+    }
+
+    /**
+     * <p>
+     * Insert or replace the path's value in the map with the provided new value. The map will be modified in-place.
+     * If the complete path does not exist in the map, it will be added to the deepest (sub-)map possible.
+     * </p>
+     * <p>
+     * For example, given the map:
+     * </p>
+     * <pre>
+     * {
+     *   "path1": {
+     *     "path2": {
+     *       "key1": "value1"
+     *     }
+     *   }
+     * }
+     * </pre>
+     * <p>
+     * And the caller wanted to insert {@code "path1.path2.path3.key2": "value2"}, the method would emit the modified map:
+     * </p>
+     * <pre>
+     * {
+     *   "path1": {
+     *     "path2": {
+     *       "key1": "value1",
+     *       "path3.key2": "value2"
+     *     }
+     *   }
+     * }
+     * </pre>
+     *
+     * @param path the value's path in the map.
+     * @param map the map to search and modify in-place.
+     * @param newValue the new value to assign to the path.
+     *
+     * @throws IllegalArgumentException If either the path cannot be fully traversed or there is ambiguity about where to insert the new
+     *                                  value.
+     */
+    static void insertValue(String path, Map<?, ?> map, Object newValue) {
+        String[] pathElements = path.split("\\.");
+        if (pathElements.length == 0) {
+            return;
+        }
+
+        List<SuffixMap> suffixMaps = extractSuffixMaps(pathElements, 0, map);
+        if (suffixMaps.isEmpty()) {
+            // This should never happen. Throw in case it does for some reason.
+            throw new IllegalStateException("extractSuffixMaps returned an empty suffix map list");
+        } else if (suffixMaps.size() == 1) {
+            SuffixMap suffixMap = suffixMaps.getFirst();
+            suffixMap.map().put(suffixMap.suffix(), newValue);
+        } else {
+            throw new IllegalArgumentException(
+                "Path [" + path + "] could be inserted in " + suffixMaps.size() + " distinct ways, it is ambiguous which one to use"
+            );
+        }
+    }
+
+    record SuffixMap(String suffix, Map<String, Object> map) {}
+
+    private static List<SuffixMap> extractSuffixMaps(String[] pathElements, int index, Object currentValue) {
+        if (currentValue instanceof List<?> valueList) {
+            List<SuffixMap> suffixMaps = new ArrayList<>(valueList.size());
+            for (Object o : valueList) {
+                suffixMaps.addAll(extractSuffixMaps(pathElements, index, o));
+            }
+
+            return suffixMaps;
+        } else if (currentValue instanceof Map<?, ?>) {
+            @SuppressWarnings("unchecked")
+            Map<String, Object> map = (Map<String, Object>) currentValue;
+            List<SuffixMap> suffixMaps = new ArrayList<>(map.size());
+
+            String key = pathElements[index];
+            while (index < pathElements.length) {
+                if (map.containsKey(key)) {
+                    if (index + 1 == pathElements.length) {
+                        // We found the complete path
+                        suffixMaps.add(new SuffixMap(key, map));
+                    } else {
+                        // We've matched that path partially, keep traversing to try to match it fully
+                        suffixMaps.addAll(extractSuffixMaps(pathElements, index + 1, map.get(key)));
+                    }
+                }
+
+                if (++index < pathElements.length) {
+                    key += "." + pathElements[index];
+                }
+            }
+
+            if (suffixMaps.isEmpty()) {
+                // We checked for all remaining elements in the path, and they do not exist. This means we found a leaf map that we should
+                // add the value to.
+                suffixMaps.add(new SuffixMap(key, map));
+            }
+
+            return suffixMaps;
+        } else {
+            throw new IllegalArgumentException(
+                "Path ["
+                    + String.join(".", Arrays.copyOfRange(pathElements, 0, index))
+                    + "] has value ["
+                    + currentValue
+                    + "] of type ["
+                    + currentValue.getClass().getSimpleName()
+                    + "], which cannot be traversed into further"
+            );
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticKnnVectorQueryRewriteInterceptor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticKnnVectorQueryRewriteInterceptor.java
new file mode 100644
index 0000000000000..57805d5277ffc
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticKnnVectorQueryRewriteInterceptor.java
@@ -0,0 +1,192 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.queries;
+
+import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.index.mapper.IndexFieldMapper;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.TermsQueryBuilder;
+import org.elasticsearch.search.vectors.KnnVectorQueryBuilder;
+import org.elasticsearch.search.vectors.QueryVectorBuilder;
+import org.elasticsearch.xpack.core.ml.vectors.TextEmbeddingQueryVectorBuilder;
+import org.elasticsearch.xpack.inference.mapper.SemanticTextField;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+public class SemanticKnnVectorQueryRewriteInterceptor extends SemanticQueryRewriteInterceptor {
+
+    public static final NodeFeature SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED = new NodeFeature(
+        "search.semantic_knn_vector_query_rewrite_interception_supported"
+    );
+
+    public SemanticKnnVectorQueryRewriteInterceptor() {}
+
+    @Override
+    protected String getFieldName(QueryBuilder queryBuilder) {
+        assert (queryBuilder instanceof KnnVectorQueryBuilder);
+        KnnVectorQueryBuilder knnVectorQueryBuilder = (KnnVectorQueryBuilder) queryBuilder;
+        return knnVectorQueryBuilder.getFieldName();
+    }
+
+    @Override
+    protected String getQuery(QueryBuilder queryBuilder) {
+        assert (queryBuilder instanceof KnnVectorQueryBuilder);
+        KnnVectorQueryBuilder knnVectorQueryBuilder = (KnnVectorQueryBuilder) queryBuilder;
+        TextEmbeddingQueryVectorBuilder queryVectorBuilder = getTextEmbeddingQueryBuilderFromQuery(knnVectorQueryBuilder);
+        return queryVectorBuilder != null ? queryVectorBuilder.getModelText() : null;
+    }
+
+    @Override
+    protected QueryBuilder buildInferenceQuery(QueryBuilder queryBuilder, InferenceIndexInformationForField indexInformation) {
+        assert (queryBuilder instanceof KnnVectorQueryBuilder);
+        KnnVectorQueryBuilder knnVectorQueryBuilder = (KnnVectorQueryBuilder) queryBuilder;
+        Map<String, List<String>> inferenceIdsIndices = indexInformation.getInferenceIdsIndices();
+        if (inferenceIdsIndices.size() == 1) {
+            // Simple case, everything uses the same inference ID
+            Map.Entry<String, List<String>> inferenceIdIndex = inferenceIdsIndices.entrySet().iterator().next();
+            String searchInferenceId = inferenceIdIndex.getKey();
+            List<String> indices = inferenceIdIndex.getValue();
+            return buildNestedQueryFromKnnVectorQuery(knnVectorQueryBuilder, indices, searchInferenceId);
+        } else {
+            // Multiple inference IDs, construct a boolean query
+            return buildInferenceQueryWithMultipleInferenceIds(knnVectorQueryBuilder, inferenceIdsIndices);
+        }
+    }
+
+    private QueryBuilder buildInferenceQueryWithMultipleInferenceIds(
+        KnnVectorQueryBuilder queryBuilder,
+        Map<String, List<String>> inferenceIdsIndices
+    ) {
+        BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
+        for (String inferenceId : inferenceIdsIndices.keySet()) {
+            boolQueryBuilder.should(
+                createSubQueryForIndices(
+                    inferenceIdsIndices.get(inferenceId),
+                    buildNestedQueryFromKnnVectorQuery(queryBuilder, inferenceIdsIndices.get(inferenceId), inferenceId)
+                )
+            );
+        }
+        return boolQueryBuilder;
+    }
+
+    @Override
+    protected QueryBuilder buildCombinedInferenceAndNonInferenceQuery(
+        QueryBuilder queryBuilder,
+        InferenceIndexInformationForField indexInformation
+    ) {
+        assert (queryBuilder instanceof KnnVectorQueryBuilder);
+        KnnVectorQueryBuilder knnVectorQueryBuilder = (KnnVectorQueryBuilder) queryBuilder;
+        Map<String, List<String>> inferenceIdsIndices = indexInformation.getInferenceIdsIndices();
+
+        BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
+        boolQueryBuilder.should(addIndexFilterToKnnVectorQuery(indexInformation.nonInferenceIndices(), knnVectorQueryBuilder));
+
+        // We always perform nested subqueries on semantic_text fields, to support knn queries using query vectors.
+        // Both pre and post filtering are required here to ensure we get the results we need without errors based on field types.
+        for (String inferenceId : inferenceIdsIndices.keySet()) {
+            boolQueryBuilder.should(
+                createSubQueryForIndices(
+                    inferenceIdsIndices.get(inferenceId),
+                    buildNestedQueryFromKnnVectorQuery(knnVectorQueryBuilder, inferenceIdsIndices.get(inferenceId), inferenceId)
+                )
+            );
+        }
+        return boolQueryBuilder;
+    }
+
+    private QueryBuilder buildNestedQueryFromKnnVectorQuery(
+        KnnVectorQueryBuilder knnVectorQueryBuilder,
+        List<String> indices,
+        String searchInferenceId
+    ) {
+        KnnVectorQueryBuilder filteredKnnVectorQueryBuilder = addIndexFilterToKnnVectorQuery(indices, knnVectorQueryBuilder);
+        TextEmbeddingQueryVectorBuilder queryVectorBuilder = getTextEmbeddingQueryBuilderFromQuery(filteredKnnVectorQueryBuilder);
+        if (queryVectorBuilder != null && queryVectorBuilder.getModelId() == null && searchInferenceId != null) {
+            // If the model ID was not specified, we infer the inference ID associated with the semantic_text field.
+            queryVectorBuilder = new TextEmbeddingQueryVectorBuilder(searchInferenceId, queryVectorBuilder.getModelText());
+        }
+        return QueryBuilders.nestedQuery(
+            SemanticTextField.getChunksFieldName(filteredKnnVectorQueryBuilder.getFieldName()),
+            buildNewKnnVectorQuery(
+                SemanticTextField.getEmbeddingsFieldName(filteredKnnVectorQueryBuilder.getFieldName()),
+                filteredKnnVectorQueryBuilder,
+                queryVectorBuilder
+            ),
+            ScoreMode.Max
+        );
+    }
+
+    private KnnVectorQueryBuilder addIndexFilterToKnnVectorQuery(Collection<String> indices, KnnVectorQueryBuilder original) {
+        KnnVectorQueryBuilder copy;
+        if (original.queryVectorBuilder() != null) {
+            copy = new KnnVectorQueryBuilder(
+                original.getFieldName(),
+                original.queryVectorBuilder(),
+                original.k(),
+                original.numCands(),
+                original.getVectorSimilarity()
+            );
+        } else {
+            copy = new KnnVectorQueryBuilder(
+                original.getFieldName(),
+                original.queryVector(),
+                original.k(),
+                original.numCands(),
+                original.rescoreVectorBuilder(),
+                original.getVectorSimilarity()
+            );
+        }
+
+        copy.addFilterQuery(new TermsQueryBuilder(IndexFieldMapper.NAME, indices));
+        return copy;
+    }
+
+    private TextEmbeddingQueryVectorBuilder getTextEmbeddingQueryBuilderFromQuery(KnnVectorQueryBuilder knnVectorQueryBuilder) {
+        QueryVectorBuilder queryVectorBuilder = knnVectorQueryBuilder.queryVectorBuilder();
+        if (queryVectorBuilder == null) {
+            return null;
+        }
+        assert (queryVectorBuilder instanceof TextEmbeddingQueryVectorBuilder);
+        return (TextEmbeddingQueryVectorBuilder) queryVectorBuilder;
+    }
+
+    private KnnVectorQueryBuilder buildNewKnnVectorQuery(
+        String fieldName,
+        KnnVectorQueryBuilder original,
+        QueryVectorBuilder queryVectorBuilder
+    ) {
+        if (original.queryVectorBuilder() != null) {
+            return new KnnVectorQueryBuilder(
+                fieldName,
+                queryVectorBuilder,
+                original.k(),
+                original.numCands(),
+                original.getVectorSimilarity()
+            );
+        } else {
+            return new KnnVectorQueryBuilder(
+                fieldName,
+                original.queryVector(),
+                original.k(),
+                original.numCands(),
+                original.rescoreVectorBuilder(),
+                original.getVectorSimilarity()
+            );
+        }
+    }
+
+    @Override
+    public String getQueryName() {
+        return KnnVectorQueryBuilder.NAME;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticMatchQueryRewriteInterceptor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticMatchQueryRewriteInterceptor.java
index a4a8123935c3e..fd1d65d00faf5 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticMatchQueryRewriteInterceptor.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticMatchQueryRewriteInterceptor.java
@@ -7,24 +7,12 @@
 
 package org.elasticsearch.xpack.inference.queries;
 
-import org.elasticsearch.action.ResolvedIndices;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
 import org.elasticsearch.features.NodeFeature;
-import org.elasticsearch.index.mapper.IndexFieldMapper;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryRewriteContext;
-import org.elasticsearch.index.query.TermQueryBuilder;
-import org.elasticsearch.index.query.TermsQueryBuilder;
-import org.elasticsearch.plugins.internal.rewriter.QueryRewriteInterceptor;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-public class SemanticMatchQueryRewriteInterceptor implements QueryRewriteInterceptor {
+public class SemanticMatchQueryRewriteInterceptor extends SemanticQueryRewriteInterceptor {
 
     public static final NodeFeature SEMANTIC_MATCH_QUERY_REWRITE_INTERCEPTION_SUPPORTED = new NodeFeature(
         "search.semantic_match_query_rewrite_interception_supported"
@@ -33,63 +21,45 @@ public class SemanticMatchQueryRewriteInterceptor implements QueryRewriteInterce
     public SemanticMatchQueryRewriteInterceptor() {}
 
     @Override
-    public QueryBuilder interceptAndRewrite(QueryRewriteContext context, QueryBuilder queryBuilder) {
+    protected String getFieldName(QueryBuilder queryBuilder) {
         assert (queryBuilder instanceof MatchQueryBuilder);
         MatchQueryBuilder matchQueryBuilder = (MatchQueryBuilder) queryBuilder;
-        QueryBuilder rewritten = queryBuilder;
-        ResolvedIndices resolvedIndices = context.getResolvedIndices();
-        if (resolvedIndices != null) {
-            Collection<IndexMetadata> indexMetadataCollection = resolvedIndices.getConcreteLocalIndicesMetadata().values();
-            List<String> inferenceIndices = new ArrayList<>();
-            List<String> nonInferenceIndices = new ArrayList<>();
-            for (IndexMetadata indexMetadata : indexMetadataCollection) {
-                String indexName = indexMetadata.getIndex().getName();
-                InferenceFieldMetadata inferenceFieldMetadata = indexMetadata.getInferenceFields().get(matchQueryBuilder.fieldName());
-                if (inferenceFieldMetadata != null) {
-                    inferenceIndices.add(indexName);
-                } else {
-                    nonInferenceIndices.add(indexName);
-                }
-            }
-
-            if (inferenceIndices.isEmpty()) {
-                return rewritten;
-            } else if (nonInferenceIndices.isEmpty() == false) {
-                BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
-                for (String inferenceIndexName : inferenceIndices) {
-                    // Add a separate clause for each semantic query, because they may be using different inference endpoints
-                    // TODO - consolidate this to a single clause once the semantic query supports multiple inference endpoints
-                    boolQueryBuilder.should(
-                        createSemanticSubQuery(inferenceIndexName, matchQueryBuilder.fieldName(), (String) matchQueryBuilder.value())
-                    );
-                }
-                boolQueryBuilder.should(createMatchSubQuery(nonInferenceIndices, matchQueryBuilder));
-                rewritten = boolQueryBuilder;
-            } else {
-                rewritten = new SemanticQueryBuilder(matchQueryBuilder.fieldName(), (String) matchQueryBuilder.value(), false);
-            }
-        }
-
-        return rewritten;
+        return matchQueryBuilder.fieldName();
+    }
 
+    @Override
+    protected String getQuery(QueryBuilder queryBuilder) {
+        assert (queryBuilder instanceof MatchQueryBuilder);
+        MatchQueryBuilder matchQueryBuilder = (MatchQueryBuilder) queryBuilder;
+        return (String) matchQueryBuilder.value();
     }
 
     @Override
-    public String getQueryName() {
-        return MatchQueryBuilder.NAME;
+    protected QueryBuilder buildInferenceQuery(QueryBuilder queryBuilder, InferenceIndexInformationForField indexInformation) {
+        return new SemanticQueryBuilder(indexInformation.fieldName(), getQuery(queryBuilder), false);
     }
 
-    private QueryBuilder createSemanticSubQuery(String indexName, String fieldName, String value) {
+    @Override
+    protected QueryBuilder buildCombinedInferenceAndNonInferenceQuery(
+        QueryBuilder queryBuilder,
+        InferenceIndexInformationForField indexInformation
+    ) {
+        assert (queryBuilder instanceof MatchQueryBuilder);
+        MatchQueryBuilder matchQueryBuilder = (MatchQueryBuilder) queryBuilder;
         BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
-        boolQueryBuilder.must(new SemanticQueryBuilder(fieldName, value, true));
-        boolQueryBuilder.filter(new TermQueryBuilder(IndexFieldMapper.NAME, indexName));
+        boolQueryBuilder.should(
+            createSemanticSubQuery(
+                indexInformation.getInferenceIndices(),
+                matchQueryBuilder.fieldName(),
+                (String) matchQueryBuilder.value()
+            )
+        );
+        boolQueryBuilder.should(createSubQueryForIndices(indexInformation.nonInferenceIndices(), matchQueryBuilder));
         return boolQueryBuilder;
     }
 
-    private QueryBuilder createMatchSubQuery(List<String> indices, MatchQueryBuilder matchQueryBuilder) {
-        BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
-        boolQueryBuilder.must(matchQueryBuilder);
-        boolQueryBuilder.filter(new TermsQueryBuilder(IndexFieldMapper.NAME, indices));
-        return boolQueryBuilder;
+    @Override
+    public String getQueryName() {
+        return MatchQueryBuilder.NAME;
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java
index 2a34651efcd9d..285739fe0936f 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.MatchNoneQueryBuilder;
@@ -51,9 +50,6 @@
 import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
 
 public class SemanticQueryBuilder extends AbstractQueryBuilder<SemanticQueryBuilder> {
-    // **** THE semantic_text.inner_hits CLUSTER FEATURE IS DEFUNCT, NEVER USE IT ****
-    public static final NodeFeature SEMANTIC_TEXT_INNER_HITS = new NodeFeature("semantic_text.inner_hits");
-
     public static final String NAME = "semantic";
 
     private static final ParseField FIELD_FIELD = new ParseField("field");
@@ -148,6 +144,14 @@ public String getWriteableName() {
         return NAME;
     }
 
+    public String getFieldName() {
+        return fieldName;
+    }
+
+    public String getQuery() {
+        return query;
+    }
+
     @Override
     public TransportVersion getMinimalSupportedVersion() {
         return TransportVersions.V_8_15_0;
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryRewriteInterceptor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryRewriteInterceptor.java
new file mode 100644
index 0000000000000..bb76ef0be24e9
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryRewriteInterceptor.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.queries;
+
+import org.elasticsearch.action.ResolvedIndices;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
+import org.elasticsearch.index.mapper.IndexFieldMapper;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.TermsQueryBuilder;
+import org.elasticsearch.plugins.internal.rewriter.QueryRewriteInterceptor;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Intercepts and adapts a query to be rewritten to work seamlessly on a semantic_text field.
+ */
+public abstract class SemanticQueryRewriteInterceptor implements QueryRewriteInterceptor {
+
+    public SemanticQueryRewriteInterceptor() {}
+
+    @Override
+    public QueryBuilder interceptAndRewrite(QueryRewriteContext context, QueryBuilder queryBuilder) {
+        String fieldName = getFieldName(queryBuilder);
+        ResolvedIndices resolvedIndices = context.getResolvedIndices();
+
+        if (resolvedIndices == null) {
+            // No resolved indices, so return the original query.
+            return queryBuilder;
+        }
+
+        InferenceIndexInformationForField indexInformation = resolveIndicesForField(fieldName, resolvedIndices);
+        if (indexInformation.getInferenceIndices().isEmpty()) {
+            // No inference fields were identified, so return the original query.
+            return queryBuilder;
+        } else if (indexInformation.nonInferenceIndices().isEmpty() == false) {
+            // Combined case where the field name requested by this query contains both
+            // semantic_text and non-inference fields, so we have to combine queries per index
+            // containing each field type.
+            return buildCombinedInferenceAndNonInferenceQuery(queryBuilder, indexInformation);
+        } else {
+            // The only fields we've identified are inference fields (e.g. semantic_text),
+            // so rewrite the entire query to work on a semantic_text field.
+            return buildInferenceQuery(queryBuilder, indexInformation);
+        }
+    }
+
+    /**
+     * @param queryBuilder {@link QueryBuilder}
+     * @return The singular field name requested by the provided query builder.
+     */
+    protected abstract String getFieldName(QueryBuilder queryBuilder);
+
+    /**
+     * @param queryBuilder {@link QueryBuilder}
+     * @return The text/query string requested by the provided query builder.
+     */
+    protected abstract String getQuery(QueryBuilder queryBuilder);
+
+    /**
+     * Builds the inference query
+     *
+     * @param queryBuilder {@link QueryBuilder}
+     * @param indexInformation {@link InferenceIndexInformationForField}
+     * @return {@link QueryBuilder}
+     */
+    protected abstract QueryBuilder buildInferenceQuery(QueryBuilder queryBuilder, InferenceIndexInformationForField indexInformation);
+
+    /**
+     * Builds a combined inference and non-inference query,
+     * which separates the different queries into appropriate indices based on field type.
+     * @param queryBuilder {@link QueryBuilder}
+     * @param indexInformation {@link InferenceIndexInformationForField}
+     * @return {@link QueryBuilder}
+     */
+    protected abstract QueryBuilder buildCombinedInferenceAndNonInferenceQuery(
+        QueryBuilder queryBuilder,
+        InferenceIndexInformationForField indexInformation
+    );
+
+    private InferenceIndexInformationForField resolveIndicesForField(String fieldName, ResolvedIndices resolvedIndices) {
+        Collection<IndexMetadata> indexMetadataCollection = resolvedIndices.getConcreteLocalIndicesMetadata().values();
+        Map<String, InferenceFieldMetadata> inferenceIndicesMetadata = new HashMap<>();
+        List<String> nonInferenceIndices = new ArrayList<>();
+        for (IndexMetadata indexMetadata : indexMetadataCollection) {
+            String indexName = indexMetadata.getIndex().getName();
+            InferenceFieldMetadata inferenceFieldMetadata = indexMetadata.getInferenceFields().get(fieldName);
+            if (inferenceFieldMetadata != null) {
+                inferenceIndicesMetadata.put(indexName, inferenceFieldMetadata);
+            } else {
+                nonInferenceIndices.add(indexName);
+            }
+        }
+
+        return new InferenceIndexInformationForField(fieldName, inferenceIndicesMetadata, nonInferenceIndices);
+    }
+
+    protected QueryBuilder createSubQueryForIndices(Collection<String> indices, QueryBuilder queryBuilder) {
+        BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
+        boolQueryBuilder.must(queryBuilder);
+        boolQueryBuilder.filter(new TermsQueryBuilder(IndexFieldMapper.NAME, indices));
+        return boolQueryBuilder;
+    }
+
+    protected QueryBuilder createSemanticSubQuery(Collection<String> indices, String fieldName, String value) {
+        BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
+        boolQueryBuilder.must(new SemanticQueryBuilder(fieldName, value, true));
+        boolQueryBuilder.filter(new TermsQueryBuilder(IndexFieldMapper.NAME, indices));
+        return boolQueryBuilder;
+    }
+
+    /**
+     * Represents the indices and associated inference information for a field.
+     */
+    public record InferenceIndexInformationForField(
+        String fieldName,
+        Map<String, InferenceFieldMetadata> inferenceIndicesMetadata,
+        List<String> nonInferenceIndices
+    ) {
+
+        public Collection<String> getInferenceIndices() {
+            return inferenceIndicesMetadata.keySet();
+        }
+
+        public Map<String, List<String>> getInferenceIdsIndices() {
+            return inferenceIndicesMetadata.entrySet()
+                .stream()
+                .collect(
+                    Collectors.groupingBy(
+                        entry -> entry.getValue().getSearchInferenceId(),
+                        Collectors.mapping(Map.Entry::getKey, Collectors.toList())
+                    )
+                );
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticSparseVectorQueryRewriteInterceptor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticSparseVectorQueryRewriteInterceptor.java
new file mode 100644
index 0000000000000..a35e83450c55a
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticSparseVectorQueryRewriteInterceptor.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.queries;
+
+import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.xpack.core.ml.search.SparseVectorQueryBuilder;
+import org.elasticsearch.xpack.inference.mapper.SemanticTextField;
+
+import java.util.List;
+import java.util.Map;
+
+public class SemanticSparseVectorQueryRewriteInterceptor extends SemanticQueryRewriteInterceptor {
+
+    public static final NodeFeature SEMANTIC_SPARSE_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED = new NodeFeature(
+        "search.semantic_sparse_vector_query_rewrite_interception_supported"
+    );
+
+    public SemanticSparseVectorQueryRewriteInterceptor() {}
+
+    @Override
+    protected String getFieldName(QueryBuilder queryBuilder) {
+        assert (queryBuilder instanceof SparseVectorQueryBuilder);
+        SparseVectorQueryBuilder sparseVectorQueryBuilder = (SparseVectorQueryBuilder) queryBuilder;
+        return sparseVectorQueryBuilder.getFieldName();
+    }
+
+    @Override
+    protected String getQuery(QueryBuilder queryBuilder) {
+        assert (queryBuilder instanceof SparseVectorQueryBuilder);
+        SparseVectorQueryBuilder sparseVectorQueryBuilder = (SparseVectorQueryBuilder) queryBuilder;
+        return sparseVectorQueryBuilder.getQuery();
+    }
+
+    @Override
+    protected QueryBuilder buildInferenceQuery(QueryBuilder queryBuilder, InferenceIndexInformationForField indexInformation) {
+        Map<String, List<String>> inferenceIdsIndices = indexInformation.getInferenceIdsIndices();
+        if (inferenceIdsIndices.size() == 1) {
+            // Simple case, everything uses the same inference ID
+            String searchInferenceId = inferenceIdsIndices.keySet().iterator().next();
+            return buildNestedQueryFromSparseVectorQuery(queryBuilder, searchInferenceId);
+        } else {
+            // Multiple inference IDs, construct a boolean query
+            return buildInferenceQueryWithMultipleInferenceIds(queryBuilder, inferenceIdsIndices);
+        }
+    }
+
+    private QueryBuilder buildInferenceQueryWithMultipleInferenceIds(
+        QueryBuilder queryBuilder,
+        Map<String, List<String>> inferenceIdsIndices
+    ) {
+        BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
+        for (String inferenceId : inferenceIdsIndices.keySet()) {
+            boolQueryBuilder.should(
+                createSubQueryForIndices(
+                    inferenceIdsIndices.get(inferenceId),
+                    buildNestedQueryFromSparseVectorQuery(queryBuilder, inferenceId)
+                )
+            );
+        }
+        return boolQueryBuilder;
+    }
+
+    @Override
+    protected QueryBuilder buildCombinedInferenceAndNonInferenceQuery(
+        QueryBuilder queryBuilder,
+        InferenceIndexInformationForField indexInformation
+    ) {
+        assert (queryBuilder instanceof SparseVectorQueryBuilder);
+        SparseVectorQueryBuilder sparseVectorQueryBuilder = (SparseVectorQueryBuilder) queryBuilder;
+        Map<String, List<String>> inferenceIdsIndices = indexInformation.getInferenceIdsIndices();
+
+        BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
+        boolQueryBuilder.should(
+            createSubQueryForIndices(
+                indexInformation.nonInferenceIndices(),
+                createSubQueryForIndices(indexInformation.nonInferenceIndices(), sparseVectorQueryBuilder)
+            )
+        );
+        // We always perform nested subqueries on semantic_text fields, to support
+        // sparse_vector queries using query vectors.
+        for (String inferenceId : inferenceIdsIndices.keySet()) {
+            boolQueryBuilder.should(
+                createSubQueryForIndices(
+                    inferenceIdsIndices.get(inferenceId),
+                    buildNestedQueryFromSparseVectorQuery(sparseVectorQueryBuilder, inferenceId)
+                )
+            );
+        }
+        return boolQueryBuilder;
+    }
+
+    private QueryBuilder buildNestedQueryFromSparseVectorQuery(QueryBuilder queryBuilder, String searchInferenceId) {
+        assert (queryBuilder instanceof SparseVectorQueryBuilder);
+        SparseVectorQueryBuilder sparseVectorQueryBuilder = (SparseVectorQueryBuilder) queryBuilder;
+        return QueryBuilders.nestedQuery(
+            SemanticTextField.getChunksFieldName(sparseVectorQueryBuilder.getFieldName()),
+            new SparseVectorQueryBuilder(
+                SemanticTextField.getEmbeddingsFieldName(sparseVectorQueryBuilder.getFieldName()),
+                sparseVectorQueryBuilder.getQueryVectors(),
+                (sparseVectorQueryBuilder.getInferenceId() == null && sparseVectorQueryBuilder.getQuery() != null)
+                    ? searchInferenceId
+                    : sparseVectorQueryBuilder.getInferenceId(),
+                sparseVectorQueryBuilder.getQuery(),
+                sparseVectorQueryBuilder.shouldPruneTokens(),
+                sparseVectorQueryBuilder.getTokenPruningConfig()
+            ),
+            ScoreMode.Max
+        );
+    }
+
+    @Override
+    public String getQueryName() {
+        return SparseVectorQueryBuilder.NAME;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java
index 134f8af0e083d..bdc833f0c3f40 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java
@@ -8,12 +8,12 @@
 package org.elasticsearch.xpack.inference.rank.random;
 
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.retriever.RetrieverBuilder;
 import org.elasticsearch.search.retriever.RetrieverParserContext;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
@@ -30,8 +30,6 @@
  */
 public class RandomRankRetrieverBuilder extends RetrieverBuilder {
 
-    public static final NodeFeature RANDOM_RERANKER_RETRIEVER_SUPPORTED = new NodeFeature("random_reranker_retriever_supported");
-
     public static final ParseField RETRIEVER_FIELD = new ParseField("retriever");
     public static final ParseField FIELD_FIELD = new ParseField("field");
     public static final ParseField RANK_WINDOW_SIZE_FIELD = new ParseField("rank_window_size");
@@ -48,7 +46,12 @@ public class RandomRankRetrieverBuilder extends RetrieverBuilder {
         });
 
     static {
-        PARSER.declareNamedObject(constructorArg(), (p, c, n) -> p.namedObject(RetrieverBuilder.class, n, c), RETRIEVER_FIELD);
+        PARSER.declareField(
+            constructorArg(),
+            RandomRankRetrieverBuilder::parseRetrieverBuilder,
+            RETRIEVER_FIELD,
+            ObjectParser.ValueType.OBJECT
+        );
         PARSER.declareString(optionalConstructorArg(), FIELD_FIELD);
         PARSER.declareInt(optionalConstructorArg(), RANK_WINDOW_SIZE_FIELD);
         PARSER.declareInt(optionalConstructorArg(), SEED_FIELD);
@@ -57,12 +60,25 @@ public class RandomRankRetrieverBuilder extends RetrieverBuilder {
     }
 
     public static RandomRankRetrieverBuilder fromXContent(XContentParser parser, RetrieverParserContext context) throws IOException {
-        if (context.clusterSupportsFeature(RANDOM_RERANKER_RETRIEVER_SUPPORTED) == false) {
-            throw new ParsingException(parser.getTokenLocation(), "unknown retriever [" + RandomRankBuilder.NAME + "]");
-        }
         return PARSER.apply(parser, context);
     }
 
+    private static RetrieverBuilder parseRetrieverBuilder(XContentParser parser, RetrieverParserContext context) throws IOException {
+        assert parser.currentToken() == XContentParser.Token.START_OBJECT;
+        parser.nextToken();
+        if (parser.currentToken() == XContentParser.Token.END_OBJECT) {
+            throw new ParsingException(parser.getTokenLocation(), "empty [" + RETRIEVER_FIELD + "] object");
+        }
+        assert parser.currentToken() == XContentParser.Token.FIELD_NAME;
+        final RetrieverBuilder builder = parser.namedObject(RetrieverBuilder.class, parser.currentName(), context);
+        parser.nextToken();
+        if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
+            throw new ParsingException(parser.getTokenLocation(), "unexpected field [" + parser.currentName() + "]");
+        }
+        assert parser.currentToken() == XContentParser.Token.END_OBJECT;
+        return builder;
+    }
+
     private final RetrieverBuilder retrieverBuilder;
     private final String field;
     private final int rankWindowSize;
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java
index f54696895a818..42248d246d3da 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.inference.rank.textsimilarity;
 
 import org.apache.lucene.search.ScoreDoc;
-import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.license.LicenseUtils;
@@ -36,18 +35,14 @@
  */
 public class TextSimilarityRankRetrieverBuilder extends CompoundRetrieverBuilder<TextSimilarityRankRetrieverBuilder> {
 
-    public static final NodeFeature TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED = new NodeFeature(
-        "text_similarity_reranker_retriever_supported"
-    );
-    public static final NodeFeature TEXT_SIMILARITY_RERANKER_COMPOSITION_SUPPORTED = new NodeFeature(
-        "text_similarity_reranker_retriever_composition_supported"
+    public static final NodeFeature TEXT_SIMILARITY_RERANKER_ALIAS_HANDLING_FIX = new NodeFeature(
+        "text_similarity_reranker_alias_handling_fix"
     );
 
     public static final ParseField RETRIEVER_FIELD = new ParseField("retriever");
     public static final ParseField INFERENCE_ID_FIELD = new ParseField("inference_id");
     public static final ParseField INFERENCE_TEXT_FIELD = new ParseField("inference_text");
     public static final ParseField FIELD_FIELD = new ParseField("field");
-    public static final ParseField RANK_WINDOW_SIZE_FIELD = new ParseField("rank_window_size");
 
     public static final ConstructingObjectParser<TextSimilarityRankRetrieverBuilder, RetrieverParserContext> PARSER =
         new ConstructingObjectParser<>(TextSimilarityRankBuilder.NAME, args -> {
@@ -78,14 +73,6 @@ public static TextSimilarityRankRetrieverBuilder fromXContent(
         RetrieverParserContext context,
         XPackLicenseState licenceState
     ) throws IOException {
-        if (context.clusterSupportsFeature(TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED) == false) {
-            throw new ParsingException(parser.getTokenLocation(), "unknown retriever [" + TextSimilarityRankBuilder.NAME + "]");
-        }
-        if (context.clusterSupportsFeature(TEXT_SIMILARITY_RERANKER_COMPOSITION_SUPPORTED) == false) {
-            throw new IllegalArgumentException(
-                "[text_similarity_reranker] retriever composition feature is not supported by all nodes in the cluster"
-            );
-        }
         if (TextSimilarityRankBuilder.TEXT_SIMILARITY_RERANKER_FEATURE.check(licenceState) == false) {
             throw LicenseUtils.newComplianceException(TextSimilarityRankBuilder.NAME);
         }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestStreamInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestStreamInferenceAction.java
index 875c288da52bd..881af435b29b6 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestStreamInferenceAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestStreamInferenceAction.java
@@ -7,13 +7,16 @@
 
 package org.elasticsearch.xpack.inference.rest;
 
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 
 import java.util.List;
+import java.util.Objects;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.xpack.inference.rest.Paths.STREAM_INFERENCE_ID_PATH;
@@ -21,6 +24,13 @@
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestStreamInferenceAction extends BaseInferenceAction {
+    private final SetOnce<ThreadPool> threadPool;
+
+    public RestStreamInferenceAction(SetOnce<ThreadPool> threadPool) {
+        super();
+        this.threadPool = Objects.requireNonNull(threadPool);
+    }
+
     @Override
     public String getName() {
         return "stream_inference_action";
@@ -38,6 +48,6 @@ protected InferenceAction.Request prepareInferenceRequest(InferenceAction.Reques
 
     @Override
     protected ActionListener<InferenceAction.Response> listener(RestChannel channel) {
-        return new ServerSentEventsRestActionListener(channel);
+        return new ServerSentEventsRestActionListener(channel, threadPool);
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceAction.java
index 5c71b560a6b9d..51f1bc48c8306 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceAction.java
@@ -7,15 +7,18 @@
 
 package org.elasticsearch.xpack.inference.rest;
 
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Objects;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.xpack.inference.rest.Paths.UNIFIED_INFERENCE_ID_PATH;
@@ -23,6 +26,13 @@
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestUnifiedCompletionInferenceAction extends BaseRestHandler {
+    private final SetOnce<ThreadPool> threadPool;
+
+    public RestUnifiedCompletionInferenceAction(SetOnce<ThreadPool> threadPool) {
+        super();
+        this.threadPool = Objects.requireNonNull(threadPool);
+    }
+
     @Override
     public String getName() {
         return "unified_inference_action";
@@ -44,6 +54,10 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
             request = UnifiedCompletionAction.Request.parseRequest(params.inferenceEntityId(), params.taskType(), inferTimeout, parser);
         }
 
-        return channel -> client.execute(UnifiedCompletionAction.INSTANCE, request, new ServerSentEventsRestActionListener(channel));
+        return channel -> client.execute(
+            UnifiedCompletionAction.INSTANCE,
+            request,
+            new ServerSentEventsRestActionListener(channel, threadPool)
+        );
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java
index 3177474ea8ca6..62cbcf902a9e0 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java
@@ -10,9 +10,11 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ContextPreservingActionListener;
 import org.elasticsearch.common.bytes.ReleasableBytesReference;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.BytesStream;
@@ -29,6 +31,7 @@
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.TaskCancelledException;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
@@ -38,11 +41,13 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Objects;
 import java.util.concurrent.Flow;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.elasticsearch.ElasticsearchException.REST_EXCEPTION_SKIP_CAUSE;
 import static org.elasticsearch.ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE;
+import static org.elasticsearch.rest.RestController.ERROR_TRACE_DEFAULT;
 
 /**
  * A version of {@link org.elasticsearch.rest.action.RestChunkedToXContentListener} that reads from a {@link Flow.Publisher} and encodes
@@ -54,6 +59,7 @@ public class ServerSentEventsRestActionListener implements ActionListener<Infere
     private final AtomicBoolean isLastPart = new AtomicBoolean(false);
     private final RestChannel channel;
     private final ToXContent.Params params;
+    private final SetOnce<ThreadPool> threadPool;
 
     /**
      * A listener for the first part of the next entry to become available for transmission.
@@ -65,13 +71,14 @@ public class ServerSentEventsRestActionListener implements ActionListener<Infere
      */
     private ActionListener<ChunkedRestResponseBodyPart> nextBodyPartListener;
 
-    public ServerSentEventsRestActionListener(RestChannel channel) {
-        this(channel, channel.request());
+    public ServerSentEventsRestActionListener(RestChannel channel, SetOnce<ThreadPool> threadPool) {
+        this(channel, channel.request(), threadPool);
     }
 
-    public ServerSentEventsRestActionListener(RestChannel channel, ToXContent.Params params) {
+    public ServerSentEventsRestActionListener(RestChannel channel, ToXContent.Params params, SetOnce<ThreadPool> threadPool) {
         this.channel = channel;
         this.params = params;
+        this.threadPool = Objects.requireNonNull(threadPool);
     }
 
     @Override
@@ -98,7 +105,7 @@ protected void ensureOpen() {
     }
 
     private void initializeStream(InferenceAction.Response response) {
-        nextBodyPartListener = ActionListener.wrap(bodyPart -> {
+        ActionListener<ChunkedRestResponseBodyPart> chunkedResponseBodyActionListener = ActionListener.wrap(bodyPart -> {
             // this is the first response, so we need to send the RestResponse to open the stream
             // all subsequent bytes will be delivered through the nextBodyPartListener
             channel.sendResponse(RestResponse.chunked(RestStatus.OK, bodyPart, this::release));
@@ -114,6 +121,12 @@ private void initializeStream(InferenceAction.Response response) {
                 )
             );
         });
+
+        nextBodyPartListener = ContextPreservingActionListener.wrapPreservingContext(
+            chunkedResponseBodyActionListener,
+            threadPool.get().getThreadContext()
+        );
+
         // subscribe will call onSubscribe, which requests the first chunk
         response.publisher().subscribe(subscriber);
     }
@@ -145,7 +158,7 @@ public void onFailure(Exception e) {
     // except we need to emit the error as SSE
     private ChunkedToXContent errorChunk(Throwable t) {
         var status = ExceptionsHelper.status(t);
-        return params -> Iterators.concat(ChunkedToXContentHelper.startObject(), ChunkedToXContentHelper.singleChunk((b, p) -> {
+        return params -> Iterators.concat(ChunkedToXContentHelper.startObject(), ChunkedToXContentHelper.chunk((b, p) -> {
             // Render the exception with a simple message
             if (channel.detailedErrorsEnabled() == false) {
                 String message = "No ElasticsearchException found";
@@ -161,7 +174,7 @@ private ChunkedToXContent errorChunk(Throwable t) {
             }
 
             var errorParams = p;
-            if (errorParams.paramAsBoolean("error_trace", false) && status != RestStatus.UNAUTHORIZED) {
+            if (errorParams.paramAsBoolean("error_trace", ERROR_TRACE_DEFAULT) && status != RestStatus.UNAUTHORIZED) {
                 errorParams = new ToXContent.DelegatingMapParams(
                     Map.of(REST_EXCEPTION_SKIP_STACK_TRACE, "false", REST_EXCEPTION_SKIP_CAUSE, "true"),
                     params
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java
index 42a276c6ee838..24f7fa182b7c2 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
 import org.elasticsearch.inference.ChunkingSettings;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
@@ -25,11 +24,8 @@
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.SimilarityMeasure;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder;
 import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker;
@@ -56,7 +52,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Stream;
 
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg;
@@ -73,6 +68,7 @@
 
 public class AlibabaCloudSearchService extends SenderService {
     public static final String NAME = AlibabaCloudSearchUtils.SERVICE_NAME;
+    private static final String SERVICE_NAME = "AlibabaCloud AI Search";
 
     private static final EnumSet<TaskType> supportedTaskTypes = EnumSet.of(
         TaskType.TEXT_EMBEDDING,
@@ -383,86 +379,62 @@ public static InferenceServiceConfiguration get() {
 
                 configurationMap.put(
                     SERVICE_ID,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN)
+                    new SettingsConfiguration.Builder().setDescription("The name of the model service to use for the {infer} task.")
                         .setLabel("Project ID")
-                        .setOrder(2)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The name of the model service to use for the {infer} task.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
-                        .setOptions(
-                            Stream.of(
-                                "ops-text-embedding-001",
-                                "ops-text-embedding-zh-001",
-                                "ops-text-embedding-en-001",
-                                "ops-text-embedding-002",
-                                "ops-text-sparse-embedding-001",
-                                "ops-bge-reranker-larger"
-                            ).map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build()).toList()
-                        )
                         .build()
                 );
 
                 configurationMap.put(
                     HOST,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription(
+                        "The name of the host address used for the {infer} task. You can find the host address at "
+                            + "https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[ the API keys section] "
+                            + "of the documentation."
+                    )
                         .setLabel("Host")
-                        .setOrder(3)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip(
-                            "The name of the host address used for the {infer} task. You can find the host address at "
-                                + "https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[ the API keys section] "
-                                + "of the documentation."
-                        )
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.put(
                     HTTP_SCHEMA_NAME,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN)
+                    new SettingsConfiguration.Builder().setDescription("")
                         .setLabel("HTTP Schema")
-                        .setOrder(4)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
-                        .setOptions(
-                            Stream.of("https", "http")
-                                .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build())
-                                .toList()
-                        )
                         .build()
                 );
 
                 configurationMap.put(
                     WORKSPACE_NAME,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("The name of the workspace used for the {infer} task.")
                         .setLabel("Workspace")
-                        .setOrder(5)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The name of the workspace used for the {infer} task.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.putAll(
-                    DefaultSecretSettings.toSettingsConfigurationWithTooltip("A valid API key for the AlibabaCloud AI Search API.")
+                    DefaultSecretSettings.toSettingsConfigurationWithDescription("A valid API key for the AlibabaCloud AI Search API.")
                 );
                 configurationMap.putAll(RateLimitSettings.toSettingsConfiguration());
 
-                return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                    Map<String, SettingsConfiguration> taskSettingsConfig;
-                    switch (t) {
-                        case TEXT_EMBEDDING -> taskSettingsConfig = AlibabaCloudSearchEmbeddingsModel.Configuration.get();
-                        case SPARSE_EMBEDDING -> taskSettingsConfig = AlibabaCloudSearchSparseModel.Configuration.get();
-                        // COMPLETION, RERANK task types have no task settings
-                        default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                    }
-                    return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                }).toList()).setConfiguration(configurationMap).build();
+                return new InferenceServiceConfiguration.Builder().setService(NAME)
+                    .setName(SERVICE_NAME)
+                    .setTaskTypes(supportedTaskTypes)
+                    .setConfigurations(configurationMap)
+                    .build();
             }
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsModel.java
index 1bcc802ab18ea..2654ee4d22ce6 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsModel.java
@@ -7,28 +7,19 @@
 
 package org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings;
 
-import org.elasticsearch.common.util.LazyInitializable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.ChunkingSettings;
 import org.elasticsearch.inference.InputType;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
-import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.alibabacloudsearch.AlibabaCloudSearchActionVisitor;
-import org.elasticsearch.xpack.inference.external.request.alibabacloudsearch.AlibabaCloudSearchEmbeddingsRequestEntity;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchModel;
 import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
-import java.util.stream.Stream;
 
 public class AlibabaCloudSearchEmbeddingsModel extends AlibabaCloudSearchModel {
     public static AlibabaCloudSearchEmbeddingsModel of(
@@ -114,35 +105,4 @@ public DefaultSecretSettings getSecretSettings() {
     public ExecutableAction accept(AlibabaCloudSearchActionVisitor visitor, Map<String, Object> taskSettings, InputType inputType) {
         return visitor.create(this, taskSettings, inputType);
     }
-
-    public static class Configuration {
-        public static Map<String, SettingsConfiguration> get() {
-            return configuration.getOrCompute();
-        }
-
-        private static final LazyInitializable<Map<String, SettingsConfiguration>, RuntimeException> configuration =
-            new LazyInitializable<>(() -> {
-                var configurationMap = new HashMap<String, SettingsConfiguration>();
-
-                configurationMap.put(
-                    AlibabaCloudSearchEmbeddingsRequestEntity.INPUT_TYPE_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN)
-                        .setLabel("Input Type")
-                        .setOrder(1)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Specifies the type of input passed to the model.")
-                        .setType(SettingsConfigurationFieldType.STRING)
-                        .setOptions(
-                            Stream.of("ingest", "search")
-                                .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build())
-                                .toList()
-                        )
-                        .setValue("")
-                        .build()
-                );
-
-                return Collections.unmodifiableMap(configurationMap);
-            });
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseModel.java
index 95bf500434c5a..0155d8fbc1f08 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseModel.java
@@ -7,28 +7,19 @@
 
 package org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse;
 
-import org.elasticsearch.common.util.LazyInitializable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.ChunkingSettings;
 import org.elasticsearch.inference.InputType;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
-import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.alibabacloudsearch.AlibabaCloudSearchActionVisitor;
-import org.elasticsearch.xpack.inference.external.request.alibabacloudsearch.AlibabaCloudSearchSparseRequestEntity;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchModel;
 import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
-import java.util.stream.Stream;
 
 public class AlibabaCloudSearchSparseModel extends AlibabaCloudSearchModel {
     public static AlibabaCloudSearchSparseModel of(
@@ -108,50 +99,4 @@ public DefaultSecretSettings getSecretSettings() {
     public ExecutableAction accept(AlibabaCloudSearchActionVisitor visitor, Map<String, Object> taskSettings, InputType inputType) {
         return visitor.create(this, taskSettings, inputType);
     }
-
-    public static class Configuration {
-        public static Map<String, SettingsConfiguration> get() {
-            return configuration.getOrCompute();
-        }
-
-        private static final LazyInitializable<Map<String, SettingsConfiguration>, RuntimeException> configuration =
-            new LazyInitializable<>(() -> {
-                var configurationMap = new HashMap<String, SettingsConfiguration>();
-
-                configurationMap.put(
-                    AlibabaCloudSearchSparseRequestEntity.INPUT_TYPE_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN)
-                        .setLabel("Input Type")
-                        .setOrder(1)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Specifies the type of input passed to the model.")
-                        .setType(SettingsConfigurationFieldType.STRING)
-                        .setOptions(
-                            Stream.of("ingest", "search")
-                                .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build())
-                                .toList()
-                        )
-                        .setValue("")
-                        .build()
-                );
-                configurationMap.put(
-                    AlibabaCloudSearchSparseRequestEntity.RETURN_TOKEN_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TOGGLE)
-                        .setLabel("Return Token")
-                        .setOrder(2)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip(
-                            "If `true`, the token name will be returned in the response. Defaults to `false` which means only the "
-                                + "token ID will be returned in the response."
-                        )
-                        .setType(SettingsConfigurationFieldType.BOOLEAN)
-                        .setValue(true)
-                        .build()
-                );
-
-                return Collections.unmodifiableMap(configurationMap);
-            });
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockProvider.java
index 340a5a65f0969..817079fd54cf4 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockProvider.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockProvider.java
@@ -17,7 +17,7 @@ public enum AmazonBedrockProvider {
     META,
     MISTRAL;
 
-    public static String NAME = "amazon_bedrock_provider";
+    public static final String NAME = "amazon_bedrock_provider";
 
     public static AmazonBedrockProvider fromString(String name) {
         return valueOf(name.trim().toUpperCase(Locale.ROOT));
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettings.java
index b5818d7e4a287..2105da235babe 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettings.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.SecretSettings;
 import org.elasticsearch.inference.SettingsConfiguration;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -129,23 +128,21 @@ public static Map<String, SettingsConfiguration> get() {
                 var configurationMap = new HashMap<String, SettingsConfiguration>();
                 configurationMap.put(
                     ACCESS_KEY_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("A valid AWS access key that has permissions to use Amazon Bedrock.")
                         .setLabel("Access Key")
-                        .setOrder(1)
                         .setRequired(true)
                         .setSensitive(true)
-                        .setTooltip("A valid AWS access key that has permissions to use Amazon Bedrock.")
+                        .setUpdatable(true)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
                 configurationMap.put(
                     SECRET_KEY_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("A valid AWS secret key that is paired with the access_key.")
                         .setLabel("Secret Key")
-                        .setOrder(2)
                         .setRequired(true)
                         .setSensitive(true)
-                        .setTooltip("A valid AWS secret key that is paired with the access_key.")
+                        .setUpdatable(true)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java
index a88881220f933..07c5e91776192 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
 import org.elasticsearch.inference.ChunkingSettings;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
@@ -26,11 +25,8 @@
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.SettingsConfiguration;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder;
 import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker;
@@ -57,7 +53,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Stream;
 
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg;
@@ -77,6 +72,7 @@
 
 public class AmazonBedrockService extends SenderService {
     public static final String NAME = "amazonbedrock";
+    private static final String SERVICE_NAME = "Amazon Bedrock";
 
     private final Sender amazonBedrockSender;
 
@@ -382,61 +378,51 @@ public static InferenceServiceConfiguration get() {
 
                 configurationMap.put(
                     PROVIDER_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN)
+                    new SettingsConfiguration.Builder().setDescription("The model provider for your deployment.")
                         .setLabel("Provider")
-                        .setOrder(3)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The model provider for your deployment.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
-                        .setOptions(
-                            Stream.of("amazontitan", "anthropic", "ai21labs", "cohere", "meta", "mistral")
-                                .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build())
-                                .toList()
-                        )
                         .build()
                 );
 
                 configurationMap.put(
                     MODEL_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription(
+                        "The base model ID or an ARN to a custom model based on a foundational model."
+                    )
                         .setLabel("Model")
-                        .setOrder(4)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The base model ID or an ARN to a custom model based on a foundational model.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.put(
                     REGION_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("The region that your model or ARN is deployed in.")
                         .setLabel("Region")
-                        .setOrder(5)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The region that your model or ARN is deployed in.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.putAll(AmazonBedrockSecretSettings.Configuration.get());
                 configurationMap.putAll(
-                    RateLimitSettings.toSettingsConfigurationWithTooltip(
+                    RateLimitSettings.toSettingsConfigurationWithDescription(
                         "By default, the amazonbedrock service sets the number of requests allowed per minute to 240."
                     )
                 );
 
-                return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                    Map<String, SettingsConfiguration> taskSettingsConfig;
-                    switch (t) {
-                        case COMPLETION -> taskSettingsConfig = AmazonBedrockChatCompletionModel.Configuration.get();
-                        // TEXT_EMBEDDING task type has no task settings
-                        default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                    }
-                    return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                }).toList()).setConfiguration(configurationMap).build();
+                return new InferenceServiceConfiguration.Builder().setService(NAME)
+                    .setName(SERVICE_NAME)
+                    .setTaskTypes(supportedTaskTypes)
+                    .setConfigurations(configurationMap)
+                    .build();
             }
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionModel.java
index 9339a8a05dc81..27dc607d671aa 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionModel.java
@@ -7,30 +7,19 @@
 
 package org.elasticsearch.xpack.inference.services.amazonbedrock.completion;
 
-import org.elasticsearch.common.util.LazyInitializable;
 import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
-import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.TaskSettings;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.amazonbedrock.AmazonBedrockActionVisitor;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockModel;
 import org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockSecretSettings;
 
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockConstants.MAX_NEW_TOKENS_FIELD;
-import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockConstants.TEMPERATURE_FIELD;
-import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockConstants.TOP_K_FIELD;
-import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockConstants.TOP_P_FIELD;
-
 public class AmazonBedrockChatCompletionModel extends AmazonBedrockModel {
 
     public static AmazonBedrockChatCompletionModel of(AmazonBedrockChatCompletionModel completionModel, Map<String, Object> taskSettings) {
@@ -91,62 +80,4 @@ public AmazonBedrockChatCompletionServiceSettings getServiceSettings() {
     public AmazonBedrockChatCompletionTaskSettings getTaskSettings() {
         return (AmazonBedrockChatCompletionTaskSettings) super.getTaskSettings();
     }
-
-    public static class Configuration {
-        public static Map<String, SettingsConfiguration> get() {
-            return configuration.getOrCompute();
-        }
-
-        private static final LazyInitializable<Map<String, SettingsConfiguration>, RuntimeException> configuration =
-            new LazyInitializable<>(() -> {
-                var configurationMap = new HashMap<String, SettingsConfiguration>();
-
-                configurationMap.put(
-                    MAX_NEW_TOKENS_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
-                        .setLabel("Max New Tokens")
-                        .setOrder(1)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Sets the maximum number for the output tokens to be generated.")
-                        .setType(SettingsConfigurationFieldType.INTEGER)
-                        .build()
-                );
-                configurationMap.put(
-                    TEMPERATURE_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
-                        .setLabel("Temperature")
-                        .setOrder(2)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("A number between 0.0 and 1.0 that controls the apparent creativity of the results.")
-                        .setType(SettingsConfigurationFieldType.INTEGER)
-                        .build()
-                );
-                configurationMap.put(
-                    TOP_P_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
-                        .setLabel("Top P")
-                        .setOrder(3)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Alternative to temperature. A number in the range of 0.0 to 1.0, to eliminate low-probability tokens.")
-                        .setType(SettingsConfigurationFieldType.INTEGER)
-                        .build()
-                );
-                configurationMap.put(
-                    TOP_K_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
-                        .setLabel("Top K")
-                        .setOrder(4)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Only available for anthropic, cohere, and mistral providers. Alternative to temperature.")
-                        .setType(SettingsConfigurationFieldType.INTEGER)
-                        .build()
-                );
-
-                return Collections.unmodifiableMap(configurationMap);
-            });
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java
index 41852e4758a8c..9dbfb0732f463 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
@@ -23,9 +22,7 @@
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.SettingsConfiguration;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.inference.external.action.anthropic.AnthropicActionCreator;
@@ -57,6 +54,7 @@
 
 public class AnthropicService extends SenderService {
     public static final String NAME = "anthropic";
+    private static final String SERVICE_NAME = "Anthropic";
 
     private static final EnumSet<TaskType> supportedTaskTypes = EnumSet.of(TaskType.COMPLETION);
 
@@ -258,31 +256,27 @@ public static InferenceServiceConfiguration get() {
 
                 configurationMap.put(
                     MODEL_ID,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("The name of the model to use for the inference task.")
                         .setLabel("Model ID")
-                        .setOrder(2)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The name of the model to use for the inference task.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration());
                 configurationMap.putAll(
-                    RateLimitSettings.toSettingsConfigurationWithTooltip(
+                    RateLimitSettings.toSettingsConfigurationWithDescription(
                         "By default, the anthropic service sets the number of requests allowed per minute to 50."
                     )
                 );
 
-                return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                    Map<String, SettingsConfiguration> taskSettingsConfig;
-                    switch (t) {
-                        case COMPLETION -> taskSettingsConfig = AnthropicChatCompletionModel.Configuration.get();
-                        default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                    }
-                    return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                }).toList()).setConfiguration(configurationMap).build();
+                return new InferenceServiceConfiguration.Builder().setService(NAME)
+                    .setName(SERVICE_NAME)
+                    .setTaskTypes(supportedTaskTypes)
+                    .setConfigurations(configurationMap)
+                    .build();
             }
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionModel.java
index df54ee4ec97c4..942cae8960daf 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionModel.java
@@ -8,14 +8,10 @@
 package org.elasticsearch.xpack.inference.services.anthropic.completion;
 
 import org.apache.http.client.utils.URIBuilder;
-import org.elasticsearch.common.util.LazyInitializable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
-import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.anthropic.AnthropicActionVisitor;
 import org.elasticsearch.xpack.inference.external.request.anthropic.AnthropicRequestUtils;
@@ -26,15 +22,8 @@
 
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.inference.services.anthropic.AnthropicServiceFields.MAX_TOKENS;
-import static org.elasticsearch.xpack.inference.services.anthropic.AnthropicServiceFields.TEMPERATURE_FIELD;
-import static org.elasticsearch.xpack.inference.services.anthropic.AnthropicServiceFields.TOP_K_FIELD;
-import static org.elasticsearch.xpack.inference.services.anthropic.AnthropicServiceFields.TOP_P_FIELD;
-
 public class AnthropicChatCompletionModel extends AnthropicModel {
 
     public static AnthropicChatCompletionModel of(AnthropicChatCompletionModel model, Map<String, Object> taskSettings) {
@@ -134,62 +123,4 @@ private static URI buildDefaultUri() throws URISyntaxException {
             .setPathSegments(AnthropicRequestUtils.API_VERSION_1, AnthropicRequestUtils.MESSAGES_PATH)
             .build();
     }
-
-    public static class Configuration {
-        public static Map<String, SettingsConfiguration> get() {
-            return configuration.getOrCompute();
-        }
-
-        private static final LazyInitializable<Map<String, SettingsConfiguration>, RuntimeException> configuration =
-            new LazyInitializable<>(() -> {
-                var configurationMap = new HashMap<String, SettingsConfiguration>();
-
-                configurationMap.put(
-                    MAX_TOKENS,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
-                        .setLabel("Max Tokens")
-                        .setOrder(1)
-                        .setRequired(true)
-                        .setSensitive(false)
-                        .setTooltip("The maximum number of tokens to generate before stopping.")
-                        .setType(SettingsConfigurationFieldType.INTEGER)
-                        .build()
-                );
-                configurationMap.put(
-                    TEMPERATURE_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
-                        .setLabel("Temperature")
-                        .setOrder(2)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("The amount of randomness injected into the response.")
-                        .setType(SettingsConfigurationFieldType.STRING)
-                        .build()
-                );
-                configurationMap.put(
-                    TOP_K_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
-                        .setLabel("Top K")
-                        .setOrder(3)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Specifies to only sample from the top K options for each subsequent token.")
-                        .setType(SettingsConfigurationFieldType.INTEGER)
-                        .build()
-                );
-                configurationMap.put(
-                    TOP_P_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
-                        .setLabel("Top P")
-                        .setOrder(4)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Specifies to use Anthropic’s nucleus sampling.")
-                        .setType(SettingsConfigurationFieldType.INTEGER)
-                        .build()
-                );
-
-                return Collections.unmodifiableMap(configurationMap);
-            });
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioEndpointType.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioEndpointType.java
index ece63f4bbf0cd..8639ba5b3aeda 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioEndpointType.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioEndpointType.java
@@ -13,7 +13,7 @@ public enum AzureAiStudioEndpointType {
     TOKEN,
     REALTIME;
 
-    public static String NAME = "azure_ai_studio_endpoint_type";
+    public static final String NAME = "azure_ai_studio_endpoint_type";
 
     public static AzureAiStudioEndpointType fromString(String name) {
         return valueOf(name.trim().toUpperCase(Locale.ROOT));
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioProvider.java
index 6b3efca0888f3..55f3f557797dd 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioProvider.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioProvider.java
@@ -17,7 +17,7 @@ public enum AzureAiStudioProvider {
     COHERE,
     DATABRICKS;
 
-    public static String NAME = "azure_ai_studio_provider";
+    public static final String NAME = "azure_ai_studio_provider";
 
     public static AzureAiStudioProvider fromString(String name) {
         return valueOf(name.trim().toUpperCase(Locale.ROOT));
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java
index bcd4a1abfbf00..649540f7efc5c 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
 import org.elasticsearch.inference.ChunkingSettings;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
@@ -26,11 +25,8 @@
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.SimilarityMeasure;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder;
 import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker;
@@ -56,7 +52,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Stream;
 
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg;
@@ -77,6 +72,7 @@ public class AzureAiStudioService extends SenderService {
 
     static final String NAME = "azureaistudio";
 
+    private static final String SERVICE_NAME = "Azure AI Studio";
     private static final EnumSet<TaskType> supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.COMPLETION);
 
     public AzureAiStudioService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) {
@@ -410,62 +406,47 @@ public static InferenceServiceConfiguration get() {
 
                 configurationMap.put(
                     TARGET_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("The target URL of your Azure AI Studio model deployment.")
                         .setLabel("Target")
-                        .setOrder(2)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The target URL of your Azure AI Studio model deployment.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.put(
                     ENDPOINT_TYPE_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN)
+                    new SettingsConfiguration.Builder().setDescription(
+                        "Specifies the type of endpoint that is used in your model deployment."
+                    )
                         .setLabel("Endpoint Type")
-                        .setOrder(3)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("Specifies the type of endpoint that is used in your model deployment.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
-                        .setOptions(
-                            Stream.of("token", "realtime")
-                                .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build())
-                                .toList()
-                        )
                         .build()
                 );
 
                 configurationMap.put(
                     PROVIDER_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN)
+                    new SettingsConfiguration.Builder().setDescription("The model provider for your deployment.")
                         .setLabel("Provider")
-                        .setOrder(3)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The model provider for your deployment.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
-                        .setOptions(
-                            Stream.of("cohere", "meta", "microsoft_phi", "mistral", "openai", "databricks")
-                                .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build())
-                                .toList()
-                        )
                         .build()
                 );
 
                 configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration());
                 configurationMap.putAll(RateLimitSettings.toSettingsConfiguration());
 
-                return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                    Map<String, SettingsConfiguration> taskSettingsConfig;
-                    switch (t) {
-                        case TEXT_EMBEDDING -> taskSettingsConfig = AzureAiStudioEmbeddingsModel.Configuration.get();
-                        case COMPLETION -> taskSettingsConfig = AzureAiStudioChatCompletionModel.Configuration.get();
-                        default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                    }
-                    return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                }).toList()).setConfiguration(configurationMap).build();
+                return new InferenceServiceConfiguration.Builder().setService(NAME)
+                    .setName(SERVICE_NAME)
+                    .setTaskTypes(supportedTaskTypes)
+                    .setConfigurations(configurationMap)
+                    .build();
             }
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionModel.java
index 0492788c2adcd..5afb3aaed61ff 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionModel.java
@@ -7,14 +7,10 @@
 
 package org.elasticsearch.xpack.inference.services.azureaistudio.completion;
 
-import org.elasticsearch.common.util.LazyInitializable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
-import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.azureaistudio.AzureAiStudioActionVisitor;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
@@ -25,12 +21,9 @@
 
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
 
 import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.COMPLETIONS_URI_PATH;
-import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.USER_FIELD;
 
 public class AzureAiStudioChatCompletionModel extends AzureAiStudioModel {
 
@@ -109,30 +102,4 @@ protected URI getEndpointUri() throws URISyntaxException {
     public ExecutableAction accept(AzureAiStudioActionVisitor creator, Map<String, Object> taskSettings) {
         return creator.create(this, taskSettings);
     }
-
-    public static class Configuration {
-        public static Map<String, SettingsConfiguration> get() {
-            return configuration.getOrCompute();
-        }
-
-        private static final LazyInitializable<Map<String, SettingsConfiguration>, RuntimeException> configuration =
-            new LazyInitializable<>(() -> {
-                var configurationMap = new HashMap<String, SettingsConfiguration>();
-
-                configurationMap.put(
-                    USER_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
-                        .setLabel("User")
-                        .setOrder(1)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Specifies the user issuing the request.")
-                        .setType(SettingsConfigurationFieldType.STRING)
-                        .setValue("")
-                        .build()
-                );
-
-                return Collections.unmodifiableMap(configurationMap);
-            });
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsModel.java
index 8b0b52c69b82c..edbefe07cff02 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsModel.java
@@ -7,15 +7,11 @@
 
 package org.elasticsearch.xpack.inference.services.azureaistudio.embeddings;
 
-import org.elasticsearch.common.util.LazyInitializable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.ChunkingSettings;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
-import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.azureaistudio.AzureAiStudioActionVisitor;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
@@ -26,15 +22,9 @@
 
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.DO_SAMPLE_FIELD;
 import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.EMBEDDINGS_URI_PATH;
-import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.MAX_NEW_TOKENS_FIELD;
-import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.TEMPERATURE_FIELD;
-import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.TOP_P_FIELD;
 
 public class AzureAiStudioEmbeddingsModel extends AzureAiStudioModel {
 
@@ -116,65 +106,4 @@ protected URI getEndpointUri() throws URISyntaxException {
     public ExecutableAction accept(AzureAiStudioActionVisitor creator, Map<String, Object> taskSettings) {
         return creator.create(this, taskSettings);
     }
-
-    public static class Configuration {
-        public static Map<String, SettingsConfiguration> get() {
-            return configuration.getOrCompute();
-        }
-
-        private static final LazyInitializable<Map<String, SettingsConfiguration>, RuntimeException> configuration =
-            new LazyInitializable<>(() -> {
-                var configurationMap = new HashMap<String, SettingsConfiguration>();
-
-                configurationMap.put(
-                    DO_SAMPLE_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
-                        .setLabel("Do Sample")
-                        .setOrder(1)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Instructs the inference process to perform sampling or not.")
-                        .setType(SettingsConfigurationFieldType.INTEGER)
-                        .build()
-                );
-                configurationMap.put(
-                    MAX_NEW_TOKENS_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
-                        .setLabel("Max New Tokens")
-                        .setOrder(2)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Provides a hint for the maximum number of output tokens to be generated.")
-                        .setType(SettingsConfigurationFieldType.INTEGER)
-                        .build()
-                );
-                configurationMap.put(
-                    TEMPERATURE_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
-                        .setLabel("Temperature")
-                        .setOrder(3)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("A number in the range of 0.0 to 2.0 that specifies the sampling temperature.")
-                        .setType(SettingsConfigurationFieldType.INTEGER)
-                        .build()
-                );
-                configurationMap.put(
-                    TOP_P_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
-                        .setLabel("Top P")
-                        .setOrder(4)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip(
-                            "A number in the range of 0.0 to 2.0 that is an alternative value to temperature. Should not be used "
-                                + "if temperature is specified."
-                        )
-                        .setType(SettingsConfigurationFieldType.INTEGER)
-                        .build()
-                );
-
-                return Collections.unmodifiableMap(configurationMap);
-            });
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java
index 70a29b28a607c..0601daf562ce9 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.SecretSettings;
 import org.elasticsearch.inference.SettingsConfiguration;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -147,23 +146,21 @@ public static Map<String, SettingsConfiguration> get() {
                 var configurationMap = new HashMap<String, SettingsConfiguration>();
                 configurationMap.put(
                     API_KEY,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("You must provide either an API key or an Entra ID.")
                         .setLabel("API Key")
-                        .setOrder(1)
                         .setRequired(false)
                         .setSensitive(true)
-                        .setTooltip("You must provide either an API key or an Entra ID.")
+                        .setUpdatable(true)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
                 configurationMap.put(
                     ENTRA_ID,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("You must provide either an API key or an Entra ID.")
                         .setLabel("Entra ID")
-                        .setOrder(2)
                         .setRequired(false)
                         .setSensitive(true)
-                        .setTooltip("You must provide either an API key or an Entra ID.")
+                        .setUpdatable(true)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java
index 0ed69604c258a..4fca5a460a12a 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
 import org.elasticsearch.inference.ChunkingSettings;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
@@ -25,9 +24,7 @@
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.SimilarityMeasure;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder;
@@ -68,6 +65,7 @@
 public class AzureOpenAiService extends SenderService {
     public static final String NAME = "azureopenai";
 
+    private static final String SERVICE_NAME = "Azure OpenAI";
     private static final EnumSet<TaskType> supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.COMPLETION);
 
     public AzureOpenAiService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) {
@@ -353,56 +351,49 @@ public static InferenceServiceConfiguration get() {
 
                 configurationMap.put(
                     RESOURCE_NAME,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("The name of your Azure OpenAI resource.")
                         .setLabel("Resource Name")
-                        .setOrder(3)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The name of your Azure OpenAI resource.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.put(
                     API_VERSION,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("The Azure API version ID to use.")
                         .setLabel("API Version")
-                        .setOrder(4)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The Azure API version ID to use.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.put(
                     DEPLOYMENT_ID,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("The deployment name of your deployed models.")
                         .setLabel("Deployment ID")
-                        .setOrder(5)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The deployment name of your deployed models.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.putAll(AzureOpenAiSecretSettings.Configuration.get());
                 configurationMap.putAll(
-                    RateLimitSettings.toSettingsConfigurationWithTooltip(
+                    RateLimitSettings.toSettingsConfigurationWithDescription(
                         "The azureopenai service sets a default number of requests allowed per minute depending on the task type."
                     )
                 );
 
-                return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                    Map<String, SettingsConfiguration> taskSettingsConfig;
-                    switch (t) {
-                        case TEXT_EMBEDDING -> taskSettingsConfig = AzureOpenAiEmbeddingsModel.Configuration.get();
-                        case COMPLETION -> taskSettingsConfig = AzureOpenAiCompletionModel.Configuration.get();
-                        default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                    }
-                    return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                }).toList()).setConfiguration(configurationMap).build();
+                return new InferenceServiceConfiguration.Builder().setService(NAME)
+                    .setName(SERVICE_NAME)
+                    .setTaskTypes(supportedTaskTypes)
+                    .setConfigurations(configurationMap)
+                    .build();
             }
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionModel.java
index 8b2846fd9ced7..ed57187adfe64 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionModel.java
@@ -7,14 +7,10 @@
 
 package org.elasticsearch.xpack.inference.services.azureopenai.completion;
 
-import org.elasticsearch.common.util.LazyInitializable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
-import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.azureopenai.AzureOpenAiActionVisitor;
 import org.elasticsearch.xpack.inference.external.request.azureopenai.AzureOpenAiUtils;
@@ -23,12 +19,8 @@
 import org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiSecretSettings;
 
 import java.net.URISyntaxException;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiServiceFields.USER;
-
 public class AzureOpenAiCompletionModel extends AzureOpenAiModel {
 
     public static AzureOpenAiCompletionModel of(AzureOpenAiCompletionModel model, Map<String, Object> taskSettings) {
@@ -127,30 +119,4 @@ public String apiVersion() {
     public String[] operationPathSegments() {
         return new String[] { AzureOpenAiUtils.CHAT_PATH, AzureOpenAiUtils.COMPLETIONS_PATH };
     }
-
-    public static class Configuration {
-        public static Map<String, SettingsConfiguration> get() {
-            return configuration.getOrCompute();
-        }
-
-        private static final LazyInitializable<Map<String, SettingsConfiguration>, RuntimeException> configuration =
-            new LazyInitializable<>(() -> {
-                var configurationMap = new HashMap<String, SettingsConfiguration>();
-
-                configurationMap.put(
-                    USER,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
-                        .setLabel("User")
-                        .setOrder(1)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Specifies the user issuing the request.")
-                        .setType(SettingsConfigurationFieldType.STRING)
-                        .setValue("")
-                        .build()
-                );
-
-                return Collections.unmodifiableMap(configurationMap);
-            });
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModel.java
index 0316804664510..dea5a8ca9e50a 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModel.java
@@ -7,15 +7,11 @@
 
 package org.elasticsearch.xpack.inference.services.azureopenai.embeddings;
 
-import org.elasticsearch.common.util.LazyInitializable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.ChunkingSettings;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
-import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.azureopenai.AzureOpenAiActionVisitor;
 import org.elasticsearch.xpack.inference.external.request.azureopenai.AzureOpenAiUtils;
@@ -24,12 +20,8 @@
 import org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiSecretSettings;
 
 import java.net.URISyntaxException;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiServiceFields.USER;
-
 public class AzureOpenAiEmbeddingsModel extends AzureOpenAiModel {
 
     public static AzureOpenAiEmbeddingsModel of(AzureOpenAiEmbeddingsModel model, Map<String, Object> taskSettings) {
@@ -131,30 +123,4 @@ public String apiVersion() {
     public String[] operationPathSegments() {
         return new String[] { AzureOpenAiUtils.EMBEDDINGS_PATH };
     }
-
-    public static class Configuration {
-        public static Map<String, SettingsConfiguration> get() {
-            return configuration.getOrCompute();
-        }
-
-        private static final LazyInitializable<Map<String, SettingsConfiguration>, RuntimeException> configuration =
-            new LazyInitializable<>(() -> {
-                var configurationMap = new HashMap<String, SettingsConfiguration>();
-
-                configurationMap.put(
-                    USER,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
-                        .setLabel("User")
-                        .setOrder(1)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Specifies the user issuing the request.")
-                        .setType(SettingsConfigurationFieldType.STRING)
-                        .setValue("")
-                        .build()
-                );
-
-                return Collections.unmodifiableMap(configurationMap);
-            });
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java
index a7d17192bfa92..60ab8ca68d5d9 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
 import org.elasticsearch.inference.ChunkingSettings;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
@@ -25,7 +24,6 @@
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.SimilarityMeasure;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder;
@@ -65,6 +63,7 @@
 public class CohereService extends SenderService {
     public static final String NAME = "cohere";
 
+    private static final String SERVICE_NAME = "Cohere";
     private static final EnumSet<TaskType> supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.COMPLETION, TaskType.RERANK);
 
     // TODO Batching - We'll instantiate a batching class within the services that want to support it and pass it through to
@@ -367,16 +366,11 @@ public static InferenceServiceConfiguration get() {
                 configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration());
                 configurationMap.putAll(RateLimitSettings.toSettingsConfiguration());
 
-                return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                    Map<String, SettingsConfiguration> taskSettingsConfig;
-                    switch (t) {
-                        case TEXT_EMBEDDING -> taskSettingsConfig = CohereEmbeddingsModel.Configuration.get();
-                        case RERANK -> taskSettingsConfig = CohereRerankModel.Configuration.get();
-                        // COMPLETION task type has no task settings
-                        default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                    }
-                    return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                }).toList()).setConfiguration(configurationMap).build();
+                return new InferenceServiceConfiguration.Builder().setService(NAME)
+                    .setName(SERVICE_NAME)
+                    .setTaskTypes(supportedTaskTypes)
+                    .setConfigurations(configurationMap)
+                    .build();
             }
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java
index 43a7bc0a5e678..0f62ab51145f4 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java
@@ -7,17 +7,12 @@
 
 package org.elasticsearch.xpack.inference.services.cohere.embeddings;
 
-import org.elasticsearch.common.util.LazyInitializable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.ChunkingSettings;
 import org.elasticsearch.inference.InputType;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
-import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.cohere.CohereActionVisitor;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
@@ -25,13 +20,7 @@
 import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
 import java.net.URI;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
-import java.util.stream.Stream;
-
-import static org.elasticsearch.xpack.inference.external.request.cohere.CohereEmbeddingsRequestEntity.INPUT_TYPE_FIELD;
-import static org.elasticsearch.xpack.inference.services.cohere.CohereServiceFields.TRUNCATE;
 
 public class CohereEmbeddingsModel extends CohereModel {
     public static CohereEmbeddingsModel of(CohereEmbeddingsModel model, Map<String, Object> taskSettings, InputType inputType) {
@@ -110,52 +99,4 @@ public ExecutableAction accept(CohereActionVisitor visitor, Map<String, Object>
     public URI uri() {
         return getServiceSettings().getCommonSettings().uri();
     }
-
-    public static class Configuration {
-        public static Map<String, SettingsConfiguration> get() {
-            return configuration.getOrCompute();
-        }
-
-        private static final LazyInitializable<Map<String, SettingsConfiguration>, RuntimeException> configuration =
-            new LazyInitializable<>(() -> {
-                var configurationMap = new HashMap<String, SettingsConfiguration>();
-
-                configurationMap.put(
-                    INPUT_TYPE_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN)
-                        .setLabel("Input Type")
-                        .setOrder(1)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Specifies the type of input passed to the model.")
-                        .setType(SettingsConfigurationFieldType.STRING)
-                        .setOptions(
-                            Stream.of("classification", "clusterning", "ingest", "search")
-                                .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build())
-                                .toList()
-                        )
-                        .setValue("")
-                        .build()
-                );
-                configurationMap.put(
-                    TRUNCATE,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN)
-                        .setLabel("Truncate")
-                        .setOrder(2)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Specifies how the API handles inputs longer than the maximum token length.")
-                        .setType(SettingsConfigurationFieldType.STRING)
-                        .setOptions(
-                            Stream.of("NONE", "START", "END")
-                                .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build())
-                                .toList()
-                        )
-                        .setValue("")
-                        .build()
-                );
-
-                return Collections.unmodifiableMap(configurationMap);
-            });
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankModel.java
index cfcfb8a3d5dae..b84b98973bbe5 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankModel.java
@@ -7,15 +7,11 @@
 
 package org.elasticsearch.xpack.inference.services.cohere.rerank;
 
-import org.elasticsearch.common.util.LazyInitializable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.InputType;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
-import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.cohere.CohereActionVisitor;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
@@ -23,13 +19,8 @@
 import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
 import java.net.URI;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankTaskSettings.RETURN_DOCUMENTS;
-import static org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankTaskSettings.TOP_N_DOCS_ONLY;
-
 public class CohereRerankModel extends CohereModel {
     public static CohereRerankModel of(CohereRerankModel model, Map<String, Object> taskSettings) {
         var requestTaskSettings = CohereRerankTaskSettings.fromMap(taskSettings);
@@ -111,41 +102,4 @@ public ExecutableAction accept(CohereActionVisitor visitor, Map<String, Object>
     public URI uri() {
         return getServiceSettings().uri();
     }
-
-    public static class Configuration {
-        public static Map<String, SettingsConfiguration> get() {
-            return configuration.getOrCompute();
-        }
-
-        private static final LazyInitializable<Map<String, SettingsConfiguration>, RuntimeException> configuration =
-            new LazyInitializable<>(() -> {
-                var configurationMap = new HashMap<String, SettingsConfiguration>();
-
-                configurationMap.put(
-                    RETURN_DOCUMENTS,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TOGGLE)
-                        .setLabel("Return Documents")
-                        .setOrder(1)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Specify whether to return doc text within the results.")
-                        .setType(SettingsConfigurationFieldType.BOOLEAN)
-                        .setValue(false)
-                        .build()
-                );
-                configurationMap.put(
-                    TOP_N_DOCS_ONLY,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
-                        .setLabel("Top N")
-                        .setOrder(2)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("The number of most relevant documents to return, defaults to the number of the documents.")
-                        .setType(SettingsConfigurationFieldType.INTEGER)
-                        .build()
-                );
-
-                return Collections.unmodifiableMap(configurationMap);
-            });
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java
index 27abc03cb6e82..48416faac6a06 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
@@ -24,9 +23,7 @@
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.SettingsConfiguration;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
@@ -34,23 +31,29 @@
 import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceError;
 import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
 import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults;
+import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.elastic.ElasticInferenceServiceActionCreator;
 import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput;
+import org.elasticsearch.xpack.inference.external.http.sender.ElasticInferenceServiceUnifiedCompletionRequestManager;
 import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
 import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs;
 import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.SenderService;
 import org.elasticsearch.xpack.inference.services.ServiceComponents;
+import org.elasticsearch.xpack.inference.services.elastic.completion.ElasticInferenceServiceCompletionModel;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
 import org.elasticsearch.xpack.inference.telemetry.TraceContext;
 
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 
 import static org.elasticsearch.xpack.core.inference.results.ResultUtils.createInvalidChunkedResultException;
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage;
 import static org.elasticsearch.xpack.inference.services.ServiceFields.MAX_INPUT_TOKENS;
 import static org.elasticsearch.xpack.inference.services.ServiceFields.MODEL_ID;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException;
@@ -58,7 +61,6 @@
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap;
-import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation;
 
 public class ElasticInferenceService extends SenderService {
 
@@ -67,7 +69,8 @@ public class ElasticInferenceService extends SenderService {
 
     private final ElasticInferenceServiceComponents elasticInferenceServiceComponents;
 
-    private static final EnumSet<TaskType> supportedTaskTypes = EnumSet.of(TaskType.SPARSE_EMBEDDING);
+    private static final EnumSet<TaskType> supportedTaskTypes = EnumSet.of(TaskType.SPARSE_EMBEDDING, TaskType.COMPLETION);
+    private static final String SERVICE_NAME = "Elastic";
 
     public ElasticInferenceService(
         HttpRequestSender.Factory factory,
@@ -78,6 +81,11 @@ public ElasticInferenceService(
         this.elasticInferenceServiceComponents = elasticInferenceServiceComponents;
     }
 
+    @Override
+    public Set<TaskType> supportedStreamingTasks() {
+        return COMPLETION_ONLY;
+    }
+
     @Override
     protected void doUnifiedCompletionInfer(
         Model model,
@@ -85,7 +93,31 @@ protected void doUnifiedCompletionInfer(
         TimeValue timeout,
         ActionListener<InferenceServiceResults> listener
     ) {
-        throwUnsupportedUnifiedCompletionOperation(NAME);
+        if (model instanceof ElasticInferenceServiceCompletionModel == false) {
+            listener.onFailure(createInvalidModelException(model));
+            return;
+        }
+
+        // We extract the trace context here as it's sufficient to propagate the trace information of the REST request,
+        // which handles the request to the inference API overall (including the outgoing request, which is started in a new thread
+        // generating a different "traceparent" as every task and every REST request creates a new span).
+        var currentTraceInfo = getCurrentTraceInfo();
+
+        var completionModel = (ElasticInferenceServiceCompletionModel) model;
+        var overriddenModel = ElasticInferenceServiceCompletionModel.of(completionModel, inputs.getRequest());
+        var errorMessage = constructFailedToSendRequestMessage(
+            overriddenModel.uri(),
+            String.format(Locale.ROOT, "%s completions", ELASTIC_INFERENCE_SERVICE_IDENTIFIER)
+        );
+
+        var requestManager = ElasticInferenceServiceUnifiedCompletionRequestManager.of(
+            overriddenModel,
+            getServiceComponents().threadPool(),
+            currentTraceInfo
+        );
+        var action = new SenderExecutableAction(getSender(), requestManager, errorMessage);
+
+        action.execute(inputs, timeout, listener);
     }
 
     @Override
@@ -97,7 +129,7 @@ protected void doInfer(
         TimeValue timeout,
         ActionListener<InferenceServiceResults> listener
     ) {
-        if (model instanceof ElasticInferenceServiceModel == false) {
+        if (model instanceof ElasticInferenceServiceExecutableActionModel == false) {
             listener.onFailure(createInvalidModelException(model));
             return;
         }
@@ -107,7 +139,7 @@ protected void doInfer(
         // generating a different "traceparent" as every task and every REST request creates a new span).
         var currentTraceInfo = getCurrentTraceInfo();
 
-        ElasticInferenceServiceModel elasticInferenceServiceModel = (ElasticInferenceServiceModel) model;
+        ElasticInferenceServiceExecutableActionModel elasticInferenceServiceModel = (ElasticInferenceServiceExecutableActionModel) model;
         var actionCreator = new ElasticInferenceServiceActionCreator(getSender(), getServiceComponents(), currentTraceInfo);
 
         var action = elasticInferenceServiceModel.accept(actionCreator, taskSettings);
@@ -199,6 +231,16 @@ private static ElasticInferenceServiceModel createModel(
                 eisServiceComponents,
                 context
             );
+            case COMPLETION -> new ElasticInferenceServiceCompletionModel(
+                inferenceEntityId,
+                taskType,
+                NAME,
+                serviceSettings,
+                taskSettings,
+                secretSettings,
+                eisServiceComponents,
+                context
+            );
             default -> throw new ElasticsearchStatusException(failureMessage, RestStatus.BAD_REQUEST);
         };
     }
@@ -317,38 +359,33 @@ public static InferenceServiceConfiguration get() {
 
                 configurationMap.put(
                     MODEL_ID,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("The name of the model to use for the inference task.")
                         .setLabel("Model ID")
-                        .setOrder(2)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The name of the model to use for the inference task.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.put(
                     MAX_INPUT_TOKENS,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
+                    new SettingsConfiguration.Builder().setDescription("Allows you to specify the maximum number of tokens per input.")
                         .setLabel("Maximum Input Tokens")
-                        .setOrder(3)
                         .setRequired(false)
                         .setSensitive(false)
-                        .setTooltip("Allows you to specify the maximum number of tokens per input.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.INTEGER)
                         .build()
                 );
 
                 configurationMap.putAll(RateLimitSettings.toSettingsConfiguration());
 
-                return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                    Map<String, SettingsConfiguration> taskSettingsConfig;
-                    switch (t) {
-                        // SPARSE_EMBEDDING task type has no task settings
-                        default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                    }
-                    return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                }).toList()).setConfiguration(configurationMap).build();
+                return new InferenceServiceConfiguration.Builder().setService(NAME)
+                    .setName(SERVICE_NAME)
+                    .setTaskTypes(supportedTaskTypes)
+                    .setConfigurations(configurationMap)
+                    .build();
             }
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceExecutableActionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceExecutableActionModel.java
new file mode 100644
index 0000000000000..3223b8b3e8d91
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceExecutableActionModel.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elastic;
+
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.ServiceSettings;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.action.elastic.ElasticInferenceServiceActionVisitor;
+
+import java.util.Map;
+
+public abstract class ElasticInferenceServiceExecutableActionModel extends ElasticInferenceServiceModel {
+
+    public ElasticInferenceServiceExecutableActionModel(
+        ModelConfigurations configurations,
+        ModelSecrets secrets,
+        ElasticInferenceServiceRateLimitServiceSettings rateLimitServiceSettings,
+        ElasticInferenceServiceComponents elasticInferenceServiceComponents
+    ) {
+        super(configurations, secrets, rateLimitServiceSettings, elasticInferenceServiceComponents);
+    }
+
+    public ElasticInferenceServiceExecutableActionModel(
+        ElasticInferenceServiceExecutableActionModel model,
+        ServiceSettings serviceSettings
+    ) {
+        super(model, serviceSettings);
+    }
+
+    public abstract ExecutableAction accept(ElasticInferenceServiceActionVisitor visitor, Map<String, Object> taskSettings);
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceModel.java
index e7809d869fec4..e03cc36e62417 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceModel.java
@@ -11,10 +11,7 @@
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.ServiceSettings;
-import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
-import org.elasticsearch.xpack.inference.external.action.elastic.ElasticInferenceServiceActionVisitor;
 
-import java.util.Map;
 import java.util.Objects;
 
 public abstract class ElasticInferenceServiceModel extends Model {
@@ -49,7 +46,4 @@ public ElasticInferenceServiceRateLimitServiceSettings rateLimitServiceSettings(
     public ElasticInferenceServiceComponents elasticInferenceServiceComponents() {
         return elasticInferenceServiceComponents;
     }
-
-    public abstract ExecutableAction accept(ElasticInferenceServiceActionVisitor visitor, Map<String, Object> taskSettings);
-
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java
index 431a3647e2879..0c1a032dc8926 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java
@@ -14,6 +14,10 @@
 import java.util.ArrayList;
 import java.util.List;
 
+/**
+ * Encapsulates settings using {@link Setting}. This does not represent service settings that are persisted
+ * via {@link org.elasticsearch.inference.ServiceSettings}, but rather Elasticsearch settings passed on startup.
+ */
 public class ElasticInferenceServiceSettings {
 
     @Deprecated
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java
index 54728a92e6254..112be95dac1fd 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.inference.services.elastic;
 
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.EmptySecretSettings;
 import org.elasticsearch.inference.EmptyTaskSettings;
@@ -15,6 +16,7 @@
 import org.elasticsearch.inference.SecretSettings;
 import org.elasticsearch.inference.TaskSettings;
 import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.elastic.ElasticInferenceServiceActionVisitor;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
@@ -27,7 +29,7 @@
 
 import static org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService.ELASTIC_INFERENCE_SERVICE_IDENTIFIER;
 
-public class ElasticInferenceServiceSparseEmbeddingsModel extends ElasticInferenceServiceModel {
+public class ElasticInferenceServiceSparseEmbeddingsModel extends ElasticInferenceServiceExecutableActionModel {
 
     private final URI uri;
 
@@ -57,12 +59,7 @@ public ElasticInferenceServiceSparseEmbeddingsModel(
         ElasticInferenceServiceSparseEmbeddingsServiceSettings serviceSettings
     ) {
         super(model, serviceSettings);
-
-        try {
-            this.uri = createUri();
-        } catch (URISyntaxException e) {
-            throw new RuntimeException(e);
-        }
+        this.uri = createUri();
     }
 
     ElasticInferenceServiceSparseEmbeddingsModel(
@@ -80,12 +77,7 @@ public ElasticInferenceServiceSparseEmbeddingsModel(
             serviceSettings,
             elasticInferenceServiceComponents
         );
-
-        try {
-            this.uri = createUri();
-        } catch (URISyntaxException e) {
-            throw new RuntimeException(e);
-        }
+        this.uri = createUri();
     }
 
     @Override
@@ -102,19 +94,42 @@ public URI uri() {
         return uri;
     }
 
-    private URI createUri() throws URISyntaxException {
+    private URI createUri() throws ElasticsearchStatusException {
         String modelId = getServiceSettings().modelId();
         String modelIdUriPath;
 
         switch (modelId) {
             case ElserModels.ELSER_V2_MODEL -> modelIdUriPath = "ELSERv2";
-            default -> throw new IllegalArgumentException(
-                String.format(Locale.ROOT, "Unsupported model for %s [%s]", ELASTIC_INFERENCE_SERVICE_IDENTIFIER, modelId)
+            default -> throw new ElasticsearchStatusException(
+                String.format(
+                    Locale.ROOT,
+                    "Unsupported model [%s] for service [%s] and task type [%s]",
+                    modelId,
+                    ELASTIC_INFERENCE_SERVICE_IDENTIFIER,
+                    TaskType.SPARSE_EMBEDDING
+                ),
+                RestStatus.BAD_REQUEST
             );
         }
 
-        return new URI(
-            elasticInferenceServiceComponents().elasticInferenceServiceUrl() + "/api/v1/sparse-text-embeddings/" + modelIdUriPath
-        );
+        try {
+            // TODO, consider transforming the base URL into a URI for better error handling.
+            return new URI(
+                elasticInferenceServiceComponents().elasticInferenceServiceUrl() + "/api/v1/sparse-text-embeddings/" + modelIdUriPath
+            );
+        } catch (URISyntaxException e) {
+            throw new ElasticsearchStatusException(
+                "Failed to create URI for service ["
+                    + this.getConfigurations().getService()
+                    + "] with taskType ["
+                    + this.getTaskType()
+                    + "] with model ["
+                    + this.getServiceSettings().modelId()
+                    + "]: "
+                    + e.getMessage(),
+                RestStatus.BAD_REQUEST,
+                e
+            );
+        }
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/completion/ElasticInferenceServiceCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/completion/ElasticInferenceServiceCompletionModel.java
new file mode 100644
index 0000000000000..84039cd7cc33c
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/completion/ElasticInferenceServiceCompletionModel.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elastic.completion;
+
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.EmptySecretSettings;
+import org.elasticsearch.inference.EmptyTaskSettings;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.SecretSettings;
+import org.elasticsearch.inference.TaskSettings;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.inference.UnifiedCompletionRequest;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceComponents;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceModel;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Map;
+import java.util.Objects;
+
+public class ElasticInferenceServiceCompletionModel extends ElasticInferenceServiceModel {
+
+    public static ElasticInferenceServiceCompletionModel of(
+        ElasticInferenceServiceCompletionModel model,
+        UnifiedCompletionRequest request
+    ) {
+        var originalModelServiceSettings = model.getServiceSettings();
+        var overriddenServiceSettings = new ElasticInferenceServiceCompletionServiceSettings(
+            Objects.requireNonNullElse(request.model(), originalModelServiceSettings.modelId()),
+            originalModelServiceSettings.rateLimitSettings()
+        );
+
+        return new ElasticInferenceServiceCompletionModel(model, overriddenServiceSettings);
+    }
+
+    private final URI uri;
+
+    public ElasticInferenceServiceCompletionModel(
+        String inferenceEntityId,
+        TaskType taskType,
+        String service,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        Map<String, Object> secrets,
+        ElasticInferenceServiceComponents elasticInferenceServiceComponents,
+        ConfigurationParseContext context
+    ) {
+        this(
+            inferenceEntityId,
+            taskType,
+            service,
+            ElasticInferenceServiceCompletionServiceSettings.fromMap(serviceSettings, context),
+            EmptyTaskSettings.INSTANCE,
+            EmptySecretSettings.INSTANCE,
+            elasticInferenceServiceComponents
+        );
+    }
+
+    public ElasticInferenceServiceCompletionModel(
+        ElasticInferenceServiceCompletionModel model,
+        ElasticInferenceServiceCompletionServiceSettings serviceSettings
+    ) {
+        super(model, serviceSettings);
+        this.uri = createUri();
+
+    }
+
+    ElasticInferenceServiceCompletionModel(
+        String inferenceEntityId,
+        TaskType taskType,
+        String service,
+        ElasticInferenceServiceCompletionServiceSettings serviceSettings,
+        @Nullable TaskSettings taskSettings,
+        @Nullable SecretSettings secretSettings,
+        ElasticInferenceServiceComponents elasticInferenceServiceComponents
+    ) {
+        super(
+            new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings),
+            new ModelSecrets(secretSettings),
+            serviceSettings,
+            elasticInferenceServiceComponents
+        );
+
+        this.uri = createUri();
+
+    }
+
+    @Override
+    public ElasticInferenceServiceCompletionServiceSettings getServiceSettings() {
+        return (ElasticInferenceServiceCompletionServiceSettings) super.getServiceSettings();
+    }
+
+    public URI uri() {
+        return uri;
+    }
+
+    private URI createUri() throws ElasticsearchStatusException {
+        try {
+            // TODO, consider transforming the base URL into a URI for better error handling.
+            return new URI(elasticInferenceServiceComponents().elasticInferenceServiceUrl() + "/api/v1/chat/completions");
+        } catch (URISyntaxException e) {
+            throw new ElasticsearchStatusException(
+                "Failed to create URI for service ["
+                    + this.getConfigurations().getService()
+                    + "] with taskType ["
+                    + this.getTaskType()
+                    + "]: "
+                    + e.getMessage(),
+                RestStatus.BAD_REQUEST,
+                e
+            );
+        }
+    }
+
+    // TODO create/refactor the Configuration class to be extensible for different task types (i.e completion, sparse embeddings).
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/completion/ElasticInferenceServiceCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/completion/ElasticInferenceServiceCompletionServiceSettings.java
new file mode 100644
index 0000000000000..3c8182a7d41a4
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/completion/ElasticInferenceServiceCompletionServiceSettings.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elastic.completion;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ServiceSettings;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceRateLimitServiceSettings;
+import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.services.ServiceFields.MODEL_ID;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredString;
+
+public class ElasticInferenceServiceCompletionServiceSettings extends FilteredXContentObject
+    implements
+        ServiceSettings,
+        ElasticInferenceServiceRateLimitServiceSettings {
+
+    public static final String NAME = "elastic_inference_service_completion_service_settings";
+
+    // TODO what value do we put here?
+    private static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(240L);
+
+    public static ElasticInferenceServiceCompletionServiceSettings fromMap(Map<String, Object> map, ConfigurationParseContext context) {
+        ValidationException validationException = new ValidationException();
+
+        String modelId = extractRequiredString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException);
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(
+            map,
+            DEFAULT_RATE_LIMIT_SETTINGS,
+            validationException,
+            ElasticInferenceService.NAME,
+            context
+        );
+
+        if (validationException.validationErrors().isEmpty() == false) {
+            throw validationException;
+        }
+
+        return new ElasticInferenceServiceCompletionServiceSettings(modelId, rateLimitSettings);
+    }
+
+    private final String modelId;
+    private final RateLimitSettings rateLimitSettings;
+
+    public ElasticInferenceServiceCompletionServiceSettings(String modelId, RateLimitSettings rateLimitSettings) {
+        this.modelId = Objects.requireNonNull(modelId);
+        this.rateLimitSettings = Objects.requireNonNullElse(rateLimitSettings, DEFAULT_RATE_LIMIT_SETTINGS);
+    }
+
+    public ElasticInferenceServiceCompletionServiceSettings(StreamInput in) throws IOException {
+        this.modelId = in.readString();
+        this.rateLimitSettings = new RateLimitSettings(in);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    public String modelId() {
+        return modelId;
+    }
+
+    @Override
+    public RateLimitSettings rateLimitSettings() {
+        return rateLimitSettings;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.ELASTIC_INFERENCE_SERVICE_UNIFIED_CHAT_COMPLETIONS_INTEGRATION;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        toXContentFragmentOfExposedFields(builder, params);
+
+        builder.endObject();
+
+        return builder;
+    }
+
+    @Override
+    protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException {
+        builder.field(MODEL_ID, modelId);
+        rateLimitSettings.toXContent(builder, params);
+
+        return builder;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(modelId);
+        rateLimitSettings.writeTo(out);
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (this == object) return true;
+        if (object == null || getClass() != object.getClass()) return false;
+        ElasticInferenceServiceCompletionServiceSettings that = (ElasticInferenceServiceCompletionServiceSettings) object;
+        return Objects.equals(modelId, that.modelId) && Objects.equals(rateLimitSettings, that.rateLimitSettings);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(modelId, rateLimitSettings);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java
index 6388bb33bb78d..b4d579b699cdf 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java
@@ -7,17 +7,7 @@
 
 package org.elasticsearch.xpack.inference.services.elasticsearch;
 
-import org.elasticsearch.common.util.LazyInitializable;
-import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.elasticsearch.xpack.inference.services.elasticsearch.RerankTaskSettings.RETURN_DOCUMENTS;
 
 public class CustomElandRerankModel extends CustomElandModel {
 
@@ -35,30 +25,4 @@ public CustomElandRerankModel(
     public CustomElandInternalServiceSettings getServiceSettings() {
         return (CustomElandInternalServiceSettings) super.getServiceSettings();
     }
-
-    public static class Configuration {
-        public static Map<String, SettingsConfiguration> get() {
-            return configuration.getOrCompute();
-        }
-
-        private static final LazyInitializable<Map<String, SettingsConfiguration>, RuntimeException> configuration =
-            new LazyInitializable<>(() -> {
-                var configurationMap = new HashMap<String, SettingsConfiguration>();
-
-                configurationMap.put(
-                    RETURN_DOCUMENTS,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TOGGLE)
-                        .setLabel("Return Documents")
-                        .setOrder(1)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Returns the document instead of only the index.")
-                        .setType(SettingsConfigurationFieldType.BOOLEAN)
-                        .setValue(true)
-                        .build()
-                );
-
-                return Collections.unmodifiableMap(configurationMap);
-            });
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java
index cb3eb5ca8e826..2931f2e23f12d 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
 import org.elasticsearch.inference.ChunkingSettings;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceExtension;
@@ -29,12 +28,9 @@
 import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.SettingsConfiguration;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.inference.UnifiedCompletionRequest;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults;
 import org.elasticsearch.xpack.core.inference.results.RankedDocsResults;
@@ -71,7 +67,6 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
 import java.util.function.Function;
-import java.util.stream.Stream;
 
 import static org.elasticsearch.xpack.core.inference.results.ResultUtils.createInvalidChunkedResultException;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMap;
@@ -82,7 +77,6 @@
 import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings.MODEL_ID;
 import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS;
 import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings.NUM_THREADS;
-import static org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.ELSER_V1_MODEL;
 import static org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.ELSER_V2_MODEL;
 import static org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.ELSER_V2_MODEL_LINUX_X86;
 
@@ -105,6 +99,7 @@ public class ElasticsearchInternalService extends BaseElasticsearchInternalServi
     public static final String DEFAULT_E5_ID = ".multilingual-e5-small-elasticsearch";
     public static final String DEFAULT_RERANK_ID = ".rerank-v1-elasticsearch";
 
+    private static final String SERVICE_NAME = "Elasticsearch";
     private static final EnumSet<TaskType> supportedTaskTypes = EnumSet.of(
         TaskType.RERANK,
         TaskType.TEXT_EMBEDDING,
@@ -138,7 +133,7 @@ public void parseRequestConfig(
         Map<String, Object> config,
         ActionListener<Model> modelListener
     ) {
-        if (inferenceEntityId.equals(DEFAULT_ELSER_ID)) {
+        if (isDefaultId(inferenceEntityId)) {
             modelListener.onFailure(
                 new ElasticsearchStatusException(
                     "[{}] is a reserved inference Id. Cannot create a new inference endpoint with a reserved Id",
@@ -1152,61 +1147,45 @@ public static InferenceServiceConfiguration get() {
 
                 configurationMap.put(
                     MODEL_ID,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN)
+                    new SettingsConfiguration.Builder().setDefaultValue(MULTILINGUAL_E5_SMALL_MODEL_ID)
+                        .setDescription("The name of the model to use for the inference task.")
                         .setLabel("Model ID")
-                        .setOrder(1)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The name of the model to use for the inference task.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
-                        .setOptions(
-                            Stream.of(
-                                ELSER_V1_MODEL,
-                                ELSER_V2_MODEL,
-                                ELSER_V2_MODEL_LINUX_X86,
-                                MULTILINGUAL_E5_SMALL_MODEL_ID,
-                                MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86
-                            ).map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build()).toList()
-                        )
-                        .setDefaultValue(MULTILINGUAL_E5_SMALL_MODEL_ID)
                         .build()
                 );
 
                 configurationMap.put(
                     NUM_ALLOCATIONS,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
+                    new SettingsConfiguration.Builder().setDefaultValue(1)
+                        .setDescription("The total number of allocations this model is assigned across machine learning nodes.")
                         .setLabel("Number Allocations")
-                        .setOrder(2)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The total number of allocations this model is assigned across machine learning nodes.")
+                        .setUpdatable(true)
                         .setType(SettingsConfigurationFieldType.INTEGER)
-                        .setDefaultValue(1)
                         .build()
                 );
 
                 configurationMap.put(
                     NUM_THREADS,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
+                    new SettingsConfiguration.Builder().setDefaultValue(2)
+                        .setDescription("Sets the number of threads used by each model allocation during inference.")
                         .setLabel("Number Threads")
-                        .setOrder(3)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("Sets the number of threads used by each model allocation during inference.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.INTEGER)
-                        .setDefaultValue(2)
                         .build()
                 );
 
-                return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                    Map<String, SettingsConfiguration> taskSettingsConfig;
-                    switch (t) {
-                        case RERANK -> taskSettingsConfig = CustomElandRerankModel.Configuration.get();
-                        // SPARSE_EMBEDDING, TEXT_EMBEDDING task types have no task settings
-                        default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                    }
-                    return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                }).toList()).setConfiguration(configurationMap).build();
+                return new InferenceServiceConfiguration.Builder().setService(NAME)
+                    .setName(SERVICE_NAME)
+                    .setTaskTypes(supportedTaskTypes)
+                    .setConfigurations(configurationMap)
+                    .build();
             }
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettings.java
index 3bcaa57827fdb..264d694703d89 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettings.java
@@ -22,7 +22,7 @@ public class ElserMlNodeTaskSettings implements TaskSettings {
 
     public static final String NAME = "elser_mlnode_task_settings";
 
-    public static ElserMlNodeTaskSettings DEFAULT = new ElserMlNodeTaskSettings();
+    public static final ElserMlNodeTaskSettings DEFAULT = new ElserMlNodeTaskSettings();
 
     public ElserMlNodeTaskSettings() {}
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserModels.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserModels.java
index 37f528ea3a750..c1764b93bfc82 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserModels.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserModels.java
@@ -16,7 +16,7 @@ public class ElserModels {
     public static final String ELSER_V2_MODEL = ".elser_model_2";
     public static final String ELSER_V2_MODEL_LINUX_X86 = ".elser_model_2_linux-x86_64";
 
-    public static Set<String> VALID_ELSER_MODEL_IDS = Set.of(
+    public static final Set<String> VALID_ELSER_MODEL_IDS = Set.of(
         ElserModels.ELSER_V1_MODEL,
         ElserModels.ELSER_V2_MODEL,
         ElserModels.ELSER_V2_MODEL_LINUX_X86
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java
index 837a001d1f8f9..1dbf2ca3e2dad 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
 import org.elasticsearch.inference.ChunkingSettings;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
@@ -25,9 +24,7 @@
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.SimilarityMeasure;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder;
@@ -72,6 +69,7 @@ public class GoogleAiStudioService extends SenderService {
 
     public static final String NAME = "googleaistudio";
 
+    private static final String SERVICE_NAME = "Google AI Studio";
     private static final EnumSet<TaskType> supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.COMPLETION);
 
     public GoogleAiStudioService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) {
@@ -353,12 +351,11 @@ public static InferenceServiceConfiguration get() {
 
                 configurationMap.put(
                     MODEL_ID,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("ID of the LLM you're using.")
                         .setLabel("Model ID")
-                        .setOrder(2)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("ID of the LLM you're using.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
@@ -366,14 +363,11 @@ public static InferenceServiceConfiguration get() {
                 configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration());
                 configurationMap.putAll(RateLimitSettings.toSettingsConfiguration());
 
-                return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                    Map<String, SettingsConfiguration> taskSettingsConfig;
-                    switch (t) {
-                        // COMPLETION, TEXT_EMBEDDING task types have no task settings
-                        default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                    }
-                    return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                }).toList()).setConfiguration(configurationMap).build();
+                return new InferenceServiceConfiguration.Builder().setService(NAME)
+                    .setName(SERVICE_NAME)
+                    .setTaskTypes(supportedTaskTypes)
+                    .setConfigurations(configurationMap)
+                    .build();
             }
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettings.java
index 272bc9eaa9a62..b185800ed75f4 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettings.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.SecretSettings;
 import org.elasticsearch.inference.SettingsConfiguration;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -123,12 +122,11 @@ public static Map<String, SettingsConfiguration> get() {
                 var configurationMap = new HashMap<String, SettingsConfiguration>();
                 configurationMap.put(
                     SERVICE_ACCOUNT_JSON,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("API Key for the provider you're connecting to.")
                         .setLabel("Credentials JSON")
-                        .setOrder(1)
                         .setRequired(true)
                         .setSensitive(true)
-                        .setTooltip("API Key for the provider you're connecting to.")
+                        .setUpdatable(true)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java
index b412f20289880..8fe9f29c73747 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
 import org.elasticsearch.inference.ChunkingSettings;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
@@ -24,9 +23,7 @@
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.SettingsConfiguration;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder;
@@ -67,6 +64,7 @@ public class GoogleVertexAiService extends SenderService {
 
     public static final String NAME = "googlevertexai";
 
+    private static final String SERVICE_NAME = "Google Vertex AI";
     private static final EnumSet<TaskType> supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.RERANK);
 
     public GoogleVertexAiService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) {
@@ -331,42 +329,39 @@ public static InferenceServiceConfiguration get() {
 
                 configurationMap.put(
                     MODEL_ID,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("ID of the LLM you're using.")
                         .setLabel("Model ID")
-                        .setOrder(2)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("ID of the LLM you're using.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.put(
                     LOCATION,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription(
+                        "Please provide the GCP region where the Vertex AI API(s) is enabled. "
+                            + "For more information, refer to the {geminiVertexAIDocs}."
+                    )
                         .setLabel("GCP Region")
-                        .setOrder(3)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip(
-                            "Please provide the GCP region where the Vertex AI API(s) is enabled. "
-                                + "For more information, refer to the {geminiVertexAIDocs}."
-                        )
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.put(
                     PROJECT_ID,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription(
+                        "The GCP Project ID which has Vertex AI API(s) enabled. For more information "
+                            + "on the URL, refer to the {geminiVertexAIDocs}."
+                    )
                         .setLabel("GCP Project")
-                        .setOrder(4)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip(
-                            "The GCP Project ID which has Vertex AI API(s) enabled. For more information "
-                                + "on the URL, refer to the {geminiVertexAIDocs}."
-                        )
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
@@ -374,15 +369,11 @@ public static InferenceServiceConfiguration get() {
                 configurationMap.putAll(GoogleVertexAiSecretSettings.Configuration.get());
                 configurationMap.putAll(RateLimitSettings.toSettingsConfiguration());
 
-                return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                    Map<String, SettingsConfiguration> taskSettingsConfig;
-                    switch (t) {
-                        case TEXT_EMBEDDING -> taskSettingsConfig = GoogleVertexAiEmbeddingsModel.Configuration.get();
-                        case RERANK -> taskSettingsConfig = GoogleVertexAiRerankModel.Configuration.get();
-                        default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                    }
-                    return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                }).toList()).setConfiguration(configurationMap).build();
+                return new InferenceServiceConfiguration.Builder().setService(NAME)
+                    .setName(SERVICE_NAME)
+                    .setTaskTypes(supportedTaskTypes)
+                    .setConfigurations(configurationMap)
+                    .build();
             }
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsModel.java
index a5acbb80b76ec..98640623ee21c 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsModel.java
@@ -8,17 +8,12 @@
 package org.elasticsearch.xpack.inference.services.googlevertexai.embeddings;
 
 import org.apache.http.client.utils.URIBuilder;
-import org.elasticsearch.common.util.LazyInitializable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.ChunkingSettings;
 import org.elasticsearch.inference.InputType;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
-import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.googlevertexai.GoogleVertexAiActionVisitor;
 import org.elasticsearch.xpack.inference.external.request.googlevertexai.GoogleVertexAiUtils;
@@ -28,14 +23,9 @@
 
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
-import java.util.stream.Stream;
 
 import static org.elasticsearch.core.Strings.format;
-import static org.elasticsearch.xpack.inference.services.googlevertexai.embeddings.GoogleVertexAiEmbeddingsTaskSettings.AUTO_TRUNCATE;
-import static org.elasticsearch.xpack.inference.services.googlevertexai.embeddings.GoogleVertexAiEmbeddingsTaskSettings.INPUT_TYPE;
 
 public class GoogleVertexAiEmbeddingsModel extends GoogleVertexAiModel {
 
@@ -165,51 +155,4 @@ public static URI buildUri(String location, String projectId, String modelId) th
             )
             .build();
     }
-
-    public static class Configuration {
-        public static Map<String, SettingsConfiguration> get() {
-            return configuration.getOrCompute();
-        }
-
-        private static final LazyInitializable<Map<String, SettingsConfiguration>, RuntimeException> configuration =
-            new LazyInitializable<>(() -> {
-                var configurationMap = new HashMap<String, SettingsConfiguration>();
-
-                configurationMap.put(
-                    INPUT_TYPE,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN)
-                        .setLabel("Input Type")
-                        .setOrder(1)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Specifies the type of input passed to the model.")
-                        .setType(SettingsConfigurationFieldType.STRING)
-                        .setOptions(
-                            Stream.of(
-                                InputType.CLASSIFICATION.toString(),
-                                InputType.CLUSTERING.toString(),
-                                InputType.INGEST.toString(),
-                                InputType.SEARCH.toString()
-                            ).map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build()).toList()
-                        )
-                        .setValue("")
-                        .build()
-                );
-
-                configurationMap.put(
-                    AUTO_TRUNCATE,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TOGGLE)
-                        .setLabel("Auto Truncate")
-                        .setOrder(2)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Specifies if the API truncates inputs longer than the maximum token length automatically.")
-                        .setType(SettingsConfigurationFieldType.BOOLEAN)
-                        .setValue(false)
-                        .build()
-                );
-
-                return Collections.unmodifiableMap(configurationMap);
-            });
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankModel.java
index e73d8d2e2613a..f522897b0d716 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankModel.java
@@ -8,15 +8,11 @@
 package org.elasticsearch.xpack.inference.services.googlevertexai.rerank;
 
 import org.apache.http.client.utils.URIBuilder;
-import org.elasticsearch.common.util.LazyInitializable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.InputType;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
-import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.googlevertexai.GoogleVertexAiActionVisitor;
 import org.elasticsearch.xpack.inference.external.request.googlevertexai.GoogleVertexAiUtils;
@@ -26,12 +22,9 @@
 
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
 
 import static org.elasticsearch.core.Strings.format;
-import static org.elasticsearch.xpack.inference.services.googlevertexai.rerank.GoogleVertexAiRerankTaskSettings.TOP_N;
 
 public class GoogleVertexAiRerankModel extends GoogleVertexAiModel {
 
@@ -140,30 +133,4 @@ public static URI buildUri(String projectId) throws URISyntaxException {
             )
             .build();
     }
-
-    public static class Configuration {
-        public static Map<String, SettingsConfiguration> get() {
-            return configuration.getOrCompute();
-        }
-
-        private static final LazyInitializable<Map<String, SettingsConfiguration>, RuntimeException> configuration =
-            new LazyInitializable<>(() -> {
-                var configurationMap = new HashMap<String, SettingsConfiguration>();
-
-                configurationMap.put(
-                    TOP_N,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TOGGLE)
-                        .setLabel("Top N")
-                        .setOrder(1)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Specifies the number of the top n documents, which should be returned.")
-                        .setType(SettingsConfigurationFieldType.BOOLEAN)
-                        .setValue(false)
-                        .build()
-                );
-
-                return Collections.unmodifiableMap(configurationMap);
-            });
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java
index acb082cd2de8d..ef6beb8ec2627 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java
@@ -16,16 +16,13 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
 import org.elasticsearch.inference.ChunkingSettings;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
 import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.SimilarityMeasure;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker;
@@ -54,6 +51,7 @@
 public class HuggingFaceService extends HuggingFaceBaseService {
     public static final String NAME = "hugging_face";
 
+    private static final String SERVICE_NAME = "Hugging Face";
     private static final EnumSet<TaskType> supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.SPARSE_EMBEDDING);
 
     public HuggingFaceService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) {
@@ -183,29 +181,24 @@ public static InferenceServiceConfiguration get() {
 
                 configurationMap.put(
                     URL,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDefaultValue("https://api.openai.com/v1/embeddings")
+                        .setDescription("The URL endpoint to use for the requests.")
                         .setLabel("URL")
-                        .setOrder(1)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The URL endpoint to use for the requests.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
-                        .setValue("https://api.openai.com/v1/embeddings")
-                        .setDefaultValue("https://api.openai.com/v1/embeddings")
                         .build()
                 );
 
                 configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration());
                 configurationMap.putAll(RateLimitSettings.toSettingsConfiguration());
 
-                return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                    Map<String, SettingsConfiguration> taskSettingsConfig;
-                    switch (t) {
-                        // SPARSE_EMBEDDING, TEXT_EMBEDDING task types have no task settings
-                        default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                    }
-                    return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                }).toList()).setConfiguration(configurationMap).build();
+                return new InferenceServiceConfiguration.Builder().setService(NAME)
+                    .setName(SERVICE_NAME)
+                    .setTaskTypes(supportedTaskTypes)
+                    .setConfigurations(configurationMap)
+                    .build();
             }
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java
index 01d2a75ebff5d..52d42f570a413 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java
@@ -17,15 +17,12 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
 import org.elasticsearch.inference.ChunkingSettings;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
 import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.SettingsConfiguration;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat;
@@ -58,6 +55,7 @@
 public class HuggingFaceElserService extends HuggingFaceBaseService {
     public static final String NAME = "hugging_face_elser";
 
+    private static final String SERVICE_NAME = "Hugging Face ELSER";
     private static final EnumSet<TaskType> supportedTaskTypes = EnumSet.of(TaskType.SPARSE_EMBEDDING);
 
     public HuggingFaceElserService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) {
@@ -178,12 +176,11 @@ public static InferenceServiceConfiguration get() {
 
                 configurationMap.put(
                     URL,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("The URL endpoint to use for the requests.")
                         .setLabel("URL")
-                        .setOrder(1)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The URL endpoint to use for the requests.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
@@ -191,14 +188,11 @@ public static InferenceServiceConfiguration get() {
                 configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration());
                 configurationMap.putAll(RateLimitSettings.toSettingsConfiguration());
 
-                return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                    Map<String, SettingsConfiguration> taskSettingsConfig;
-                    switch (t) {
-                        // SPARSE_EMBEDDING task type has no task settings
-                        default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                    }
-                    return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                }).toList()).setConfiguration(configurationMap).build();
+                return new InferenceServiceConfiguration.Builder().setService(NAME)
+                    .setName(SERVICE_NAME)
+                    .setTaskTypes(supportedTaskTypes)
+                    .setConfigurations(configurationMap)
+                    .build();
             }
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java
index 482554e060a47..dd368f88a993c 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
 import org.elasticsearch.inference.ChunkingSettings;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
@@ -25,9 +24,7 @@
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.SimilarityMeasure;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder;
@@ -68,6 +65,7 @@ public class IbmWatsonxService extends SenderService {
 
     public static final String NAME = "watsonxai";
 
+    private static final String SERVICE_NAME = "IBM Watsonx";
     private static final EnumSet<TaskType> supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING);
 
     public IbmWatsonxService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) {
@@ -326,72 +324,64 @@ public static InferenceServiceConfiguration get() {
 
                 configurationMap.put(
                     API_VERSION,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("The IBM Watsonx API version ID to use.")
                         .setLabel("API Version")
-                        .setOrder(1)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The IBM Watsonx API version ID to use.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.put(
                     PROJECT_ID,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("")
                         .setLabel("Project ID")
-                        .setOrder(2)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.put(
                     MODEL_ID,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("The name of the model to use for the inference task.")
                         .setLabel("Model ID")
-                        .setOrder(3)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The name of the model to use for the inference task.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.put(
                     URL,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("")
                         .setLabel("URL")
-                        .setOrder(4)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.put(
                     MAX_INPUT_TOKENS,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
+                    new SettingsConfiguration.Builder().setDescription("Allows you to specify the maximum number of tokens per input.")
                         .setLabel("Maximum Input Tokens")
-                        .setOrder(5)
                         .setRequired(false)
                         .setSensitive(false)
-                        .setTooltip("Allows you to specify the maximum number of tokens per input.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.INTEGER)
                         .build()
                 );
 
-                return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                    Map<String, SettingsConfiguration> taskSettingsConfig;
-                    switch (t) {
-                        // TEXT_EMBEDDING task type has no task settings
-                        default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                    }
-                    return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                }).toList()).setConfiguration(configurationMap).build();
+                return new InferenceServiceConfiguration.Builder().setService(NAME)
+                    .setName(SERVICE_NAME)
+                    .setTaskTypes(supportedTaskTypes)
+                    .setConfigurations(configurationMap)
+                    .build();
             }
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIModel.java
new file mode 100644
index 0000000000000..bfd8235e3da48
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIModel.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai;
+
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.Model;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.ServiceSettings;
+import org.elasticsearch.inference.TaskSettings;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.action.jinaai.JinaAIActionVisitor;
+import org.elasticsearch.xpack.inference.services.ServiceUtils;
+import org.elasticsearch.xpack.inference.services.settings.ApiKeySecrets;
+
+import java.net.URI;
+import java.util.Map;
+import java.util.Objects;
+
+public abstract class JinaAIModel extends Model {
+    private final SecureString apiKey;
+    private final JinaAIRateLimitServiceSettings rateLimitServiceSettings;
+
+    public JinaAIModel(
+        ModelConfigurations configurations,
+        ModelSecrets secrets,
+        @Nullable ApiKeySecrets apiKeySecrets,
+        JinaAIRateLimitServiceSettings rateLimitServiceSettings
+    ) {
+        super(configurations, secrets);
+
+        this.rateLimitServiceSettings = Objects.requireNonNull(rateLimitServiceSettings);
+        apiKey = ServiceUtils.apiKey(apiKeySecrets);
+    }
+
+    protected JinaAIModel(JinaAIModel model, TaskSettings taskSettings) {
+        super(model, taskSettings);
+
+        rateLimitServiceSettings = model.rateLimitServiceSettings();
+        apiKey = model.apiKey();
+    }
+
+    protected JinaAIModel(JinaAIModel model, ServiceSettings serviceSettings) {
+        super(model, serviceSettings);
+
+        rateLimitServiceSettings = model.rateLimitServiceSettings();
+        apiKey = model.apiKey();
+    }
+
+    public SecureString apiKey() {
+        return apiKey;
+    }
+
+    public JinaAIRateLimitServiceSettings rateLimitServiceSettings() {
+        return rateLimitServiceSettings;
+    }
+
+    public abstract ExecutableAction accept(JinaAIActionVisitor creator, Map<String, Object> taskSettings, InputType inputType);
+
+    public abstract URI uri();
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIRateLimitServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIRateLimitServiceSettings.java
new file mode 100644
index 0000000000000..ac65ad1c9d714
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIRateLimitServiceSettings.java
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai;
+
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+
+public interface JinaAIRateLimitServiceSettings {
+    RateLimitSettings rateLimitSettings();
+
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java
new file mode 100644
index 0000000000000..ed76df5875562
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java
@@ -0,0 +1,353 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai;
+
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.util.LazyInitializable;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.ChunkedInference;
+import org.elasticsearch.inference.ChunkingSettings;
+import org.elasticsearch.inference.InferenceServiceConfiguration;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.Model;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.SettingsConfiguration;
+import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder;
+import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker;
+import org.elasticsearch.xpack.inference.external.action.jinaai.JinaAIActionCreator;
+import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput;
+import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
+import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs;
+import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.SenderService;
+import org.elasticsearch.xpack.inference.services.ServiceComponents;
+import org.elasticsearch.xpack.inference.services.ServiceUtils;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModel;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsServiceSettings;
+import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankModel;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+import org.elasticsearch.xpack.inference.services.validation.ModelValidatorBuilder;
+
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMap;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation;
+import static org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceFields.EMBEDDING_MAX_BATCH_SIZE;
+
+public class JinaAIService extends SenderService {
+    public static final String NAME = "jinaai";
+
+    private static final String SERVICE_NAME = "Jina AI";
+    private static final EnumSet<TaskType> supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.RERANK);
+
+    public JinaAIService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) {
+        super(factory, serviceComponents);
+    }
+
+    @Override
+    public String name() {
+        return NAME;
+    }
+
+    @Override
+    public void parseRequestConfig(
+        String inferenceEntityId,
+        TaskType taskType,
+        Map<String, Object> config,
+        ActionListener<Model> parsedModelListener
+    ) {
+        try {
+            Map<String, Object> serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS);
+            Map<String, Object> taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS);
+
+            ChunkingSettings chunkingSettings = null;
+            if (TaskType.TEXT_EMBEDDING.equals(taskType)) {
+                chunkingSettings = ChunkingSettingsBuilder.fromMap(
+                    removeFromMapOrDefaultEmpty(config, ModelConfigurations.CHUNKING_SETTINGS)
+                );
+            }
+            JinaAIModel model = createModel(
+                inferenceEntityId,
+                taskType,
+                serviceSettingsMap,
+                taskSettingsMap,
+                chunkingSettings,
+                serviceSettingsMap,
+                TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME),
+                ConfigurationParseContext.REQUEST
+            );
+
+            throwIfNotEmptyMap(config, NAME);
+            throwIfNotEmptyMap(serviceSettingsMap, NAME);
+            throwIfNotEmptyMap(taskSettingsMap, NAME);
+
+            parsedModelListener.onResponse(model);
+        } catch (Exception e) {
+            parsedModelListener.onFailure(e);
+        }
+    }
+
+    private static JinaAIModel createModelFromPersistent(
+        String inferenceEntityId,
+        TaskType taskType,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        ChunkingSettings chunkingSettings,
+        @Nullable Map<String, Object> secretSettings,
+        String failureMessage
+    ) {
+        return createModel(
+            inferenceEntityId,
+            taskType,
+            serviceSettings,
+            taskSettings,
+            chunkingSettings,
+            secretSettings,
+            failureMessage,
+            ConfigurationParseContext.PERSISTENT
+        );
+    }
+
+    private static JinaAIModel createModel(
+        String inferenceEntityId,
+        TaskType taskType,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        ChunkingSettings chunkingSettings,
+        @Nullable Map<String, Object> secretSettings,
+        String failureMessage,
+        ConfigurationParseContext context
+    ) {
+        return switch (taskType) {
+            case TEXT_EMBEDDING -> new JinaAIEmbeddingsModel(
+                inferenceEntityId,
+                NAME,
+                serviceSettings,
+                taskSettings,
+                chunkingSettings,
+                secretSettings,
+                context
+            );
+            case RERANK -> new JinaAIRerankModel(inferenceEntityId, NAME, serviceSettings, taskSettings, secretSettings, context);
+            default -> throw new ElasticsearchStatusException(failureMessage, RestStatus.BAD_REQUEST);
+        };
+    }
+
+    @Override
+    public JinaAIModel parsePersistedConfigWithSecrets(
+        String inferenceEntityId,
+        TaskType taskType,
+        Map<String, Object> config,
+        Map<String, Object> secrets
+    ) {
+        Map<String, Object> serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS);
+        Map<String, Object> taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS);
+        Map<String, Object> secretSettingsMap = removeFromMapOrThrowIfNull(secrets, ModelSecrets.SECRET_SETTINGS);
+
+        ChunkingSettings chunkingSettings = null;
+        if (TaskType.TEXT_EMBEDDING.equals(taskType)) {
+            chunkingSettings = ChunkingSettingsBuilder.fromMap(removeFromMap(config, ModelConfigurations.CHUNKING_SETTINGS));
+        }
+
+        return createModelFromPersistent(
+            inferenceEntityId,
+            taskType,
+            serviceSettingsMap,
+            taskSettingsMap,
+            chunkingSettings,
+            secretSettingsMap,
+            parsePersistedConfigErrorMsg(inferenceEntityId, NAME)
+        );
+    }
+
+    @Override
+    public JinaAIModel parsePersistedConfig(String inferenceEntityId, TaskType taskType, Map<String, Object> config) {
+        Map<String, Object> serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS);
+        Map<String, Object> taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS);
+
+        ChunkingSettings chunkingSettings = null;
+        if (TaskType.TEXT_EMBEDDING.equals(taskType)) {
+            chunkingSettings = ChunkingSettingsBuilder.fromMap(removeFromMap(config, ModelConfigurations.CHUNKING_SETTINGS));
+        }
+
+        return createModelFromPersistent(
+            inferenceEntityId,
+            taskType,
+            serviceSettingsMap,
+            taskSettingsMap,
+            chunkingSettings,
+            null,
+            parsePersistedConfigErrorMsg(inferenceEntityId, NAME)
+        );
+    }
+
+    @Override
+    public InferenceServiceConfiguration getConfiguration() {
+        return Configuration.get();
+    }
+
+    @Override
+    public EnumSet<TaskType> supportedTaskTypes() {
+        return supportedTaskTypes;
+    }
+
+    @Override
+    protected void doUnifiedCompletionInfer(
+        Model model,
+        UnifiedChatInput inputs,
+        TimeValue timeout,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+        throwUnsupportedUnifiedCompletionOperation(NAME);
+    }
+
+    @Override
+    public void doInfer(
+        Model model,
+        InferenceInputs inputs,
+        Map<String, Object> taskSettings,
+        InputType inputType,
+        TimeValue timeout,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+        if (model instanceof JinaAIModel == false) {
+            listener.onFailure(createInvalidModelException(model));
+            return;
+        }
+
+        JinaAIModel jinaaiModel = (JinaAIModel) model;
+        var actionCreator = new JinaAIActionCreator(getSender(), getServiceComponents());
+
+        var action = jinaaiModel.accept(actionCreator, taskSettings, inputType);
+        action.execute(inputs, timeout, listener);
+    }
+
+    @Override
+    protected void doChunkedInfer(
+        Model model,
+        DocumentsOnlyInput inputs,
+        Map<String, Object> taskSettings,
+        InputType inputType,
+        TimeValue timeout,
+        ActionListener<List<ChunkedInference>> listener
+    ) {
+        if (model instanceof JinaAIModel == false) {
+            listener.onFailure(createInvalidModelException(model));
+            return;
+        }
+
+        JinaAIModel jinaaiModel = (JinaAIModel) model;
+        var actionCreator = new JinaAIActionCreator(getSender(), getServiceComponents());
+
+        List<EmbeddingRequestChunker.BatchRequestAndListener> batchedRequests = new EmbeddingRequestChunker(
+            inputs.getInputs(),
+            EMBEDDING_MAX_BATCH_SIZE,
+            EmbeddingRequestChunker.EmbeddingType.fromDenseVectorElementType(model.getServiceSettings().elementType()),
+            jinaaiModel.getConfigurations().getChunkingSettings()
+        ).batchRequestsWithListeners(listener);
+
+        for (var request : batchedRequests) {
+            var action = jinaaiModel.accept(actionCreator, taskSettings, inputType);
+            action.execute(new DocumentsOnlyInput(request.batch().inputs()), timeout, request.listener());
+        }
+    }
+
+    /**
+     * For text embedding models get the embedding size and
+     * update the service settings.
+     *
+     * @param model The new model
+     * @param listener The listener
+     */
+    @Override
+    public void checkModelConfig(Model model, ActionListener<Model> listener) {
+        ModelValidatorBuilder.buildModelValidator(model.getTaskType()).validate(this, model, listener);
+    }
+
+    @Override
+    public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) {
+        if (model instanceof JinaAIEmbeddingsModel embeddingsModel) {
+            var serviceSettings = embeddingsModel.getServiceSettings();
+            var similarityFromModel = serviceSettings.similarity();
+            var similarityToUse = similarityFromModel == null ? defaultSimilarity() : similarityFromModel;
+            var maxInputTokens = serviceSettings.maxInputTokens();
+
+            var updatedServiceSettings = new JinaAIEmbeddingsServiceSettings(
+                new JinaAIServiceSettings(
+                    serviceSettings.getCommonSettings().uri(),
+                    serviceSettings.getCommonSettings().modelId(),
+                    serviceSettings.getCommonSettings().rateLimitSettings()
+                ),
+                similarityToUse,
+                embeddingSize,
+                maxInputTokens
+            );
+
+            return new JinaAIEmbeddingsModel(embeddingsModel, updatedServiceSettings);
+        } else {
+            throw ServiceUtils.invalidModelTypeForUpdateModelWithEmbeddingDetails(model.getClass());
+        }
+    }
+
+    /**
+     * Return the default similarity measure for the embedding type.
+     * JinaAI embeddings are normalized to unit vectors therefore Dot
+     * Product similarity can be used and is the default for all JinaAI
+     * models.
+     *
+     * @return The default similarity.
+     */
+    static SimilarityMeasure defaultSimilarity() {
+        return SimilarityMeasure.DOT_PRODUCT;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.JINA_AI_INTEGRATION_ADDED;
+    }
+
+    public static class Configuration {
+        public static InferenceServiceConfiguration get() {
+            return configuration.getOrCompute();
+        }
+
+        private static final LazyInitializable<InferenceServiceConfiguration, RuntimeException> configuration = new LazyInitializable<>(
+            () -> {
+                var configurationMap = new HashMap<String, SettingsConfiguration>();
+
+                configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration());
+                configurationMap.putAll(RateLimitSettings.toSettingsConfiguration());
+
+                return new InferenceServiceConfiguration.Builder().setService(NAME)
+                    .setName(SERVICE_NAME)
+                    .setTaskTypes(supportedTaskTypes)
+                    .setConfigurations(configurationMap)
+                    .build();
+            }
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceFields.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceFields.java
new file mode 100644
index 0000000000000..2df8f1440e471
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceFields.java
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai;
+
+public class JinaAIServiceFields {
+
+    static final int EMBEDDING_MAX_BATCH_SIZE = 2048;
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceSettings.java
new file mode 100644
index 0000000000000..66c6193f653f1
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceSettings.java
@@ -0,0 +1,159 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ServiceSettings;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.services.ServiceFields.URL;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.convertToUri;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.createOptionalUri;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalString;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredString;
+
+public class JinaAIServiceSettings extends FilteredXContentObject implements ServiceSettings, JinaAIRateLimitServiceSettings {
+
+    public static final String NAME = "jinaai_service_settings";
+    public static final String MODEL_ID = "model_id";
+    private static final Logger logger = LogManager.getLogger(JinaAIServiceSettings.class);
+    // See https://jina.ai/contact-sales/#rate-limit
+    public static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(2_000);
+
+    public static JinaAIServiceSettings fromMap(Map<String, Object> map, ConfigurationParseContext context) {
+        ValidationException validationException = new ValidationException();
+
+        String url = extractOptionalString(map, URL, ModelConfigurations.SERVICE_SETTINGS, validationException);
+        URI uri = convertToUri(url, URL, ModelConfigurations.SERVICE_SETTINGS, validationException);
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(
+            map,
+            DEFAULT_RATE_LIMIT_SETTINGS,
+            validationException,
+            JinaAIService.NAME,
+            context
+        );
+
+        String modelId = extractRequiredString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException);
+
+        if (validationException.validationErrors().isEmpty() == false) {
+            throw validationException;
+        }
+
+        return new JinaAIServiceSettings(uri, modelId, rateLimitSettings);
+    }
+
+    private final URI uri;
+    private final String modelId;
+    private final RateLimitSettings rateLimitSettings;
+
+    public JinaAIServiceSettings(@Nullable URI uri, String modelId, @Nullable RateLimitSettings rateLimitSettings) {
+        this.uri = uri;
+        this.modelId = Objects.requireNonNull(modelId);
+        this.rateLimitSettings = Objects.requireNonNullElse(rateLimitSettings, DEFAULT_RATE_LIMIT_SETTINGS);
+    }
+
+    public JinaAIServiceSettings(@Nullable String url, String modelId, @Nullable RateLimitSettings rateLimitSettings) {
+        this(createOptionalUri(url), modelId, rateLimitSettings);
+    }
+
+    public JinaAIServiceSettings(StreamInput in) throws IOException {
+        uri = createOptionalUri(in.readOptionalString());
+        modelId = in.readOptionalString();
+        rateLimitSettings = new RateLimitSettings(in);
+    }
+
+    @Override
+    public RateLimitSettings rateLimitSettings() {
+        return rateLimitSettings;
+    }
+
+    public URI uri() {
+        return uri;
+    }
+
+    @Override
+    public String modelId() {
+        return modelId;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        toXContentFragment(builder, params);
+
+        builder.endObject();
+        return builder;
+    }
+
+    public XContentBuilder toXContentFragment(XContentBuilder builder, Params params) throws IOException {
+        return toXContentFragmentOfExposedFields(builder, params);
+    }
+
+    @Override
+    public XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException {
+        if (uri != null) {
+            builder.field(URL, uri.toString());
+        }
+        if (modelId != null) {
+            builder.field(MODEL_ID, modelId);
+        }
+        rateLimitSettings.toXContent(builder, params);
+
+        return builder;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.JINA_AI_INTEGRATION_ADDED;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        var uriToWrite = uri != null ? uri.toString() : null;
+        out.writeOptionalString(uriToWrite);
+        out.writeOptionalString(modelId);
+        rateLimitSettings.writeTo(out);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        JinaAIServiceSettings that = (JinaAIServiceSettings) o;
+        return Objects.equals(uri, that.uri)
+            && Objects.equals(modelId, that.modelId)
+            && Objects.equals(rateLimitSettings, that.rateLimitSettings);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(uri, modelId, rateLimitSettings);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModel.java
new file mode 100644
index 0000000000000..2c2a24a75d098
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModel.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai.embeddings;
+
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.ChunkingSettings;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.action.jinaai.JinaAIActionVisitor;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.jinaai.JinaAIModel;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
+
+import java.net.URI;
+import java.util.Map;
+
+public class JinaAIEmbeddingsModel extends JinaAIModel {
+    public static JinaAIEmbeddingsModel of(JinaAIEmbeddingsModel model, Map<String, Object> taskSettings, InputType inputType) {
+        var requestTaskSettings = JinaAIEmbeddingsTaskSettings.fromMap(taskSettings);
+        return new JinaAIEmbeddingsModel(model, JinaAIEmbeddingsTaskSettings.of(model.getTaskSettings(), requestTaskSettings, inputType));
+    }
+
+    public JinaAIEmbeddingsModel(
+        String inferenceId,
+        String service,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        ChunkingSettings chunkingSettings,
+        @Nullable Map<String, Object> secrets,
+        ConfigurationParseContext context
+    ) {
+        this(
+            inferenceId,
+            service,
+            JinaAIEmbeddingsServiceSettings.fromMap(serviceSettings, context),
+            JinaAIEmbeddingsTaskSettings.fromMap(taskSettings),
+            chunkingSettings,
+            DefaultSecretSettings.fromMap(secrets)
+        );
+    }
+
+    // should only be used for testing
+    JinaAIEmbeddingsModel(
+        String modelId,
+        String service,
+        JinaAIEmbeddingsServiceSettings serviceSettings,
+        JinaAIEmbeddingsTaskSettings taskSettings,
+        ChunkingSettings chunkingSettings,
+        @Nullable DefaultSecretSettings secretSettings
+    ) {
+        super(
+            new ModelConfigurations(modelId, TaskType.TEXT_EMBEDDING, service, serviceSettings, taskSettings, chunkingSettings),
+            new ModelSecrets(secretSettings),
+            secretSettings,
+            serviceSettings.getCommonSettings()
+        );
+    }
+
+    private JinaAIEmbeddingsModel(JinaAIEmbeddingsModel model, JinaAIEmbeddingsTaskSettings taskSettings) {
+        super(model, taskSettings);
+    }
+
+    public JinaAIEmbeddingsModel(JinaAIEmbeddingsModel model, JinaAIEmbeddingsServiceSettings serviceSettings) {
+        super(model, serviceSettings);
+    }
+
+    @Override
+    public JinaAIEmbeddingsServiceSettings getServiceSettings() {
+        return (JinaAIEmbeddingsServiceSettings) super.getServiceSettings();
+    }
+
+    @Override
+    public JinaAIEmbeddingsTaskSettings getTaskSettings() {
+        return (JinaAIEmbeddingsTaskSettings) super.getTaskSettings();
+    }
+
+    @Override
+    public DefaultSecretSettings getSecretSettings() {
+        return (DefaultSecretSettings) super.getSecretSettings();
+    }
+
+    @Override
+    public ExecutableAction accept(JinaAIActionVisitor visitor, Map<String, Object> taskSettings, InputType inputType) {
+        return visitor.create(this, taskSettings, inputType);
+    }
+
+    @Override
+    public URI uri() {
+        return getServiceSettings().getCommonSettings().uri();
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsServiceSettings.java
new file mode 100644
index 0000000000000..449da72674be4
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsServiceSettings.java
@@ -0,0 +1,162 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai.embeddings;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ServiceSettings;
+import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettings;
+import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.services.ServiceFields.DIMENSIONS;
+import static org.elasticsearch.xpack.inference.services.ServiceFields.MAX_INPUT_TOKENS;
+import static org.elasticsearch.xpack.inference.services.ServiceFields.SIMILARITY;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractSimilarity;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeAsType;
+
+public class JinaAIEmbeddingsServiceSettings extends FilteredXContentObject implements ServiceSettings {
+    public static final String NAME = "jinaai_embeddings_service_settings";
+
+    public static JinaAIEmbeddingsServiceSettings fromMap(Map<String, Object> map, ConfigurationParseContext context) {
+        ValidationException validationException = new ValidationException();
+        var commonServiceSettings = JinaAIServiceSettings.fromMap(map, context);
+        SimilarityMeasure similarity = extractSimilarity(map, ModelConfigurations.SERVICE_SETTINGS, validationException);
+        Integer dims = removeAsType(map, DIMENSIONS, Integer.class);
+        Integer maxInputTokens = removeAsType(map, MAX_INPUT_TOKENS, Integer.class);
+
+        if (validationException.validationErrors().isEmpty() == false) {
+            throw validationException;
+        }
+
+        return new JinaAIEmbeddingsServiceSettings(commonServiceSettings, similarity, dims, maxInputTokens);
+    }
+
+    private final JinaAIServiceSettings commonSettings;
+    private final SimilarityMeasure similarity;
+    private final Integer dimensions;
+    private final Integer maxInputTokens;
+
+    public JinaAIEmbeddingsServiceSettings(
+        JinaAIServiceSettings commonSettings,
+        @Nullable SimilarityMeasure similarity,
+        @Nullable Integer dimensions,
+        @Nullable Integer maxInputTokens
+    ) {
+        this.commonSettings = commonSettings;
+        this.similarity = similarity;
+        this.dimensions = dimensions;
+        this.maxInputTokens = maxInputTokens;
+    }
+
+    public JinaAIEmbeddingsServiceSettings(StreamInput in) throws IOException {
+        this.commonSettings = new JinaAIServiceSettings(in);
+        this.similarity = in.readOptionalEnum(SimilarityMeasure.class);
+        this.dimensions = in.readOptionalVInt();
+        this.maxInputTokens = in.readOptionalVInt();
+    }
+
+    public JinaAIServiceSettings getCommonSettings() {
+        return commonSettings;
+    }
+
+    @Override
+    public SimilarityMeasure similarity() {
+        return similarity;
+    }
+
+    @Override
+    public Integer dimensions() {
+        return dimensions;
+    }
+
+    public Integer maxInputTokens() {
+        return maxInputTokens;
+    }
+
+    @Override
+    public String modelId() {
+        return commonSettings.modelId();
+    }
+
+    @Override
+    public DenseVectorFieldMapper.ElementType elementType() {
+        return DenseVectorFieldMapper.ElementType.FLOAT;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        builder = commonSettings.toXContentFragment(builder, params);
+        if (similarity != null) {
+            builder.field(SIMILARITY, similarity);
+        }
+        if (dimensions != null) {
+            builder.field(DIMENSIONS, dimensions);
+        }
+        if (maxInputTokens != null) {
+            builder.field(MAX_INPUT_TOKENS, maxInputTokens);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException {
+        commonSettings.toXContentFragmentOfExposedFields(builder, params);
+
+        return builder;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.JINA_AI_INTEGRATION_ADDED;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        commonSettings.writeTo(out);
+        out.writeOptionalEnum(SimilarityMeasure.translateSimilarity(similarity, out.getTransportVersion()));
+        out.writeOptionalVInt(dimensions);
+        out.writeOptionalVInt(maxInputTokens);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        JinaAIEmbeddingsServiceSettings that = (JinaAIEmbeddingsServiceSettings) o;
+        return Objects.equals(commonSettings, that.commonSettings)
+            && Objects.equals(similarity, that.similarity)
+            && Objects.equals(dimensions, that.dimensions)
+            && Objects.equals(maxInputTokens, that.maxInputTokens);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(commonSettings, similarity, dimensions, maxInputTokens);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsTaskSettings.java
new file mode 100644
index 0000000000000..77150b5097aa6
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsTaskSettings.java
@@ -0,0 +1,183 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai.embeddings;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.TaskSettings;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalEnum;
+
+/**
+ * Defines the task settings for the JinaAI text embeddings service.
+ *
+ */
+public class JinaAIEmbeddingsTaskSettings implements TaskSettings {
+
+    public static final String NAME = "jinaai_embeddings_task_settings";
+    public static final JinaAIEmbeddingsTaskSettings EMPTY_SETTINGS = new JinaAIEmbeddingsTaskSettings((InputType) null);
+    static final String INPUT_TYPE = "input_type";
+    static final EnumSet<InputType> VALID_REQUEST_VALUES = EnumSet.of(
+        InputType.INGEST,
+        InputType.SEARCH,
+        InputType.CLASSIFICATION,
+        InputType.CLUSTERING
+    );
+
+    public static JinaAIEmbeddingsTaskSettings fromMap(Map<String, Object> map) {
+        if (map == null || map.isEmpty()) {
+            return EMPTY_SETTINGS;
+        }
+
+        ValidationException validationException = new ValidationException();
+
+        InputType inputType = extractOptionalEnum(
+            map,
+            INPUT_TYPE,
+            ModelConfigurations.TASK_SETTINGS,
+            InputType::fromString,
+            VALID_REQUEST_VALUES,
+            validationException
+        );
+
+        if (validationException.validationErrors().isEmpty() == false) {
+            throw validationException;
+        }
+
+        return new JinaAIEmbeddingsTaskSettings(inputType);
+    }
+
+    /**
+     * Creates a new {@link JinaAIEmbeddingsTaskSettings} by preferring non-null fields from the provided parameters.
+     * For the input type, preference is given to requestInputType if it is not null and not UNSPECIFIED.
+     * Then preference is given to the requestTaskSettings and finally to originalSettings even if the value is null.
+     *
+     * Similarly, for the truncation field preference is given to requestTaskSettings if it is not null and then to
+     * originalSettings.
+     * @param originalSettings the settings stored as part of the inference entity configuration
+     * @param requestTaskSettings the settings passed in within the task_settings field of the request
+     * @param requestInputType the input type passed in the request parameters
+     * @return a constructed {@link JinaAIEmbeddingsTaskSettings}
+     */
+    public static JinaAIEmbeddingsTaskSettings of(
+        JinaAIEmbeddingsTaskSettings originalSettings,
+        JinaAIEmbeddingsTaskSettings requestTaskSettings,
+        InputType requestInputType
+    ) {
+        var inputTypeToUse = getValidInputType(originalSettings, requestTaskSettings, requestInputType);
+
+        return new JinaAIEmbeddingsTaskSettings(inputTypeToUse);
+    }
+
+    private static InputType getValidInputType(
+        JinaAIEmbeddingsTaskSettings originalSettings,
+        JinaAIEmbeddingsTaskSettings requestTaskSettings,
+        InputType requestInputType
+    ) {
+        InputType inputTypeToUse = originalSettings.inputType;
+
+        if (VALID_REQUEST_VALUES.contains(requestInputType)) {
+            inputTypeToUse = requestInputType;
+        } else if (requestTaskSettings.inputType != null) {
+            inputTypeToUse = requestTaskSettings.inputType;
+        }
+
+        return inputTypeToUse;
+    }
+
+    private final InputType inputType;
+
+    public JinaAIEmbeddingsTaskSettings(StreamInput in) throws IOException {
+        this(in.readOptionalEnum(InputType.class));
+    }
+
+    public JinaAIEmbeddingsTaskSettings(@Nullable InputType inputType) {
+        validateInputType(inputType);
+        this.inputType = inputType;
+    }
+
+    private static void validateInputType(InputType inputType) {
+        if (inputType == null) {
+            return;
+        }
+
+        assert VALID_REQUEST_VALUES.contains(inputType) : invalidInputTypeMessage(inputType);
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return inputType == null;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (inputType != null) {
+            builder.field(INPUT_TYPE, inputType);
+        }
+
+        builder.endObject();
+        return builder;
+    }
+
+    public InputType getInputType() {
+        return inputType;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.JINA_AI_INTEGRATION_ADDED;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalEnum(inputType);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        JinaAIEmbeddingsTaskSettings that = (JinaAIEmbeddingsTaskSettings) o;
+        return Objects.equals(inputType, that.inputType);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(inputType);
+    }
+
+    public static String invalidInputTypeMessage(InputType inputType) {
+        return Strings.format("received invalid input type value [%s]", inputType.toString());
+    }
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        JinaAIEmbeddingsTaskSettings updatedSettings = JinaAIEmbeddingsTaskSettings.fromMap(new HashMap<>(newSettings));
+        return of(this, updatedSettings, updatedSettings.inputType != null ? updatedSettings.inputType : this.inputType);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModel.java
new file mode 100644
index 0000000000000..b072d0508c8ee
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModel.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai.rerank;
+
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.action.jinaai.JinaAIActionVisitor;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.jinaai.JinaAIModel;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
+
+import java.net.URI;
+import java.util.Map;
+
+public class JinaAIRerankModel extends JinaAIModel {
+    public static JinaAIRerankModel of(JinaAIRerankModel model, Map<String, Object> taskSettings) {
+        var requestTaskSettings = JinaAIRerankTaskSettings.fromMap(taskSettings);
+        return new JinaAIRerankModel(model, JinaAIRerankTaskSettings.of(model.getTaskSettings(), requestTaskSettings));
+    }
+
+    public JinaAIRerankModel(
+        String inferenceId,
+        String service,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        @Nullable Map<String, Object> secrets,
+        ConfigurationParseContext context
+    ) {
+        this(
+            inferenceId,
+            service,
+            JinaAIRerankServiceSettings.fromMap(serviceSettings, context),
+            JinaAIRerankTaskSettings.fromMap(taskSettings),
+            DefaultSecretSettings.fromMap(secrets)
+        );
+    }
+
+    // should only be used for testing
+    JinaAIRerankModel(
+        String modelId,
+        String service,
+        JinaAIRerankServiceSettings serviceSettings,
+        JinaAIRerankTaskSettings taskSettings,
+        @Nullable DefaultSecretSettings secretSettings
+    ) {
+        super(
+            new ModelConfigurations(modelId, TaskType.RERANK, service, serviceSettings, taskSettings),
+            new ModelSecrets(secretSettings),
+            secretSettings,
+            serviceSettings.getCommonSettings()
+        );
+    }
+
+    private JinaAIRerankModel(JinaAIRerankModel model, JinaAIRerankTaskSettings taskSettings) {
+        super(model, taskSettings);
+    }
+
+    public JinaAIRerankModel(JinaAIRerankModel model, JinaAIRerankServiceSettings serviceSettings) {
+        super(model, serviceSettings);
+    }
+
+    @Override
+    public JinaAIRerankServiceSettings getServiceSettings() {
+        return (JinaAIRerankServiceSettings) super.getServiceSettings();
+    }
+
+    @Override
+    public JinaAIRerankTaskSettings getTaskSettings() {
+        return (JinaAIRerankTaskSettings) super.getTaskSettings();
+    }
+
+    @Override
+    public DefaultSecretSettings getSecretSettings() {
+        return (DefaultSecretSettings) super.getSecretSettings();
+    }
+
+    /**
+     * Accepts a visitor to create an executable action. The returned action will not return documents in the response.
+     * @param visitor _
+     * @param taskSettings _
+     * @param inputType ignored for rerank task
+     * @return the rerank action
+     */
+    @Override
+    public ExecutableAction accept(JinaAIActionVisitor visitor, Map<String, Object> taskSettings, InputType inputType) {
+        return visitor.create(this, taskSettings);
+    }
+
+    @Override
+    public URI uri() {
+        return getServiceSettings().getCommonSettings().uri();
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankServiceSettings.java
new file mode 100644
index 0000000000000..a9e492c2738c2
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankServiceSettings.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai.rerank;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.inference.ServiceSettings;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.jinaai.JinaAIRateLimitServiceSettings;
+import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettings;
+import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+public class JinaAIRerankServiceSettings extends FilteredXContentObject implements ServiceSettings, JinaAIRateLimitServiceSettings {
+    public static final String NAME = "jinaai_rerank_service_settings";
+
+    private static final Logger logger = LogManager.getLogger(JinaAIRerankServiceSettings.class);
+
+    public static JinaAIRerankServiceSettings fromMap(Map<String, Object> map, ConfigurationParseContext context) {
+        ValidationException validationException = new ValidationException();
+
+        if (validationException.validationErrors().isEmpty() == false) {
+            throw validationException;
+        }
+
+        var commonServiceSettings = JinaAIServiceSettings.fromMap(map, context);
+
+        return new JinaAIRerankServiceSettings(commonServiceSettings);
+    }
+
+    private final JinaAIServiceSettings commonSettings;
+
+    public JinaAIRerankServiceSettings(JinaAIServiceSettings commonSettings) {
+        this.commonSettings = commonSettings;
+    }
+
+    public JinaAIRerankServiceSettings(StreamInput in) throws IOException {
+        this.commonSettings = new JinaAIServiceSettings(in);
+    }
+
+    public JinaAIServiceSettings getCommonSettings() {
+        return commonSettings;
+    }
+
+    @Override
+    public String modelId() {
+        return commonSettings.modelId();
+    }
+
+    @Override
+    public RateLimitSettings rateLimitSettings() {
+        return commonSettings.rateLimitSettings();
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        builder = commonSettings.toXContentFragment(builder, params);
+
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException {
+        commonSettings.toXContentFragmentOfExposedFields(builder, params);
+        return builder;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.JINA_AI_INTEGRATION_ADDED;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        commonSettings.writeTo(out);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        JinaAIRerankServiceSettings that = (JinaAIRerankServiceSettings) o;
+        return Objects.equals(commonSettings, that.commonSettings);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(commonSettings);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankTaskSettings.java
new file mode 100644
index 0000000000000..8dd93d89b6fb5
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankTaskSettings.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai.rerank;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.TaskSettings;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalBoolean;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalPositiveInteger;
+
+/**
+ * Defines the task settings for the JinaAI rerank service.
+ *
+ */
+public class JinaAIRerankTaskSettings implements TaskSettings {
+
+    public static final String NAME = "jinaai_rerank_task_settings";
+    public static final String RETURN_DOCUMENTS = "return_documents";
+    public static final String TOP_N_DOCS_ONLY = "top_n";
+
+    public static final JinaAIRerankTaskSettings EMPTY_SETTINGS = new JinaAIRerankTaskSettings(null, null);
+
+    public static JinaAIRerankTaskSettings fromMap(Map<String, Object> map) {
+        ValidationException validationException = new ValidationException();
+
+        if (map == null || map.isEmpty()) {
+            return EMPTY_SETTINGS;
+        }
+
+        Boolean returnDocuments = extractOptionalBoolean(map, RETURN_DOCUMENTS, validationException);
+        Integer topNDocumentsOnly = extractOptionalPositiveInteger(
+            map,
+            TOP_N_DOCS_ONLY,
+            ModelConfigurations.TASK_SETTINGS,
+            validationException
+        );
+
+        if (validationException.validationErrors().isEmpty() == false) {
+            throw validationException;
+        }
+
+        return of(topNDocumentsOnly, returnDocuments);
+    }
+
+    /**
+     * Creates a new {@link JinaAIRerankTaskSettings} by preferring non-null fields from the request settings over the original settings.
+     *
+     * @param originalSettings    the settings stored as part of the inference entity configuration
+     * @param requestTaskSettings the settings passed in within the task_settings field of the request
+     * @return a constructed {@link JinaAIRerankTaskSettings}
+     */
+    public static JinaAIRerankTaskSettings of(JinaAIRerankTaskSettings originalSettings, JinaAIRerankTaskSettings requestTaskSettings) {
+        return new JinaAIRerankTaskSettings(
+            requestTaskSettings.getTopNDocumentsOnly() != null
+                ? requestTaskSettings.getTopNDocumentsOnly()
+                : originalSettings.getTopNDocumentsOnly(),
+            requestTaskSettings.getReturnDocuments() != null
+                ? requestTaskSettings.getReturnDocuments()
+                : originalSettings.getReturnDocuments()
+        );
+    }
+
+    public static JinaAIRerankTaskSettings of(Integer topNDocumentsOnly, Boolean returnDocuments) {
+        return new JinaAIRerankTaskSettings(topNDocumentsOnly, returnDocuments);
+    }
+
+    private final Integer topNDocumentsOnly;
+    private final Boolean returnDocuments;
+
+    public JinaAIRerankTaskSettings(StreamInput in) throws IOException {
+        this(in.readOptionalInt(), in.readOptionalBoolean());
+    }
+
+    public JinaAIRerankTaskSettings(@Nullable Integer topNDocumentsOnly, @Nullable Boolean doReturnDocuments) {
+        this.topNDocumentsOnly = topNDocumentsOnly;
+        this.returnDocuments = doReturnDocuments;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return topNDocumentsOnly == null && returnDocuments == null;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (topNDocumentsOnly != null) {
+            builder.field(TOP_N_DOCS_ONLY, topNDocumentsOnly);
+        }
+        if (returnDocuments != null) {
+            builder.field(RETURN_DOCUMENTS, returnDocuments);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.JINA_AI_INTEGRATION_ADDED;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalInt(topNDocumentsOnly);
+        out.writeOptionalBoolean(returnDocuments);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        JinaAIRerankTaskSettings that = (JinaAIRerankTaskSettings) o;
+        return Objects.equals(returnDocuments, that.returnDocuments) && Objects.equals(topNDocumentsOnly, that.topNDocumentsOnly);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(returnDocuments, topNDocumentsOnly);
+    }
+
+    public static String invalidInputTypeMessage(InputType inputType) {
+        return Strings.format("received invalid input type value [%s]", inputType.toString());
+    }
+
+    public Boolean getDoesReturnDocuments() {
+        return returnDocuments;
+    }
+
+    public Integer getTopNDocumentsOnly() {
+        return topNDocumentsOnly;
+    }
+
+    public Boolean getReturnDocuments() {
+        return returnDocuments;
+    }
+
+    @Override
+    public TaskSettings updatedTaskSettings(Map<String, Object> newSettings) {
+        JinaAIRerankTaskSettings updatedSettings = JinaAIRerankTaskSettings.fromMap(new HashMap<>(newSettings));
+        return JinaAIRerankTaskSettings.of(this, updatedSettings);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java
index dc0576651aeba..129d9023a1ebc 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
 import org.elasticsearch.inference.ChunkingSettings;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
@@ -25,9 +24,7 @@
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.SimilarityMeasure;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder;
@@ -65,6 +62,7 @@
 public class MistralService extends SenderService {
     public static final String NAME = "mistral";
 
+    private static final String SERVICE_NAME = "Mistral";
     private static final EnumSet<TaskType> supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING);
 
     public MistralService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) {
@@ -320,24 +318,24 @@ public static InferenceServiceConfiguration get() {
 
                 configurationMap.put(
                     MODEL_FIELD,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription(
+                        "Refer to the Mistral models documentation for the list of available text embedding models."
+                    )
                         .setLabel("Model")
-                        .setOrder(2)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("Refer to the Mistral models documentation for the list of available text embedding models.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.put(
                     MAX_INPUT_TOKENS,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
+                    new SettingsConfiguration.Builder().setDescription("Allows you to specify the maximum number of tokens per input.")
                         .setLabel("Maximum Input Tokens")
-                        .setOrder(3)
                         .setRequired(false)
                         .setSensitive(false)
-                        .setTooltip("Allows you to specify the maximum number of tokens per input.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.INTEGER)
                         .build()
                 );
@@ -345,14 +343,11 @@ public static InferenceServiceConfiguration get() {
                 configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration());
                 configurationMap.putAll(RateLimitSettings.toSettingsConfiguration());
 
-                return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                    Map<String, SettingsConfiguration> taskSettingsConfig;
-                    switch (t) {
-                        // TEXT_EMBEDDING task type has no task settings
-                        default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                    }
-                    return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                }).toList()).setConfiguration(configurationMap).build();
+                return new InferenceServiceConfiguration.Builder().setService(NAME)
+                    .setName(SERVICE_NAME)
+                    .setTaskTypes(supportedTaskTypes)
+                    .setConfigurations(configurationMap)
+                    .build();
             }
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java
index ed7a01e829dd2..ba9dea8ace8ee 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
 import org.elasticsearch.inference.ChunkingSettings;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
@@ -25,9 +24,7 @@
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.SimilarityMeasure;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder;
@@ -72,6 +69,7 @@
 public class OpenAiService extends SenderService {
     public static final String NAME = "openai";
 
+    private static final String SERVICE_NAME = "OpenAI";
     private static final EnumSet<TaskType> supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.COMPLETION);
 
     public OpenAiService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) {
@@ -397,65 +395,58 @@ public static InferenceServiceConfiguration get() {
 
                 configurationMap.put(
                     MODEL_ID,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("The name of the model to use for the inference task.")
                         .setLabel("Model ID")
-                        .setOrder(2)
                         .setRequired(true)
                         .setSensitive(false)
-                        .setTooltip("The name of the model to use for the inference task.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.put(
                     ORGANIZATION,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+                    new SettingsConfiguration.Builder().setDescription("The unique identifier of your organization.")
                         .setLabel("Organization ID")
-                        .setOrder(3)
                         .setRequired(false)
                         .setSensitive(false)
-                        .setTooltip("The unique identifier of your organization.")
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
                         .build()
                 );
 
                 configurationMap.put(
                     URL,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
-                        .setLabel("URL")
-                        .setOrder(4)
-                        .setRequired(true)
-                        .setSensitive(false)
-                        .setTooltip(
+                    new SettingsConfiguration.Builder().setDefaultValue("https://api.openai.com/v1/chat/completions")
+                        .setDescription(
                             "The OpenAI API endpoint URL. For more information on the URL, refer to the "
                                 + "https://platform.openai.com/docs/api-reference."
                         )
+                        .setLabel("URL")
+                        .setRequired(true)
+                        .setSensitive(false)
+                        .setUpdatable(false)
                         .setType(SettingsConfigurationFieldType.STRING)
-                        .setDefaultValue("https://api.openai.com/v1/chat/completions")
                         .build()
                 );
 
                 configurationMap.putAll(
-                    DefaultSecretSettings.toSettingsConfigurationWithTooltip(
+                    DefaultSecretSettings.toSettingsConfigurationWithDescription(
                         "The OpenAI API authentication key. For more details about generating OpenAI API keys, "
                             + "refer to the https://platform.openai.com/account/api-keys."
                     )
                 );
                 configurationMap.putAll(
-                    RateLimitSettings.toSettingsConfigurationWithTooltip(
+                    RateLimitSettings.toSettingsConfigurationWithDescription(
                         "Default number of requests allowed per minute. For text_embedding is 3000. For completion is 500."
                     )
                 );
 
-                return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> {
-                    Map<String, SettingsConfiguration> taskSettingsConfig;
-                    switch (t) {
-                        case TEXT_EMBEDDING -> taskSettingsConfig = OpenAiEmbeddingsModel.Configuration.get();
-                        case COMPLETION -> taskSettingsConfig = OpenAiChatCompletionModel.Configuration.get();
-                        default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                    }
-                    return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                }).toList()).setConfiguration(configurationMap).build();
+                return new InferenceServiceConfiguration.Builder().setService(NAME)
+                    .setName(SERVICE_NAME)
+                    .setTaskTypes(supportedTaskTypes)
+                    .setConfigurations(configurationMap)
+                    .build();
             }
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java
index 7d79d64b3a771..dea703b9ce243 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java
@@ -7,28 +7,20 @@
 
 package org.elasticsearch.xpack.inference.services.openai.completion;
 
-import org.elasticsearch.common.util.LazyInitializable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
-import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.inference.UnifiedCompletionRequest;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.openai.OpenAiActionVisitor;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.openai.OpenAiModel;
 import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.xpack.inference.services.openai.OpenAiServiceFields.USER;
-
 public class OpenAiChatCompletionModel extends OpenAiModel {
 
     public static OpenAiChatCompletionModel of(OpenAiChatCompletionModel model, Map<String, Object> taskSettings) {
@@ -118,30 +110,4 @@ public DefaultSecretSettings getSecretSettings() {
     public ExecutableAction accept(OpenAiActionVisitor creator, Map<String, Object> taskSettings) {
         return creator.create(this, taskSettings);
     }
-
-    public static class Configuration {
-        public static Map<String, SettingsConfiguration> get() {
-            return configuration.getOrCompute();
-        }
-
-        private static final LazyInitializable<Map<String, SettingsConfiguration>, RuntimeException> configuration =
-            new LazyInitializable<>(() -> {
-                var configurationMap = new HashMap<String, SettingsConfiguration>();
-
-                configurationMap.put(
-                    USER,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
-                        .setLabel("User")
-                        .setOrder(1)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Specifies the user issuing the request.")
-                        .setType(SettingsConfigurationFieldType.STRING)
-                        .setValue("")
-                        .build()
-                );
-
-                return Collections.unmodifiableMap(configurationMap);
-            });
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModel.java
index cab2a82fc86c6..5659c46050ad8 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModel.java
@@ -7,27 +7,19 @@
 
 package org.elasticsearch.xpack.inference.services.openai.embeddings;
 
-import org.elasticsearch.common.util.LazyInitializable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.ChunkingSettings;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
-import org.elasticsearch.inference.SettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
-import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.openai.OpenAiActionVisitor;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.openai.OpenAiModel;
 import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.inference.services.openai.OpenAiServiceFields.USER;
-
 public class OpenAiEmbeddingsModel extends OpenAiModel {
 
     public static OpenAiEmbeddingsModel of(OpenAiEmbeddingsModel model, Map<String, Object> taskSettings) {
@@ -105,30 +97,4 @@ public DefaultSecretSettings getSecretSettings() {
     public ExecutableAction accept(OpenAiActionVisitor creator, Map<String, Object> taskSettings) {
         return creator.create(this, taskSettings);
     }
-
-    public static class Configuration {
-        public static Map<String, SettingsConfiguration> get() {
-            return configuration.getOrCompute();
-        }
-
-        private static final LazyInitializable<Map<String, SettingsConfiguration>, RuntimeException> configuration =
-            new LazyInitializable<>(() -> {
-                var configurationMap = new HashMap<String, SettingsConfiguration>();
-
-                configurationMap.put(
-                    USER,
-                    new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
-                        .setLabel("User")
-                        .setOrder(1)
-                        .setRequired(false)
-                        .setSensitive(false)
-                        .setTooltip("Specifies the user issuing the request.")
-                        .setType(SettingsConfigurationFieldType.STRING)
-                        .setValue("")
-                        .build()
-                );
-
-                return Collections.unmodifiableMap(configurationMap);
-            });
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java
index 771d2308a502f..15e8128969ddb 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.SecretSettings;
 import org.elasticsearch.inference.SettingsConfiguration;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -52,16 +51,15 @@ public static DefaultSecretSettings fromMap(@Nullable Map<String, Object> map) {
         return new DefaultSecretSettings(secureApiToken);
     }
 
-    public static Map<String, SettingsConfiguration> toSettingsConfigurationWithTooltip(String tooltip) {
+    public static Map<String, SettingsConfiguration> toSettingsConfigurationWithDescription(String description) {
         var configurationMap = new HashMap<String, SettingsConfiguration>();
         configurationMap.put(
             API_KEY,
-            new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX)
+            new SettingsConfiguration.Builder().setDescription(description)
                 .setLabel("API Key")
-                .setOrder(1)
                 .setRequired(true)
                 .setSensitive(true)
-                .setTooltip(tooltip)
+                .setUpdatable(true)
                 .setType(SettingsConfigurationFieldType.STRING)
                 .build()
         );
@@ -69,7 +67,7 @@ public static Map<String, SettingsConfiguration> toSettingsConfigurationWithTool
     }
 
     public static Map<String, SettingsConfiguration> toSettingsConfiguration() {
-        return DefaultSecretSettings.toSettingsConfigurationWithTooltip("API Key for the provider you're connecting to.");
+        return DefaultSecretSettings.toSettingsConfigurationWithDescription("API Key for the provider you're connecting to.");
     }
 
     public DefaultSecretSettings {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/RateLimitSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/RateLimitSettings.java
index 416d5bff12ce9..30147a6d24a96 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/RateLimitSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/RateLimitSettings.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.inference.SettingsConfiguration;
-import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType;
 import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -52,16 +51,15 @@ public static RateLimitSettings of(
         return requestsPerMinute == null ? defaultValue : new RateLimitSettings(requestsPerMinute);
     }
 
-    public static Map<String, SettingsConfiguration> toSettingsConfigurationWithTooltip(String tooltip) {
+    public static Map<String, SettingsConfiguration> toSettingsConfigurationWithDescription(String description) {
         var configurationMap = new HashMap<String, SettingsConfiguration>();
         configurationMap.put(
             FIELD_NAME + "." + REQUESTS_PER_MINUTE_FIELD,
-            new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC)
+            new SettingsConfiguration.Builder().setDescription(description)
                 .setLabel("Rate Limit")
-                .setOrder(6)
                 .setRequired(false)
                 .setSensitive(false)
-                .setTooltip(tooltip)
+                .setUpdatable(false)
                 .setType(SettingsConfigurationFieldType.INTEGER)
                 .build()
         );
@@ -69,7 +67,7 @@ public static Map<String, SettingsConfiguration> toSettingsConfigurationWithTool
     }
 
     public static Map<String, SettingsConfiguration> toSettingsConfiguration() {
-        return RateLimitSettings.toSettingsConfigurationWithTooltip("Minimize the number of rate limit errors.");
+        return RateLimitSettings.toSettingsConfigurationWithDescription("Minimize the number of rate limit errors.");
     }
 
     /**
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/TraceContextHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/TraceContextHandler.java
new file mode 100644
index 0000000000000..92fe214d821db
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/TraceContextHandler.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.telemetry;
+
+import org.apache.http.client.methods.HttpPost;
+import org.elasticsearch.tasks.Task;
+
+public record TraceContextHandler(TraceContext traceContext) {
+
+    public void propagateTraceContext(HttpPost httpPost) {
+        if (traceContext == null) {
+            return;
+        }
+
+        var traceParent = traceContext.traceParent();
+        var traceState = traceContext.traceState();
+
+        if (traceParent != null) {
+            httpPost.setHeader(Task.TRACE_PARENT_HTTP_HEADER, traceParent);
+        }
+
+        if (traceState != null) {
+            httpPost.setHeader(Task.TRACE_STATE, traceState);
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/inference/src/main/plugin-metadata/entitlement-policy.yaml
new file mode 100644
index 0000000000000..41383d0b6736a
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/plugin-metadata/entitlement-policy.yaml
@@ -0,0 +1,2 @@
+com.google.api.client:
+  - set_https_connection_properties
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/cluster/metadata/SemanticTextClusterMetadataTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/cluster/metadata/SemanticTextClusterMetadataTests.java
index 61033a0211065..e49d98d48eb0c 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/cluster/metadata/SemanticTextClusterMetadataTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/cluster/metadata/SemanticTextClusterMetadataTests.java
@@ -14,8 +14,7 @@
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.elasticsearch.xpack.core.XPackPlugin;
-import org.elasticsearch.xpack.inference.InferencePlugin;
+import org.elasticsearch.xpack.inference.LocalStateInferencePlugin;
 import org.hamcrest.Matchers;
 
 import java.util.Arrays;
@@ -29,7 +28,7 @@ public class SemanticTextClusterMetadataTests extends ESSingleNodeTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return List.of(XPackPlugin.class, InferencePlugin.class);
+        return List.of(LocalStateInferencePlugin.class);
     }
 
     public void testCreateIndexWithSemanticTextField() {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/index/query/SemanticKnnVectorQueryRewriteInterceptorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/index/query/SemanticKnnVectorQueryRewriteInterceptorTests.java
new file mode 100644
index 0000000000000..073bf8f5afb9a
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/index/query/SemanticKnnVectorQueryRewriteInterceptorTests.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.index.query;
+
+import org.elasticsearch.action.MockResolvedIndices;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.ResolvedIndices;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.plugins.internal.rewriter.QueryRewriteInterceptor;
+import org.elasticsearch.search.vectors.KnnVectorQueryBuilder;
+import org.elasticsearch.search.vectors.QueryVectorBuilder;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.client.NoOpClient;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.xpack.core.ml.vectors.TextEmbeddingQueryVectorBuilder;
+import org.elasticsearch.xpack.inference.mapper.SemanticTextField;
+import org.elasticsearch.xpack.inference.queries.SemanticKnnVectorQueryRewriteInterceptor;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class SemanticKnnVectorQueryRewriteInterceptorTests extends ESTestCase {
+
+    private TestThreadPool threadPool;
+    private NoOpClient client;
+    private Index index;
+
+    private static final String FIELD_NAME = "fieldName";
+    private static final String INFERENCE_ID = "inferenceId";
+    private static final String QUERY = "query";
+
+    @Before
+    public void setup() {
+        threadPool = createThreadPool();
+        client = new NoOpClient(threadPool);
+        index = new Index(randomAlphaOfLength(10), randomAlphaOfLength(10));
+    }
+
+    @After
+    public void cleanup() {
+        threadPool.close();
+    }
+
+    public void testKnnQueryWithVectorBuilderIsInterceptedAndRewritten() throws IOException {
+        Map<String, InferenceFieldMetadata> inferenceFields = Map.of(
+            FIELD_NAME,
+            new InferenceFieldMetadata(index.getName(), INFERENCE_ID, new String[] { FIELD_NAME })
+        );
+        QueryRewriteContext context = createQueryRewriteContext(inferenceFields);
+        QueryVectorBuilder queryVectorBuilder = new TextEmbeddingQueryVectorBuilder(INFERENCE_ID, QUERY);
+        KnnVectorQueryBuilder original = new KnnVectorQueryBuilder(FIELD_NAME, queryVectorBuilder, 10, 100, null);
+        testRewrittenInferenceQuery(context, original);
+    }
+
+    public void testKnnWithQueryBuilderWithoutInferenceIdIsInterceptedAndRewritten() throws IOException {
+        Map<String, InferenceFieldMetadata> inferenceFields = Map.of(
+            FIELD_NAME,
+            new InferenceFieldMetadata(index.getName(), INFERENCE_ID, new String[] { FIELD_NAME })
+        );
+        QueryRewriteContext context = createQueryRewriteContext(inferenceFields);
+        QueryVectorBuilder queryVectorBuilder = new TextEmbeddingQueryVectorBuilder(null, QUERY);
+        KnnVectorQueryBuilder original = new KnnVectorQueryBuilder(FIELD_NAME, queryVectorBuilder, 10, 100, null);
+        testRewrittenInferenceQuery(context, original);
+    }
+
+    private void testRewrittenInferenceQuery(QueryRewriteContext context, KnnVectorQueryBuilder original) throws IOException {
+        QueryBuilder rewritten = original.rewrite(context);
+        assertTrue(
+            "Expected query to be intercepted, but was [" + rewritten.getClass().getName() + "]",
+            rewritten instanceof InterceptedQueryBuilderWrapper
+        );
+        InterceptedQueryBuilderWrapper intercepted = (InterceptedQueryBuilderWrapper) rewritten;
+        assertTrue(intercepted.queryBuilder instanceof NestedQueryBuilder);
+        NestedQueryBuilder nestedQueryBuilder = (NestedQueryBuilder) intercepted.queryBuilder;
+        assertEquals(SemanticTextField.getChunksFieldName(FIELD_NAME), nestedQueryBuilder.path());
+        QueryBuilder innerQuery = nestedQueryBuilder.query();
+        assertTrue(innerQuery instanceof KnnVectorQueryBuilder);
+        KnnVectorQueryBuilder knnVectorQueryBuilder = (KnnVectorQueryBuilder) innerQuery;
+        assertEquals(SemanticTextField.getEmbeddingsFieldName(FIELD_NAME), knnVectorQueryBuilder.getFieldName());
+        assertTrue(knnVectorQueryBuilder.queryVectorBuilder() instanceof TextEmbeddingQueryVectorBuilder);
+        TextEmbeddingQueryVectorBuilder textEmbeddingQueryVectorBuilder = (TextEmbeddingQueryVectorBuilder) knnVectorQueryBuilder
+            .queryVectorBuilder();
+        assertEquals(QUERY, textEmbeddingQueryVectorBuilder.getModelText());
+        assertEquals(INFERENCE_ID, textEmbeddingQueryVectorBuilder.getModelId());
+    }
+
+    public void testKnnVectorQueryOnNonInferenceFieldRemainsUnchanged() throws IOException {
+        QueryRewriteContext context = createQueryRewriteContext(Map.of()); // No inference fields
+        QueryVectorBuilder queryVectorBuilder = new TextEmbeddingQueryVectorBuilder(null, QUERY);
+        QueryBuilder original = new KnnVectorQueryBuilder(FIELD_NAME, queryVectorBuilder, 10, 100, null);
+        QueryBuilder rewritten = original.rewrite(context);
+        assertTrue(
+            "Expected query to remain knn but was [" + rewritten.getClass().getName() + "]",
+            rewritten instanceof KnnVectorQueryBuilder
+        );
+        assertEquals(original, rewritten);
+    }
+
+    private QueryRewriteContext createQueryRewriteContext(Map<String, InferenceFieldMetadata> inferenceFields) {
+        IndexMetadata indexMetadata = IndexMetadata.builder(index.getName())
+            .settings(
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                    .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID())
+            )
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .putInferenceFields(inferenceFields)
+            .build();
+
+        ResolvedIndices resolvedIndices = new MockResolvedIndices(
+            Map.of(),
+            new OriginalIndices(new String[] { index.getName() }, IndicesOptions.DEFAULT),
+            Map.of(index, indexMetadata)
+        );
+
+        return new QueryRewriteContext(null, client, null, resolvedIndices, null, createRewriteInterceptor());
+    }
+
+    private QueryRewriteInterceptor createRewriteInterceptor() {
+        return new SemanticKnnVectorQueryRewriteInterceptor();
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/index/query/SemanticMatchQueryRewriteInterceptorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/index/query/SemanticMatchQueryRewriteInterceptorTests.java
new file mode 100644
index 0000000000000..47705c14d5941
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/index/query/SemanticMatchQueryRewriteInterceptorTests.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.index.query;
+
+import org.elasticsearch.action.MockResolvedIndices;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.ResolvedIndices;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.plugins.internal.rewriter.QueryRewriteInterceptor;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.client.NoOpClient;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.xpack.inference.queries.SemanticMatchQueryRewriteInterceptor;
+import org.elasticsearch.xpack.inference.queries.SemanticQueryBuilder;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class SemanticMatchQueryRewriteInterceptorTests extends ESTestCase {
+
+    private TestThreadPool threadPool;
+    private NoOpClient client;
+    private Index index;
+
+    private static final String FIELD_NAME = "fieldName";
+    private static final String VALUE = "value";
+
+    @Before
+    public void setup() {
+        threadPool = createThreadPool();
+        client = new NoOpClient(threadPool);
+        index = new Index(randomAlphaOfLength(10), randomAlphaOfLength(10));
+    }
+
+    @After
+    public void cleanup() {
+        threadPool.close();
+    }
+
+    public void testMatchQueryOnInferenceFieldIsInterceptedAndRewrittenToSemanticQuery() throws IOException {
+        Map<String, InferenceFieldMetadata> inferenceFields = Map.of(
+            FIELD_NAME,
+            new InferenceFieldMetadata(index.getName(), "inferenceId", new String[] { FIELD_NAME })
+        );
+        QueryRewriteContext context = createQueryRewriteContext(inferenceFields);
+        QueryBuilder original = createTestQueryBuilder();
+        QueryBuilder rewritten = original.rewrite(context);
+        assertTrue(
+            "Expected query to be intercepted, but was [" + rewritten.getClass().getName() + "]",
+            rewritten instanceof InterceptedQueryBuilderWrapper
+        );
+        InterceptedQueryBuilderWrapper intercepted = (InterceptedQueryBuilderWrapper) rewritten;
+        assertTrue(intercepted.queryBuilder instanceof SemanticQueryBuilder);
+        SemanticQueryBuilder semanticQueryBuilder = (SemanticQueryBuilder) intercepted.queryBuilder;
+        assertEquals(FIELD_NAME, semanticQueryBuilder.getFieldName());
+        assertEquals(VALUE, semanticQueryBuilder.getQuery());
+    }
+
+    public void testMatchQueryOnNonInferenceFieldRemainsMatchQuery() throws IOException {
+        QueryRewriteContext context = createQueryRewriteContext(Map.of()); // No inference fields
+        QueryBuilder original = createTestQueryBuilder();
+        QueryBuilder rewritten = original.rewrite(context);
+        assertTrue(
+            "Expected query to remain match but was [" + rewritten.getClass().getName() + "]",
+            rewritten instanceof MatchQueryBuilder
+        );
+        assertEquals(original, rewritten);
+    }
+
+    private MatchQueryBuilder createTestQueryBuilder() {
+        return new MatchQueryBuilder(FIELD_NAME, VALUE);
+    }
+
+    private QueryRewriteContext createQueryRewriteContext(Map<String, InferenceFieldMetadata> inferenceFields) {
+        IndexMetadata indexMetadata = IndexMetadata.builder(index.getName())
+            .settings(
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                    .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID())
+            )
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .putInferenceFields(inferenceFields)
+            .build();
+
+        ResolvedIndices resolvedIndices = new MockResolvedIndices(
+            Map.of(),
+            new OriginalIndices(new String[] { index.getName() }, IndicesOptions.DEFAULT),
+            Map.of(index, indexMetadata)
+        );
+
+        return new QueryRewriteContext(null, client, null, resolvedIndices, null, createRewriteInterceptor());
+    }
+
+    private QueryRewriteInterceptor createRewriteInterceptor() {
+        return new SemanticMatchQueryRewriteInterceptor();
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/index/query/SemanticSparseVectorQueryRewriteInterceptorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/index/query/SemanticSparseVectorQueryRewriteInterceptorTests.java
new file mode 100644
index 0000000000000..1adad1df7b29b
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/index/query/SemanticSparseVectorQueryRewriteInterceptorTests.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.index.query;
+
+import org.elasticsearch.action.MockResolvedIndices;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.ResolvedIndices;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.plugins.internal.rewriter.QueryRewriteInterceptor;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.client.NoOpClient;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.xpack.core.ml.search.SparseVectorQueryBuilder;
+import org.elasticsearch.xpack.inference.mapper.SemanticTextField;
+import org.elasticsearch.xpack.inference.queries.SemanticSparseVectorQueryRewriteInterceptor;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class SemanticSparseVectorQueryRewriteInterceptorTests extends ESTestCase {
+
+    private TestThreadPool threadPool;
+    private NoOpClient client;
+    private Index index;
+
+    private static final String FIELD_NAME = "fieldName";
+    private static final String INFERENCE_ID = "inferenceId";
+    private static final String QUERY = "query";
+
+    @Before
+    public void setup() {
+        threadPool = createThreadPool();
+        client = new NoOpClient(threadPool);
+        index = new Index(randomAlphaOfLength(10), randomAlphaOfLength(10));
+    }
+
+    @After
+    public void cleanup() {
+        threadPool.close();
+    }
+
+    public void testSparseVectorQueryOnInferenceFieldIsInterceptedAndRewritten() throws IOException {
+        Map<String, InferenceFieldMetadata> inferenceFields = Map.of(
+            FIELD_NAME,
+            new InferenceFieldMetadata(index.getName(), "inferenceId", new String[] { FIELD_NAME })
+        );
+        QueryRewriteContext context = createQueryRewriteContext(inferenceFields);
+        QueryBuilder original = new SparseVectorQueryBuilder(FIELD_NAME, INFERENCE_ID, QUERY);
+        QueryBuilder rewritten = original.rewrite(context);
+        assertTrue(
+            "Expected query to be intercepted, but was [" + rewritten.getClass().getName() + "]",
+            rewritten instanceof InterceptedQueryBuilderWrapper
+        );
+        InterceptedQueryBuilderWrapper intercepted = (InterceptedQueryBuilderWrapper) rewritten;
+        assertTrue(intercepted.queryBuilder instanceof NestedQueryBuilder);
+        NestedQueryBuilder nestedQueryBuilder = (NestedQueryBuilder) intercepted.queryBuilder;
+        assertEquals(SemanticTextField.getChunksFieldName(FIELD_NAME), nestedQueryBuilder.path());
+        QueryBuilder innerQuery = nestedQueryBuilder.query();
+        assertTrue(innerQuery instanceof SparseVectorQueryBuilder);
+        SparseVectorQueryBuilder sparseVectorQueryBuilder = (SparseVectorQueryBuilder) innerQuery;
+        assertEquals(SemanticTextField.getEmbeddingsFieldName(FIELD_NAME), sparseVectorQueryBuilder.getFieldName());
+        assertEquals(INFERENCE_ID, sparseVectorQueryBuilder.getInferenceId());
+        assertEquals(QUERY, sparseVectorQueryBuilder.getQuery());
+    }
+
+    public void testSparseVectorQueryOnInferenceFieldWithoutInferenceIdIsInterceptedAndRewritten() throws IOException {
+        Map<String, InferenceFieldMetadata> inferenceFields = Map.of(
+            FIELD_NAME,
+            new InferenceFieldMetadata(index.getName(), "inferenceId", new String[] { FIELD_NAME })
+        );
+        QueryRewriteContext context = createQueryRewriteContext(inferenceFields);
+        QueryBuilder original = new SparseVectorQueryBuilder(FIELD_NAME, null, QUERY);
+        QueryBuilder rewritten = original.rewrite(context);
+        assertTrue(
+            "Expected query to be intercepted, but was [" + rewritten.getClass().getName() + "]",
+            rewritten instanceof InterceptedQueryBuilderWrapper
+        );
+        InterceptedQueryBuilderWrapper intercepted = (InterceptedQueryBuilderWrapper) rewritten;
+        assertTrue(intercepted.queryBuilder instanceof NestedQueryBuilder);
+        NestedQueryBuilder nestedQueryBuilder = (NestedQueryBuilder) intercepted.queryBuilder;
+        assertEquals(SemanticTextField.getChunksFieldName(FIELD_NAME), nestedQueryBuilder.path());
+        QueryBuilder innerQuery = nestedQueryBuilder.query();
+        assertTrue(innerQuery instanceof SparseVectorQueryBuilder);
+        SparseVectorQueryBuilder sparseVectorQueryBuilder = (SparseVectorQueryBuilder) innerQuery;
+        assertEquals(SemanticTextField.getEmbeddingsFieldName(FIELD_NAME), sparseVectorQueryBuilder.getFieldName());
+        assertEquals(INFERENCE_ID, sparseVectorQueryBuilder.getInferenceId());
+        assertEquals(QUERY, sparseVectorQueryBuilder.getQuery());
+    }
+
+    public void testSparseVectorQueryOnNonInferenceFieldRemainsUnchanged() throws IOException {
+        QueryRewriteContext context = createQueryRewriteContext(Map.of()); // No inference fields
+        QueryBuilder original = new SparseVectorQueryBuilder(FIELD_NAME, INFERENCE_ID, QUERY);
+        QueryBuilder rewritten = original.rewrite(context);
+        assertTrue(
+            "Expected query to remain sparse_vector but was [" + rewritten.getClass().getName() + "]",
+            rewritten instanceof SparseVectorQueryBuilder
+        );
+        assertEquals(original, rewritten);
+    }
+
+    private QueryRewriteContext createQueryRewriteContext(Map<String, InferenceFieldMetadata> inferenceFields) {
+        IndexMetadata indexMetadata = IndexMetadata.builder(index.getName())
+            .settings(
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                    .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID())
+            )
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .putInferenceFields(inferenceFields)
+            .build();
+
+        ResolvedIndices resolvedIndices = new MockResolvedIndices(
+            Map.of(),
+            new OriginalIndices(new String[] { index.getName() }, IndicesOptions.DEFAULT),
+            Map.of(index, indexMetadata)
+        );
+
+        return new QueryRewriteContext(null, client, null, resolvedIndices, null, createRewriteInterceptor());
+    }
+
+    private QueryRewriteInterceptor createRewriteInterceptor() {
+        return new SemanticSparseVectorQueryRewriteInterceptor();
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java
index 0f322e64755be..6d9cbf8f9160e 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.inference.Model;
@@ -20,6 +21,9 @@
 import org.elasticsearch.inference.SimilarityMeasure;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.threadpool.ScalingExecutorBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.inference.results.ChatCompletionResults;
 import org.elasticsearch.xpack.inference.common.Truncator;
 import org.elasticsearch.xpack.inference.external.http.HttpClientManager;
@@ -32,6 +36,7 @@
 import org.elasticsearch.xpack.inference.registry.ModelRegistry;
 import org.hamcrest.Matchers;
 
+import java.io.IOException;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
@@ -46,6 +51,7 @@
 import static org.elasticsearch.xpack.inference.InferencePlugin.UTILITY_THREAD_POOL_NAME;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
@@ -237,4 +243,14 @@ public static ActionListener<Model> getModelListenerForException(Class<?> except
             assertThat(e.getMessage(), is(expectedMessage));
         });
     }
+
+    public static void assertJsonEquals(String actual, String expected) throws IOException {
+        var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE);
+        try (
+            var actualParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, actual);
+            var expectedParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, expected);
+        ) {
+            assertThat(actualParser.mapOrdered(), equalTo(expectedParser.mapOrdered()));
+        }
+    }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java
index 47f3a0e0b57aa..a723e5a9dffdf 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java
@@ -18,11 +18,13 @@
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.inference.UnparsedModel;
+import org.elasticsearch.license.MockLicenseState;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.inference.action.BaseInferenceActionRequest;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
+import org.elasticsearch.xpack.inference.InferencePlugin;
 import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager;
 import org.elasticsearch.xpack.inference.registry.ModelRegistry;
 import org.elasticsearch.xpack.inference.telemetry.InferenceStats;
@@ -49,6 +51,7 @@
 import static org.mockito.Mockito.when;
 
 public abstract class BaseTransportInferenceActionTestCase<Request extends BaseInferenceActionRequest> extends ESTestCase {
+    private MockLicenseState licenseState;
     private ModelRegistry modelRegistry;
     private StreamingTaskManager streamingTaskManager;
     private BaseTransportInferenceAction<Request> action;
@@ -64,16 +67,28 @@ public void setUp() throws Exception {
         super.setUp();
         TransportService transportService = mock();
         ActionFilters actionFilters = mock();
+        licenseState = mock();
         modelRegistry = mock();
         serviceRegistry = mock();
         inferenceStats = new InferenceStats(mock(), mock());
         streamingTaskManager = mock();
-        action = createAction(transportService, actionFilters, modelRegistry, serviceRegistry, inferenceStats, streamingTaskManager);
+        action = createAction(
+            transportService,
+            actionFilters,
+            licenseState,
+            modelRegistry,
+            serviceRegistry,
+            inferenceStats,
+            streamingTaskManager
+        );
+
+        mockValidLicenseState();
     }
 
     protected abstract BaseTransportInferenceAction<Request> createAction(
         TransportService transportService,
         ActionFilters actionFilters,
+        MockLicenseState licenseState,
         ModelRegistry modelRegistry,
         InferenceServiceRegistry serviceRegistry,
         InferenceStats inferenceStats,
@@ -361,4 +376,8 @@ protected void mockModelAndServiceRegistry(InferenceService service) {
 
         when(serviceRegistry.getService(any())).thenReturn(Optional.of(service));
     }
+
+    protected void mockValidLicenseState(){
+        when(licenseState.isAllowed(InferencePlugin.INFERENCE_API_FEATURE)).thenReturn(true);
+    }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java
index e54175cb27009..a5efe04c22c04 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.inference.InferenceServiceRegistry;
+import org.elasticsearch.license.MockLicenseState;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager;
@@ -23,6 +24,7 @@ public class TransportInferenceActionTests extends BaseTransportInferenceActionT
     protected BaseTransportInferenceAction<InferenceAction.Request> createAction(
         TransportService transportService,
         ActionFilters actionFilters,
+        MockLicenseState licenseState,
         ModelRegistry modelRegistry,
         InferenceServiceRegistry serviceRegistry,
         InferenceStats inferenceStats,
@@ -31,6 +33,7 @@ protected BaseTransportInferenceAction<InferenceAction.Request> createAction(
         return new TransportInferenceAction(
             transportService,
             actionFilters,
+            licenseState,
             modelRegistry,
             serviceRegistry,
             inferenceStats,
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java
index 4c943599ce523..3856a3d111b6e 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.inference.InferenceServiceRegistry;
 import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.license.MockLicenseState;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction;
@@ -36,6 +37,7 @@ public class TransportUnifiedCompletionActionTests extends BaseTransportInferenc
     protected BaseTransportInferenceAction<UnifiedCompletionAction.Request> createAction(
         TransportService transportService,
         ActionFilters actionFilters,
+        MockLicenseState licenseState,
         ModelRegistry modelRegistry,
         InferenceServiceRegistry serviceRegistry,
         InferenceStats inferenceStats,
@@ -44,6 +46,7 @@ protected BaseTransportInferenceAction<UnifiedCompletionAction.Request> createAc
         return new TransportUnifiedCompletionInferenceAction(
             transportService,
             actionFilters,
+            licenseState,
             modelRegistry,
             serviceRegistry,
             inferenceStats,
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java
index 0b7d136ffb04c..0432a2ff3fc9e 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java
@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.inference.action.filter;
 
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.bulk.BulkItemRequest;
@@ -16,10 +18,19 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.ActionFilterChain;
 import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.inference.ChunkedInference;
 import org.elasticsearch.inference.InferenceService;
@@ -57,6 +68,8 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.awaitLatch;
 import static org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilter.DEFAULT_BATCH_SIZE;
 import static org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilter.getIndexRequestOrNull;
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getChunksFieldName;
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getOriginalTextFieldName;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomChunkedInferenceEmbeddingSparse;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomSemanticText;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomSemanticTextInput;
@@ -65,14 +78,27 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
 import static org.mockito.Mockito.any;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 public class ShardBulkInferenceActionFilterTests extends ESTestCase {
+    private static final Object EXPLICIT_NULL = new Object();
+
+    private final boolean useLegacyFormat;
     private ThreadPool threadPool;
 
+    public ShardBulkInferenceActionFilterTests(boolean useLegacyFormat) {
+        this.useLegacyFormat = useLegacyFormat;
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return List.of(new Object[] { true }, new Object[] { false });
+    }
+
     @Before
     public void setupThreadPool() {
         threadPool = new TestThreadPool(getTestName());
@@ -85,7 +111,7 @@ public void tearDownThreadPool() throws Exception {
 
     @SuppressWarnings({ "unchecked", "rawtypes" })
     public void testFilterNoop() throws Exception {
-        ShardBulkInferenceActionFilter filter = createFilter(threadPool, Map.of(), DEFAULT_BATCH_SIZE);
+        ShardBulkInferenceActionFilter filter = createFilter(threadPool, Map.of(), DEFAULT_BATCH_SIZE, useLegacyFormat);
         CountDownLatch chainExecuted = new CountDownLatch(1);
         ActionFilterChain actionFilterChain = (task, action, request, listener) -> {
             try {
@@ -114,7 +140,8 @@ public void testInferenceNotFound() throws Exception {
         ShardBulkInferenceActionFilter filter = createFilter(
             threadPool,
             Map.of(model.getInferenceEntityId(), model),
-            randomIntBetween(1, 10)
+            randomIntBetween(1, 10),
+            useLegacyFormat
         );
         CountDownLatch chainExecuted = new CountDownLatch(1);
         ActionFilterChain actionFilterChain = (task, action, request, listener) -> {
@@ -144,7 +171,7 @@ public void testInferenceNotFound() throws Exception {
         );
         BulkItemRequest[] items = new BulkItemRequest[10];
         for (int i = 0; i < items.length; i++) {
-            items[i] = randomBulkItemRequest(Map.of(), inferenceFieldMap)[0];
+            items[i] = randomBulkItemRequest(useLegacyFormat, Map.of(), inferenceFieldMap)[0];
         }
         BulkShardRequest request = new BulkShardRequest(new ShardId("test", "test", 0), WriteRequest.RefreshPolicy.NONE, items);
         request.setInferenceFieldMap(inferenceFieldMap);
@@ -155,10 +182,12 @@ public void testInferenceNotFound() throws Exception {
     @SuppressWarnings({ "unchecked", "rawtypes" })
     public void testItemFailures() throws Exception {
         StaticModel model = StaticModel.createRandomInstance();
+
         ShardBulkInferenceActionFilter filter = createFilter(
             threadPool,
             Map.of(model.getInferenceEntityId(), model),
-            randomIntBetween(1, 10)
+            randomIntBetween(1, 10),
+            useLegacyFormat
         );
         model.putResult("I am a failure", new ChunkedInferenceError(new IllegalArgumentException("boom")));
         model.putResult("I am a success", randomChunkedInferenceEmbeddingSparse(List.of("I am a success")));
@@ -178,7 +207,15 @@ public void testItemFailures() throws Exception {
                 // item 1 is a success
                 assertNull(bulkShardRequest.items()[1].getPrimaryResponse());
                 IndexRequest actualRequest = getIndexRequestOrNull(bulkShardRequest.items()[1].request());
-                assertThat(XContentMapValues.extractValue("field1.text", actualRequest.sourceAsMap()), equalTo("I am a success"));
+                assertThat(
+                    XContentMapValues.extractValue(useLegacyFormat ? "field1.text" : "field1", actualRequest.sourceAsMap()),
+                    equalTo("I am a success")
+                );
+                if (useLegacyFormat == false) {
+                    assertNotNull(
+                        XContentMapValues.extractValue(InferenceMetadataFieldsMapper.NAME + ".field1", actualRequest.sourceAsMap())
+                    );
+                }
 
                 // item 2 is a failure
                 assertNotNull(bulkShardRequest.items()[2].getPrimaryResponse());
@@ -206,6 +243,79 @@ public void testItemFailures() throws Exception {
         awaitLatch(chainExecuted, 10, TimeUnit.SECONDS);
     }
 
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    public void testExplicitNull() throws Exception {
+        StaticModel model = StaticModel.createRandomInstance();
+        model.putResult("I am a failure", new ChunkedInferenceError(new IllegalArgumentException("boom")));
+        model.putResult("I am a success", randomChunkedInferenceEmbeddingSparse(List.of("I am a success")));
+
+        ShardBulkInferenceActionFilter filter = createFilter(
+            threadPool,
+            Map.of(model.getInferenceEntityId(), model),
+            randomIntBetween(1, 10),
+            useLegacyFormat
+        );
+
+        CountDownLatch chainExecuted = new CountDownLatch(1);
+        ActionFilterChain actionFilterChain = (task, action, request, listener) -> {
+            try {
+                BulkShardRequest bulkShardRequest = (BulkShardRequest) request;
+                assertNull(bulkShardRequest.getInferenceFieldMap());
+                assertThat(bulkShardRequest.items().length, equalTo(5));
+
+                // item 0
+                assertNull(bulkShardRequest.items()[0].getPrimaryResponse());
+                IndexRequest actualRequest = getIndexRequestOrNull(bulkShardRequest.items()[0].request());
+                assertThat(XContentMapValues.extractValue("obj.field1", actualRequest.sourceAsMap(), EXPLICIT_NULL), is(EXPLICIT_NULL));
+                assertNull(XContentMapValues.extractValue(InferenceMetadataFieldsMapper.NAME, actualRequest.sourceAsMap(), EXPLICIT_NULL));
+
+                // item 1 is a success
+                assertNull(bulkShardRequest.items()[1].getPrimaryResponse());
+                actualRequest = getIndexRequestOrNull(bulkShardRequest.items()[1].request());
+                assertInferenceResults(useLegacyFormat, actualRequest, "obj.field1", "I am a success", 1);
+
+                // item 2 is a failure
+                assertNotNull(bulkShardRequest.items()[2].getPrimaryResponse());
+                assertTrue(bulkShardRequest.items()[2].getPrimaryResponse().isFailed());
+                var failure = bulkShardRequest.items()[2].getPrimaryResponse().getFailure();
+                assertThat(failure.getCause().getCause().getMessage(), containsString("boom"));
+
+                // item 3
+                assertNull(bulkShardRequest.items()[3].getPrimaryResponse());
+                actualRequest = getIndexRequestOrNull(bulkShardRequest.items()[3].request());
+                assertInferenceResults(useLegacyFormat, actualRequest, "obj.field1", EXPLICIT_NULL, 0);
+
+                // item 4
+                assertNull(bulkShardRequest.items()[4].getPrimaryResponse());
+                actualRequest = getIndexRequestOrNull(bulkShardRequest.items()[4].request());
+                assertNull(XContentMapValues.extractValue("obj.field1", actualRequest.sourceAsMap(), EXPLICIT_NULL));
+                assertNull(XContentMapValues.extractValue(InferenceMetadataFieldsMapper.NAME, actualRequest.sourceAsMap(), EXPLICIT_NULL));
+            } finally {
+                chainExecuted.countDown();
+            }
+        };
+        ActionListener actionListener = mock(ActionListener.class);
+        Task task = mock(Task.class);
+
+        Map<String, InferenceFieldMetadata> inferenceFieldMap = Map.of(
+            "obj.field1",
+            new InferenceFieldMetadata("obj.field1", model.getInferenceEntityId(), new String[] { "obj.field1" })
+        );
+        Map<String, Object> sourceWithNull = new HashMap<>();
+        sourceWithNull.put("field1", null);
+
+        BulkItemRequest[] items = new BulkItemRequest[5];
+        items[0] = new BulkItemRequest(0, new IndexRequest("index").source(Map.of("obj", sourceWithNull)));
+        items[1] = new BulkItemRequest(1, new IndexRequest("index").source("obj.field1", "I am a success"));
+        items[2] = new BulkItemRequest(2, new IndexRequest("index").source("obj.field1", "I am a failure"));
+        items[3] = new BulkItemRequest(3, new UpdateRequest().doc(new IndexRequest("index").source(Map.of("obj", sourceWithNull))));
+        items[4] = new BulkItemRequest(4, new UpdateRequest().doc(new IndexRequest("index").source(Map.of("field2", "value"))));
+        BulkShardRequest request = new BulkShardRequest(new ShardId("test", "test", 0), WriteRequest.RefreshPolicy.NONE, items);
+        request.setInferenceFieldMap(inferenceFieldMap);
+        filter.apply(task, TransportShardBulkAction.ACTION_NAME, request, actionListener, actionFilterChain);
+        awaitLatch(chainExecuted, 10, TimeUnit.SECONDS);
+    }
+
     @SuppressWarnings({ "unchecked", "rawtypes" })
     public void testManyRandomDocs() throws Exception {
         Map<String, StaticModel> inferenceModelMap = new HashMap<>();
@@ -227,12 +337,12 @@ public void testManyRandomDocs() throws Exception {
         BulkItemRequest[] originalRequests = new BulkItemRequest[numRequests];
         BulkItemRequest[] modifiedRequests = new BulkItemRequest[numRequests];
         for (int id = 0; id < numRequests; id++) {
-            BulkItemRequest[] res = randomBulkItemRequest(inferenceModelMap, inferenceFieldMap);
+            BulkItemRequest[] res = randomBulkItemRequest(useLegacyFormat, inferenceModelMap, inferenceFieldMap);
             originalRequests[id] = res[0];
             modifiedRequests[id] = res[1];
         }
 
-        ShardBulkInferenceActionFilter filter = createFilter(threadPool, inferenceModelMap, randomIntBetween(10, 30));
+        ShardBulkInferenceActionFilter filter = createFilter(threadPool, inferenceModelMap, randomIntBetween(10, 30), useLegacyFormat);
         CountDownLatch chainExecuted = new CountDownLatch(1);
         ActionFilterChain actionFilterChain = (task, action, request, listener) -> {
             try {
@@ -263,7 +373,12 @@ public void testManyRandomDocs() throws Exception {
     }
 
     @SuppressWarnings("unchecked")
-    private static ShardBulkInferenceActionFilter createFilter(ThreadPool threadPool, Map<String, StaticModel> modelMap, int batchSize) {
+    private static ShardBulkInferenceActionFilter createFilter(
+        ThreadPool threadPool,
+        Map<String, StaticModel> modelMap,
+        int batchSize,
+        boolean useLegacyFormat
+    ) {
         ModelRegistry modelRegistry = mock(ModelRegistry.class);
         Answer<?> unparsedModelAnswer = invocationOnMock -> {
             String id = (String) invocationOnMock.getArguments()[0];
@@ -319,24 +434,50 @@ private static ShardBulkInferenceActionFilter createFilter(ThreadPool threadPool
 
         InferenceServiceRegistry inferenceServiceRegistry = mock(InferenceServiceRegistry.class);
         when(inferenceServiceRegistry.getService(any())).thenReturn(Optional.of(inferenceService));
-        ShardBulkInferenceActionFilter filter = new ShardBulkInferenceActionFilter(inferenceServiceRegistry, modelRegistry, batchSize);
-        return filter;
+
+        return new ShardBulkInferenceActionFilter(
+            createClusterService(useLegacyFormat),
+            inferenceServiceRegistry,
+            modelRegistry,
+            batchSize
+        );
+    }
+
+    private static ClusterService createClusterService(boolean useLegacyFormat) {
+        IndexMetadata indexMetadata = mock(IndexMetadata.class);
+        var settings = Settings.builder()
+            .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current())
+            .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat)
+            .build();
+        when(indexMetadata.getSettings()).thenReturn(settings);
+
+        Metadata metadata = mock(Metadata.class);
+        when(metadata.index(any(String.class))).thenReturn(indexMetadata);
+
+        ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(metadata).build();
+        ClusterService clusterService = mock(ClusterService.class);
+        when(clusterService.state()).thenReturn(clusterState);
+
+        return clusterService;
     }
 
     private static BulkItemRequest[] randomBulkItemRequest(
+        boolean useLegacyFormat,
         Map<String, StaticModel> modelMap,
         Map<String, InferenceFieldMetadata> fieldInferenceMap
     ) throws IOException {
         Map<String, Object> docMap = new LinkedHashMap<>();
         Map<String, Object> expectedDocMap = new LinkedHashMap<>();
         XContentType requestContentType = randomFrom(XContentType.values());
+
+        Map<String, Object> inferenceMetadataFields = new HashMap<>();
         for (var entry : fieldInferenceMap.values()) {
             String field = entry.getName();
             var model = modelMap.get(entry.getInferenceId());
             Object inputObject = randomSemanticTextInput();
             String inputText = inputObject.toString();
             docMap.put(field, inputObject);
-            expectedDocMap.put(field, inputText);
+            expectedDocMap.put(field, useLegacyFormat ? inputText : inputObject);
             if (model == null) {
                 // ignore results, the doc should fail with a resource not found exception
                 continue;
@@ -349,6 +490,7 @@ private static BulkItemRequest[] randomBulkItemRequest(
             if (model.hasResult(inputText)) {
                 var results = model.getResults(inputText);
                 semanticTextField = semanticTextFieldFromChunkedInferenceResults(
+                    useLegacyFormat,
                     field,
                     model,
                     List.of(inputText),
@@ -356,11 +498,19 @@ private static BulkItemRequest[] randomBulkItemRequest(
                     requestContentType
                 );
             } else {
-                semanticTextField = randomSemanticText(field, model, List.of(inputText), requestContentType);
-                model.putResult(inputText, toChunkedResult(semanticTextField));
+                Map<String, List<String>> inputTextMap = Map.of(field, List.of(inputText));
+                semanticTextField = randomSemanticText(useLegacyFormat, field, model, List.of(inputText), requestContentType);
+                model.putResult(inputText, toChunkedResult(useLegacyFormat, inputTextMap, semanticTextField));
             }
 
-            expectedDocMap.put(field, semanticTextField);
+            if (useLegacyFormat) {
+                expectedDocMap.put(field, semanticTextField);
+            } else {
+                inferenceMetadataFields.put(field, semanticTextField);
+            }
+        }
+        if (useLegacyFormat == false) {
+            expectedDocMap.put(InferenceMetadataFieldsMapper.NAME, inferenceMetadataFields);
         }
 
         int requestId = randomIntBetween(0, Integer.MAX_VALUE);
@@ -369,6 +519,53 @@ private static BulkItemRequest[] randomBulkItemRequest(
             new BulkItemRequest(requestId, new IndexRequest("index").source(expectedDocMap, requestContentType)) };
     }
 
+    @SuppressWarnings({ "unchecked" })
+    private static void assertInferenceResults(
+        boolean useLegacyFormat,
+        IndexRequest request,
+        String fieldName,
+        Object expectedOriginalValue,
+        int expectedChunkCount
+    ) {
+        final Map<String, Object> requestMap = request.sourceAsMap();
+        if (useLegacyFormat) {
+            assertThat(
+                XContentMapValues.extractValue(getOriginalTextFieldName(fieldName), requestMap, EXPLICIT_NULL),
+                equalTo(expectedOriginalValue)
+            );
+
+            List<Object> chunks = (List<Object>) XContentMapValues.extractValue(getChunksFieldName(fieldName), requestMap);
+            if (expectedChunkCount > 0) {
+                assertNotNull(chunks);
+                assertThat(chunks.size(), equalTo(expectedChunkCount));
+            } else {
+                // If the expected chunk count is 0, we expect that no inference has been performed. In this case, the source should not be
+                // transformed, and thus the semantic text field structure should not be created.
+                assertNull(chunks);
+            }
+        } else {
+            assertThat(XContentMapValues.extractValue(fieldName, requestMap, EXPLICIT_NULL), equalTo(expectedOriginalValue));
+
+            Map<String, Object> inferenceMetadataFields = (Map<String, Object>) XContentMapValues.extractValue(
+                InferenceMetadataFieldsMapper.NAME,
+                requestMap,
+                EXPLICIT_NULL
+            );
+            assertNotNull(inferenceMetadataFields);
+
+            // When using the inference metadata fields format, chunks are mapped by source field. We handle clearing inference results for
+            // a field by emitting an empty chunk list for it. This is done to prevent the clear operation from clearing inference results
+            // for other source fields.
+            List<Object> chunks = (List<Object>) XContentMapValues.extractValue(
+                getChunksFieldName(fieldName) + "." + fieldName,
+                inferenceMetadataFields,
+                EXPLICIT_NULL
+            );
+            assertNotNull(chunks);
+            assertThat(chunks.size(), equalTo(expectedChunkCount));
+        }
+    }
+
     private static class StaticModel extends TestModel {
         private final Map<String, ChunkedInference> resultMap;
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java
index b4fa5c9122258..a4228d50eae4b 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java
@@ -56,7 +56,7 @@ public class WordBoundaryChunkerTests extends ESTestCase {
             + " خليفہ المومنين يا خليفہ المسلمين يا صحابی يا رضي الله عنه چئي۔ (ب) آنحضور ﷺ جي گھروارين کان علاوه ڪنھن کي ام المومنين "
             + "چئي۔ (ج) آنحضور ﷺ جي خاندان جي اھل بيت کان علاوہڍه ڪنھن کي اھل بيت چئي۔ (د) پنھنجي عبادت گاھ کي مسجد چئي۔" };
 
-    public static int NUM_WORDS_IN_TEST_TEXT;
+    public static final int NUM_WORDS_IN_TEST_TEXT;
     static {
         var wordIterator = BreakIterator.getWordInstance(Locale.ROOT);
         wordIterator.setText(TEST_TEXT);
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInputTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInputTests.java
index 42e1b18168aec..1c0643739d410 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInputTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInputTests.java
@@ -24,20 +24,8 @@ public void testConvertsStringInputToMessages() {
             Matchers.is(
                 UnifiedCompletionRequest.of(
                     List.of(
-                        new UnifiedCompletionRequest.Message(
-                            new UnifiedCompletionRequest.ContentString("hello"),
-                            "a role",
-                            null,
-                            null,
-                            null
-                        ),
-                        new UnifiedCompletionRequest.Message(
-                            new UnifiedCompletionRequest.ContentString("awesome"),
-                            "a role",
-                            null,
-                            null,
-                            null
-                        )
+                        new UnifiedCompletionRequest.Message(new UnifiedCompletionRequest.ContentString("hello"), "a role", null, null),
+                        new UnifiedCompletionRequest.Message(new UnifiedCompletionRequest.ContentString("awesome"), "a role", null, null)
                     )
                 )
             )
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIResponseHandlerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIResponseHandlerTests.java
new file mode 100644
index 0000000000000..4c18915e0187b
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIResponseHandlerTests.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.jinaai;
+
+import org.apache.http.Header;
+import org.apache.http.HeaderElement;
+import org.apache.http.HttpResponse;
+import org.apache.http.StatusLine;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.http.retry.RetryException;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.hamcrest.MatcherAssert;
+
+import java.nio.charset.StandardCharsets;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.core.Is.is;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class JinaAIResponseHandlerTests extends ESTestCase {
+    public void testCheckForFailureStatusCode_DoesNotThrowForStatusCodesBetween200And299() {
+        callCheckForFailureStatusCode(randomIntBetween(200, 299), "id");
+    }
+
+    public void testCheckForFailureStatusCode_ThrowsFor503() {
+        var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(503, "id"));
+        assertFalse(exception.shouldRetry());
+        MatcherAssert.assertThat(
+            exception.getCause().getMessage(),
+            containsString("Received a server error status code for request from inference entity id [id] status [503]")
+        );
+        MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.BAD_REQUEST));
+    }
+
+    public void testCheckForFailureStatusCode_ThrowsFor500_WithShouldRetryTrue() {
+        var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(500, "id"));
+        assertTrue(exception.shouldRetry());
+        MatcherAssert.assertThat(
+            exception.getCause().getMessage(),
+            containsString("Received a server error status code for request from inference entity id [id] status [500]")
+        );
+        MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.BAD_REQUEST));
+    }
+
+    public void testCheckForFailureStatusCode_ThrowsFor429_WithShouldRetryTrue() {
+        var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(429, "id"));
+        assertTrue(exception.shouldRetry());
+        MatcherAssert.assertThat(
+            exception.getCause().getMessage(),
+            containsString("Received a rate limit status code for request from inference entity id [id] status [429]")
+        );
+        MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.TOO_MANY_REQUESTS));
+    }
+
+    public void testCheckForFailureStatusCode_ThrowsFor400() {
+        var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(400, "id"));
+        assertFalse(exception.shouldRetry());
+        MatcherAssert.assertThat(
+            exception.getCause().getMessage(),
+            containsString("Received an input validation error response for request from inference entity id [id] status [400]")
+        );
+        MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.BAD_REQUEST));
+    }
+
+    public void testCheckForFailureStatusCode_ThrowsFor400_InputsTooLarge() {
+        var exception = expectThrows(
+            RetryException.class,
+            () -> callCheckForFailureStatusCode(400, "\"input\" length 2049 is larger than the largest allowed size 2048", "id")
+        );
+        assertFalse(exception.shouldRetry());
+        MatcherAssert.assertThat(
+            exception.getCause().getMessage(),
+            containsString("Received an input validation error response for request from inference entity id [id] status [400]")
+        );
+        MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.BAD_REQUEST));
+    }
+
+    public void testCheckForFailureStatusCode_ThrowsFor401() {
+        var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(401, "inferenceEntityId"));
+        assertFalse(exception.shouldRetry());
+        MatcherAssert.assertThat(
+            exception.getCause().getMessage(),
+            containsString(
+                "Received an authentication error status code for request from inference entity id [inferenceEntityId] status [401]"
+            )
+        );
+        MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.UNAUTHORIZED));
+    }
+
+    public void testCheckForFailureStatusCode_ThrowsFor402() {
+        var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(402, "inferenceEntityId"));
+        assertFalse(exception.shouldRetry());
+        MatcherAssert.assertThat(exception.getCause().getMessage(), containsString("Payment required"));
+        MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.PAYMENT_REQUIRED));
+    }
+
+    private static void callCheckForFailureStatusCode(int statusCode, String modelId) {
+        callCheckForFailureStatusCode(statusCode, null, modelId);
+    }
+
+    private static void callCheckForFailureStatusCode(int statusCode, @Nullable String errorMessage, String modelId) {
+        var statusLine = mock(StatusLine.class);
+        when(statusLine.getStatusCode()).thenReturn(statusCode);
+
+        var httpResponse = mock(HttpResponse.class);
+        when(httpResponse.getStatusLine()).thenReturn(statusLine);
+        var header = mock(Header.class);
+        when(header.getElements()).thenReturn(new HeaderElement[] {});
+        when(httpResponse.getFirstHeader(anyString())).thenReturn(header);
+
+        String escapedErrorMessage = errorMessage != null ? errorMessage.replace("\\", "\\\\").replace("\"", "\\\"") : errorMessage;
+
+        String responseJson = Strings.format("""
+                {
+                    "detail": "%s"
+                }
+            """, escapedErrorMessage);
+
+        var mockRequest = mock(Request.class);
+        when(mockRequest.getInferenceEntityId()).thenReturn(modelId);
+        var httpResult = new HttpResult(httpResponse, errorMessage == null ? new byte[] {} : responseJson.getBytes(StandardCharsets.UTF_8));
+        var handler = new JinaAIResponseHandler("", (request, result) -> null);
+
+        handler.checkForFailureStatusCode(mockRequest, httpResult);
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java
index 6aaab219c331d..39eeef3a95520 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java
@@ -50,7 +50,7 @@ public void testXContent_WritesNoOptionalFields_WhenTheyAreNotDefined() throws I
     public void testConvertToString_ThrowsAssertionFailure_WhenInputTypeIsUnspecified() {
         var thrownException = expectThrows(
             AssertionError.class,
-            () -> AlibabaCloudSearchEmbeddingsRequestEntity.covertToString(InputType.UNSPECIFIED)
+            () -> AlibabaCloudSearchEmbeddingsRequestEntity.convertToString(InputType.UNSPECIFIED)
         );
         MatcherAssert.assertThat(thrownException.getMessage(), is("received invalid input type value [unspecified]"));
     }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntityTests.java
index 0690bf56893ca..8c8aeba4a0a07 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntityTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntityTests.java
@@ -84,7 +84,10 @@ public void testXContent_WritesNoOptionalFields_WhenTheyAreNotDefined() throws I
     }
 
     public void testConvertToString_ThrowsAssertionFailure_WhenInputTypeIsUnspecified() {
-        var thrownException = expectThrows(AssertionError.class, () -> CohereEmbeddingsRequestEntity.covertToString(InputType.UNSPECIFIED));
+        var thrownException = expectThrows(
+            AssertionError.class,
+            () -> CohereEmbeddingsRequestEntity.convertToString(InputType.UNSPECIFIED)
+        );
         MatcherAssert.assertThat(thrownException.getMessage(), is("received invalid input type value [unspecified]"));
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceUnifiedChatCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceUnifiedChatCompletionRequestEntityTests.java
new file mode 100644
index 0000000000000..15b4898650784
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceUnifiedChatCompletionRequestEntityTests.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.elastic;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.inference.UnifiedCompletionRequest;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.json.JsonXContent;
+import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput;
+import org.elasticsearch.xpack.inference.external.request.openai.OpenAiUnifiedChatCompletionRequestEntity;
+import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModel;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+import static org.elasticsearch.xpack.inference.Utils.assertJsonEquals;
+import static org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModelTests.createChatCompletionModel;
+
+public class ElasticInferenceServiceUnifiedChatCompletionRequestEntityTests extends ESTestCase {
+
+    private static final String ROLE = "user";
+
+    public void testModelUserFieldsSerialization() throws IOException {
+        UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message(
+            new UnifiedCompletionRequest.ContentString("Hello, world!"),
+            ROLE,
+            null,
+            null
+        );
+        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
+        messageList.add(message);
+
+        var unifiedRequest = UnifiedCompletionRequest.of(messageList);
+
+        UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
+        OpenAiChatCompletionModel model = createChatCompletionModel("test-url", "organizationId", "api-key", "test-endpoint", null);
+
+        OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
+
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
+
+        String jsonString = Strings.toString(builder);
+        String expectedJson = """
+            {
+                "messages": [
+                    {
+                        "content": "Hello, world!",
+                        "role": "user"
+                    }
+                ],
+                "model": "test-endpoint",
+                "n": 1,
+                "stream": true,
+                "stream_options": {
+                    "include_usage": true
+                }
+            }
+            """;
+        assertJsonEquals(jsonString, expectedJson);
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestEntityTests.java
new file mode 100644
index 0000000000000..7f3f6e5cdeb82
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestEntityTests.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.jinaai;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettings;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.is;
+
+public class JinaAIEmbeddingsRequestEntityTests extends ESTestCase {
+    public void testXContent_WritesAllFields_WhenTheyAreDefined() throws IOException {
+        var entity = new JinaAIEmbeddingsRequestEntity(List.of("abc"), new JinaAIEmbeddingsTaskSettings(InputType.INGEST), "model");
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        MatcherAssert.assertThat(xContentResult, is("""
+            {"input":["abc"],"model":"model","task":"retrieval.passage"}"""));
+    }
+
+    public void testXContent_WritesNoOptionalFields_WhenTheyAreNotDefined() throws IOException {
+        var entity = new JinaAIEmbeddingsRequestEntity(List.of("abc"), JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, "model");
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        MatcherAssert.assertThat(xContentResult, is("""
+            {"input":["abc"],"model":"model"}"""));
+    }
+
+    public void testConvertToString_ThrowsAssertionFailure_WhenInputTypeIsUnspecified() {
+        var thrownException = expectThrows(
+            AssertionError.class,
+            () -> JinaAIEmbeddingsRequestEntity.convertToString(InputType.UNSPECIFIED)
+        );
+        MatcherAssert.assertThat(thrownException.getMessage(), is("received invalid input type value [unspecified]"));
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestTests.java
new file mode 100644
index 0000000000000..05194ceb0de9e
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestTests.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.jinaai;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.HttpPost;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModel;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModelTests;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettings;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+
+public class JinaAIEmbeddingsRequestTests extends ESTestCase {
+    public void testCreateRequest_UrlDefined() throws IOException {
+        var request = createRequest(
+            List.of("abc"),
+            JinaAIEmbeddingsModelTests.createModel("url", "secret", JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, null, null, "model")
+        );
+
+        var httpRequest = request.createHttpRequest();
+        MatcherAssert.assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class));
+
+        var httpPost = (HttpPost) httpRequest.httpRequestBase();
+
+        MatcherAssert.assertThat(httpPost.getURI().toString(), is("url"));
+        MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType()));
+        MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer secret"));
+        MatcherAssert.assertThat(
+            httpPost.getLastHeader(JinaAIUtils.REQUEST_SOURCE_HEADER).getValue(),
+            is(JinaAIUtils.ELASTIC_REQUEST_SOURCE)
+        );
+
+        var requestMap = entityAsMap(httpPost.getEntity().getContent());
+        MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"), "model", "model")));
+    }
+
+    public void testCreateRequest_AllOptionsDefined() throws IOException {
+        var request = createRequest(
+            List.of("abc"),
+            JinaAIEmbeddingsModelTests.createModel("url", "secret", new JinaAIEmbeddingsTaskSettings(InputType.INGEST), null, null, "model")
+        );
+
+        var httpRequest = request.createHttpRequest();
+        MatcherAssert.assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class));
+
+        var httpPost = (HttpPost) httpRequest.httpRequestBase();
+
+        MatcherAssert.assertThat(httpPost.getURI().toString(), is("url"));
+        MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType()));
+        MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer secret"));
+        MatcherAssert.assertThat(
+            httpPost.getLastHeader(JinaAIUtils.REQUEST_SOURCE_HEADER).getValue(),
+            is(JinaAIUtils.ELASTIC_REQUEST_SOURCE)
+        );
+
+        var requestMap = entityAsMap(httpPost.getEntity().getContent());
+        MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"), "model", "model", "task", "retrieval.passage")));
+    }
+
+    public void testCreateRequest_InputTypeSearch() throws IOException {
+        var request = createRequest(
+            List.of("abc"),
+            JinaAIEmbeddingsModelTests.createModel("url", "secret", new JinaAIEmbeddingsTaskSettings(InputType.SEARCH), null, null, "model")
+        );
+
+        var httpRequest = request.createHttpRequest();
+        MatcherAssert.assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class));
+
+        var httpPost = (HttpPost) httpRequest.httpRequestBase();
+
+        MatcherAssert.assertThat(httpPost.getURI().toString(), is("url"));
+        MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType()));
+        MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer secret"));
+        MatcherAssert.assertThat(
+            httpPost.getLastHeader(JinaAIUtils.REQUEST_SOURCE_HEADER).getValue(),
+            is(JinaAIUtils.ELASTIC_REQUEST_SOURCE)
+        );
+
+        var requestMap = entityAsMap(httpPost.getEntity().getContent());
+        MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"), "model", "model", "task", "retrieval.query")));
+    }
+
+    public static JinaAIEmbeddingsRequest createRequest(List<String> input, JinaAIEmbeddingsModel model) {
+        return new JinaAIEmbeddingsRequest(input, model);
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRequestTests.java
new file mode 100644
index 0000000000000..031b44225628c
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRequestTests.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.jinaai;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.HttpPost;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.external.jinaai.JinaAIAccount;
+
+import java.net.URI;
+
+import static org.hamcrest.Matchers.is;
+
+public class JinaAIRequestTests extends ESTestCase {
+
+    public void testDecorateWithAuthHeader() {
+        var request = new HttpPost("http://www.abc.com");
+
+        JinaAIRequest.decorateWithAuthHeader(
+            request,
+            new JinaAIAccount(URI.create("http://www.abc.com"), new SecureString(new char[] { 'a', 'b', 'c' }))
+        );
+
+        assertThat(request.getFirstHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType()));
+        assertThat(request.getFirstHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer abc"));
+        assertThat(request.getFirstHeader(JinaAIUtils.REQUEST_SOURCE_HEADER).getValue(), is(JinaAIUtils.ELASTIC_REQUEST_SOURCE));
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestEntityTests.java
new file mode 100644
index 0000000000000..7fd738fa2a8e4
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestEntityTests.java
@@ -0,0 +1,140 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.jinaai;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankTaskSettings;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xpack.inference.MatchersUtils.equalToIgnoringWhitespaceInJsonString;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+public class JinaAIRerankRequestEntityTests extends ESTestCase {
+    public void testXContent_SingleRequest_WritesModelAndTopNIfDefined() throws IOException {
+        var entity = new JinaAIRerankRequestEntity("query", List.of("abc"), new JinaAIRerankTaskSettings(8, null), "model");
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString("""
+            {
+                "model": "model",
+                "query": "query",
+                "documents": [
+                    "abc"
+                ],
+                "top_n": 8
+            }
+            """));
+    }
+
+    public void testXContent_SingleRequest_WritesModelAndTopNIfDefined_ReturnDocumentsTrue() throws IOException {
+        var entity = new JinaAIRerankRequestEntity("query", List.of("abc"), new JinaAIRerankTaskSettings(8, true), "model");
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString("""
+            {
+                "model": "model",
+                "query": "query",
+                "documents": [
+                    "abc"
+                ],
+                "top_n": 8,
+                "return_documents": true
+            }
+            """));
+    }
+
+    public void testXContent_SingleRequest_WritesModelAndTopNIfDefined_ReturnDocumentsFalse() throws IOException {
+        var entity = new JinaAIRerankRequestEntity("query", List.of("abc"), new JinaAIRerankTaskSettings(8, false), "model");
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString("""
+            {
+                "model": "model",
+                "query": "query",
+                "documents": [
+                    "abc"
+                ],
+                "top_n": 8,
+                "return_documents": false
+            }
+            """));
+    }
+
+    public void testXContent_SingleRequest_DoesNotWriteTopNIfNull() throws IOException {
+        var entity = new JinaAIRerankRequestEntity("query", List.of("abc"), null, "model");
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString("""
+            {
+                "model": "model",
+                "query": "query",
+                "documents": [
+                    "abc"
+                ]
+            }
+            """));
+    }
+
+    public void testXContent_MultipleRequests_WritesModelAndTopNIfDefined() throws IOException {
+        var entity = new JinaAIRerankRequestEntity("query", List.of("abc", "def"), new JinaAIRerankTaskSettings(8, null), "model");
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString("""
+            {
+                "model": "model",
+                "query": "query",
+                "documents": [
+                    "abc",
+                    "def"
+                ],
+                "top_n": 8
+            }
+            """));
+    }
+
+    public void testXContent_MultipleRequests_DoesNotWriteTopNIfNull() throws IOException {
+        var entity = new JinaAIRerankRequestEntity("query", List.of("abc", "def"), null, "model");
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString("""
+            {
+                "model": "model",
+                "query": "query",
+                "documents": [
+                   "abc",
+                   "def"
+                ]
+            }
+            """));
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestTests.java
new file mode 100644
index 0000000000000..819362d397ba5
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestTests.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.jinaai;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.HttpPost;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankModelTests;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
+import static org.hamcrest.Matchers.aMapWithSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.sameInstance;
+
+public class JinaAIRerankRequestTests extends ESTestCase {
+
+    private static final String API_KEY = "foo";
+
+    public void testCreateRequest_WithoutModelSet_And_WithoutTopNSet() throws IOException {
+        var input = "input";
+        var query = "query";
+        var modelId = "model";
+
+        var request = createRequest(query, input, modelId, null);
+        var httpRequest = request.createHttpRequest();
+
+        assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class));
+        var httpPost = (HttpPost) httpRequest.httpRequestBase();
+
+        assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType()));
+        assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer " + API_KEY));
+
+        var requestMap = entityAsMap(httpPost.getEntity().getContent());
+
+        assertThat(requestMap, aMapWithSize(3));
+        assertThat(requestMap.get("documents"), is(List.of(input)));
+        assertThat(requestMap.get("query"), is(query));
+        assertThat(requestMap.get("model"), is(modelId));
+    }
+
+    public void testCreateRequest_WithTopNSet() throws IOException {
+        var input = "input";
+        var query = "query";
+        var topN = 1;
+        var modelId = "model";
+
+        var request = createRequest(query, input, modelId, topN);
+        var httpRequest = request.createHttpRequest();
+
+        assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class));
+        var httpPost = (HttpPost) httpRequest.httpRequestBase();
+
+        assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType()));
+        assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer " + API_KEY));
+
+        var requestMap = entityAsMap(httpPost.getEntity().getContent());
+
+        assertThat(requestMap, aMapWithSize(4));
+        assertThat(requestMap.get("documents"), is(List.of(input)));
+        assertThat(requestMap.get("query"), is(query));
+        assertThat(requestMap.get("top_n"), is(topN));
+        assertThat(requestMap.get("model"), is(modelId));
+    }
+
+    public void testCreateRequest_WithModelSet() throws IOException {
+        var input = "input";
+        var query = "query";
+        var modelId = "model";
+
+        var request = createRequest(query, input, modelId, null);
+        var httpRequest = request.createHttpRequest();
+
+        assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class));
+        var httpPost = (HttpPost) httpRequest.httpRequestBase();
+
+        assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType()));
+        assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer " + API_KEY));
+
+        var requestMap = entityAsMap(httpPost.getEntity().getContent());
+
+        assertThat(requestMap, aMapWithSize(3));
+        assertThat(requestMap.get("documents"), is(List.of(input)));
+        assertThat(requestMap.get("query"), is(query));
+        assertThat(requestMap.get("model"), is(modelId));
+    }
+
+    public void testTruncate_DoesNotTruncate() {
+        var request = createRequest("query", "input", "null", null);
+        var truncatedRequest = request.truncate();
+
+        assertThat(truncatedRequest, sameInstance(request));
+    }
+
+    private static JinaAIRerankRequest createRequest(String query, String input, @Nullable String modelId, @Nullable Integer topN) {
+        var rerankModel = JinaAIRerankModelTests.createModel(API_KEY, modelId, topN);
+        return new JinaAIRerankRequest(query, List.of(input), rerankModel);
+
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIUtilsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIUtilsTests.java
new file mode 100644
index 0000000000000..e3b4cfbed20ef
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIUtilsTests.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.jinaai;
+
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.is;
+
+public class JinaAIUtilsTests extends ESTestCase {
+
+    public void testCreateRequestSourceHeader() {
+        var requestSourceHeader = JinaAIUtils.createRequestSourceHeader();
+
+        assertThat(requestSourceHeader.getName(), is("Request-Source"));
+        assertThat(requestSourceHeader.getValue(), is("unspecified:elasticsearch"));
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestEntityTests.java
index f945c154ea234..b0c58f3e94af8 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestEntityTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestEntityTests.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.request.openai;
 
-import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.inference.UnifiedCompletionRequest;
 import org.elasticsearch.test.ESTestCase;
@@ -19,218 +18,35 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Random;
 
+import static org.elasticsearch.xpack.inference.Utils.assertJsonEquals;
 import static org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModelTests.createChatCompletionModel;
-import static org.hamcrest.Matchers.equalTo;
 
 public class OpenAiUnifiedChatCompletionRequestEntityTests extends ESTestCase {
 
-    // 1. Basic Serialization
-    // Test with minimal required fields to ensure basic serialization works.
-    public void testBasicSerialization() throws IOException {
-        UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message(
-            new UnifiedCompletionRequest.ContentString("Hello, world!"),
-            OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD,
-            null,
-            null,
-            null
-        );
-        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
-        messageList.add(message);
-        UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(messageList, null, null, null, null, null, null, null);
-
-        UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
-        OpenAiChatCompletionModel model = createChatCompletionModel("test-url", "organizationId", "api-key", "test-endpoint", null);
-
-        OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
-
-        XContentBuilder builder = JsonXContent.contentBuilder();
-        entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
-
-        String jsonString = Strings.toString(builder);
-        String expectedJson = """
-            {
-                "messages": [
-                    {
-                        "content": "Hello, world!",
-                        "role": "user"
-                    }
-                ],
-                "model": "test-endpoint",
-                "n": 1,
-                "stream": true,
-                "stream_options": {
-                    "include_usage": true
-                }
-            }
-            """;
-        assertJsonEquals(jsonString, expectedJson);
-    }
-
-    // 2. Serialization with All Fields
-    // Test with all possible fields populated to ensure complete serialization.
-    public void testSerializationWithAllFields() throws IOException {
-        // Create a message with all fields populated
-        UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message(
-            new UnifiedCompletionRequest.ContentString("Hello, world!"),
-            OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD,
-            "name",
-            "tool_call_id",
-            Collections.singletonList(
-                new UnifiedCompletionRequest.ToolCall(
-                    "id",
-                    new UnifiedCompletionRequest.ToolCall.FunctionField("arguments", "function_name"),
-                    "type"
-                )
-            )
-        );
-
-        // Create a tool with all fields populated
-        UnifiedCompletionRequest.Tool tool = new UnifiedCompletionRequest.Tool(
-            "type",
-            new UnifiedCompletionRequest.Tool.FunctionField(
-                "Fetches the weather in the given location",
-                "get_weather",
-                createParameters(),
-                true
-            )
-        );
-        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
-        messageList.add(message);
-        // Create the unified request with all fields populated
-        UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(
-            messageList,
-            "model",
-            100L, // maxCompletionTokens
-            Collections.singletonList("stop"),
-            0.9f, // temperature
-            new UnifiedCompletionRequest.ToolChoiceString("tool_choice"),
-            Collections.singletonList(tool),
-            0.8f // topP
-        );
-
-        // Create the unified chat input
-        UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
-
-        OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null);
-
-        // Create the entity
-        OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
-
-        // Serialize to XContent
-        XContentBuilder builder = JsonXContent.contentBuilder();
-        entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
-
-        // Convert to string and verify
-        String jsonString = Strings.toString(builder);
-        String expectedJson = """
-            {
-                "messages": [
-                    {
-                        "content": "Hello, world!",
-                        "role": "user",
-                        "name": "name",
-                        "tool_call_id": "tool_call_id",
-                        "tool_calls": [
-                            {
-                                "id": "id",
-                                "function": {
-                                    "arguments": "arguments",
-                                    "name": "function_name"
-                                },
-                                "type": "type"
-                            }
-                        ]
-                    }
-                ],
-                "model": "model-name",
-                "max_completion_tokens": 100,
-                "n": 1,
-                "stop": ["stop"],
-                "temperature": 0.9,
-                "tool_choice": "tool_choice",
-                "tools": [
-                    {
-                        "type": "type",
-                        "function": {
-                            "description": "Fetches the weather in the given location",
-                            "name": "get_weather",
-                            "parameters": {
-                                "type": "object",
-                                "properties": {
-                                    "location": {
-                                        "description": "The location to get the weather for",
-                                        "type": "string"
-                                    },
-                                    "unit": {
-                                        "description": "The unit to return the temperature in",
-                                        "type": "string",
-                                        "enum": ["F", "C"]
-                                    }
-                                },
-                                "additionalProperties": false,
-                                "required": ["location", "unit"]
-                            },
-                            "strict": true
-                        }
-                    }
-                ],
-                "top_p": 0.8,
-                "stream": true,
-                "stream_options": {
-                    "include_usage": true
-                }
-            }
-            """;
-        assertJsonEquals(jsonString, expectedJson);
-
-    }
+    private static final String ROLE = "user";
+    private static final String USER = "a_user";
 
-    // 3. Serialization with Null Optional Fields
-    // Test with optional fields set to null to ensure they are correctly omitted from the output.
-    public void testSerializationWithNullOptionalFields() throws IOException {
-        // Create a message with minimal required fields
+    public void testModelUserFieldsSerialization() throws IOException {
         UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message(
             new UnifiedCompletionRequest.ContentString("Hello, world!"),
-            OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD,
-            null,
+            ROLE,
             null,
             null
         );
         var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
         messageList.add(message);
 
-        // Create the unified request with optional fields set to null
-        UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(
-            messageList,
-            null, // model
-            null, // maxCompletionTokens
-            null, // stop
-            null, // temperature
-            null, // toolChoice
-            null, // tools
-            null  // topP
-        );
+        var unifiedRequest = UnifiedCompletionRequest.of(messageList);
 
-        // Create the unified chat input
         UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
+        OpenAiChatCompletionModel model = createChatCompletionModel("test-url", "organizationId", "api-key", "test-endpoint", USER);
 
-        OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null);
-
-        // Create the entity
         OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
 
-        // Serialize to XContent
         XContentBuilder builder = JsonXContent.contentBuilder();
         entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
 
-        // Convert to string and verify
         String jsonString = Strings.toString(builder);
         String expectedJson = """
             {
@@ -240,617 +56,15 @@ public void testSerializationWithNullOptionalFields() throws IOException {
                         "role": "user"
                     }
                 ],
-                "model": "model-name",
-                "n": 1,
-                "stream": true,
-                "stream_options": {
-                    "include_usage": true
-                }
-            }
-            """;
-        assertJsonEquals(jsonString, expectedJson);
-    }
-
-    // 4. Serialization with Empty Lists
-    // Test with fields that are lists set to empty lists to ensure they are correctly serialized.
-    public void testSerializationWithEmptyLists() throws IOException {
-        // Create a message with minimal required fields
-        UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message(
-            new UnifiedCompletionRequest.ContentString("Hello, world!"),
-            OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD,
-            null,
-            null,
-            Collections.emptyList() // empty toolCalls list
-        );
-        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
-        messageList.add(message);
-        // Create the unified request with empty lists
-        UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(
-            messageList,
-            null, // model
-            null, // maxCompletionTokens
-            Collections.emptyList(), // empty stop list
-            null, // temperature
-            null, // toolChoice
-            Collections.emptyList(), // empty tools list
-            null  // topP
-        );
-
-        // Create the unified chat input
-        UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
-
-        OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null);
-
-        // Create the entity
-        OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
-
-        // Serialize to XContent
-        XContentBuilder builder = JsonXContent.contentBuilder();
-        entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
-
-        // Convert to string and verify
-        String jsonString = Strings.toString(builder);
-        String expectedJson = """
-            {
-                "messages": [
-                    {
-                        "content": "Hello, world!",
-                        "role": "user",
-                        "tool_calls": []
-                    }
-                ],
-                "model": "model-name",
-                "n": 1,
-                "stream": true,
-                "stream_options": {
-                    "include_usage": true
-                }
-            }
-            """;
-        assertJsonEquals(jsonString, expectedJson);
-    }
-
-    // 5. Serialization with Nested Objects
-    // Test with nested objects (e.g., toolCalls, toolChoice, tool) to ensure they are correctly serialized.
-    public void testSerializationWithNestedObjects() throws IOException {
-        Random random = Randomness.get();
-
-        // Generate random values
-        String randomContent = "Hello, world! " + random.nextInt(1000);
-        String randomName = "name" + random.nextInt(1000);
-        String randomToolCallId = "tool_call_id" + random.nextInt(1000);
-        String randomArguments = "arguments" + random.nextInt(1000);
-        String randomFunctionName = "function_name" + random.nextInt(1000);
-        String randomType = "type" + random.nextInt(1000);
-        String randomModel = "model" + random.nextInt(1000);
-        String randomStop = "stop" + random.nextInt(1000);
-        float randomTemperature = (float) ((float) Math.round(0.5d + (double) random.nextFloat() * 0.5d * 100000d) / 100000d);
-        float randomTopP = (float) ((float) Math.round(0.5d + (double) random.nextFloat() * 0.5d * 100000d) / 100000d);
-
-        // Create a message with nested toolCalls
-        UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message(
-            new UnifiedCompletionRequest.ContentString(randomContent),
-            OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD,
-            randomName,
-            randomToolCallId,
-            Collections.singletonList(
-                new UnifiedCompletionRequest.ToolCall(
-                    "id",
-                    new UnifiedCompletionRequest.ToolCall.FunctionField(randomArguments, randomFunctionName),
-                    randomType
-                )
-            )
-        );
-
-        // Create a tool with nested function fields
-        UnifiedCompletionRequest.Tool tool = new UnifiedCompletionRequest.Tool(
-            randomType,
-            new UnifiedCompletionRequest.Tool.FunctionField(
-                "Fetches the weather in the given location",
-                "get_weather",
-                createParameters(),
-                true
-            )
-        );
-        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
-        messageList.add(message);
-        // Create the unified request with nested objects
-        UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(
-            messageList,
-            randomModel,
-            100L, // maxCompletionTokens
-            Collections.singletonList(randomStop),
-            randomTemperature, // temperature
-            new UnifiedCompletionRequest.ToolChoiceObject(
-                randomType,
-                new UnifiedCompletionRequest.ToolChoiceObject.FunctionField(randomFunctionName)
-            ),
-            Collections.singletonList(tool),
-            randomTopP // topP
-        );
-
-        // Create the unified chat input
-        UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
-
-        OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", randomModel, null);
-
-        // Create the entity
-        OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
-
-        // Serialize to XContent
-        XContentBuilder builder = JsonXContent.contentBuilder();
-        entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
-
-        // Convert to string and verify
-        String jsonString = Strings.toString(builder);
-        // Expected JSON should be dynamically generated based on random values
-        String expectedJson = String.format(
-            Locale.US,
-            """
-                {
-                    "messages": [
-                        {
-                            "content": "%s",
-                            "role": "user",
-                            "name": "%s",
-                            "tool_call_id": "%s",
-                            "tool_calls": [
-                                {
-                                    "id": "id",
-                                    "function": {
-                                        "arguments": "%s",
-                                        "name": "%s"
-                                    },
-                                    "type": "%s"
-                                }
-                            ]
-                        }
-                    ],
-                    "model": "%s",
-                    "max_completion_tokens": 100,
-                    "n": 1,
-                    "stop": ["%s"],
-                    "temperature": %.5f,
-                    "tool_choice": {
-                        "type": "%s",
-                        "function": {
-                            "name": "%s"
-                        }
-                    },
-                    "tools": [
-                        {
-                            "type": "%s",
-                            "function": {
-                                "description": "Fetches the weather in the given location",
-                                "name": "get_weather",
-                                "parameters": {
-                                    "type": "object",
-                                    "properties": {
-                                        "unit": {
-                                            "description": "The unit to return the temperature in",
-                                            "type": "string",
-                                            "enum": ["F", "C"]
-                                        },
-                                        "location": {
-                                            "description": "The location to get the weather for",
-                                            "type": "string"
-                                        }
-                                    },
-                                    "additionalProperties": false,
-                                    "required": ["location", "unit"]
-                                },
-                                "strict": true
-                            }
-                        }
-                    ],
-                    "top_p": %.5f,
-                    "stream": true,
-                    "stream_options": {
-                        "include_usage": true
-                    }
-                }
-                """,
-            randomContent,
-            randomName,
-            randomToolCallId,
-            randomArguments,
-            randomFunctionName,
-            randomType,
-            randomModel,
-            randomStop,
-            randomTemperature,
-            randomType,
-            randomFunctionName,
-            randomType,
-            randomTopP
-        );
-        assertJsonEquals(jsonString, expectedJson);
-    }
-
-    // 6. Serialization with Different Content Types
-    // Test with different content types in messages (e.g., ContentString, ContentObjects) to ensure they are correctly serialized.
-    public void testSerializationWithDifferentContentTypes() throws IOException {
-        Random random = Randomness.get();
-
-        // Generate random values for ContentString
-        String randomContentString = "Hello, world! " + random.nextInt(1000);
-
-        // Generate random values for ContentObjects
-        String randomText = "Random text " + random.nextInt(1000);
-        String randomType = "type" + random.nextInt(1000);
-        UnifiedCompletionRequest.ContentObject contentObject = new UnifiedCompletionRequest.ContentObject(randomText, randomType);
-
-        var contentObjectsList = new ArrayList<UnifiedCompletionRequest.ContentObject>();
-        contentObjectsList.add(contentObject);
-        UnifiedCompletionRequest.ContentObjects contentObjects = new UnifiedCompletionRequest.ContentObjects(contentObjectsList);
-
-        // Create messages with different content types
-        UnifiedCompletionRequest.Message messageWithString = new UnifiedCompletionRequest.Message(
-            new UnifiedCompletionRequest.ContentString(randomContentString),
-            OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD,
-            null,
-            null,
-            null
-        );
-
-        UnifiedCompletionRequest.Message messageWithObjects = new UnifiedCompletionRequest.Message(
-            contentObjects,
-            OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD,
-            null,
-            null,
-            null
-        );
-        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
-        messageList.add(messageWithString);
-        messageList.add(messageWithObjects);
-
-        // Create the unified request with both types of messages
-        UnifiedCompletionRequest unifiedRequest = UnifiedCompletionRequest.of(messageList);
-
-        // Create the unified chat input
-        UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
-
-        OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null);
-
-        // Create the entity
-        OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
-
-        // Serialize to XContent
-        XContentBuilder builder = JsonXContent.contentBuilder();
-        entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
-
-        // Convert to string and verify
-        String jsonString = Strings.toString(builder);
-        String expectedJson = String.format(Locale.US, """
-            {
-                "messages": [
-                    {
-                        "content": "%s",
-                        "role": "user"
-                    },
-                    {
-                        "content": [
-                            {
-                                "text": "%s",
-                                "type": "%s"
-                            }
-                        ],
-                        "role": "user"
-                    }
-                ],
-                "model": "model-name",
-                "n": 1,
-                "stream": true,
-                "stream_options": {
-                    "include_usage": true
-                }
-            }
-            """, randomContentString, randomText, randomType);
-        assertJsonEquals(jsonString, expectedJson);
-    }
-
-    // 7. Serialization with Special Characters
-    // Test with special characters in string fields to ensure they are correctly escaped and serialized.
-    public void testSerializationWithSpecialCharacters() throws IOException {
-        // Create a message with special characters
-        UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message(
-            new UnifiedCompletionRequest.ContentString("Hello, world! \n \"Special\" characters: \t \\ /"),
-            OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD,
-            "name\nwith\nnewlines",
-            "tool_call_id\twith\ttabs",
-            Collections.singletonList(
-                new UnifiedCompletionRequest.ToolCall(
-                    "id\\with\\backslashes",
-                    new UnifiedCompletionRequest.ToolCall.FunctionField("arguments\"with\"quotes", "function_name/with/slashes"),
-                    "type"
-                )
-            )
-        );
-        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
-        messageList.add(message);
-        // Create the unified request
-        UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(
-            messageList,
-            null, // model
-            null, // maxCompletionTokens
-            null, // stop
-            null, // temperature
-            null, // toolChoice
-            null, // tools
-            null  // topP
-        );
-
-        // Create the unified chat input
-        UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
-
-        OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null);
-
-        // Create the entity
-        OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
-
-        // Serialize to XContent
-        XContentBuilder builder = JsonXContent.contentBuilder();
-        entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
-
-        // Convert to string and verify
-        String jsonString = Strings.toString(builder);
-        String expectedJson = """
-            {
-                "messages": [
-                    {
-                        "content": "Hello, world! \\n \\"Special\\" characters: \\t \\\\ /",
-                        "role": "user",
-                        "name": "name\\nwith\\nnewlines",
-                        "tool_call_id": "tool_call_id\\twith\\ttabs",
-                        "tool_calls": [
-                            {
-                                "id": "id\\\\with\\\\backslashes",
-                                "function": {
-                                    "arguments": "arguments\\"with\\"quotes",
-                                    "name": "function_name/with/slashes"
-                                },
-                                "type": "type"
-                            }
-                        ]
-                    }
-                ],
-                "model": "model-name",
+                "model": "test-endpoint",
                 "n": 1,
                 "stream": true,
                 "stream_options": {
                     "include_usage": true
-                }
+                },
+                "user": "a_user"
             }
             """;
         assertJsonEquals(jsonString, expectedJson);
     }
-
-    // 8. Serialization with Boolean Fields
-    // Test with boolean fields (stream) set to both true and false to ensure they are correctly serialized.
-    public void testSerializationWithBooleanFields() throws IOException {
-        // Create a message with minimal required fields
-        UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message(
-            new UnifiedCompletionRequest.ContentString("Hello, world!"),
-            OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD,
-            null,
-            null,
-            null
-        );
-        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
-        messageList.add(message);
-        // Create the unified request
-        UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(
-            messageList,
-            null, // model
-            null, // maxCompletionTokens
-            null, // stop
-            null, // temperature
-            null, // toolChoice
-            null, // tools
-            null  // topP
-        );
-
-        OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null);
-
-        // Test with stream set to true
-        UnifiedChatInput unifiedChatInputTrue = new UnifiedChatInput(unifiedRequest, true);
-        OpenAiUnifiedChatCompletionRequestEntity entityTrue = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInputTrue, model);
-
-        XContentBuilder builderTrue = JsonXContent.contentBuilder();
-        entityTrue.toXContent(builderTrue, ToXContent.EMPTY_PARAMS);
-
-        String jsonStringTrue = Strings.toString(builderTrue);
-        String expectedJsonTrue = """
-            {
-                "messages": [
-                    {
-                        "content": "Hello, world!",
-                        "role": "user"
-                    }
-                ],
-                "model": "model-name",
-                "n": 1,
-                "stream": true,
-                "stream_options": {
-                    "include_usage": true
-                }
-            }
-            """;
-        assertJsonEquals(expectedJsonTrue, jsonStringTrue);
-
-        // Test with stream set to false
-        UnifiedChatInput unifiedChatInputFalse = new UnifiedChatInput(unifiedRequest, false);
-        OpenAiUnifiedChatCompletionRequestEntity entityFalse = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInputFalse, model);
-
-        XContentBuilder builderFalse = JsonXContent.contentBuilder();
-        entityFalse.toXContent(builderFalse, ToXContent.EMPTY_PARAMS);
-
-        String jsonStringFalse = Strings.toString(builderFalse);
-        String expectedJsonFalse = """
-            {
-                "messages": [
-                    {
-                        "content": "Hello, world!",
-                        "role": "user"
-                    }
-                ],
-                "model": "model-name",
-                "n": 1,
-                "stream": false
-            }
-            """;
-        assertJsonEquals(expectedJsonFalse, jsonStringFalse);
-    }
-
-    // 9. Serialization with Missing Required Fields
-    // Test with missing required fields to ensure appropriate exceptions are thrown.
-    public void testSerializationWithMissingRequiredFields() {
-        // Create a message with missing content (required field)
-        UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message(
-            null, // missing content
-            OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD,
-            null,
-            null,
-            null
-        );
-        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
-        messageList.add(message);
-        // Create the unified request
-        UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(
-            messageList,
-            null, // model
-            null, // maxCompletionTokens
-            null, // stop
-            null, // temperature
-            null, // toolChoice
-            null, // tools
-            null  // topP
-        );
-
-        // Create the unified chat input
-        UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
-
-        OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null);
-
-        // Create the entity
-        OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
-
-        // Attempt to serialize to XContent and expect an exception
-        try {
-            XContentBuilder builder = JsonXContent.contentBuilder();
-            entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            fail("Expected an exception due to missing required fields");
-        } catch (NullPointerException | IOException e) {
-            // Expected exception
-        }
-    }
-
-    // 10. Serialization with Mixed Valid and Invalid Data
-    // Test with a mix of valid and invalid data to ensure the serializer handles it gracefully.
-    public void testSerializationWithMixedValidAndInvalidData() throws IOException {
-        // Create a valid message
-        UnifiedCompletionRequest.Message validMessage = new UnifiedCompletionRequest.Message(
-            new UnifiedCompletionRequest.ContentString("Valid content"),
-            OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD,
-            "validName",
-            "validToolCallId",
-            Collections.singletonList(
-                new UnifiedCompletionRequest.ToolCall(
-                    "validId",
-                    new UnifiedCompletionRequest.ToolCall.FunctionField("validArguments", "validFunctionName"),
-                    "validType"
-                )
-            )
-        );
-
-        // Create an invalid message with null content
-        UnifiedCompletionRequest.Message invalidMessage = new UnifiedCompletionRequest.Message(
-            null, // invalid content
-            OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD,
-            "invalidName",
-            "invalidToolCallId",
-            Collections.singletonList(
-                new UnifiedCompletionRequest.ToolCall(
-                    "invalidId",
-                    new UnifiedCompletionRequest.ToolCall.FunctionField("invalidArguments", "invalidFunctionName"),
-                    "invalidType"
-                )
-            )
-        );
-        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
-        messageList.add(validMessage);
-        messageList.add(invalidMessage);
-        // Create the unified request with both valid and invalid messages
-        UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(
-            messageList,
-            "model-name",
-            100L, // maxCompletionTokens
-            Collections.singletonList("stop"),
-            0.9f, // temperature
-            new UnifiedCompletionRequest.ToolChoiceString("tool_choice"),
-            Collections.singletonList(
-                new UnifiedCompletionRequest.Tool(
-                    "type",
-                    new UnifiedCompletionRequest.Tool.FunctionField(
-                        "Fetches the weather in the given location",
-                        "get_weather",
-                        createParameters(),
-                        true
-                    )
-                )
-            ),
-            0.8f // topP
-        );
-
-        // Create the unified chat input
-        UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
-
-        OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null);
-
-        // Create the entity
-        OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
-
-        // Serialize to XContent and verify
-        try {
-            XContentBuilder builder = JsonXContent.contentBuilder();
-            entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            fail("Expected an exception due to invalid data");
-        } catch (NullPointerException | IOException e) {
-            // Expected exception
-        }
-    }
-
-    public static Map<String, Object> createParameters() {
-        Map<String, Object> parameters = new LinkedHashMap<>();
-        parameters.put("type", "object");
-
-        Map<String, Object> properties = new HashMap<>();
-
-        Map<String, Object> location = new HashMap<>();
-        location.put("type", "string");
-        location.put("description", "The location to get the weather for");
-        properties.put("location", location);
-
-        Map<String, Object> unit = new HashMap<>();
-        unit.put("type", "string");
-        unit.put("description", "The unit to return the temperature in");
-        unit.put("enum", new String[] { "F", "C" });
-        properties.put("unit", unit);
-
-        parameters.put("properties", properties);
-        parameters.put("additionalProperties", false);
-        parameters.put("required", new String[] { "location", "unit" });
-
-        return parameters;
-    }
-
-    private void assertJsonEquals(String actual, String expected) throws IOException {
-        try (
-            var actualParser = createParser(JsonXContent.jsonXContent, actual);
-            var expectedParser = createParser(JsonXContent.jsonXContent, expected)
-        ) {
-            assertThat(actualParser.mapOrdered(), equalTo(expectedParser.mapOrdered()));
-        }
-    }
-
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntityTests.java
new file mode 100644
index 0000000000000..7dbb9d5441a4a
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntityTests.java
@@ -0,0 +1,397 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.jinaai;
+
+import org.apache.http.HttpResponse;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.request.Request;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
+
+public class JinaAIEmbeddingsResponseEntityTests extends ESTestCase {
+    public void testFromResponse_CreatesResultsForASingleItem() throws IOException {
+        String responseJson = """
+            {
+              "object": "list",
+              "data": [
+                  {
+                      "object": "embedding",
+                      "index": 0,
+                      "embedding": [
+                          0.014539449,
+                          -0.015288644
+                      ]
+                  }
+              ],
+              "model": "jina-embeddings-v3",
+              "usage": {
+                  "prompt_tokens": 8,
+                  "total_tokens": 8
+              }
+            }
+            """;
+
+        InferenceTextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse(
+            mock(Request.class),
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertThat(
+            parsedResults.embeddings(),
+            is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.014539449F, -0.015288644F })))
+        );
+    }
+
+    public void testFromResponse_CreatesResultsForMultipleItems() throws IOException {
+        String responseJson = """
+            {
+              "object": "list",
+              "data": [
+                  {
+                      "object": "embedding",
+                      "index": 0,
+                      "embedding": [
+                          0.014539449,
+                          -0.015288644
+                      ]
+                  },
+                  {
+                      "object": "embedding",
+                      "index": 1,
+                      "embedding": [
+                          0.0123,
+                          -0.0123
+                      ]
+                  }
+              ],
+              "model": "jina-embeddings-v3",
+              "usage": {
+                  "prompt_tokens": 8,
+                  "total_tokens": 8
+              }
+            }
+            """;
+
+        InferenceTextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse(
+            mock(Request.class),
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertThat(
+            parsedResults.embeddings(),
+            is(
+                List.of(
+                    new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.014539449F, -0.015288644F }),
+                    new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.0123F, -0.0123F })
+                )
+            )
+        );
+    }
+
+    public void testFromResponse_FailsWhenDataFieldIsNotPresent() {
+        String responseJson = """
+            {
+              "object": "list",
+              "not_data": [
+                  {
+                      "object": "embedding",
+                      "index": 0,
+                      "embedding": [
+                          0.014539449,
+                          -0.015288644
+                      ]
+                  }
+              ],
+              "model": "jina-embeddings-v3",
+              "usage": {
+                  "prompt_tokens": 8,
+                  "total_tokens": 8
+              }
+            }
+            """;
+
+        var thrownException = expectThrows(
+            IllegalStateException.class,
+            () -> JinaAIEmbeddingsResponseEntity.fromResponse(
+                mock(Request.class),
+                new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+            )
+        );
+
+        assertThat(thrownException.getMessage(), is("Failed to find required field [data] in JinaAI embeddings response"));
+    }
+
+    public void testFromResponse_FailsWhenDataFieldNotAnArray() {
+        String responseJson = """
+            {
+              "object": "list",
+              "data": {
+                  "test": {
+                      "object": "embedding",
+                      "index": 0,
+                      "embedding": [
+                          0.014539449,
+                          -0.015288644
+                      ]
+                  }
+              },
+              "model": "jina-embeddings-v3",
+              "usage": {
+                  "prompt_tokens": 8,
+                  "total_tokens": 8
+              }
+            }
+            """;
+
+        var thrownException = expectThrows(
+            ParsingException.class,
+            () -> JinaAIEmbeddingsResponseEntity.fromResponse(
+                mock(Request.class),
+                new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+            )
+        );
+
+        assertThat(
+            thrownException.getMessage(),
+            is("Failed to parse object: expecting token of type [START_ARRAY] but found [START_OBJECT]")
+        );
+    }
+
+    public void testFromResponse_FailsWhenEmbeddingsDoesNotExist() {
+        String responseJson = """
+            {
+              "object": "list",
+              "data": [
+                  {
+                      "object": "embedding",
+                      "index": 0,
+                      "embeddingzzz": [
+                          0.014539449,
+                          -0.015288644
+                      ]
+                  }
+              ],
+              "model": "jina-embeddings-v3",
+              "usage": {
+                  "prompt_tokens": 8,
+                  "total_tokens": 8
+              }
+            }
+            """;
+
+        var thrownException = expectThrows(
+            IllegalStateException.class,
+            () -> JinaAIEmbeddingsResponseEntity.fromResponse(
+                mock(Request.class),
+                new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+            )
+        );
+
+        assertThat(thrownException.getMessage(), is("Failed to find required field [embedding] in JinaAI embeddings response"));
+    }
+
+    public void testFromResponse_FailsWhenEmbeddingValueIsAString() {
+        String responseJson = """
+            {
+              "object": "list",
+              "data": [
+                  {
+                      "object": "embedding",
+                      "index": 0,
+                      "embedding": [
+                          "abc"
+                      ]
+                  }
+              ],
+              "model": "jina-embeddings-v3",
+              "usage": {
+                  "prompt_tokens": 8,
+                  "total_tokens": 8
+              }
+            }
+            """;
+
+        var thrownException = expectThrows(
+            ParsingException.class,
+            () -> JinaAIEmbeddingsResponseEntity.fromResponse(
+                mock(Request.class),
+                new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+            )
+        );
+
+        assertThat(
+            thrownException.getMessage(),
+            is("Failed to parse object: expecting token of type [VALUE_NUMBER] but found [VALUE_STRING]")
+        );
+    }
+
+    public void testFromResponse_SucceedsWhenEmbeddingValueIsInt() throws IOException {
+        String responseJson = """
+            {
+              "object": "list",
+              "data": [
+                  {
+                      "object": "embedding",
+                      "index": 0,
+                      "embedding": [
+                          1
+                      ]
+                  }
+              ],
+              "model": "jina-embeddings-v3",
+              "usage": {
+                  "prompt_tokens": 8,
+                  "total_tokens": 8
+              }
+            }
+            """;
+
+        InferenceTextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse(
+            mock(Request.class),
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertThat(
+            parsedResults.embeddings(),
+            is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 1.0F })))
+        );
+    }
+
+    public void testFromResponse_SucceedsWhenEmbeddingValueIsLong() throws IOException {
+        String responseJson = """
+            {
+              "object": "list",
+              "data": [
+                  {
+                      "object": "embedding",
+                      "index": 0,
+                      "embedding": [
+                          40294967295
+                      ]
+                  }
+              ],
+              "model": "jina-embeddings-v3",
+              "usage": {
+                  "prompt_tokens": 8,
+                  "total_tokens": 8
+              }
+            }
+            """;
+
+        InferenceTextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse(
+            mock(Request.class),
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertThat(
+            parsedResults.embeddings(),
+            is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 4.0294965E10F })))
+        );
+    }
+
+    public void testFromResponse_FailsWhenEmbeddingValueIsAnObject() {
+        String responseJson = """
+            {
+              "object": "list",
+              "data": [
+                  {
+                      "object": "embedding",
+                      "index": 0,
+                      "embedding": [
+                          {}
+                      ]
+                  }
+              ],
+              "model": "jina-embeddings-v3",
+              "usage": {
+                  "prompt_tokens": 8,
+                  "total_tokens": 8
+              }
+            }
+            """;
+
+        var thrownException = expectThrows(
+            ParsingException.class,
+            () -> JinaAIEmbeddingsResponseEntity.fromResponse(
+                mock(Request.class),
+                new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+            )
+        );
+
+        assertThat(
+            thrownException.getMessage(),
+            is("Failed to parse object: expecting token of type [VALUE_NUMBER] but found [START_OBJECT]")
+        );
+    }
+
+    public void testFieldsInDifferentOrderServer() throws IOException {
+        // The fields of the objects in the data array are reordered
+        String response = """
+            {
+                "object": "list",
+                "id": "6667830b-716b-4796-9a61-33b67b5cc81d",
+                "model": "jina-embeddings-v3",
+                "data": [
+                    {
+                        "embedding": [
+                            -0.9,
+                            0.5,
+                            0.3
+                        ],
+                        "index": 0,
+                        "object": "embedding"
+                    },
+                    {
+                        "index": 0,
+                        "embedding": [
+                            0.1,
+                            0.5
+                        ],
+                        "object": "embedding"
+                    },
+                    {
+                        "object": "embedding",
+                        "index": 0,
+                        "embedding": [
+                            0.5,
+                            0.5
+                        ]
+                    }
+                ],
+                "usage": {
+                    "prompt_tokens": 0,
+                    "completion_tokens": 0,
+                    "total_tokens": 0
+                }
+            }""";
+
+        InferenceTextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse(
+            mock(Request.class),
+            new HttpResult(mock(HttpResponse.class), response.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertThat(
+            parsedResults.embeddings(),
+            is(
+                List.of(
+                    new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { -0.9F, 0.5F, 0.3F }),
+                    new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.1F, 0.5F }),
+                    new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.5F, 0.5F })
+                )
+            )
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIErrorResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIErrorResponseEntityTests.java
new file mode 100644
index 0000000000000..ce3bd10566cd8
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIErrorResponseEntityTests.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.jinaai;
+
+import org.apache.http.HttpResponse;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse;
+import org.hamcrest.MatcherAssert;
+
+import java.nio.charset.StandardCharsets;
+
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
+
+public class JinaAIErrorResponseEntityTests extends ESTestCase {
+    public void testFromResponse() {
+        String message = "\"input\" length 2049 is larger than the largest allowed size 2048";
+        String escapedMessage = message.replace("\\", "\\\\").replace("\"", "\\\"");
+        String responseJson = Strings.format("""
+            {
+                "detail": "%s"
+            }
+            """, escapedMessage);
+
+        ErrorResponse errorResponse = JinaAIErrorResponseEntity.fromResponse(
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+        assertNotNull(errorResponse);
+        MatcherAssert.assertThat(errorResponse.getErrorMessage(), is(message));
+    }
+
+    public void testFromResponse_noMessage() {
+        String responseJson = """
+            {
+                "error": "abc"
+            }
+            """;
+
+        ErrorResponse errorResponse = JinaAIErrorResponseEntity.fromResponse(
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+        MatcherAssert.assertThat(errorResponse, is(ErrorResponse.UNDEFINED_ERROR));
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIRerankResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIRerankResponseEntityTests.java
new file mode 100644
index 0000000000000..33fe9819bd88a
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIRerankResponseEntityTests.java
@@ -0,0 +1,180 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.jinaai;
+
+import org.apache.http.HttpResponse;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.inference.results.RankedDocsResults;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
+
+public class JinaAIRerankResponseEntityTests extends ESTestCase {
+
+    public void testResponseLiteral() throws IOException {
+        InferenceServiceResults parsedResults = JinaAIRerankResponseEntity.fromResponse(
+            new HttpResult(mock(HttpResponse.class), responseLiteral.getBytes(StandardCharsets.UTF_8))
+        );
+
+        MatcherAssert.assertThat(parsedResults, instanceOf(RankedDocsResults.class));
+        List<RankedDocsResults.RankedDoc> expected = responseLiteralDocs();
+        for (int i = 0; i < ((RankedDocsResults) parsedResults).getRankedDocs().size(); i++) {
+            assertEquals(((RankedDocsResults) parsedResults).getRankedDocs().get(i).index(), expected.get(i).index());
+        }
+    }
+
+    public void testGeneratedResponse() throws IOException {
+        int numDocs = randomIntBetween(1, 10);
+
+        List<RankedDocsResults.RankedDoc> expected = new ArrayList<>(numDocs);
+        StringBuilder responseBuilder = new StringBuilder();
+
+        responseBuilder.append("{");
+        responseBuilder.append("\"model\": \"model\",");
+        responseBuilder.append("\"index\":\"").append(randomAlphaOfLength(36)).append("\",");
+        responseBuilder.append("\"results\": [");
+        List<Integer> indices = linear(numDocs);
+        List<Float> scores = linearFloats(numDocs);
+        for (int i = 0; i < numDocs; i++) {
+            int index = indices.remove(randomInt(indices.size() - 1));
+
+            responseBuilder.append("{");
+            responseBuilder.append("\"index\":").append(index).append(",");
+            responseBuilder.append("\"relevance_score\":").append(scores.get(i).toString()).append("}");
+            expected.add(new RankedDocsResults.RankedDoc(index, scores.get(i), null));
+            if (i < numDocs - 1) {
+                responseBuilder.append(",");
+            }
+        }
+        responseBuilder.append("],");
+        responseBuilder.append("\"usage\": {");
+        responseBuilder.append("\"total_tokens\": 15}");
+        responseBuilder.append("}");
+
+        InferenceServiceResults parsedResults = JinaAIRerankResponseEntity.fromResponse(
+            new HttpResult(mock(HttpResponse.class), responseBuilder.toString().getBytes(StandardCharsets.UTF_8))
+        );
+        MatcherAssert.assertThat(parsedResults, instanceOf(RankedDocsResults.class));
+        for (int i = 0; i < ((RankedDocsResults) parsedResults).getRankedDocs().size(); i++) {
+            assertEquals(((RankedDocsResults) parsedResults).getRankedDocs().get(i).index(), expected.get(i).index());
+        }
+    }
+
+    private ArrayList<RankedDocsResults.RankedDoc> responseLiteralDocs() {
+        var list = new ArrayList<RankedDocsResults.RankedDoc>();
+
+        list.add(new RankedDocsResults.RankedDoc(2, 0.98005307F, null));
+        list.add(new RankedDocsResults.RankedDoc(3, 0.27904198F, null));
+        list.add(new RankedDocsResults.RankedDoc(0, 0.10194652F, null));
+        return list;
+
+    };
+
+    private final String responseLiteral = """
+        {
+            "model": "model",
+            "results": [
+                {
+                    "index": 2,
+                    "relevance_score": 0.98005307
+                },
+                {
+                    "index": 3,
+                    "relevance_score": 0.27904198
+                },
+                {
+                    "index": 0,
+                    "relevance_score": 0.10194652
+                }
+            ],
+            "usage": {
+                "total_tokens": 15
+            }
+        }
+        """;
+
+    public void testResponseLiteralWithDocuments() throws IOException {
+        InferenceServiceResults parsedResults = JinaAIRerankResponseEntity.fromResponse(
+            new HttpResult(mock(HttpResponse.class), responseLiteralWithDocuments.getBytes(StandardCharsets.UTF_8))
+        );
+
+        MatcherAssert.assertThat(parsedResults, instanceOf(RankedDocsResults.class));
+        MatcherAssert.assertThat(((RankedDocsResults) parsedResults).getRankedDocs(), is(responseLiteralDocsWithText));
+    }
+
+    private final String responseLiteralWithDocuments = """
+        {
+            "model": "model",
+            "results": [
+                {
+                    "document": {
+                        "text": "Washington, D.C.."
+                    },
+                    "index": 2,
+                    "relevance_score": 0.98005307
+                },
+                {
+                    "document": {
+                        "text": "Capital punishment has existed in the United States since beforethe United States was a country. "
+                    },
+                    "index": 3,
+                    "relevance_score": 0.27904198
+                },
+                {
+                    "document": {
+                        "text": "Carson City is the capital city of the American state of Nevada."
+                    },
+                    "index": 0,
+                    "relevance_score": 0.10194652
+                }
+            ],
+            "usage": {
+                "total_tokens": 15
+            }
+        }
+        """;
+
+    private final List<RankedDocsResults.RankedDoc> responseLiteralDocsWithText = List.of(
+        new RankedDocsResults.RankedDoc(2, 0.98005307F, "Washington, D.C.."),
+        new RankedDocsResults.RankedDoc(
+            3,
+            0.27904198F,
+            "Capital punishment has existed in the United States since beforethe United States was a country. "
+        ),
+        new RankedDocsResults.RankedDoc(0, 0.10194652F, "Carson City is the capital city of the American state of Nevada.")
+    );
+
+    private ArrayList<Integer> linear(int n) {
+        ArrayList<Integer> list = new ArrayList<>();
+        for (int i = 0; i <= n; i++) {
+            list.add(i);
+        }
+        return list;
+    }
+
+    // creates a list of doubles of monotonically decreasing magnitude
+    private ArrayList<Float> linearFloats(int n) {
+        ArrayList<Float> list = new ArrayList<>();
+        float startValue = 1.0f;
+        float decrement = startValue / n + 1;
+        for (int i = 0; i <= n; i++) {
+            list.add(startValue - (i * decrement));
+        }
+        return list;
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/unified/UnifiedChatCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/unified/UnifiedChatCompletionRequestEntityTests.java
new file mode 100644
index 0000000000000..d9388cab0e1ec
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/unified/UnifiedChatCompletionRequestEntityTests.java
@@ -0,0 +1,705 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.unified;
+
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.inference.UnifiedCompletionRequest;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.json.JsonXContent;
+import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput;
+import org.elasticsearch.xpack.inference.external.request.openai.OpenAiUnifiedChatCompletionRequestEntity;
+import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModel;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Random;
+
+import static org.elasticsearch.xpack.inference.Utils.assertJsonEquals;
+import static org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModelTests.createChatCompletionModel;
+
+public class UnifiedChatCompletionRequestEntityTests extends ESTestCase {
+
+    private static final String ROLE = "user";
+
+    public void testBasicSerialization() throws IOException {
+        UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message(
+            new UnifiedCompletionRequest.ContentString("Hello, world!"),
+            ROLE,
+            null,
+            null
+        );
+        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
+        messageList.add(message);
+        UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(messageList, null, null, null, null, null, null, null);
+
+        UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
+        OpenAiChatCompletionModel model = createChatCompletionModel("test-url", "organizationId", "api-key", "test-endpoint", null);
+
+        OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
+
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
+
+        String jsonString = Strings.toString(builder);
+        String expectedJson = """
+            {
+                "messages": [
+                    {
+                        "content": "Hello, world!",
+                        "role": "user"
+                    }
+                ],
+                "model": "test-endpoint",
+                "n": 1,
+                "stream": true,
+                "stream_options": {
+                    "include_usage": true
+                }
+            }
+            """;
+        assertJsonEquals(jsonString, expectedJson);
+    }
+
+    public void testSerializationWithAllFields() throws IOException {
+        UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message(
+            new UnifiedCompletionRequest.ContentString("Hello, world!"),
+            ROLE,
+            "tool_call_id",
+            Collections.singletonList(
+                new UnifiedCompletionRequest.ToolCall(
+                    "id",
+                    new UnifiedCompletionRequest.ToolCall.FunctionField("arguments", "function_name"),
+                    "type"
+                )
+            )
+        );
+
+        UnifiedCompletionRequest.Tool tool = new UnifiedCompletionRequest.Tool(
+            "type",
+            new UnifiedCompletionRequest.Tool.FunctionField(
+                "Fetches the weather in the given location",
+                "get_weather",
+                createParameters(),
+                true
+            )
+        );
+        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
+        messageList.add(message);
+        UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(
+            messageList,
+            "model",
+            100L, // maxCompletionTokens
+            Collections.singletonList("stop"),
+            0.9f, // temperature
+            new UnifiedCompletionRequest.ToolChoiceString("tool_choice"),
+            Collections.singletonList(tool),
+            0.8f // topP
+        );
+
+        UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
+
+        OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null);
+
+        OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
+
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
+
+        String jsonString = Strings.toString(builder);
+        String expectedJson = """
+            {
+                "messages": [
+                    {
+                        "content": "Hello, world!",
+                        "role": "user",
+                        "tool_call_id": "tool_call_id",
+                        "tool_calls": [
+                            {
+                                "id": "id",
+                                "function": {
+                                    "arguments": "arguments",
+                                    "name": "function_name"
+                                },
+                                "type": "type"
+                            }
+                        ]
+                    }
+                ],
+                "model": "model-name",
+                "max_completion_tokens": 100,
+                "n": 1,
+                "stop": ["stop"],
+                "temperature": 0.9,
+                "tool_choice": "tool_choice",
+                "tools": [
+                    {
+                        "type": "type",
+                        "function": {
+                            "description": "Fetches the weather in the given location",
+                            "name": "get_weather",
+                            "parameters": {
+                                "type": "object",
+                                "properties": {
+                                    "location": {
+                                        "description": "The location to get the weather for",
+                                        "type": "string"
+                                    },
+                                    "unit": {
+                                        "description": "The unit to return the temperature in",
+                                        "type": "string",
+                                        "enum": ["F", "C"]
+                                    }
+                                },
+                                "additionalProperties": false,
+                                "required": ["location", "unit"]
+                            },
+                            "strict": true
+                        }
+                    }
+                ],
+                "top_p": 0.8,
+                "stream": true,
+                "stream_options": {
+                    "include_usage": true
+                }
+            }
+            """;
+        assertJsonEquals(jsonString, expectedJson);
+
+    }
+
+    public void testSerializationWithNullOptionalFields() throws IOException {
+        UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message(
+            new UnifiedCompletionRequest.ContentString("Hello, world!"),
+            ROLE,
+            null,
+            null
+        );
+        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
+        messageList.add(message);
+
+        UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(
+            messageList,
+            null, // model
+            null, // maxCompletionTokens
+            null, // stop
+            null, // temperature
+            null, // toolChoice
+            null, // tools
+            null  // topP
+        );
+
+        UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
+
+        OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null);
+
+        OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
+
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
+
+        String jsonString = Strings.toString(builder);
+        String expectedJson = """
+            {
+                "messages": [
+                    {
+                        "content": "Hello, world!",
+                        "role": "user"
+                    }
+                ],
+                "model": "model-name",
+                "n": 1,
+                "stream": true,
+                "stream_options": {
+                    "include_usage": true
+                }
+            }
+            """;
+        assertJsonEquals(jsonString, expectedJson);
+    }
+
+    public void testSerializationWithEmptyLists() throws IOException {
+        UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message(
+            new UnifiedCompletionRequest.ContentString("Hello, world!"),
+            ROLE,
+            null,
+            Collections.emptyList() // empty toolCalls list
+        );
+        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
+        messageList.add(message);
+        UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(
+            messageList,
+            null, // model
+            null, // maxCompletionTokens
+            Collections.emptyList(), // empty stop list
+            null, // temperature
+            null, // toolChoice
+            Collections.emptyList(), // empty tools list
+            null  // topP
+        );
+
+        UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
+
+        OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null);
+
+        OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
+
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
+
+        String jsonString = Strings.toString(builder);
+        String expectedJson = """
+            {
+                "messages": [
+                    {
+                        "content": "Hello, world!",
+                        "role": "user",
+                        "tool_calls": []
+                    }
+                ],
+                "model": "model-name",
+                "n": 1,
+                "stream": true,
+                "stream_options": {
+                    "include_usage": true
+                }
+            }
+            """;
+        assertJsonEquals(jsonString, expectedJson);
+    }
+
+    public void testSerializationWithNestedObjects() throws IOException {
+        Random random = Randomness.get();
+
+        String randomContent = "Hello, world! " + random.nextInt(1000);
+        String randomToolCallId = "tool_call_id" + random.nextInt(1000);
+        String randomArguments = "arguments" + random.nextInt(1000);
+        String randomFunctionName = "function_name" + random.nextInt(1000);
+        String randomType = "type" + random.nextInt(1000);
+        String randomModel = "model" + random.nextInt(1000);
+        String randomStop = "stop" + random.nextInt(1000);
+        float randomTemperature = (float) ((float) Math.round(0.5d + (double) random.nextFloat() * 0.5d * 100000d) / 100000d);
+        float randomTopP = (float) ((float) Math.round(0.5d + (double) random.nextFloat() * 0.5d * 100000d) / 100000d);
+
+        UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message(
+            new UnifiedCompletionRequest.ContentString(randomContent),
+            ROLE,
+            randomToolCallId,
+            Collections.singletonList(
+                new UnifiedCompletionRequest.ToolCall(
+                    "id",
+                    new UnifiedCompletionRequest.ToolCall.FunctionField(randomArguments, randomFunctionName),
+                    randomType
+                )
+            )
+        );
+
+        UnifiedCompletionRequest.Tool tool = new UnifiedCompletionRequest.Tool(
+            randomType,
+            new UnifiedCompletionRequest.Tool.FunctionField(
+                "Fetches the weather in the given location",
+                "get_weather",
+                createParameters(),
+                true
+            )
+        );
+        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
+        messageList.add(message);
+        UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(
+            messageList,
+            randomModel,
+            100L, // maxCompletionTokens
+            Collections.singletonList(randomStop),
+            randomTemperature, // temperature
+            new UnifiedCompletionRequest.ToolChoiceObject(
+                randomType,
+                new UnifiedCompletionRequest.ToolChoiceObject.FunctionField(randomFunctionName)
+            ),
+            Collections.singletonList(tool),
+            randomTopP // topP
+        );
+
+        UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
+
+        OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", randomModel, null);
+
+        OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
+
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
+
+        String jsonString = Strings.toString(builder);
+        String expectedJson = String.format(
+            Locale.US,
+            """
+                {
+                    "messages": [
+                        {
+                            "content": "%s",
+                            "role": "user",
+                            "tool_call_id": "%s",
+                            "tool_calls": [
+                                {
+                                    "id": "id",
+                                    "function": {
+                                        "arguments": "%s",
+                                        "name": "%s"
+                                    },
+                                    "type": "%s"
+                                }
+                            ]
+                        }
+                    ],
+                    "model": "%s",
+                    "max_completion_tokens": 100,
+                    "n": 1,
+                    "stop": ["%s"],
+                    "temperature": %.5f,
+                    "tool_choice": {
+                        "type": "%s",
+                        "function": {
+                            "name": "%s"
+                        }
+                    },
+                    "tools": [
+                        {
+                            "type": "%s",
+                            "function": {
+                                "description": "Fetches the weather in the given location",
+                                "name": "get_weather",
+                                "parameters": {
+                                    "type": "object",
+                                    "properties": {
+                                        "unit": {
+                                            "description": "The unit to return the temperature in",
+                                            "type": "string",
+                                            "enum": ["F", "C"]
+                                        },
+                                        "location": {
+                                            "description": "The location to get the weather for",
+                                            "type": "string"
+                                        }
+                                    },
+                                    "additionalProperties": false,
+                                    "required": ["location", "unit"]
+                                },
+                                "strict": true
+                            }
+                        }
+                    ],
+                    "top_p": %.5f,
+                    "stream": true,
+                    "stream_options": {
+                        "include_usage": true
+                    }
+                }
+                """,
+            randomContent,
+            randomToolCallId,
+            randomArguments,
+            randomFunctionName,
+            randomType,
+            randomModel,
+            randomStop,
+            randomTemperature,
+            randomType,
+            randomFunctionName,
+            randomType,
+            randomTopP
+        );
+        assertJsonEquals(jsonString, expectedJson);
+    }
+
+    public void testSerializationWithDifferentContentTypes() throws IOException {
+        Random random = Randomness.get();
+
+        String randomContentString = "Hello, world! " + random.nextInt(1000);
+
+        String randomText = "Random text " + random.nextInt(1000);
+        String randomType = "type" + random.nextInt(1000);
+        UnifiedCompletionRequest.ContentObject contentObject = new UnifiedCompletionRequest.ContentObject(randomText, randomType);
+
+        var contentObjectsList = new ArrayList<UnifiedCompletionRequest.ContentObject>();
+        contentObjectsList.add(contentObject);
+        UnifiedCompletionRequest.ContentObjects contentObjects = new UnifiedCompletionRequest.ContentObjects(contentObjectsList);
+
+        UnifiedCompletionRequest.Message messageWithString = new UnifiedCompletionRequest.Message(
+            new UnifiedCompletionRequest.ContentString(randomContentString),
+            ROLE,
+            null,
+            null
+        );
+
+        UnifiedCompletionRequest.Message messageWithObjects = new UnifiedCompletionRequest.Message(contentObjects, ROLE, null, null);
+        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
+        messageList.add(messageWithString);
+        messageList.add(messageWithObjects);
+
+        UnifiedCompletionRequest unifiedRequest = UnifiedCompletionRequest.of(messageList);
+
+        UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
+
+        OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null);
+
+        OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
+
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
+
+        String jsonString = Strings.toString(builder);
+        String expectedJson = String.format(Locale.US, """
+            {
+                "messages": [
+                    {
+                        "content": "%s",
+                        "role": "user"
+                    },
+                    {
+                        "content": [
+                            {
+                                "text": "%s",
+                                "type": "%s"
+                            }
+                        ],
+                        "role": "user"
+                    }
+                ],
+                "model": "model-name",
+                "n": 1,
+                "stream": true,
+                "stream_options": {
+                    "include_usage": true
+                }
+            }
+            """, randomContentString, randomText, randomType);
+        assertJsonEquals(jsonString, expectedJson);
+    }
+
+    public void testSerializationWithSpecialCharacters() throws IOException {
+        UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message(
+            new UnifiedCompletionRequest.ContentString("Hello, world! \n \"Special\" characters: \t \\ /"),
+            ROLE,
+            "tool_call_id\twith\ttabs",
+            Collections.singletonList(
+                new UnifiedCompletionRequest.ToolCall(
+                    "id\\with\\backslashes",
+                    new UnifiedCompletionRequest.ToolCall.FunctionField("arguments\"with\"quotes", "function_name/with/slashes"),
+                    "type"
+                )
+            )
+        );
+        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
+        messageList.add(message);
+        UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(
+            messageList,
+            null, // model
+            null, // maxCompletionTokens
+            null, // stop
+            null, // temperature
+            null, // toolChoice
+            null, // tools
+            null  // topP
+        );
+
+        UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
+
+        OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null);
+
+        OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
+
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
+
+        String jsonString = Strings.toString(builder);
+        String expectedJson = """
+            {
+                "messages": [
+                    {
+                        "content": "Hello, world! \\n \\"Special\\" characters: \\t \\\\ /",
+                        "role": "user",
+                        "tool_call_id": "tool_call_id\\twith\\ttabs",
+                        "tool_calls": [
+                            {
+                                "id": "id\\\\with\\\\backslashes",
+                                "function": {
+                                    "arguments": "arguments\\"with\\"quotes",
+                                    "name": "function_name/with/slashes"
+                                },
+                                "type": "type"
+                            }
+                        ]
+                    }
+                ],
+                "model": "model-name",
+                "n": 1,
+                "stream": true,
+                "stream_options": {
+                    "include_usage": true
+                }
+            }
+            """;
+        assertJsonEquals(jsonString, expectedJson);
+    }
+
+    public void testSerializationWithBooleanFields() throws IOException {
+        UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message(
+            new UnifiedCompletionRequest.ContentString("Hello, world!"),
+            ROLE,
+            null,
+            null
+        );
+        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
+        messageList.add(message);
+        UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(
+            messageList,
+            null, // model
+            null, // maxCompletionTokens
+            null, // stop
+            null, // temperature
+            null, // toolChoice
+            null, // tools
+            null  // topP
+        );
+
+        OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null);
+
+        UnifiedChatInput unifiedChatInputTrue = new UnifiedChatInput(unifiedRequest, true);
+        OpenAiUnifiedChatCompletionRequestEntity entityTrue = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInputTrue, model);
+
+        XContentBuilder builderTrue = JsonXContent.contentBuilder();
+        entityTrue.toXContent(builderTrue, ToXContent.EMPTY_PARAMS);
+
+        String jsonStringTrue = Strings.toString(builderTrue);
+        String expectedJsonTrue = """
+            {
+                "messages": [
+                    {
+                        "content": "Hello, world!",
+                        "role": "user"
+                    }
+                ],
+                "model": "model-name",
+                "n": 1,
+                "stream": true,
+                "stream_options": {
+                    "include_usage": true
+                }
+            }
+            """;
+        assertJsonEquals(expectedJsonTrue, jsonStringTrue);
+
+        UnifiedChatInput unifiedChatInputFalse = new UnifiedChatInput(unifiedRequest, false);
+        OpenAiUnifiedChatCompletionRequestEntity entityFalse = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInputFalse, model);
+
+        XContentBuilder builderFalse = JsonXContent.contentBuilder();
+        entityFalse.toXContent(builderFalse, ToXContent.EMPTY_PARAMS);
+
+        String jsonStringFalse = Strings.toString(builderFalse);
+        String expectedJsonFalse = """
+            {
+                "messages": [
+                    {
+                        "content": "Hello, world!",
+                        "role": "user"
+                    }
+                ],
+                "model": "model-name",
+                "n": 1,
+                "stream": false
+            }
+            """;
+        assertJsonEquals(expectedJsonFalse, jsonStringFalse);
+    }
+
+    public void testSerializationWithoutContentField() throws IOException {
+        UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message(
+            null,
+            "assistant",
+            "tool_call_id\twith\ttabs",
+            Collections.singletonList(
+                new UnifiedCompletionRequest.ToolCall(
+                    "id\\with\\backslashes",
+                    new UnifiedCompletionRequest.ToolCall.FunctionField("arguments\"with\"quotes", "function_name/with/slashes"),
+                    "type"
+                )
+            )
+        );
+        var messageList = new ArrayList<UnifiedCompletionRequest.Message>();
+        messageList.add(message);
+        UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(messageList, null, null, null, null, null, null, null);
+
+        UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true);
+        OpenAiChatCompletionModel model = createChatCompletionModel("test-url", "organizationId", "api-key", "test-endpoint", null);
+
+        OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model);
+
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        entity.toXContent(builder, ToXContent.EMPTY_PARAMS);
+
+        String jsonString = Strings.toString(builder);
+        String expectedJson = """
+            {
+                "messages": [
+                    {
+                        "role": "assistant",
+                        "tool_call_id": "tool_call_id\\twith\\ttabs",
+                        "tool_calls": [
+                            {
+                                "id": "id\\\\with\\\\backslashes",
+                                "function": {
+                                    "arguments": "arguments\\"with\\"quotes",
+                                    "name": "function_name/with/slashes"
+                                },
+                                "type": "type"
+                            }
+                        ]
+                   }
+                ],
+                "model": "test-endpoint",
+                "n": 1,
+                "stream": true,
+                "stream_options": {
+                    "include_usage": true
+                }
+            }
+            """;
+        assertJsonEquals(jsonString, expectedJson);
+    }
+
+    public static Map<String, Object> createParameters() {
+        Map<String, Object> parameters = new LinkedHashMap<>();
+        parameters.put("type", "object");
+
+        Map<String, Object> properties = new HashMap<>();
+
+        Map<String, Object> location = new HashMap<>();
+        location.put("type", "string");
+        location.put("description", "The location to get the weather for");
+        properties.put("location", location);
+
+        Map<String, Object> unit = new HashMap<>();
+        unit.put("type", "string");
+        unit.put("description", "The unit to return the temperature in");
+        unit.put("enum", new String[] { "F", "C" });
+        properties.put("unit", unit);
+
+        parameters.put("properties", properties);
+        parameters.put("additionalProperties", false);
+        parameters.put("required", new String[] { "location", "unit" });
+
+        return parameters;
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java
index 78743409ca178..b808dce4f9750 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java
@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.inference.highlight;
 
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriterConfig;
@@ -26,6 +28,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperServiceTestCase;
 import org.elasticsearch.index.mapper.SourceToParse;
@@ -50,7 +53,6 @@
 import org.elasticsearch.xpack.core.ml.search.WeightedToken;
 import org.elasticsearch.xpack.inference.InferencePlugin;
 import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper;
-import org.junit.Before;
 import org.mockito.Mockito;
 
 import java.io.IOException;
@@ -70,31 +72,35 @@ public class SemanticTextHighlighterTests extends MapperServiceTestCase {
     private static final String SEMANTIC_FIELD_E5 = "body-e5";
     private static final String SEMANTIC_FIELD_ELSER = "body-elser";
 
-    private Map<String, Object> queries;
+    private final boolean useLegacyFormat;
+    private final Map<String, Object> queries;
 
-    @Override
-    protected Collection<? extends Plugin> getPlugins() {
-        return List.of(new InferencePlugin(Settings.EMPTY));
+    public SemanticTextHighlighterTests(boolean useLegacyFormat) throws IOException {
+        this.useLegacyFormat = useLegacyFormat;
+        var input = Streams.readFully(SemanticTextHighlighterTests.class.getResourceAsStream("queries.json"));
+        this.queries = XContentHelper.convertToMap(input, false, XContentType.JSON).v2();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return List.of(new Object[] { true }, new Object[] { false });
     }
 
     @Override
-    @Before
-    public void setUp() throws Exception {
-        super.setUp();
-        var input = Streams.readFully(SemanticTextHighlighterTests.class.getResourceAsStream("queries.json"));
-        this.queries = XContentHelper.convertToMap(input, false, XContentType.JSON).v2();
+    protected Collection<? extends Plugin> getPlugins() {
+        return List.of(new InferencePlugin(Settings.EMPTY));
     }
 
     @SuppressWarnings("unchecked")
     public void testDenseVector() throws Exception {
-        var mapperService = createDefaultMapperService();
+        var mapperService = createDefaultMapperService(useLegacyFormat);
         Map<String, Object> queryMap = (Map<String, Object>) queries.get("dense_vector_1");
         float[] vector = readDenseVector(queryMap.get("embeddings"));
         var fieldType = (SemanticTextFieldMapper.SemanticTextFieldType) mapperService.mappingLookup().getFieldType(SEMANTIC_FIELD_E5);
         KnnVectorQueryBuilder knnQuery = new KnnVectorQueryBuilder(fieldType.getEmbeddingsField().fullPath(), vector, 10, 10, null, null);
         NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder(fieldType.getChunksField().fullPath(), knnQuery, ScoreMode.Max);
         var shardRequest = createShardSearchRequest(nestedQueryBuilder);
-        var sourceToParse = new SourceToParse("0", readSampleDoc("sample-doc.json.gz"), XContentType.JSON);
+        var sourceToParse = new SourceToParse("0", readSampleDoc(useLegacyFormat), XContentType.JSON);
 
         String[] expectedScorePassages = ((List<String>) queryMap.get("expected_by_score")).toArray(String[]::new);
         for (int i = 0; i < expectedScorePassages.length; i++) {
@@ -123,7 +129,7 @@ public void testDenseVector() throws Exception {
 
     @SuppressWarnings("unchecked")
     public void testSparseVector() throws Exception {
-        var mapperService = createDefaultMapperService();
+        var mapperService = createDefaultMapperService(useLegacyFormat);
         Map<String, Object> queryMap = (Map<String, Object>) queries.get("sparse_vector_1");
         List<WeightedToken> tokens = readSparseVector(queryMap.get("embeddings"));
         var fieldType = (SemanticTextFieldMapper.SemanticTextFieldType) mapperService.mappingLookup().getFieldType(SEMANTIC_FIELD_ELSER);
@@ -137,7 +143,7 @@ public void testSparseVector() throws Exception {
         );
         NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder(fieldType.getChunksField().fullPath(), sparseQuery, ScoreMode.Max);
         var shardRequest = createShardSearchRequest(nestedQueryBuilder);
-        var sourceToParse = new SourceToParse("0", readSampleDoc("sample-doc.json.gz"), XContentType.JSON);
+        var sourceToParse = new SourceToParse("0", readSampleDoc(useLegacyFormat), XContentType.JSON);
 
         String[] expectedScorePassages = ((List<String>) queryMap.get("expected_by_score")).toArray(String[]::new);
         for (int i = 0; i < expectedScorePassages.length; i++) {
@@ -164,9 +170,12 @@ public void testSparseVector() throws Exception {
         );
     }
 
-    private MapperService createDefaultMapperService() throws IOException {
+    private MapperService createDefaultMapperService(boolean useLegacyFormat) throws IOException {
         var mappings = Streams.readFully(SemanticTextHighlighterTests.class.getResourceAsStream("mappings.json"));
-        return createMapperService(mappings.utf8ToString());
+        var settings = Settings.builder()
+            .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat)
+            .build();
+        return createMapperService(settings, mappings.utf8ToString());
     }
 
     private float[] readDenseVector(Object value) {
@@ -280,7 +289,8 @@ private ShardSearchRequest createShardSearchRequest(QueryBuilder queryBuilder) {
         return new ShardSearchRequest(OriginalIndices.NONE, request, new ShardId("index", "index", 0), 0, 1, AliasFilter.EMPTY, 1, 0, null);
     }
 
-    private BytesReference readSampleDoc(String fileName) throws IOException {
+    private BytesReference readSampleDoc(boolean useLegacyFormat) throws IOException {
+        String fileName = useLegacyFormat ? "sample-doc-legacy.json.gz" : "sample-doc.json.gz";
         try (var in = new GZIPInputStream(SemanticTextHighlighterTests.class.getResourceAsStream(fileName))) {
             return new BytesArray(new BytesRef(in.readAllBytes()));
         }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldMapperTests.java
new file mode 100644
index 0000000000000..8fcc0df0093ce
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldMapperTests.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.mapper;
+
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperServiceTestCase;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.index.IndexVersionUtils;
+import org.elasticsearch.xpack.inference.InferencePlugin;
+
+import java.util.Collection;
+import java.util.Collections;
+
+public class SemanticInferenceMetadataFieldMapperTests extends MapperServiceTestCase {
+    @Override
+    protected Collection<? extends Plugin> getPlugins() {
+        return Collections.singletonList(new InferencePlugin(Settings.EMPTY));
+    }
+
+    public void testIsEnabled() {
+        var settings = Settings.builder()
+            .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), getRandomCompatibleIndexVersion(true))
+            .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), true)
+            .build();
+        assertFalse(InferenceMetadataFieldsMapper.isEnabled(settings));
+
+        settings = Settings.builder()
+            .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), getRandomCompatibleIndexVersion(true))
+            .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), false)
+            .build();
+        assertFalse(InferenceMetadataFieldsMapper.isEnabled(settings));
+
+        settings = Settings.builder()
+            .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), getRandomCompatibleIndexVersion(false))
+            .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), true)
+            .build();
+        assertFalse(InferenceMetadataFieldsMapper.isEnabled(settings));
+
+        settings = Settings.builder()
+            .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), getRandomCompatibleIndexVersion(false))
+            .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), false)
+            .build();
+        assertTrue(InferenceMetadataFieldsMapper.isEnabled(settings));
+    }
+
+    @Override
+    public void testFieldHasValue() {
+        assertTrue(
+            getMappedFieldType().fieldHasValue(
+                new FieldInfos(new FieldInfo[] { getFieldInfoWithName(SemanticInferenceMetadataFieldsMapper.NAME) })
+            )
+        );
+    }
+
+    @Override
+    public void testFieldHasValueWithEmptyFieldInfos() {
+        assertFalse(getMappedFieldType().fieldHasValue(FieldInfos.EMPTY));
+    }
+
+    @Override
+    public MappedFieldType getMappedFieldType() {
+        return new SemanticInferenceMetadataFieldsMapper.FieldType();
+    }
+
+    static IndexVersion getRandomCompatibleIndexVersion(boolean useLegacyFormat) {
+        if (useLegacyFormat) {
+            if (randomBoolean()) {
+                return IndexVersionUtils.randomVersionBetween(
+                    random(),
+                    IndexVersions.UPGRADE_TO_LUCENE_10_0_0,
+                    IndexVersionUtils.getPreviousVersion(IndexVersions.INFERENCE_METADATA_FIELDS)
+                );
+            }
+            return IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT);
+        } else {
+            if (randomBoolean()) {
+                return IndexVersionUtils.randomVersionBetween(random(), IndexVersions.INFERENCE_METADATA_FIELDS, IndexVersion.current());
+            }
+            return IndexVersionUtils.randomVersionBetween(
+                random(),
+                IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT,
+                IndexVersionUtils.getPreviousVersion(IndexVersions.UPGRADE_TO_LUCENE_10_0_0)
+            );
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java
index c6a492dfcf4e9..b056f5880b21a 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java
@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.inference.mapper;
 
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexableField;
@@ -22,6 +24,7 @@
 import org.apache.lucene.search.join.QueryBitSetProducer;
 import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.CheckedBiFunction;
 import org.elasticsearch.common.Strings;
@@ -29,10 +32,12 @@
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentParsingException;
 import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.LuceneDocument;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -57,35 +62,30 @@
 import org.elasticsearch.search.NestedDocuments;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentFactory;
-import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xcontent.json.JsonXContent;
+import org.elasticsearch.xpack.core.XPackClientPlugin;
 import org.elasticsearch.xpack.core.ml.search.SparseVectorQueryWrapper;
 import org.elasticsearch.xpack.inference.InferencePlugin;
 import org.elasticsearch.xpack.inference.model.TestModel;
 import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
 import java.util.function.BiConsumer;
-import java.util.stream.Stream;
 
-import static java.util.Collections.singletonList;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKED_EMBEDDINGS_FIELD;
-import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKED_TEXT_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKS_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.INFERENCE_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.INFERENCE_ID_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.MODEL_SETTINGS_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.SEARCH_INFERENCE_ID_FIELD;
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.TEXT_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getChunksFieldName;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getEmbeddingsFieldName;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.DEFAULT_ELSER_2_INFERENCE_ID;
@@ -95,9 +95,39 @@
 import static org.hamcrest.Matchers.instanceOf;
 
 public class SemanticTextFieldMapperTests extends MapperTestCase {
+    private final boolean useLegacyFormat;
+
+    public SemanticTextFieldMapperTests(boolean useLegacyFormat) {
+        this.useLegacyFormat = useLegacyFormat;
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return List.of(new Object[] { true }, new Object[] { false });
+    }
+
     @Override
     protected Collection<? extends Plugin> getPlugins() {
-        return singletonList(new InferencePlugin(Settings.EMPTY));
+        return List.of(new InferencePlugin(Settings.EMPTY), new XPackClientPlugin());
+    }
+
+    private MapperService createMapperService(XContentBuilder mappings, boolean useLegacyFormat) throws IOException {
+        var settings = Settings.builder()
+            .put(
+                IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(),
+                SemanticInferenceMetadataFieldMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat)
+            )
+            .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat)
+            .build();
+        return createMapperService(settings, mappings);
+    }
+
+    @Override
+    protected Settings getIndexSettings() {
+        return Settings.builder()
+            .put(super.getIndexSettings())
+            .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat)
+            .build();
     }
 
     @Override
@@ -158,7 +188,8 @@ public MappedFieldType getMappedFieldType() {
             null,
             null,
             null,
-            IndexVersion.current(),
+            getVersion(),
+            false,
             Map.of()
         );
     }
@@ -175,7 +206,7 @@ public void testDefaults() throws Exception {
         final XContentBuilder fieldMapping = fieldMapping(this::minimalMapping);
         final XContentBuilder expectedMapping = fieldMapping(this::metaMapping);
 
-        MapperService mapperService = createMapperService(fieldMapping);
+        MapperService mapperService = createMapperService(fieldMapping, useLegacyFormat);
         DocumentMapper mapper = mapperService.documentMapper();
         assertEquals(Strings.toString(expectedMapping), mapper.mappingSource().toString());
         assertSemanticTextField(mapperService, fieldName, false);
@@ -207,7 +238,7 @@ public void testSetInferenceEndpoints() throws IOException {
 
         {
             final XContentBuilder fieldMapping = fieldMapping(b -> b.field("type", "semantic_text").field(INFERENCE_ID_FIELD, inferenceId));
-            final MapperService mapperService = createMapperService(fieldMapping);
+            final MapperService mapperService = createMapperService(fieldMapping, useLegacyFormat);
             assertSemanticTextField(mapperService, fieldName, false);
             assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId);
             assertSerialization.accept(fieldMapping, mapperService);
@@ -221,7 +252,7 @@ public void testSetInferenceEndpoints() throws IOException {
                     .field(INFERENCE_ID_FIELD, DEFAULT_ELSER_2_INFERENCE_ID)
                     .field(SEARCH_INFERENCE_ID_FIELD, searchInferenceId)
             );
-            final MapperService mapperService = createMapperService(fieldMapping);
+            final MapperService mapperService = createMapperService(fieldMapping, useLegacyFormat);
             assertSemanticTextField(mapperService, fieldName, false);
             assertInferenceEndpoints(mapperService, fieldName, DEFAULT_ELSER_2_INFERENCE_ID, searchInferenceId);
             assertSerialization.accept(expectedMapping, mapperService);
@@ -232,7 +263,7 @@ public void testSetInferenceEndpoints() throws IOException {
                     .field(INFERENCE_ID_FIELD, inferenceId)
                     .field(SEARCH_INFERENCE_ID_FIELD, searchInferenceId)
             );
-            MapperService mapperService = createMapperService(fieldMapping);
+            MapperService mapperService = createMapperService(fieldMapping, useLegacyFormat);
             assertSemanticTextField(mapperService, fieldName, false);
             assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId);
             assertSerialization.accept(fieldMapping, mapperService);
@@ -243,7 +274,10 @@ public void testInvalidInferenceEndpoints() {
         {
             Exception e = expectThrows(
                 MapperParsingException.class,
-                () -> createMapperService(fieldMapping(b -> b.field("type", "semantic_text").field(INFERENCE_ID_FIELD, (String) null)))
+                () -> createMapperService(
+                    fieldMapping(b -> b.field("type", "semantic_text").field(INFERENCE_ID_FIELD, (String) null)),
+                    useLegacyFormat
+                )
             );
             assertThat(
                 e.getMessage(),
@@ -253,14 +287,20 @@ public void testInvalidInferenceEndpoints() {
         {
             Exception e = expectThrows(
                 MapperParsingException.class,
-                () -> createMapperService(fieldMapping(b -> b.field("type", "semantic_text").field(INFERENCE_ID_FIELD, "")))
+                () -> createMapperService(
+                    fieldMapping(b -> b.field("type", "semantic_text").field(INFERENCE_ID_FIELD, "")),
+                    useLegacyFormat
+                )
             );
             assertThat(e.getMessage(), containsString("[inference_id] on mapper [field] of type [semantic_text] must not be empty"));
         }
         {
             Exception e = expectThrows(
                 MapperParsingException.class,
-                () -> createMapperService(fieldMapping(b -> b.field("type", "semantic_text").field(SEARCH_INFERENCE_ID_FIELD, "")))
+                () -> createMapperService(
+                    fieldMapping(b -> b.field("type", "semantic_text").field(SEARCH_INFERENCE_ID_FIELD, "")),
+                    useLegacyFormat
+                )
             );
             assertThat(e.getMessage(), containsString("[search_inference_id] on mapper [field] of type [semantic_text] must not be empty"));
         }
@@ -275,14 +315,15 @@ public void testCannotBeUsedInMultiFields() {
             b.field("inference_id", "my_inference_id");
             b.endObject();
             b.endObject();
-        })));
+        }), useLegacyFormat));
         assertThat(e.getMessage(), containsString("Field [semantic] of type [semantic_text] can't be used in multifields"));
     }
 
     public void testUpdatesToInferenceIdNotSupported() throws IOException {
         String fieldName = randomAlphaOfLengthBetween(5, 15);
         MapperService mapperService = createMapperService(
-            mapping(b -> b.startObject(fieldName).field("type", "semantic_text").field("inference_id", "test_model").endObject())
+            mapping(b -> b.startObject(fieldName).field("type", "semantic_text").field("inference_id", "test_model").endObject()),
+            useLegacyFormat
         );
         assertSemanticTextField(mapperService, fieldName, false);
         Exception e = expectThrows(
@@ -326,7 +367,8 @@ public void testUpdateModelSettings() throws IOException {
         for (int depth = 1; depth < 5; depth++) {
             String fieldName = randomFieldName(depth);
             MapperService mapperService = createMapperService(
-                mapping(b -> b.startObject(fieldName).field("type", "semantic_text").field("inference_id", "test_model").endObject())
+                mapping(b -> b.startObject(fieldName).field("type", "semantic_text").field("inference_id", "test_model").endObject()),
+                useLegacyFormat
             );
             assertSemanticTextField(mapperService, fieldName, false);
             {
@@ -415,7 +457,7 @@ public void testUpdateSearchInferenceId() throws IOException {
 
         for (int depth = 1; depth < 5; depth++) {
             String fieldName = randomFieldName(depth);
-            MapperService mapperService = createMapperService(buildMapping.apply(fieldName, null));
+            MapperService mapperService = createMapperService(buildMapping.apply(fieldName, null), useLegacyFormat);
             assertSemanticTextField(mapperService, fieldName, false);
             assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId);
 
@@ -471,12 +513,20 @@ private static void assertSemanticTextField(MapperService mapperService, String
             .get(getChunksFieldName(fieldName));
         assertThat(chunksMapper, equalTo(semanticFieldMapper.fieldType().getChunksField()));
         assertThat(chunksMapper.fullPath(), equalTo(getChunksFieldName(fieldName)));
-        Mapper textMapper = chunksMapper.getMapper(CHUNKED_TEXT_FIELD);
-        assertNotNull(textMapper);
-        assertThat(textMapper, instanceOf(KeywordFieldMapper.class));
-        KeywordFieldMapper textFieldMapper = (KeywordFieldMapper) textMapper;
-        assertFalse(textFieldMapper.fieldType().isIndexed());
-        assertFalse(textFieldMapper.fieldType().hasDocValues());
+
+        Mapper textMapper = chunksMapper.getMapper(TEXT_FIELD);
+        if (semanticTextFieldType.useLegacyFormat()) {
+            assertNotNull(textMapper);
+            assertThat(textMapper, instanceOf(KeywordFieldMapper.class));
+            KeywordFieldMapper textFieldMapper = (KeywordFieldMapper) textMapper;
+            assertFalse(textFieldMapper.fieldType().isIndexed());
+            assertFalse(textFieldMapper.fieldType().hasDocValues());
+        } else {
+            assertNull(textMapper);
+            var offsetMapper = semanticTextFieldType.getOffsetsField();
+            assertThat(offsetMapper, instanceOf(OffsetSourceFieldMapper.class));
+        }
+
         if (expectedModelSettings) {
             assertNotNull(semanticFieldMapper.fieldType().getModelSettings());
             Mapper embeddingsMapper = chunksMapper.getMapper(CHUNKED_EMBEDDINGS_FIELD);
@@ -486,7 +536,11 @@ private static void assertSemanticTextField(MapperService mapperService, String
             assertTrue(embeddingsFieldMapper.fieldType() == mapperService.mappingLookup().getFieldType(getEmbeddingsFieldName(fieldName)));
             assertThat(embeddingsMapper.fullPath(), equalTo(getEmbeddingsFieldName(fieldName)));
             switch (semanticFieldMapper.fieldType().getModelSettings().taskType()) {
-                case SPARSE_EMBEDDING -> assertThat(embeddingsMapper, instanceOf(SparseVectorFieldMapper.class));
+                case SPARSE_EMBEDDING -> {
+                    assertThat(embeddingsMapper, instanceOf(SparseVectorFieldMapper.class));
+                    SparseVectorFieldMapper sparseMapper = (SparseVectorFieldMapper) embeddingsMapper;
+                    assertEquals(sparseMapper.fieldType().isStored(), semanticTextFieldType.useLegacyFormat() == false);
+                }
                 case TEXT_EMBEDDING -> assertThat(embeddingsMapper, instanceOf(DenseVectorFieldMapper.class));
                 default -> throw new AssertionError("Invalid task type");
             }
@@ -523,7 +577,7 @@ public void testSuccessfulParse() throws IOException {
                 addSemanticTextMapping(b, fieldName2, model2.getInferenceEntityId(), setSearchInferenceId ? searchInferenceId : null);
             });
 
-            MapperService mapperService = createMapperService(mapping);
+            MapperService mapperService = createMapperService(mapping, useLegacyFormat);
             assertSemanticTextField(mapperService, fieldName1, false);
             assertInferenceEndpoints(
                 mapperService,
@@ -543,10 +597,11 @@ public void testSuccessfulParse() throws IOException {
             ParsedDocument doc = documentMapper.parse(
                 source(
                     b -> addSemanticTextInferenceResults(
+                        useLegacyFormat,
                         b,
                         List.of(
-                            randomSemanticText(fieldName1, model1, List.of("a b", "c"), XContentType.JSON),
-                            randomSemanticText(fieldName2, model2, List.of("d e f"), XContentType.JSON)
+                            randomSemanticText(useLegacyFormat, fieldName1, model1, List.of("a b", "c"), XContentType.JSON),
+                            randomSemanticText(useLegacyFormat, fieldName2, model2, List.of("d e f"), XContentType.JSON)
                         )
                     )
                 )
@@ -626,52 +681,64 @@ public void testSuccessfulParse() throws IOException {
     }
 
     public void testMissingInferenceId() throws IOException {
-        DocumentMapper documentMapper = createDocumentMapper(mapping(b -> addSemanticTextMapping(b, "field", "my_id", null)));
+        final MapperService mapperService = createMapperService(
+            mapping(b -> addSemanticTextMapping(b, "field", "my_id", null)),
+            useLegacyFormat
+        );
+
         IllegalArgumentException ex = expectThrows(
             DocumentParsingException.class,
             IllegalArgumentException.class,
-            () -> documentMapper.parse(
-                source(
-                    b -> b.startObject("field")
-                        .startObject(INFERENCE_FIELD)
-                        .field(MODEL_SETTINGS_FIELD, new SemanticTextField.ModelSettings(TaskType.SPARSE_EMBEDDING, null, null, null))
-                        .field(CHUNKS_FIELD, List.of())
-                        .endObject()
-                        .endObject()
+            () -> mapperService.documentMapper()
+                .parse(
+                    semanticTextInferenceSource(
+                        useLegacyFormat,
+                        b -> b.startObject("field")
+                            .startObject(INFERENCE_FIELD)
+                            .field(MODEL_SETTINGS_FIELD, new SemanticTextField.ModelSettings(TaskType.SPARSE_EMBEDDING, null, null, null))
+                            .field(CHUNKS_FIELD, useLegacyFormat ? List.of() : Map.of())
+                            .endObject()
+                            .endObject()
+                    )
                 )
-            )
         );
         assertThat(ex.getCause().getMessage(), containsString("Required [inference_id]"));
     }
 
     public void testMissingModelSettings() throws IOException {
-        DocumentMapper documentMapper = createDocumentMapper(mapping(b -> addSemanticTextMapping(b, "field", "my_id", null)));
+        MapperService mapperService = createMapperService(mapping(b -> addSemanticTextMapping(b, "field", "my_id", null)), useLegacyFormat);
         IllegalArgumentException ex = expectThrows(
             DocumentParsingException.class,
             IllegalArgumentException.class,
-            () -> documentMapper.parse(
-                source(b -> b.startObject("field").startObject(INFERENCE_FIELD).field(INFERENCE_ID_FIELD, "my_id").endObject().endObject())
-            )
+            () -> mapperService.documentMapper()
+                .parse(
+                    semanticTextInferenceSource(
+                        useLegacyFormat,
+                        b -> b.startObject("field").startObject(INFERENCE_FIELD).field(INFERENCE_ID_FIELD, "my_id").endObject().endObject()
+                    )
+                )
         );
         assertThat(ex.getCause().getMessage(), containsString("Required [model_settings, chunks]"));
     }
 
     public void testMissingTaskType() throws IOException {
-        DocumentMapper documentMapper = createDocumentMapper(mapping(b -> addSemanticTextMapping(b, "field", "my_id", null)));
+        MapperService mapperService = createMapperService(mapping(b -> addSemanticTextMapping(b, "field", "my_id", null)), useLegacyFormat);
         IllegalArgumentException ex = expectThrows(
             DocumentParsingException.class,
             IllegalArgumentException.class,
-            () -> documentMapper.parse(
-                source(
-                    b -> b.startObject("field")
-                        .startObject(INFERENCE_FIELD)
-                        .field(INFERENCE_ID_FIELD, "my_id")
-                        .startObject(MODEL_SETTINGS_FIELD)
-                        .endObject()
-                        .endObject()
-                        .endObject()
+            () -> mapperService.documentMapper()
+                .parse(
+                    semanticTextInferenceSource(
+                        useLegacyFormat,
+                        b -> b.startObject("field")
+                            .startObject(INFERENCE_FIELD)
+                            .field(INFERENCE_ID_FIELD, "my_id")
+                            .startObject(MODEL_SETTINGS_FIELD)
+                            .endObject()
+                            .endObject()
+                            .endObject()
+                    )
                 )
-            )
         );
         assertThat(ex.getCause().getMessage(), containsString("failed to parse field [model_settings]"));
     }
@@ -712,6 +779,35 @@ public void testDenseVectorElementType() throws IOException {
         assertMapperService.accept(byteMapperService, DenseVectorFieldMapper.ElementType.BYTE);
     }
 
+    public void testModelSettingsRequiredWithChunks() throws IOException {
+        // Create inference results where model settings are set to null and chunks are provided
+        Model model = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING);
+        SemanticTextField randomSemanticText = randomSemanticText(useLegacyFormat, "field", model, List.of("a"), XContentType.JSON);
+        SemanticTextField inferenceResults = new SemanticTextField(
+            randomSemanticText.useLegacyFormat(),
+            randomSemanticText.fieldName(),
+            randomSemanticText.originalValues(),
+            new SemanticTextField.InferenceResult(
+                randomSemanticText.inference().inferenceId(),
+                null,
+                randomSemanticText.inference().chunks()
+            ),
+            randomSemanticText.contentType()
+        );
+
+        MapperService mapperService = createMapperService(
+            mapping(b -> addSemanticTextMapping(b, "field", model.getInferenceEntityId(), null)),
+            useLegacyFormat
+        );
+        SourceToParse source = source(b -> addSemanticTextInferenceResults(useLegacyFormat, b, List.of(inferenceResults)));
+        DocumentParsingException ex = expectThrows(
+            DocumentParsingException.class,
+            DocumentParsingException.class,
+            () -> mapperService.documentMapper().parse(source)
+        );
+        assertThat(ex.getMessage(), containsString("[model_settings] must be set for field [field] when chunks are provided"));
+    }
+
     private MapperService mapperServiceForFieldWithModelSettings(
         String fieldName,
         String inferenceId,
@@ -731,7 +827,7 @@ private MapperService mapperServiceForFieldWithModelSettings(
             mappingParams += ",search_inference_id=" + searchInferenceId;
         }
 
-        MapperService mapperService = createMapperService(mapping(b -> {}));
+        MapperService mapperService = createMapperService(mapping(b -> {}), useLegacyFormat);
         mapperService.merge(
             "_doc",
             new CompressedXContent(Strings.toString(PutMappingRequest.simpleMapping(fieldName, mappingParams))),
@@ -739,14 +835,19 @@ private MapperService mapperServiceForFieldWithModelSettings(
         );
 
         SemanticTextField semanticTextField = new SemanticTextField(
+            useLegacyFormat,
             fieldName,
             List.of(),
-            new SemanticTextField.InferenceResult(inferenceId, modelSettings, List.of()),
+            new SemanticTextField.InferenceResult(inferenceId, modelSettings, Map.of()),
             XContentType.JSON
         );
         XContentBuilder builder = JsonXContent.contentBuilder().startObject();
-        builder.field(semanticTextField.fieldName());
-        builder.value(semanticTextField);
+        if (useLegacyFormat) {
+            builder.field(semanticTextField.fieldName());
+            builder.value(semanticTextField);
+        } else {
+            builder.field(InferenceMetadataFieldsMapper.NAME, Map.of(semanticTextField.fieldName(), semanticTextField));
+        }
         builder.endObject();
 
         SourceToParse sourceToParse = new SourceToParse("test", BytesReference.bytes(builder), XContentType.JSON);
@@ -798,266 +899,6 @@ public void testExistsQueryDenseVector() throws IOException {
         assertThat(existsQuery, instanceOf(ESToParentBlockJoinQuery.class));
     }
 
-    public void testInsertValueMapTraversal() throws IOException {
-        {
-            XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("test", "value").endObject();
-
-            Map<String, Object> map = toSourceMap(Strings.toString(builder));
-            SemanticTextFieldMapper.insertValue("test", map, "value2");
-            assertThat(getMapValue(map, "test"), equalTo("value2"));
-            SemanticTextFieldMapper.insertValue("something.else", map, "something_else_value");
-            assertThat(getMapValue(map, "something\\.else"), equalTo("something_else_value"));
-        }
-        {
-            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
-            builder.startObject("path1").startObject("path2").field("test", "value").endObject().endObject();
-            builder.endObject();
-
-            Map<String, Object> map = toSourceMap(Strings.toString(builder));
-            SemanticTextFieldMapper.insertValue("path1.path2.test", map, "value2");
-            assertThat(getMapValue(map, "path1.path2.test"), equalTo("value2"));
-            SemanticTextFieldMapper.insertValue("path1.path2.test_me", map, "test_me_value");
-            assertThat(getMapValue(map, "path1.path2.test_me"), equalTo("test_me_value"));
-            SemanticTextFieldMapper.insertValue("path1.non_path2.test", map, "test_value");
-            assertThat(getMapValue(map, "path1.non_path2\\.test"), equalTo("test_value"));
-
-            SemanticTextFieldMapper.insertValue("path1.path2", map, Map.of("path3", "bar"));
-            assertThat(getMapValue(map, "path1.path2"), equalTo(Map.of("path3", "bar")));
-
-            SemanticTextFieldMapper.insertValue("path1", map, "baz");
-            assertThat(getMapValue(map, "path1"), equalTo("baz"));
-
-            SemanticTextFieldMapper.insertValue("path3.path4", map, Map.of("test", "foo"));
-            assertThat(getMapValue(map, "path3\\.path4"), equalTo(Map.of("test", "foo")));
-        }
-        {
-            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
-            builder.startObject("path1").array("test", "value1", "value2").endObject();
-            builder.endObject();
-            Map<String, Object> map = toSourceMap(Strings.toString(builder));
-
-            SemanticTextFieldMapper.insertValue("path1.test", map, List.of("value3", "value4", "value5"));
-            assertThat(getMapValue(map, "path1.test"), equalTo(List.of("value3", "value4", "value5")));
-
-            SemanticTextFieldMapper.insertValue("path2.test", map, List.of("value6", "value7", "value8"));
-            assertThat(getMapValue(map, "path2\\.test"), equalTo(List.of("value6", "value7", "value8")));
-        }
-    }
-
-    public void testInsertValueListTraversal() throws IOException {
-        {
-            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
-            {
-                builder.startObject("path1");
-                {
-                    builder.startArray("path2");
-                    builder.startObject().field("test", "value1").endObject();
-                    builder.endArray();
-                }
-                builder.endObject();
-            }
-            {
-                builder.startObject("path3");
-                {
-                    builder.startArray("path4");
-                    builder.startObject().field("test", "value1").endObject();
-                    builder.endArray();
-                }
-                builder.endObject();
-            }
-            builder.endObject();
-            Map<String, Object> map = toSourceMap(Strings.toString(builder));
-
-            SemanticTextFieldMapper.insertValue("path1.path2.test", map, "value2");
-            assertThat(getMapValue(map, "path1.path2.test"), equalTo("value2"));
-            SemanticTextFieldMapper.insertValue("path1.path2.test2", map, "value3");
-            assertThat(getMapValue(map, "path1.path2.test2"), equalTo("value3"));
-            assertThat(getMapValue(map, "path1.path2"), equalTo(List.of(Map.of("test", "value2", "test2", "value3"))));
-
-            SemanticTextFieldMapper.insertValue("path3.path4.test", map, "value4");
-            assertThat(getMapValue(map, "path3.path4.test"), equalTo("value4"));
-        }
-        {
-            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
-            {
-                builder.startObject("path1");
-                {
-                    builder.startArray("path2");
-                    builder.startArray();
-                    builder.startObject().field("test", "value1").endObject();
-                    builder.endArray();
-                    builder.endArray();
-                }
-                builder.endObject();
-            }
-            builder.endObject();
-            Map<String, Object> map = toSourceMap(Strings.toString(builder));
-
-            SemanticTextFieldMapper.insertValue("path1.path2.test", map, "value2");
-            assertThat(getMapValue(map, "path1.path2.test"), equalTo("value2"));
-            SemanticTextFieldMapper.insertValue("path1.path2.test2", map, "value3");
-            assertThat(getMapValue(map, "path1.path2.test2"), equalTo("value3"));
-            assertThat(getMapValue(map, "path1.path2"), equalTo(List.of(List.of(Map.of("test", "value2", "test2", "value3")))));
-        }
-    }
-
-    public void testInsertValueFieldsWithDots() throws IOException {
-        {
-            XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("xxx.yyy", "value1").endObject();
-            Map<String, Object> map = toSourceMap(Strings.toString(builder));
-
-            SemanticTextFieldMapper.insertValue("xxx.yyy", map, "value2");
-            assertThat(getMapValue(map, "xxx\\.yyy"), equalTo("value2"));
-
-            SemanticTextFieldMapper.insertValue("xxx", map, "value3");
-            assertThat(getMapValue(map, "xxx"), equalTo("value3"));
-        }
-        {
-            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
-            {
-                builder.startObject("path1.path2");
-                {
-                    builder.startObject("path3.path4");
-                    builder.field("test", "value1");
-                    builder.endObject();
-                }
-                builder.endObject();
-            }
-            builder.endObject();
-            Map<String, Object> map = toSourceMap(Strings.toString(builder));
-
-            SemanticTextFieldMapper.insertValue("path1.path2.path3.path4.test", map, "value2");
-            assertThat(getMapValue(map, "path1\\.path2.path3\\.path4.test"), equalTo("value2"));
-
-            SemanticTextFieldMapper.insertValue("path1.path2.path3.path4.test2", map, "value3");
-            assertThat(getMapValue(map, "path1\\.path2.path3\\.path4.test2"), equalTo("value3"));
-            assertThat(getMapValue(map, "path1\\.path2.path3\\.path4"), equalTo(Map.of("test", "value2", "test2", "value3")));
-        }
-    }
-
-    public void testInsertValueAmbiguousPath() throws IOException {
-        // Mixed dotted object notation
-        {
-            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
-            {
-                builder.startObject("path1.path2");
-                {
-                    builder.startObject("path3");
-                    builder.field("test1", "value1");
-                    builder.endObject();
-                }
-                builder.endObject();
-            }
-            {
-                builder.startObject("path1");
-                {
-                    builder.startObject("path2.path3");
-                    builder.field("test2", "value2");
-                    builder.endObject();
-                }
-                builder.endObject();
-            }
-            builder.endObject();
-            Map<String, Object> map = toSourceMap(Strings.toString(builder));
-            final Map<String, Object> originalMap = Collections.unmodifiableMap(toSourceMap(Strings.toString(builder)));
-
-            IllegalArgumentException ex = assertThrows(
-                IllegalArgumentException.class,
-                () -> SemanticTextFieldMapper.insertValue("path1.path2.path3.test1", map, "value3")
-            );
-            assertThat(
-                ex.getMessage(),
-                equalTo("Path [path1.path2.path3.test1] could be inserted in 2 distinct ways, it is ambiguous which one to use")
-            );
-
-            ex = assertThrows(
-                IllegalArgumentException.class,
-                () -> SemanticTextFieldMapper.insertValue("path1.path2.path3.test3", map, "value4")
-            );
-            assertThat(
-                ex.getMessage(),
-                equalTo("Path [path1.path2.path3.test3] could be inserted in 2 distinct ways, it is ambiguous which one to use")
-            );
-
-            assertThat(map, equalTo(originalMap));
-        }
-
-        // traversal through lists
-        {
-            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
-            {
-                builder.startObject("path1.path2");
-                {
-                    builder.startArray("path3");
-                    builder.startObject().field("test1", "value1").endObject();
-                    builder.endArray();
-                }
-                builder.endObject();
-            }
-            {
-                builder.startObject("path1");
-                {
-                    builder.startArray("path2.path3");
-                    builder.startObject().field("test2", "value2").endObject();
-                    builder.endArray();
-                }
-                builder.endObject();
-            }
-            builder.endObject();
-            Map<String, Object> map = toSourceMap(Strings.toString(builder));
-            final Map<String, Object> originalMap = Collections.unmodifiableMap(toSourceMap(Strings.toString(builder)));
-
-            IllegalArgumentException ex = assertThrows(
-                IllegalArgumentException.class,
-                () -> SemanticTextFieldMapper.insertValue("path1.path2.path3.test1", map, "value3")
-            );
-            assertThat(
-                ex.getMessage(),
-                equalTo("Path [path1.path2.path3.test1] could be inserted in 2 distinct ways, it is ambiguous which one to use")
-            );
-
-            ex = assertThrows(
-                IllegalArgumentException.class,
-                () -> SemanticTextFieldMapper.insertValue("path1.path2.path3.test3", map, "value4")
-            );
-            assertThat(
-                ex.getMessage(),
-                equalTo("Path [path1.path2.path3.test3] could be inserted in 2 distinct ways, it is ambiguous which one to use")
-            );
-
-            assertThat(map, equalTo(originalMap));
-        }
-    }
-
-    public void testInsertValueCannotTraversePath() throws IOException {
-        XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
-        {
-            builder.startObject("path1");
-            {
-                builder.startArray("path2");
-                builder.startArray();
-                builder.startObject().field("test", "value1").endObject();
-                builder.endArray();
-                builder.endArray();
-            }
-            builder.endObject();
-        }
-        builder.endObject();
-        Map<String, Object> map = toSourceMap(Strings.toString(builder));
-        final Map<String, Object> originalMap = Collections.unmodifiableMap(toSourceMap(Strings.toString(builder)));
-
-        IllegalArgumentException ex = assertThrows(
-            IllegalArgumentException.class,
-            () -> SemanticTextFieldMapper.insertValue("path1.path2.test.test2", map, "value2")
-        );
-        assertThat(
-            ex.getMessage(),
-            equalTo("Path [path1.path2.test] has value [value1] of type [String], which cannot be traversed into further")
-        );
-
-        assertThat(map, equalTo(originalMap));
-    }
-
     @Override
     protected void assertExistsQuery(MappedFieldType fieldType, Query query, LuceneDocument fields) {
         // Until a doc is indexed, the query is rewritten as match no docs
@@ -1079,11 +920,23 @@ private static void addSemanticTextMapping(
         mappingBuilder.endObject();
     }
 
-    private static void addSemanticTextInferenceResults(XContentBuilder sourceBuilder, List<SemanticTextField> semanticTextInferenceResults)
-        throws IOException {
-        for (var field : semanticTextInferenceResults) {
-            sourceBuilder.field(field.fieldName());
-            sourceBuilder.value(field);
+    private static void addSemanticTextInferenceResults(
+        boolean useLegacyFormat,
+        XContentBuilder sourceBuilder,
+        List<SemanticTextField> semanticTextInferenceResults
+    ) throws IOException {
+        if (useLegacyFormat) {
+            for (var field : semanticTextInferenceResults) {
+                sourceBuilder.field(field.fieldName());
+                sourceBuilder.value(field);
+            }
+        } else {
+            // Use a linked hash map to maintain insertion-order iteration over the inference fields
+            Map<String, Object> inferenceMetadataFields = new LinkedHashMap<>();
+            for (var field : semanticTextInferenceResults) {
+                inferenceMetadataFields.put(field.fieldName(), field);
+            }
+            sourceBuilder.field(InferenceMetadataFieldsMapper.NAME, inferenceMetadataFields);
         }
     }
 
@@ -1119,6 +972,19 @@ private static Query generateNestedTermSparseVectorQuery(NestedLookup nestedLook
         );
     }
 
+    private static SourceToParse semanticTextInferenceSource(boolean useLegacyFormat, CheckedConsumer<XContentBuilder, IOException> build)
+        throws IOException {
+        return source(b -> {
+            if (useLegacyFormat == false) {
+                b.startObject(InferenceMetadataFieldsMapper.NAME);
+            }
+            build.accept(b);
+            if (useLegacyFormat == false) {
+                b.endObject();
+            }
+        });
+    }
+
     private static void assertChildLeafNestedDocument(
         LeafNestedDocuments leaf,
         int advanceToDoc,
@@ -1143,68 +1009,4 @@ private static void assertSparseFeatures(LuceneDocument doc, String fieldName, i
         }
         assertThat(count, equalTo(expectedCount));
     }
-
-    private Map<String, Object> toSourceMap(String source) throws IOException {
-        try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
-            return parser.map();
-        }
-    }
-
-    private static Object getMapValue(Map<String, Object> map, String key) {
-        // Split the path on unescaped "." chars and then unescape the escaped "." chars
-        final String[] pathElements = Arrays.stream(key.split("(?<!\\\\)\\.")).map(k -> k.replace("\\.", ".")).toArray(String[]::new);
-
-        Object value = null;
-        Object nextLayer = map;
-        for (int i = 0; i < pathElements.length; i++) {
-            if (nextLayer instanceof Map<?, ?> nextMap) {
-                value = nextMap.get(pathElements[i]);
-            } else if (nextLayer instanceof List<?> nextList) {
-                final String pathElement = pathElements[i];
-                List<?> values = nextList.stream().flatMap(v -> {
-                    Stream.Builder<Object> streamBuilder = Stream.builder();
-                    if (v instanceof List<?> innerList) {
-                        traverseList(innerList, streamBuilder);
-                    } else {
-                        streamBuilder.add(v);
-                    }
-                    return streamBuilder.build();
-                }).filter(v -> v instanceof Map<?, ?>).map(v -> ((Map<?, ?>) v).get(pathElement)).filter(Objects::nonNull).toList();
-
-                if (values.isEmpty()) {
-                    return null;
-                } else if (values.size() > 1) {
-                    throw new AssertionError("List " + nextList + " contains multiple values for [" + pathElement + "]");
-                } else {
-                    value = values.getFirst();
-                }
-            } else if (nextLayer == null) {
-                break;
-            } else {
-                throw new AssertionError(
-                    "Path ["
-                        + String.join(".", Arrays.copyOfRange(pathElements, 0, i))
-                        + "] has value ["
-                        + value
-                        + "] of type ["
-                        + value.getClass().getSimpleName()
-                        + "], which cannot be traversed into further"
-                );
-            }
-
-            nextLayer = value;
-        }
-
-        return value;
-    }
-
-    private static void traverseList(List<?> list, Stream.Builder<Object> streamBuilder) {
-        for (Object value : list) {
-            if (value instanceof List<?> innerList) {
-                traverseList(innerList, streamBuilder);
-            } else {
-                streamBuilder.add(value);
-            }
-        }
-    }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java
index dcdd9b3d42341..29ca71d38e1b2 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java
@@ -7,9 +7,10 @@
 
 package org.elasticsearch.xpack.inference.mapper;
 
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.inference.ChunkedInference;
 import org.elasticsearch.inference.Model;
@@ -27,18 +28,31 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
+import java.util.ListIterator;
 import java.util.Map;
 import java.util.function.Predicate;
 
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKED_EMBEDDINGS_FIELD;
-import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.toSemanticTextFieldChunks;
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.toSemanticTextFieldChunk;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
 public class SemanticTextFieldTests extends AbstractXContentTestCase<SemanticTextField> {
     private static final String NAME = "field";
 
+    private final boolean useLegacyFormat;
+
+    public SemanticTextFieldTests(boolean useLegacyFormat) {
+        this.useLegacyFormat = useLegacyFormat;
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return List.of(new Object[] { true }, new Object[] { false });
+    }
+
     @Override
     protected Predicate<String> getRandomFieldsExcludeFilter() {
         return n -> n.endsWith(CHUNKED_EMBEDDINGS_FIELD);
@@ -46,39 +60,45 @@ protected Predicate<String> getRandomFieldsExcludeFilter() {
 
     @Override
     protected void assertEqualInstances(SemanticTextField expectedInstance, SemanticTextField newInstance) {
+        assertThat(newInstance.useLegacyFormat(), equalTo(newInstance.useLegacyFormat()));
         assertThat(newInstance.fieldName(), equalTo(expectedInstance.fieldName()));
         assertThat(newInstance.originalValues(), equalTo(expectedInstance.originalValues()));
         assertThat(newInstance.inference().modelSettings(), equalTo(expectedInstance.inference().modelSettings()));
         assertThat(newInstance.inference().chunks().size(), equalTo(expectedInstance.inference().chunks().size()));
         SemanticTextField.ModelSettings modelSettings = newInstance.inference().modelSettings();
-        for (int i = 0; i < newInstance.inference().chunks().size(); i++) {
-            assertThat(newInstance.inference().chunks().get(i).text(), equalTo(expectedInstance.inference().chunks().get(i).text()));
-            switch (modelSettings.taskType()) {
-                case TEXT_EMBEDDING -> {
-                    double[] expectedVector = parseDenseVector(
-                        expectedInstance.inference().chunks().get(i).rawEmbeddings(),
-                        modelSettings.dimensions(),
-                        expectedInstance.contentType()
-                    );
-                    double[] newVector = parseDenseVector(
-                        newInstance.inference().chunks().get(i).rawEmbeddings(),
-                        modelSettings.dimensions(),
-                        newInstance.contentType()
-                    );
-                    assertArrayEquals(expectedVector, newVector, 0.0000001f);
+        for (var entry : newInstance.inference().chunks().entrySet()) {
+            var expectedChunks = expectedInstance.inference().chunks().get(entry.getKey());
+            assertNotNull(expectedChunks);
+            assertThat(entry.getValue().size(), equalTo(expectedChunks.size()));
+            for (int i = 0; i < entry.getValue().size(); i++) {
+                var actualChunk = entry.getValue().get(i);
+                assertThat(actualChunk.text(), equalTo(expectedChunks.get(i).text()));
+                assertThat(actualChunk.startOffset(), equalTo(expectedChunks.get(i).startOffset()));
+                assertThat(actualChunk.endOffset(), equalTo(expectedChunks.get(i).endOffset()));
+                switch (modelSettings.taskType()) {
+                    case TEXT_EMBEDDING -> {
+                        double[] expectedVector = parseDenseVector(
+                            expectedChunks.get(i).rawEmbeddings(),
+                            modelSettings.dimensions(),
+                            expectedInstance.contentType()
+                        );
+                        double[] newVector = parseDenseVector(
+                            actualChunk.rawEmbeddings(),
+                            modelSettings.dimensions(),
+                            newInstance.contentType()
+                        );
+                        assertArrayEquals(expectedVector, newVector, 0.0000001f);
+                    }
+                    case SPARSE_EMBEDDING -> {
+                        List<WeightedToken> expectedTokens = parseWeightedTokens(
+                            expectedChunks.get(i).rawEmbeddings(),
+                            expectedInstance.contentType()
+                        );
+                        List<WeightedToken> newTokens = parseWeightedTokens(actualChunk.rawEmbeddings(), newInstance.contentType());
+                        assertThat(newTokens, equalTo(expectedTokens));
+                    }
+                    default -> throw new AssertionError("Invalid task type " + modelSettings.taskType());
                 }
-                case SPARSE_EMBEDDING -> {
-                    List<WeightedToken> expectedTokens = parseWeightedTokens(
-                        expectedInstance.inference().chunks().get(i).rawEmbeddings(),
-                        expectedInstance.contentType()
-                    );
-                    List<WeightedToken> newTokens = parseWeightedTokens(
-                        newInstance.inference().chunks().get(i).rawEmbeddings(),
-                        newInstance.contentType()
-                    );
-                    assertThat(newTokens, equalTo(expectedTokens));
-                }
-                default -> throw new AssertionError("Invalid task type " + modelSettings.taskType());
             }
         }
     }
@@ -87,7 +107,13 @@ protected void assertEqualInstances(SemanticTextField expectedInstance, Semantic
     protected SemanticTextField createTestInstance() {
         List<String> rawValues = randomList(1, 5, () -> randomSemanticTextInput().toString());
         try { // try catch required for override
-            return randomSemanticText(NAME, TestModel.createRandomInstance(), rawValues, randomFrom(XContentType.values()));
+            return randomSemanticText(
+                useLegacyFormat,
+                NAME,
+                TestModel.createRandomInstance(),
+                rawValues,
+                randomFrom(XContentType.values())
+            );
         } catch (IOException e) {
             fail("Failed to create random SemanticTextField instance");
         }
@@ -96,12 +122,12 @@ protected SemanticTextField createTestInstance() {
 
     @Override
     protected SemanticTextField doParseInstance(XContentParser parser) throws IOException {
-        return SemanticTextField.parse(parser, new Tuple<>(NAME, parser.contentType()));
+        return SemanticTextField.parse(parser, new SemanticTextField.ParserContext(useLegacyFormat, NAME, parser.contentType()));
     }
 
     @Override
     protected boolean supportsUnknownFields() {
-        return true;
+        return false;
     }
 
     public void testModelSettingsValidation() {
@@ -185,30 +211,60 @@ public static ChunkedInferenceEmbeddingSparse randomChunkedInferenceEmbeddingSpa
         return new ChunkedInferenceEmbeddingSparse(chunks);
     }
 
-    public static SemanticTextField randomSemanticText(String fieldName, Model model, List<String> inputs, XContentType contentType)
-        throws IOException {
+    public static SemanticTextField randomSemanticText(
+        boolean useLegacyFormat,
+        String fieldName,
+        Model model,
+        List<String> inputs,
+        XContentType contentType
+    ) throws IOException {
         ChunkedInference results = switch (model.getTaskType()) {
             case TEXT_EMBEDDING -> randomChunkedInferenceEmbeddingFloat(model, inputs);
             case SPARSE_EMBEDDING -> randomChunkedInferenceEmbeddingSparse(inputs);
             default -> throw new AssertionError("invalid task type: " + model.getTaskType().name());
         };
-        return semanticTextFieldFromChunkedInferenceResults(fieldName, model, inputs, results, contentType);
+        return semanticTextFieldFromChunkedInferenceResults(useLegacyFormat, fieldName, model, inputs, results, contentType);
     }
 
     public static SemanticTextField semanticTextFieldFromChunkedInferenceResults(
+        boolean useLegacyFormat,
         String fieldName,
         Model model,
         List<String> inputs,
         ChunkedInference results,
         XContentType contentType
     ) throws IOException {
+
+        // In this test framework, we don't perform "real" chunking; each input generates one chunk. Thus, we can assume there is a
+        // one-to-one relationship between inputs and chunks. Iterate over the inputs and chunks to match each input with its
+        // corresponding chunk.
+        final List<SemanticTextField.Chunk> chunks = new ArrayList<>(inputs.size());
+        int offsetAdjustment = 0;
+        Iterator<String> inputsIt = inputs.iterator();
+        Iterator<ChunkedInference.Chunk> chunkIt = results.chunksAsMatchedTextAndByteReference(contentType.xContent());
+        while (inputsIt.hasNext() && chunkIt.hasNext()) {
+            String input = inputsIt.next();
+            var chunk = chunkIt.next();
+            chunks.add(toSemanticTextFieldChunk(input, offsetAdjustment, chunk, useLegacyFormat));
+
+            // When using the inference metadata fields format, all the input values are concatenated so that the
+            // chunk text offsets are expressed in the context of a single string. Calculate the offset adjustment
+            // to apply to account for this.
+            offsetAdjustment = input.length() + 1; // Add one for separator char length
+        }
+
+        if (inputsIt.hasNext() || chunkIt.hasNext()) {
+            throw new IllegalArgumentException("Input list size and chunk count do not match");
+        }
+
         return new SemanticTextField(
+            useLegacyFormat,
             fieldName,
-            inputs,
+            useLegacyFormat ? inputs : null,
             new SemanticTextField.InferenceResult(
                 model.getInferenceEntityId(),
                 new SemanticTextField.ModelSettings(model),
-                toSemanticTextFieldChunks(List.of(results), contentType)
+                Map.of(fieldName, chunks)
             ),
             contentType
         );
@@ -232,37 +288,53 @@ public static Object randomSemanticTextInput() {
         }
     }
 
-    public static ChunkedInference toChunkedResult(SemanticTextField field) throws IOException {
+    public static ChunkedInference toChunkedResult(
+        boolean useLegacyFormat,
+        Map<String, List<String>> matchedTextMap,
+        SemanticTextField field
+    ) {
         switch (field.inference().modelSettings().taskType()) {
             case SPARSE_EMBEDDING -> {
                 List<ChunkedInferenceEmbeddingSparse.SparseEmbeddingChunk> chunks = new ArrayList<>();
-                for (var chunk : field.inference().chunks()) {
-                    var tokens = parseWeightedTokens(chunk.rawEmbeddings(), field.contentType());
-                    chunks.add(
-                        new ChunkedInferenceEmbeddingSparse.SparseEmbeddingChunk(
-                            tokens,
-                            chunk.text(),
-                            new ChunkedInference.TextOffset(0, chunk.text().length())
-                        )
-                    );
+                for (var entry : field.inference().chunks().entrySet()) {
+                    String entryField = entry.getKey();
+                    List<SemanticTextField.Chunk> entryChunks = entry.getValue();
+                    List<String> entryFieldMatchedText = validateAndGetMatchedTextForField(matchedTextMap, entryField, entryChunks.size());
+
+                    ListIterator<String> matchedTextIt = entryFieldMatchedText.listIterator();
+                    for (var chunk : entryChunks) {
+                        String matchedText = matchedTextIt.next();
+                        ChunkedInference.TextOffset offset = createOffset(useLegacyFormat, chunk, matchedText);
+                        var tokens = parseWeightedTokens(chunk.rawEmbeddings(), field.contentType());
+                        chunks.add(new ChunkedInferenceEmbeddingSparse.SparseEmbeddingChunk(tokens, matchedText, offset));
+                    }
                 }
                 return new ChunkedInferenceEmbeddingSparse(chunks);
             }
             case TEXT_EMBEDDING -> {
                 List<ChunkedInferenceEmbeddingFloat.FloatEmbeddingChunk> chunks = new ArrayList<>();
-                for (var chunk : field.inference().chunks()) {
-                    double[] values = parseDenseVector(
-                        chunk.rawEmbeddings(),
-                        field.inference().modelSettings().dimensions(),
-                        field.contentType()
-                    );
-                    chunks.add(
-                        new ChunkedInferenceEmbeddingFloat.FloatEmbeddingChunk(
-                            FloatConversionUtils.floatArrayOf(values),
-                            chunk.text(),
-                            new ChunkedInference.TextOffset(0, chunk.text().length())
-                        )
-                    );
+                for (var entry : field.inference().chunks().entrySet()) {
+                    String entryField = entry.getKey();
+                    List<SemanticTextField.Chunk> entryChunks = entry.getValue();
+                    List<String> entryFieldMatchedText = validateAndGetMatchedTextForField(matchedTextMap, entryField, entryChunks.size());
+
+                    ListIterator<String> matchedTextIt = entryFieldMatchedText.listIterator();
+                    for (var chunk : entryChunks) {
+                        String matchedText = matchedTextIt.next();
+                        ChunkedInference.TextOffset offset = createOffset(useLegacyFormat, chunk, matchedText);
+                        double[] values = parseDenseVector(
+                            chunk.rawEmbeddings(),
+                            field.inference().modelSettings().dimensions(),
+                            field.contentType()
+                        );
+                        chunks.add(
+                            new ChunkedInferenceEmbeddingFloat.FloatEmbeddingChunk(
+                                FloatConversionUtils.floatArrayOf(values),
+                                matchedText,
+                                offset
+                            )
+                        );
+                    }
                 }
                 return new ChunkedInferenceEmbeddingFloat(chunks);
             }
@@ -270,6 +342,38 @@ public static ChunkedInference toChunkedResult(SemanticTextField field) throws I
         }
     }
 
+    private static List<String> validateAndGetMatchedTextForField(
+        Map<String, List<String>> matchedTextMap,
+        String fieldName,
+        int chunkCount
+    ) {
+        List<String> fieldMatchedText = matchedTextMap.get(fieldName);
+        if (fieldMatchedText == null) {
+            throw new IllegalStateException("No matched text list exists for field [" + fieldName + "]");
+        } else if (fieldMatchedText.size() != chunkCount) {
+            throw new IllegalStateException("Matched text list size does not equal chunk count for field [" + fieldName + "]");
+        }
+
+        return fieldMatchedText;
+    }
+
+    /**
+     * Create a {@link ChunkedInference.TextOffset} instance with valid offset values. When using the legacy semantic text format, the
+     * offset values are not written to {@link SemanticTextField.Chunk}, so we cannot read them from there. Instead, use the knowledge that
+     * the matched text corresponds to one complete input value (i.e. one input value -> one chunk) to calculate the offset values.
+     *
+     * @param useLegacyFormat Whether the old format should be used
+     * @param chunk The chunk to get/calculate offset values for
+     * @param matchedText The matched text to calculate offset values for
+     * @return A {@link ChunkedInference.TextOffset} instance with valid offset values
+     */
+    private static ChunkedInference.TextOffset createOffset(boolean useLegacyFormat, SemanticTextField.Chunk chunk, String matchedText) {
+        final int startOffset = useLegacyFormat ? 0 : chunk.startOffset();
+        final int endOffset = useLegacyFormat ? matchedText.length() : chunk.endOffset();
+
+        return new ChunkedInference.TextOffset(startOffset, endOffset);
+    }
+
     private static double[] parseDenseVector(BytesReference value, int numDims, XContentType contentType) {
         try (XContentParser parser = XContentHelper.createParserNotCompressed(XContentParserConfiguration.EMPTY, value, contentType)) {
             parser.nextToken();
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextUtilsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextUtilsTests.java
new file mode 100644
index 0000000000000..e334335d6c78d
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextUtilsTests.java
@@ -0,0 +1,351 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.mapper;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Stream;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class SemanticTextUtilsTests extends ESTestCase {
+    public void testInsertValueMapTraversal() throws IOException {
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("test", "value").endObject();
+
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+            SemanticTextUtils.insertValue("test", map, "value2");
+            assertThat(getMapValue(map, "test"), equalTo("value2"));
+            SemanticTextUtils.insertValue("something.else", map, "something_else_value");
+            assertThat(getMapValue(map, "something\\.else"), equalTo("something_else_value"));
+        }
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            builder.startObject("path1").startObject("path2").field("test", "value").endObject().endObject();
+            builder.endObject();
+
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+            SemanticTextUtils.insertValue("path1.path2.test", map, "value2");
+            assertThat(getMapValue(map, "path1.path2.test"), equalTo("value2"));
+            SemanticTextUtils.insertValue("path1.path2.test_me", map, "test_me_value");
+            assertThat(getMapValue(map, "path1.path2.test_me"), equalTo("test_me_value"));
+            SemanticTextUtils.insertValue("path1.non_path2.test", map, "test_value");
+            assertThat(getMapValue(map, "path1.non_path2\\.test"), equalTo("test_value"));
+
+            SemanticTextUtils.insertValue("path1.path2", map, Map.of("path3", "bar"));
+            assertThat(getMapValue(map, "path1.path2"), equalTo(Map.of("path3", "bar")));
+
+            SemanticTextUtils.insertValue("path1", map, "baz");
+            assertThat(getMapValue(map, "path1"), equalTo("baz"));
+
+            SemanticTextUtils.insertValue("path3.path4", map, Map.of("test", "foo"));
+            assertThat(getMapValue(map, "path3\\.path4"), equalTo(Map.of("test", "foo")));
+        }
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            builder.startObject("path1").array("test", "value1", "value2").endObject();
+            builder.endObject();
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+
+            SemanticTextUtils.insertValue("path1.test", map, List.of("value3", "value4", "value5"));
+            assertThat(getMapValue(map, "path1.test"), equalTo(List.of("value3", "value4", "value5")));
+
+            SemanticTextUtils.insertValue("path2.test", map, List.of("value6", "value7", "value8"));
+            assertThat(getMapValue(map, "path2\\.test"), equalTo(List.of("value6", "value7", "value8")));
+        }
+    }
+
+    public void testInsertValueListTraversal() throws IOException {
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            {
+                builder.startObject("path1");
+                {
+                    builder.startArray("path2");
+                    builder.startObject().field("test", "value1").endObject();
+                    builder.endArray();
+                }
+                builder.endObject();
+            }
+            {
+                builder.startObject("path3");
+                {
+                    builder.startArray("path4");
+                    builder.startObject().field("test", "value1").endObject();
+                    builder.endArray();
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+
+            SemanticTextUtils.insertValue("path1.path2.test", map, "value2");
+            assertThat(getMapValue(map, "path1.path2.test"), equalTo("value2"));
+            SemanticTextUtils.insertValue("path1.path2.test2", map, "value3");
+            assertThat(getMapValue(map, "path1.path2.test2"), equalTo("value3"));
+            assertThat(getMapValue(map, "path1.path2"), equalTo(List.of(Map.of("test", "value2", "test2", "value3"))));
+
+            SemanticTextUtils.insertValue("path3.path4.test", map, "value4");
+            assertThat(getMapValue(map, "path3.path4.test"), equalTo("value4"));
+        }
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            {
+                builder.startObject("path1");
+                {
+                    builder.startArray("path2");
+                    builder.startArray();
+                    builder.startObject().field("test", "value1").endObject();
+                    builder.endArray();
+                    builder.endArray();
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+
+            SemanticTextUtils.insertValue("path1.path2.test", map, "value2");
+            assertThat(getMapValue(map, "path1.path2.test"), equalTo("value2"));
+            SemanticTextUtils.insertValue("path1.path2.test2", map, "value3");
+            assertThat(getMapValue(map, "path1.path2.test2"), equalTo("value3"));
+            assertThat(getMapValue(map, "path1.path2"), equalTo(List.of(List.of(Map.of("test", "value2", "test2", "value3")))));
+        }
+    }
+
+    public void testInsertValueFieldsWithDots() throws IOException {
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("xxx.yyy", "value1").endObject();
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+
+            SemanticTextUtils.insertValue("xxx.yyy", map, "value2");
+            assertThat(getMapValue(map, "xxx\\.yyy"), equalTo("value2"));
+
+            SemanticTextUtils.insertValue("xxx", map, "value3");
+            assertThat(getMapValue(map, "xxx"), equalTo("value3"));
+        }
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            {
+                builder.startObject("path1.path2");
+                {
+                    builder.startObject("path3.path4");
+                    builder.field("test", "value1");
+                    builder.endObject();
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+
+            SemanticTextUtils.insertValue("path1.path2.path3.path4.test", map, "value2");
+            assertThat(getMapValue(map, "path1\\.path2.path3\\.path4.test"), equalTo("value2"));
+
+            SemanticTextUtils.insertValue("path1.path2.path3.path4.test2", map, "value3");
+            assertThat(getMapValue(map, "path1\\.path2.path3\\.path4.test2"), equalTo("value3"));
+            assertThat(getMapValue(map, "path1\\.path2.path3\\.path4"), equalTo(Map.of("test", "value2", "test2", "value3")));
+        }
+    }
+
+    public void testInsertValueAmbiguousPath() throws IOException {
+        // Mixed dotted object notation
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            {
+                builder.startObject("path1.path2");
+                {
+                    builder.startObject("path3");
+                    builder.field("test1", "value1");
+                    builder.endObject();
+                }
+                builder.endObject();
+            }
+            {
+                builder.startObject("path1");
+                {
+                    builder.startObject("path2.path3");
+                    builder.field("test2", "value2");
+                    builder.endObject();
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+            final Map<String, Object> originalMap = Collections.unmodifiableMap(toSourceMap(Strings.toString(builder)));
+
+            IllegalArgumentException ex = assertThrows(
+                IllegalArgumentException.class,
+                () -> SemanticTextUtils.insertValue("path1.path2.path3.test1", map, "value3")
+            );
+            assertThat(
+                ex.getMessage(),
+                equalTo("Path [path1.path2.path3.test1] could be inserted in 2 distinct ways, it is ambiguous which one to use")
+            );
+
+            ex = assertThrows(
+                IllegalArgumentException.class,
+                () -> SemanticTextUtils.insertValue("path1.path2.path3.test3", map, "value4")
+            );
+            assertThat(
+                ex.getMessage(),
+                equalTo("Path [path1.path2.path3.test3] could be inserted in 2 distinct ways, it is ambiguous which one to use")
+            );
+
+            assertThat(map, equalTo(originalMap));
+        }
+
+        // traversal through lists
+        {
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            {
+                builder.startObject("path1.path2");
+                {
+                    builder.startArray("path3");
+                    builder.startObject().field("test1", "value1").endObject();
+                    builder.endArray();
+                }
+                builder.endObject();
+            }
+            {
+                builder.startObject("path1");
+                {
+                    builder.startArray("path2.path3");
+                    builder.startObject().field("test2", "value2").endObject();
+                    builder.endArray();
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            Map<String, Object> map = toSourceMap(Strings.toString(builder));
+            final Map<String, Object> originalMap = Collections.unmodifiableMap(toSourceMap(Strings.toString(builder)));
+
+            IllegalArgumentException ex = assertThrows(
+                IllegalArgumentException.class,
+                () -> SemanticTextUtils.insertValue("path1.path2.path3.test1", map, "value3")
+            );
+            assertThat(
+                ex.getMessage(),
+                equalTo("Path [path1.path2.path3.test1] could be inserted in 2 distinct ways, it is ambiguous which one to use")
+            );
+
+            ex = assertThrows(
+                IllegalArgumentException.class,
+                () -> SemanticTextUtils.insertValue("path1.path2.path3.test3", map, "value4")
+            );
+            assertThat(
+                ex.getMessage(),
+                equalTo("Path [path1.path2.path3.test3] could be inserted in 2 distinct ways, it is ambiguous which one to use")
+            );
+
+            assertThat(map, equalTo(originalMap));
+        }
+    }
+
+    public void testInsertValueCannotTraversePath() throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+        {
+            builder.startObject("path1");
+            {
+                builder.startArray("path2");
+                builder.startArray();
+                builder.startObject().field("test", "value1").endObject();
+                builder.endArray();
+                builder.endArray();
+            }
+            builder.endObject();
+        }
+        builder.endObject();
+        Map<String, Object> map = toSourceMap(Strings.toString(builder));
+        final Map<String, Object> originalMap = Collections.unmodifiableMap(toSourceMap(Strings.toString(builder)));
+
+        IllegalArgumentException ex = assertThrows(
+            IllegalArgumentException.class,
+            () -> SemanticTextUtils.insertValue("path1.path2.test.test2", map, "value2")
+        );
+        assertThat(
+            ex.getMessage(),
+            equalTo("Path [path1.path2.test] has value [value1] of type [String], which cannot be traversed into further")
+        );
+
+        assertThat(map, equalTo(originalMap));
+    }
+
+    private Map<String, Object> toSourceMap(String source) throws IOException {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
+            return parser.map();
+        }
+    }
+
+    private static Object getMapValue(Map<String, Object> map, String key) {
+        // Split the path on unescaped "." chars and then unescape the escaped "." chars
+        final String[] pathElements = Arrays.stream(key.split("(?<!\\\\)\\.")).map(k -> k.replace("\\.", ".")).toArray(String[]::new);
+
+        Object value = null;
+        Object nextLayer = map;
+        for (int i = 0; i < pathElements.length; i++) {
+            if (nextLayer instanceof Map<?, ?> nextMap) {
+                value = nextMap.get(pathElements[i]);
+            } else if (nextLayer instanceof List<?> nextList) {
+                final String pathElement = pathElements[i];
+                List<?> values = nextList.stream().flatMap(v -> {
+                    Stream.Builder<Object> streamBuilder = Stream.builder();
+                    if (v instanceof List<?> innerList) {
+                        traverseList(innerList, streamBuilder);
+                    } else {
+                        streamBuilder.add(v);
+                    }
+                    return streamBuilder.build();
+                }).filter(v -> v instanceof Map<?, ?>).map(v -> ((Map<?, ?>) v).get(pathElement)).filter(Objects::nonNull).toList();
+
+                if (values.isEmpty()) {
+                    return null;
+                } else if (values.size() > 1) {
+                    throw new AssertionError("List " + nextList + " contains multiple values for [" + pathElement + "]");
+                } else {
+                    value = values.getFirst();
+                }
+            } else if (nextLayer == null) {
+                break;
+            } else {
+                throw new AssertionError(
+                    "Path ["
+                        + String.join(".", Arrays.copyOfRange(pathElements, 0, i))
+                        + "] has value ["
+                        + value
+                        + "] of type ["
+                        + value.getClass().getSimpleName()
+                        + "], which cannot be traversed into further"
+                );
+            }
+
+            nextLayer = value;
+        }
+
+        return value;
+    }
+
+    private static void traverseList(List<?> list, Stream.Builder<Object> streamBuilder) {
+        for (Object value : list) {
+            if (value instanceof List<?> innerList) {
+                traverseList(innerList, streamBuilder);
+            } else {
+                streamBuilder.add(value);
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java
index 36aa2200eceae..d5042643013e6 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java
@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.inference.queries;
 
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
@@ -27,6 +29,7 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
@@ -41,7 +44,6 @@
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.AbstractQueryTestCase;
-import org.elasticsearch.test.index.IndexVersionUtils;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xcontent.json.JsonXContent;
@@ -65,11 +67,11 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 
 import static org.apache.lucene.search.BooleanClause.Occur.FILTER;
 import static org.apache.lucene.search.BooleanClause.Occur.MUST;
 import static org.apache.lucene.search.BooleanClause.Occur.SHOULD;
-import static org.elasticsearch.index.IndexVersions.NEW_SPARSE_VECTOR;
 import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfig.DEFAULT_RESULTS_FIELD;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -86,6 +88,7 @@ public class SemanticQueryBuilderTests extends AbstractQueryTestCase<SemanticQue
     private static InferenceResultType inferenceResultType;
     private static DenseVectorFieldMapper.ElementType denseVectorElementType;
     private static boolean useSearchInferenceId;
+    private final boolean useLegacyFormat;
 
     private enum InferenceResultType {
         NONE,
@@ -95,6 +98,15 @@ private enum InferenceResultType {
 
     private Integer queryTokenCount;
 
+    public SemanticQueryBuilderTests(boolean useLegacyFormat) {
+        this.useLegacyFormat = useLegacyFormat;
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return List.of(new Object[] { true }, new Object[] { false });
+    }
+
     @BeforeClass
     public static void setInferenceResultType() {
         // These are class variables because they are used when initializing additional mappings, which happens once per test suite run in
@@ -121,12 +133,10 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
 
     @Override
     protected Settings createTestIndexSettings() {
-        // Randomize index version within compatible range
-        // we have to prefer CURRENT since with the range of versions we support it's rather unlikely to get the current actually.
-        IndexVersion indexVersionCreated = randomBoolean()
-            ? IndexVersion.current()
-            : IndexVersionUtils.randomVersionBetween(random(), NEW_SPARSE_VECTOR, IndexVersion.current());
-        return Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, indexVersionCreated).build();
+        return Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+            .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat)
+            .build();
     }
 
     @Override
@@ -148,7 +158,11 @@ protected void initializeAdditionalMappings(MapperService mapperService) throws
     private void applyRandomInferenceResults(MapperService mapperService) throws IOException {
         // Parse random inference results (or no inference results) to set up the dynamic inference result mappings under the semantic text
         // field
-        SourceToParse sourceToParse = buildSemanticTextFieldWithInferenceResults(inferenceResultType, denseVectorElementType);
+        SourceToParse sourceToParse = buildSemanticTextFieldWithInferenceResults(
+            inferenceResultType,
+            denseVectorElementType,
+            useLegacyFormat
+        );
         if (sourceToParse != null) {
             ParsedDocument parsedDocument = mapperService.documentMapper().parse(sourceToParse);
             mapperService.merge(
@@ -198,7 +212,7 @@ private void assertSparseEmbeddingLuceneQuery(Query query) {
         Query innerQuery = assertOuterBooleanQuery(query);
         assertThat(innerQuery, instanceOf(SparseVectorQueryWrapper.class));
         var sparseQuery = (SparseVectorQueryWrapper) innerQuery;
-        assertThat(((SparseVectorQueryWrapper) innerQuery).getTermsQuery(), instanceOf(BooleanQuery.class));
+        assertThat(sparseQuery.getTermsQuery(), instanceOf(BooleanQuery.class));
 
         BooleanQuery innerBooleanQuery = (BooleanQuery) sparseQuery.getTermsQuery();
         assertThat(innerBooleanQuery.clauses().size(), equalTo(queryTokenCount));
@@ -334,7 +348,8 @@ public void testSerializingQueryWhenNoInferenceId() throws IOException {
 
     private static SourceToParse buildSemanticTextFieldWithInferenceResults(
         InferenceResultType inferenceResultType,
-        DenseVectorFieldMapper.ElementType denseVectorElementType
+        DenseVectorFieldMapper.ElementType denseVectorElementType,
+        boolean useLegacyFormat
     ) throws IOException {
         SemanticTextField.ModelSettings modelSettings = switch (inferenceResultType) {
             case NONE -> null;
@@ -350,15 +365,21 @@ private static SourceToParse buildSemanticTextFieldWithInferenceResults(
         SourceToParse sourceToParse = null;
         if (modelSettings != null) {
             SemanticTextField semanticTextField = new SemanticTextField(
+                useLegacyFormat,
                 SEMANTIC_TEXT_FIELD,
-                List.of(),
-                new SemanticTextField.InferenceResult(INFERENCE_ID, modelSettings, List.of()),
+                null,
+                new SemanticTextField.InferenceResult(INFERENCE_ID, modelSettings, Map.of(SEMANTIC_TEXT_FIELD, List.of())),
                 XContentType.JSON
             );
 
             XContentBuilder builder = JsonXContent.contentBuilder().startObject();
-            builder.field(semanticTextField.fieldName());
-            builder.value(semanticTextField);
+            if (useLegacyFormat == false) {
+                builder.startObject(InferenceMetadataFieldsMapper.NAME);
+            }
+            builder.field(semanticTextField.fieldName(), semanticTextField);
+            if (useLegacyFormat == false) {
+                builder.endObject();
+            }
             builder.endObject();
             sourceToParse = new SourceToParse("test", BytesReference.bytes(builder), XContentType.JSON);
         }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java
index c0ef4e45f101f..2d91877da97ba 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.inference.rank.random;
 
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.core.Predicates;
 import org.elasticsearch.search.retriever.RetrieverBuilder;
 import org.elasticsearch.search.retriever.RetrieverParserContext;
 import org.elasticsearch.search.retriever.TestRetrieverBuilder;
@@ -15,6 +16,7 @@
 import org.elasticsearch.usage.SearchUsage;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentParseException;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.json.JsonXContent;
 
@@ -23,6 +25,7 @@
 import java.util.List;
 
 import static org.elasticsearch.search.rank.RankBuilder.DEFAULT_RANK_WINDOW_SIZE;
+import static org.hamcrest.Matchers.containsString;
 
 public class RandomRankRetrieverBuilderTests extends AbstractXContentTestCase<RandomRankRetrieverBuilder> {
 
@@ -49,10 +52,7 @@ protected RandomRankRetrieverBuilder createTestInstance() {
     protected RandomRankRetrieverBuilder doParseInstance(XContentParser parser) throws IOException {
         return (RandomRankRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder(
             parser,
-            new RetrieverParserContext(
-                new SearchUsage(),
-                nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED || nf == RandomRankRetrieverBuilder.RANDOM_RERANKER_RETRIEVER_SUPPORTED
-            )
+            new RetrieverParserContext(new SearchUsage(), Predicates.never())
         );
     }
 
@@ -99,4 +99,65 @@ public void testParserDefaults() throws IOException {
         }
     }
 
+    public void testParserEmptyRetriever() throws IOException {
+        String json = """
+            {
+              "retriever": {
+              },
+              "field": "my-field"
+            }""";
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) {
+            XContentParseException ex = expectThrows(
+                XContentParseException.class,
+                () -> RandomRankRetrieverBuilder.PARSER.parse(parser, null)
+            );
+            assertThat(ex.getMessage(), containsString("[random_reranker] failed to parse field [retriever]"));
+            assertThat(ex.getCause().getMessage(), containsString("empty [retriever] object"));
+        }
+    }
+
+    public void testParserWrongRetrieverName() throws IOException {
+        String json = """
+            {
+              "retriever": {
+                "test2": {
+                  "value": "my-test-retriever"
+                }
+              },
+              "field": "my-field"
+            }""";
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) {
+            XContentParseException ex = expectThrows(
+                XContentParseException.class,
+                () -> RandomRankRetrieverBuilder.PARSER.parse(parser, null)
+            );
+            assertThat(ex.getMessage(), containsString("[random_reranker] failed to parse field [retriever]"));
+            assertThat(ex.getCause().getMessage(), containsString("unknown field [test2]"));
+        }
+    }
+
+    public void testExtraContent() throws IOException {
+        String json = """
+            {
+              "retriever": {
+                "test": {
+                  "value": "my-test-retriever"
+                },
+                "field2": "my-field"
+              },
+              "field": "my-field"
+            }""";
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) {
+            XContentParseException ex = expectThrows(
+                XContentParseException.class,
+                () -> RandomRankRetrieverBuilder.PARSER.parse(parser, null)
+            );
+            assertThat(ex.getMessage(), containsString("[random_reranker] failed to parse field [retriever]"));
+            assertThat(ex.getCause().getMessage(), containsString("unexpected field [field2]"));
+        }
+    }
+
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java
index 478f3b2f33c93..b6d455dd233ba 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.core.Predicates;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -68,12 +69,7 @@ protected TextSimilarityRankRetrieverBuilder createTestInstance() {
     protected TextSimilarityRankRetrieverBuilder doParseInstance(XContentParser parser) throws IOException {
         return (TextSimilarityRankRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder(
             parser,
-            new RetrieverParserContext(
-                new SearchUsage(),
-                nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED
-                    || nf == TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED
-                    || nf == TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_COMPOSITION_SUPPORTED
-            )
+            new RetrieverParserContext(new SearchUsage(), Predicates.never())
         );
     }
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestStreamInferenceActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestStreamInferenceActionTests.java
index b999e2c9b72f0..f67680ef6b625 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestStreamInferenceActionTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestStreamInferenceActionTests.java
@@ -12,8 +12,11 @@
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.test.rest.FakeRestRequest;
 import org.elasticsearch.test.rest.RestActionTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
+import org.junit.After;
 import org.junit.Before;
 
 import static org.elasticsearch.xpack.inference.rest.BaseInferenceActionTests.createResponse;
@@ -22,10 +25,18 @@
 import static org.hamcrest.Matchers.instanceOf;
 
 public class RestStreamInferenceActionTests extends RestActionTestCase {
+    private final SetOnce<ThreadPool> threadPool = new SetOnce<>();
 
     @Before
     public void setUpAction() {
-        controller().registerHandler(new RestStreamInferenceAction());
+        threadPool.set(new TestThreadPool(getTestName()));
+        controller().registerHandler(new RestStreamInferenceAction(threadPool));
+    }
+
+    @After
+    public void tearDownAction() {
+        terminate(threadPool.get());
+
     }
 
     public void testStreamIsTrue() {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceActionTests.java
index 5acfe67b175df..9dc23c890c14d 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceActionTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceActionTests.java
@@ -17,8 +17,11 @@
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.test.rest.FakeRestRequest;
 import org.elasticsearch.test.rest.RestActionTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction;
+import org.junit.After;
 import org.junit.Before;
 
 import static org.elasticsearch.xpack.inference.rest.BaseInferenceActionTests.createResponse;
@@ -27,10 +30,17 @@
 import static org.hamcrest.Matchers.instanceOf;
 
 public class RestUnifiedCompletionInferenceActionTests extends RestActionTestCase {
+    private final SetOnce<ThreadPool> threadPool = new SetOnce<>();
 
     @Before
     public void setUpAction() {
-        controller().registerHandler(new RestUnifiedCompletionInferenceAction());
+        threadPool.set(new TestThreadPool(getTestName()));
+        controller().registerHandler(new RestUnifiedCompletionInferenceAction(threadPool));
+    }
+
+    @After
+    public void tearDownAction() {
+        terminate(threadPool.get());
     }
 
     public void testStreamIsTrue() {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java
index f8a5bd8a54d31..69b26585d8d49 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java
@@ -12,13 +12,10 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.ChunkedInference;
-import org.elasticsearch.inference.EmptySettingsConfiguration;
 import org.elasticsearch.inference.InferenceServiceConfiguration;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
 import org.elasticsearch.inference.Model;
-import org.elasticsearch.inference.SettingsConfiguration;
-import org.elasticsearch.inference.TaskSettingsConfiguration;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -177,16 +174,10 @@ public TransportVersion getMinimalSupportedVersion() {
 
         @Override
         public InferenceServiceConfiguration getConfiguration() {
-            return new InferenceServiceConfiguration.Builder().setProvider("test service")
-                .setTaskTypes(supportedTaskTypes().stream().map(t -> {
-                    Map<String, SettingsConfiguration> taskSettingsConfig;
-                    switch (t) {
-                        // no task settings
-                        default -> taskSettingsConfig = EmptySettingsConfiguration.get();
-                    }
-                    return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build();
-                }).toList())
-                .setConfiguration(new HashMap<>())
+            return new InferenceServiceConfiguration.Builder().setService("test service")
+                .setName("Test")
+                .setTaskTypes(supportedTaskTypes())
+                .setConfigurations(new HashMap<>())
                 .build();
         }
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java
index 46c3a062f7db0..e2cb93f731162 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java
@@ -438,209 +438,57 @@ public void testGetConfiguration() throws Exception {
             String content = XContentHelper.stripWhitespace(
                 """
                     {
-                       "provider": "alibabacloud-ai-search",
-                       "task_types": [
-                             {
-                                 "task_type": "text_embedding",
-                                 "configuration": {
-                                     "input_type": {
-                                         "default_value": null,
-                                         "depends_on": [],
-                                         "display": "dropdown",
-                                         "label": "Input Type",
-                                         "options": [
-                                             {
-                                                 "label": "ingest",
-                                                 "value": "ingest"
-                                             },
-                                             {
-                                                 "label": "search",
-                                                 "value": "search"
-                                             }
-                                         ],
-                                         "order": 1,
-                                         "required": false,
-                                         "sensitive": false,
-                                         "tooltip": "Specifies the type of input passed to the model.",
-                                         "type": "str",
-                                         "ui_restrictions": [],
-                                         "validations": [],
-                                         "value": ""
-                                     }
-                                 }
-                             },
-                             {
-                                 "task_type": "sparse_embedding",
-                                 "configuration": {
-                                     "return_token": {
-                                         "default_value": null,
-                                         "depends_on": [],
-                                         "display": "toggle",
-                                         "label": "Return Token",
-                                         "order": 2,
-                                         "required": false,
-                                         "sensitive": false,
-                                         "tooltip": "If `true`, the token name will be returned in the response. Defaults to `false` which means only the token ID will be returned in the response.",
-                                         "type": "bool",
-                                         "ui_restrictions": [],
-                                         "validations": [],
-                                         "value": true
-                                     },
-                                     "input_type": {
-                                         "default_value": null,
-                                         "depends_on": [],
-                                         "display": "dropdown",
-                                         "label": "Input Type",
-                                         "options": [
-                                             {
-                                                 "label": "ingest",
-                                                 "value": "ingest"
-                                             },
-                                             {
-                                                 "label": "search",
-                                                 "value": "search"
-                                             }
-                                         ],
-                                         "order": 1,
-                                         "required": false,
-                                         "sensitive": false,
-                                         "tooltip": "Specifies the type of input passed to the model.",
-                                         "type": "str",
-                                         "ui_restrictions": [],
-                                         "validations": [],
-                                         "value": ""
-                                     }
-                                 }
-                             },
-                             {
-                                 "task_type": "rerank",
-                                 "configuration": {}
-                             },
-                             {
-                                 "task_type": "completion",
-                                 "configuration": {}
-                             }
-                       ],
-                       "configuration": {
+                       "service": "alibabacloud-ai-search",
+                       "name": "AlibabaCloud AI Search",
+                       "task_types": ["text_embedding", "sparse_embedding", "rerank", "completion"],
+                       "configurations": {
                          "workspace": {
-                           "default_value": null,
-                           "depends_on": [],
-                           "display": "textbox",
+                           "description": "The name of the workspace used for the {infer} task.",
                            "label": "Workspace",
-                           "order": 5,
                            "required": true,
                            "sensitive": false,
-                           "tooltip": "The name of the workspace used for the {infer} task.",
-                           "type": "str",
-                           "ui_restrictions": [],
-                           "validations": [],
-                           "value": null
+                           "updatable": false,
+                           "type": "str"
                          },
                          "api_key": {
-                           "default_value": null,
-                           "depends_on": [],
-                           "display": "textbox",
+                           "description": "A valid API key for the AlibabaCloud AI Search API.",
                            "label": "API Key",
-                           "order": 1,
                            "required": true,
                            "sensitive": true,
-                           "tooltip": "A valid API key for the AlibabaCloud AI Search API.",
-                           "type": "str",
-                           "ui_restrictions": [],
-                           "validations": [],
-                           "value": null
+                           "updatable": true,
+                           "type": "str"
                          },
                          "service_id": {
-                           "default_value": null,
-                           "depends_on": [],
-                           "display": "dropdown",
+                           "description": "The name of the model service to use for the {infer} task.",
                            "label": "Project ID",
-                           "options": [
-                             {
-                               "label": "ops-text-embedding-001",
-                               "value": "ops-text-embedding-001"
-                             },
-                             {
-                               "label": "ops-text-embedding-zh-001",
-                               "value": "ops-text-embedding-zh-001"
-                             },
-                             {
-                               "label": "ops-text-embedding-en-001",
-                               "value": "ops-text-embedding-en-001"
-                             },
-                             {
-                               "label": "ops-text-embedding-002",
-                               "value": "ops-text-embedding-002"
-                             },
-                             {
-                               "label": "ops-text-sparse-embedding-001",
-                               "value": "ops-text-sparse-embedding-001"
-                             },
-                             {
-                               "label": "ops-bge-reranker-larger",
-                               "value": "ops-bge-reranker-larger"
-                             }
-                           ],
-                           "order": 2,
                            "required": true,
                            "sensitive": false,
-                           "tooltip": "The name of the model service to use for the {infer} task.",
-                           "type": "str",
-                           "ui_restrictions": [],
-                           "validations": [],
-                           "value": null
+                           "updatable": false,
+                           "type": "str"
                          },
                          "host": {
-                           "default_value": null,
-                           "depends_on": [],
-                           "display": "textbox",
+                           "description": "The name of the host address used for the {infer} task. You can find the host address at https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[ the API keys section] of the documentation.",
                            "label": "Host",
-                           "order": 3,
                            "required": true,
                            "sensitive": false,
-                           "tooltip": "The name of the host address used for the {infer} task. You can find the host address at https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[ the API keys section] of the documentation.",
-                           "type": "str",
-                           "ui_restrictions": [],
-                           "validations": [],
-                           "value": null
+                           "updatable": false,
+                           "type": "str"
                          },
                          "rate_limit.requests_per_minute": {
-                           "default_value": null,
-                           "depends_on": [],
-                           "display": "numeric",
+                           "description": "Minimize the number of rate limit errors.",
                            "label": "Rate Limit",
-                           "order": 6,
                            "required": false,
                            "sensitive": false,
-                           "tooltip": "Minimize the number of rate limit errors.",
-                           "type": "int",
-                           "ui_restrictions": [],
-                           "validations": [],
-                           "value": null
+                           "updatable": false,
+                           "type": "int"
                          },
                          "http_schema": {
-                           "default_value": null,
-                           "depends_on": [],
-                           "display": "dropdown",
+                           "description": "",
                            "label": "HTTP Schema",
-                           "options": [
-                             {
-                               "label": "https",
-                               "value": "https"
-                             },
-                             {
-                               "label": "http",
-                               "value": "http"
-                             }
-                           ],
-                           "order": 4,
                            "required": true,
                            "sensitive": false,
-                           "tooltip": "",
-                           "type": "str",
-                           "ui_restrictions": [],
-                           "validations": [],
-                           "value": null
+                           "updatable": false,
+                           "type": "str"
                          }
                        }
                     }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionServiceSettingsTests.java
index 3a5f56c0b4247..167a3f2688292 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionServiceSettingsTests.java
@@ -8,9 +8,7 @@
 package org.elasticsearch.xpack.inference.services.alibabacloudsearch.completion;
 
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.inference.SimilarityMeasure;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xpack.inference.services.ServiceFields;
 import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceSettings;
 import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceSettingsTests;
 import org.hamcrest.MatcherAssert;
@@ -29,10 +27,6 @@ public static AlibabaCloudSearchCompletionServiceSettings createRandom() {
     }
 
     public void testFromMap() {
-        var url = "https://www.abc.com";
-        var similarity = SimilarityMeasure.DOT_PRODUCT.toString();
-        var dims = 1536;
-        var maxInputTokens = 512;
         var model = "model";
         var host = "host";
         var workspaceName = "default";
@@ -40,14 +34,6 @@ public void testFromMap() {
         var serviceSettings = AlibabaCloudSearchCompletionServiceSettings.fromMap(
             new HashMap<>(
                 Map.of(
-                    ServiceFields.URL,
-                    url,
-                    ServiceFields.SIMILARITY,
-                    similarity,
-                    ServiceFields.DIMENSIONS,
-                    dims,
-                    ServiceFields.MAX_INPUT_TOKENS,
-                    maxInputTokens,
                     AlibabaCloudSearchServiceSettings.HOST,
                     host,
                     AlibabaCloudSearchServiceSettings.SERVICE_ID,
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java
index 7ca66d54d8ac3..ce4d928458dca 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java
@@ -154,192 +154,63 @@ public void testParseRequestConfig_ThrowsUnsupportedModelType() throws IOExcepti
     @SuppressWarnings("checkstyle:LineLength")
     public void testGetConfiguration() throws Exception {
         try (var service = createAmazonBedrockService()) {
-            String content = XContentHelper.stripWhitespace(
-                """
-                    {
-                         "provider": "amazonbedrock",
-                         "task_types": [
-                               {
-                                   "task_type": "text_embedding",
-                                   "configuration": {}
-                               },
-                               {
-                                   "task_type": "completion",
-                                   "configuration": {
-                                       "top_p": {
-                                           "default_value": null,
-                                           "depends_on": [],
-                                           "display": "numeric",
-                                           "label": "Top P",
-                                           "order": 3,
-                                           "required": false,
-                                           "sensitive": false,
-                                           "tooltip": "Alternative to temperature. A number in the range of 0.0 to 1.0, to eliminate low-probability tokens.",
-                                           "type": "int",
-                                           "ui_restrictions": [],
-                                           "validations": [],
-                                           "value": null
-                                       },
-                                       "max_new_tokens": {
-                                           "default_value": null,
-                                           "depends_on": [],
-                                           "display": "numeric",
-                                           "label": "Max New Tokens",
-                                           "order": 1,
-                                           "required": false,
-                                           "sensitive": false,
-                                           "tooltip": "Sets the maximum number for the output tokens to be generated.",
-                                           "type": "int",
-                                           "ui_restrictions": [],
-                                           "validations": [],
-                                           "value": null
-                                       },
-                                       "top_k": {
-                                           "default_value": null,
-                                           "depends_on": [],
-                                           "display": "numeric",
-                                           "label": "Top K",
-                                           "order": 4,
-                                           "required": false,
-                                           "sensitive": false,
-                                           "tooltip": "Only available for anthropic, cohere, and mistral providers. Alternative to temperature.",
-                                           "type": "int",
-                                           "ui_restrictions": [],
-                                           "validations": [],
-                                           "value": null
-                                       },
-                                       "temperature": {
-                                           "default_value": null,
-                                           "depends_on": [],
-                                           "display": "numeric",
-                                           "label": "Temperature",
-                                           "order": 2,
-                                           "required": false,
-                                           "sensitive": false,
-                                           "tooltip": "A number between 0.0 and 1.0 that controls the apparent creativity of the results.",
-                                           "type": "int",
-                                           "ui_restrictions": [],
-                                           "validations": [],
-                                           "value": null
-                                       }
-                                   }
-                               }
-                         ],
-                         "configuration": {
-                             "secret_key": {
-                                 "default_value": null,
-                                 "depends_on": [],
-                                 "display": "textbox",
-                                 "label": "Secret Key",
-                                 "order": 2,
-                                 "required": true,
-                                 "sensitive": true,
-                                 "tooltip": "A valid AWS secret key that is paired with the access_key.",
-                                 "type": "str",
-                                 "ui_restrictions": [],
-                                 "validations": [],
-                                 "value": null
-                             },
-                             "provider": {
-                                 "default_value": null,
-                                 "depends_on": [],
-                                 "display": "dropdown",
-                                 "label": "Provider",
-                                 "options": [
-                                     {
-                                         "label": "amazontitan",
-                                         "value": "amazontitan"
-                                     },
-                                     {
-                                         "label": "anthropic",
-                                         "value": "anthropic"
-                                     },
-                                     {
-                                         "label": "ai21labs",
-                                         "value": "ai21labs"
-                                     },
-                                     {
-                                         "label": "cohere",
-                                         "value": "cohere"
-                                     },
-                                     {
-                                         "label": "meta",
-                                         "value": "meta"
-                                     },
-                                     {
-                                         "label": "mistral",
-                                         "value": "mistral"
-                                     }
-                                 ],
-                                 "order": 3,
-                                 "required": true,
-                                 "sensitive": false,
-                                 "tooltip": "The model provider for your deployment.",
-                                 "type": "str",
-                                 "ui_restrictions": [],
-                                 "validations": [],
-                                 "value": null
-                             },
-                             "access_key": {
-                                 "default_value": null,
-                                 "depends_on": [],
-                                 "display": "textbox",
-                                 "label": "Access Key",
-                                 "order": 1,
-                                 "required": true,
-                                 "sensitive": true,
-                                 "tooltip": "A valid AWS access key that has permissions to use Amazon Bedrock.",
-                                 "type": "str",
-                                 "ui_restrictions": [],
-                                 "validations": [],
-                                 "value": null
-                             },
-                             "model": {
-                                 "default_value": null,
-                                 "depends_on": [],
-                                 "display": "textbox",
-                                 "label": "Model",
-                                 "order": 4,
-                                 "required": true,
-                                 "sensitive": false,
-                                 "tooltip": "The base model ID or an ARN to a custom model based on a foundational model.",
-                                 "type": "str",
-                                 "ui_restrictions": [],
-                                 "validations": [],
-                                 "value": null
-                             },
-                             "rate_limit.requests_per_minute": {
-                                 "default_value": null,
-                                 "depends_on": [],
-                                 "display": "numeric",
-                                 "label": "Rate Limit",
-                                 "order": 6,
-                                 "required": false,
-                                 "sensitive": false,
-                                 "tooltip": "By default, the amazonbedrock service sets the number of requests allowed per minute to 240.",
-                                 "type": "int",
-                                 "ui_restrictions": [],
-                                 "validations": [],
-                                 "value": null
-                             },
-                             "region": {
-                                 "default_value": null,
-                                 "depends_on": [],
-                                 "display": "textbox",
-                                 "label": "Region",
-                                 "order": 5,
-                                 "required": true,
-                                 "sensitive": false,
-                                 "tooltip": "The region that your model or ARN is deployed in.",
-                                 "type": "str",
-                                 "ui_restrictions": [],
-                                 "validations": [],
-                                 "value": null
-                             }
+            String content = XContentHelper.stripWhitespace("""
+                {
+                     "service": "amazonbedrock",
+                     "name": "Amazon Bedrock",
+                     "task_types": ["text_embedding", "completion"],
+                     "configurations": {
+                         "secret_key": {
+                             "description": "A valid AWS secret key that is paired with the access_key.",
+                             "label": "Secret Key",
+                             "required": true,
+                             "sensitive": true,
+                             "updatable": true,
+                             "type": "str"
+                         },
+                         "provider": {
+                             "description": "The model provider for your deployment.",
+                             "label": "Provider",
+                             "required": true,
+                             "sensitive": false,
+                             "updatable": false,
+                             "type": "str"
+                         },
+                         "access_key": {
+                             "description": "A valid AWS access key that has permissions to use Amazon Bedrock.",
+                             "label": "Access Key",
+                             "required": true,
+                             "sensitive": true,
+                             "updatable": true,
+                             "type": "str"
+                         },
+                         "model": {
+                             "description": "The base model ID or an ARN to a custom model based on a foundational model.",
+                             "label": "Model",
+                             "required": true,
+                             "sensitive": false,
+                             "updatable": false,
+                             "type": "str"
+                         },
+                         "rate_limit.requests_per_minute": {
+                             "description": "By default, the amazonbedrock service sets the number of requests allowed per minute to 240.",
+                             "label": "Rate Limit",
+                             "required": false,
+                             "sensitive": false,
+                             "updatable": false,
+                             "type": "int"
+                         },
+                         "region": {
+                             "description": "The region that your model or ARN is deployed in.",
+                             "label": "Region",
+                             "required": true,
+                             "sensitive": false,
+                             "updatable": false,
+                             "type": "str"
                          }
                      }
-                    """
-            );
+                 }
+                """);
             InferenceServiceConfiguration configuration = InferenceServiceConfiguration.fromXContentBytes(
                 new BytesArray(content),
                 XContentType.JSON
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java
index 0f802637c6700..7eb7ad1d0a19e 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java
@@ -604,112 +604,33 @@ public void testGetConfiguration() throws Exception {
         try (var service = createServiceWithMockSender()) {
             String content = XContentHelper.stripWhitespace("""
                 {
-                      "provider": "anthropic",
-                      "task_types": [
-                           {
-                               "task_type": "completion",
-                               "configuration": {
-                                   "top_p": {
-                                       "default_value": null,
-                                       "depends_on": [],
-                                       "display": "numeric",
-                                       "label": "Top P",
-                                       "order": 4,
-                                       "required": false,
-                                       "sensitive": false,
-                                       "tooltip": "Specifies to use Anthropic’s nucleus sampling.",
-                                       "type": "int",
-                                       "ui_restrictions": [],
-                                       "validations": [],
-                                       "value": null
-                                   },
-                                   "max_tokens": {
-                                       "default_value": null,
-                                       "depends_on": [],
-                                       "display": "numeric",
-                                       "label": "Max Tokens",
-                                       "order": 1,
-                                       "required": true,
-                                       "sensitive": false,
-                                       "tooltip": "The maximum number of tokens to generate before stopping.",
-                                       "type": "int",
-                                       "ui_restrictions": [],
-                                       "validations": [],
-                                       "value": null
-                                   },
-                                   "top_k": {
-                                       "default_value": null,
-                                       "depends_on": [],
-                                       "display": "numeric",
-                                       "label": "Top K",
-                                       "order": 3,
-                                       "required": false,
-                                       "sensitive": false,
-                                       "tooltip": "Specifies to only sample from the top K options for each subsequent token.",
-                                       "type": "int",
-                                       "ui_restrictions": [],
-                                       "validations": [],
-                                       "value": null
-                                   },
-                                   "temperature": {
-                                       "default_value": null,
-                                       "depends_on": [],
-                                       "display": "textbox",
-                                       "label": "Temperature",
-                                       "order": 2,
-                                       "required": false,
-                                       "sensitive": false,
-                                       "tooltip": "The amount of randomness injected into the response.",
-                                       "type": "str",
-                                       "ui_restrictions": [],
-                                       "validations": [],
-                                       "value": null
-                                   }
-                               }
-                           }
-                      ],
-                      "configuration": {
+                      "service": "anthropic",
+                      "name": "Anthropic",
+                      "task_types": ["completion"],
+                      "configurations": {
                           "api_key": {
-                              "default_value": null,
-                              "depends_on": [],
-                              "display": "textbox",
+                              "description": "API Key for the provider you're connecting to.",
                               "label": "API Key",
-                              "order": 1,
                               "required": true,
                               "sensitive": true,
-                              "tooltip": "API Key for the provider you're connecting to.",
-                              "type": "str",
-                              "ui_restrictions": [],
-                              "validations": [],
-                              "value": null
+                              "updatable": true,
+                              "type": "str"
                           },
                           "rate_limit.requests_per_minute": {
-                              "default_value": null,
-                              "depends_on": [],
-                              "display": "numeric",
+                              "description": "By default, the anthropic service sets the number of requests allowed per minute to 50.",
                               "label": "Rate Limit",
-                              "order": 6,
                               "required": false,
                               "sensitive": false,
-                              "tooltip": "By default, the anthropic service sets the number of requests allowed per minute to 50.",
-                              "type": "int",
-                              "ui_restrictions": [],
-                              "validations": [],
-                              "value": null
+                              "updatable": false,
+                              "type": "int"
                           },
                           "model_id": {
-                              "default_value": null,
-                              "depends_on": [],
-                              "display": "textbox",
+                              "description": "The name of the model to use for the inference task.",
                               "label": "Model ID",
-                              "order": 2,
                               "required": true,
                               "sensitive": false,
-                              "tooltip": "The name of the model to use for the inference task.",
-                              "type": "str",
-                              "ui_restrictions": [],
-                              "validations": [],
-                              "value": null
+                              "updatable": false,
+                              "type": "str"
                           }
                       }
                   }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java
index fe60eaa760e69..72ebd0b96bdc1 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java
@@ -1389,203 +1389,55 @@ public void testInfer_StreamRequest_ErrorResponse() throws Exception {
     @SuppressWarnings("checkstyle:LineLength")
     public void testGetConfiguration() throws Exception {
         try (var service = createService()) {
-            String content = XContentHelper.stripWhitespace(
-                """
-                    {
-                        "provider": "azureaistudio",
-                        "task_types": [
-                             {
-                                 "task_type": "text_embedding",
-                                 "configuration": {
-                                     "top_p": {
-                                         "default_value": null,
-                                         "depends_on": [],
-                                         "display": "numeric",
-                                         "label": "Top P",
-                                         "order": 4,
-                                         "required": false,
-                                         "sensitive": false,
-                                         "tooltip": "A number in the range of 0.0 to 2.0 that is an alternative value to temperature. Should not be used if temperature is specified.",
-                                         "type": "int",
-                                         "ui_restrictions": [],
-                                         "validations": [],
-                                         "value": null
-                                     },
-                                     "max_new_tokens": {
-                                         "default_value": null,
-                                         "depends_on": [],
-                                         "display": "numeric",
-                                         "label": "Max New Tokens",
-                                         "order": 2,
-                                         "required": false,
-                                         "sensitive": false,
-                                         "tooltip": "Provides a hint for the maximum number of output tokens to be generated.",
-                                         "type": "int",
-                                         "ui_restrictions": [],
-                                         "validations": [],
-                                         "value": null
-                                     },
-                                     "temperature": {
-                                         "default_value": null,
-                                         "depends_on": [],
-                                         "display": "numeric",
-                                         "label": "Temperature",
-                                         "order": 3,
-                                         "required": false,
-                                         "sensitive": false,
-                                         "tooltip": "A number in the range of 0.0 to 2.0 that specifies the sampling temperature.",
-                                         "type": "int",
-                                         "ui_restrictions": [],
-                                         "validations": [],
-                                         "value": null
-                                     },
-                                     "do_sample": {
-                                         "default_value": null,
-                                         "depends_on": [],
-                                         "display": "numeric",
-                                         "label": "Do Sample",
-                                         "order": 1,
-                                         "required": false,
-                                         "sensitive": false,
-                                         "tooltip": "Instructs the inference process to perform sampling or not.",
-                                         "type": "int",
-                                         "ui_restrictions": [],
-                                         "validations": [],
-                                         "value": null
-                                     }
-                                 }
-                             },
-                             {
-                                 "task_type": "completion",
-                                 "configuration": {
-                                     "user": {
-                                         "default_value": null,
-                                         "depends_on": [],
-                                         "display": "textbox",
-                                         "label": "User",
-                                         "order": 1,
-                                         "required": false,
-                                         "sensitive": false,
-                                         "tooltip": "Specifies the user issuing the request.",
-                                         "type": "str",
-                                         "ui_restrictions": [],
-                                         "validations": [],
-                                         "value": ""
-                                     }
-                                 }
-                             }
-                        ],
-                        "configuration": {
-                            "endpoint_type": {
-                                "default_value": null,
-                                "depends_on": [],
-                                "display": "dropdown",
-                                "label": "Endpoint Type",
-                                "options": [
-                                    {
-                                        "label": "token",
-                                        "value": "token"
-                                    },
-                                    {
-                                        "label": "realtime",
-                                        "value": "realtime"
-                                    }
-                                ],
-                                "order": 3,
-                                "required": true,
-                                "sensitive": false,
-                                "tooltip": "Specifies the type of endpoint that is used in your model deployment.",
-                                "type": "str",
-                                "ui_restrictions": [],
-                                "validations": [],
-                                "value": null
-                            },
-                            "provider": {
-                                "default_value": null,
-                                "depends_on": [],
-                                "display": "dropdown",
-                                "label": "Provider",
-                                "options": [
-                                    {
-                                        "label": "cohere",
-                                        "value": "cohere"
-                                    },
-                                    {
-                                        "label": "meta",
-                                        "value": "meta"
-                                    },
-                                    {
-                                        "label": "microsoft_phi",
-                                        "value": "microsoft_phi"
-                                    },
-                                    {
-                                        "label": "mistral",
-                                        "value": "mistral"
-                                    },
-                                    {
-                                        "label": "openai",
-                                        "value": "openai"
-                                    },
-                                    {
-                                        "label": "databricks",
-                                        "value": "databricks"
-                                    }
-                                ],
-                                "order": 3,
-                                "required": true,
-                                "sensitive": false,
-                                "tooltip": "The model provider for your deployment.",
-                                "type": "str",
-                                "ui_restrictions": [],
-                                "validations": [],
-                                "value": null
-                            },
-                            "api_key": {
-                                "default_value": null,
-                                "depends_on": [],
-                                "display": "textbox",
-                                "label": "API Key",
-                                "order": 1,
-                                "required": true,
-                                "sensitive": true,
-                                "tooltip": "API Key for the provider you're connecting to.",
-                                "type": "str",
-                                "ui_restrictions": [],
-                                "validations": [],
-                                "value": null
-                            },
-                            "rate_limit.requests_per_minute": {
-                                "default_value": null,
-                                "depends_on": [],
-                                "display": "numeric",
-                                "label": "Rate Limit",
-                                "order": 6,
-                                "required": false,
-                                "sensitive": false,
-                                "tooltip": "Minimize the number of rate limit errors.",
-                                "type": "int",
-                                "ui_restrictions": [],
-                                "validations": [],
-                                "value": null
-                            },
-                            "target": {
-                                "default_value": null,
-                                "depends_on": [],
-                                "display": "textbox",
-                                "label": "Target",
-                                "order": 2,
-                                "required": true,
-                                "sensitive": false,
-                                "tooltip": "The target URL of your Azure AI Studio model deployment.",
-                                "type": "str",
-                                "ui_restrictions": [],
-                                "validations": [],
-                                "value": null
-                            }
+            String content = XContentHelper.stripWhitespace("""
+                {
+                    "service": "azureaistudio",
+                    "name": "Azure AI Studio",
+                    "task_types": ["text_embedding", "completion"],
+                    "configurations": {
+                        "endpoint_type": {
+                            "description": "Specifies the type of endpoint that is used in your model deployment.",
+                            "label": "Endpoint Type",
+                            "required": true,
+                            "sensitive": false,
+                            "updatable": false,
+                            "type": "str"
+                        },
+                        "provider": {
+                            "description": "The model provider for your deployment.",
+                            "label": "Provider",
+                            "required": true,
+                            "sensitive": false,
+                            "updatable": false,
+                            "type": "str"
+                        },
+                        "api_key": {
+                            "description": "API Key for the provider you're connecting to.",
+                            "label": "API Key",
+                            "required": true,
+                            "sensitive": true,
+                            "updatable": true,
+                            "type": "str"
+                        },
+                        "rate_limit.requests_per_minute": {
+                            "description": "Minimize the number of rate limit errors.",
+                            "label": "Rate Limit",
+                            "required": false,
+                            "sensitive": false,
+                            "updatable": false,
+                            "type": "int"
+                        },
+                        "target": {
+                            "description": "The target URL of your Azure AI Studio model deployment.",
+                            "label": "Target",
+                            "required": true,
+                            "sensitive": false,
+                            "updatable": false,
+                            "type": "str"
                         }
                     }
-                    """
-            );
+                }
+                """);
             InferenceServiceConfiguration configuration = InferenceServiceConfiguration.fromXContentBytes(
                 new BytesArray(content),
                 XContentType.JSON
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java
index cf79d70749fda..5c69815cbb0ab 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java
@@ -1460,131 +1460,57 @@ public void testGetConfiguration() throws Exception {
             String content = XContentHelper.stripWhitespace(
                 """
                     {
-                            "provider": "azureopenai",
-                            "task_types": [
-                                 {
-                                     "task_type": "text_embedding",
-                                     "configuration": {
-                                         "user": {
-                                             "default_value": null,
-                                             "depends_on": [],
-                                             "display": "textbox",
-                                             "label": "User",
-                                             "order": 1,
-                                             "required": false,
-                                             "sensitive": false,
-                                             "tooltip": "Specifies the user issuing the request.",
-                                             "type": "str",
-                                             "ui_restrictions": [],
-                                             "validations": [],
-                                             "value": ""
-                                         }
-                                     }
-                                 },
-                                 {
-                                     "task_type": "completion",
-                                     "configuration": {
-                                         "user": {
-                                             "default_value": null,
-                                             "depends_on": [],
-                                             "display": "textbox",
-                                             "label": "User",
-                                             "order": 1,
-                                             "required": false,
-                                             "sensitive": false,
-                                             "tooltip": "Specifies the user issuing the request.",
-                                             "type": "str",
-                                             "ui_restrictions": [],
-                                             "validations": [],
-                                             "value": ""
-                                         }
-                                     }
-                                 }
-                            ],
-                            "configuration": {
+                            "service": "azureopenai",
+                            "name": "Azure OpenAI",
+                            "task_types": ["text_embedding", "completion"],
+                            "configurations": {
                                 "api_key": {
-                                    "default_value": null,
-                                    "depends_on": [],
-                                    "display": "textbox",
+                                    "description": "You must provide either an API key or an Entra ID.",
                                     "label": "API Key",
-                                    "order": 1,
                                     "required": false,
                                     "sensitive": true,
-                                    "tooltip": "You must provide either an API key or an Entra ID.",
-                                    "type": "str",
-                                    "ui_restrictions": [],
-                                    "validations": [],
-                                    "value": null
+                                    "updatable": true,
+                                    "type": "str"
                                 },
                                 "entra_id": {
-                                    "default_value": null,
-                                    "depends_on": [],
-                                    "display": "textbox",
+                                    "description": "You must provide either an API key or an Entra ID.",
                                     "label": "Entra ID",
-                                    "order": 2,
                                     "required": false,
                                     "sensitive": true,
-                                    "tooltip": "You must provide either an API key or an Entra ID.",
-                                    "type": "str",
-                                    "ui_restrictions": [],
-                                    "validations": [],
-                                    "value": null
+                                    "updatable": true,
+                                    "type": "str"
                                 },
                                 "rate_limit.requests_per_minute": {
-                                    "default_value": null,
-                                    "depends_on": [],
-                                    "display": "numeric",
+                                    "description": "The azureopenai service sets a default number of requests allowed per minute depending on the task type.",
                                     "label": "Rate Limit",
-                                    "order": 6,
                                     "required": false,
                                     "sensitive": false,
-                                    "tooltip": "The azureopenai service sets a default number of requests allowed per minute depending on the task type.",
-                                    "type": "int",
-                                    "ui_restrictions": [],
-                                    "validations": [],
-                                    "value": null
+                                    "updatable": false,
+                                    "type": "int"
                                 },
                                 "deployment_id": {
-                                    "default_value": null,
-                                    "depends_on": [],
-                                    "display": "textbox",
+                                    "description": "The deployment name of your deployed models.",
                                     "label": "Deployment ID",
-                                    "order": 5,
                                     "required": true,
                                     "sensitive": false,
-                                    "tooltip": "The deployment name of your deployed models.",
-                                    "type": "str",
-                                    "ui_restrictions": [],
-                                    "validations": [],
-                                    "value": null
+                                    "updatable": false,
+                                    "type": "str"
                                 },
                                 "resource_name": {
-                                    "default_value": null,
-                                    "depends_on": [],
-                                    "display": "textbox",
+                                    "description": "The name of your Azure OpenAI resource.",
                                     "label": "Resource Name",
-                                    "order": 3,
                                     "required": true,
                                     "sensitive": false,
-                                    "tooltip": "The name of your Azure OpenAI resource.",
-                                    "type": "str",
-                                    "ui_restrictions": [],
-                                    "validations": [],
-                                    "value": null
+                                    "updatable": false,
+                                    "type": "str"
                                 },
                                 "api_version": {
-                                    "default_value": null,
-                                    "depends_on": [],
-                                    "display": "textbox",
+                                    "description": "The Azure API version ID to use.",
                                     "label": "API Version",
-                                    "order": 4,
                                     "required": true,
                                     "sensitive": false,
-                                    "tooltip": "The Azure API version ID to use.",
-                                    "type": "str",
-                                    "ui_restrictions": [],
-                                    "validations": [],
-                                    "value": null
+                                    "updatable": false,
+                                    "type": "str"
                                 }
                             }
                         }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java
index bf67db3b2a117..dbad76fd46fc4 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java
@@ -1633,147 +1633,31 @@ public void testInfer_StreamRequest_ErrorResponse() throws Exception {
     @SuppressWarnings("checkstyle:LineLength")
     public void testGetConfiguration() throws Exception {
         try (var service = createCohereService()) {
-            String content = XContentHelper.stripWhitespace(
-                """
-                    {
-                            "provider": "cohere",
-                            "task_types": [
-                                 {
-                                     "task_type": "text_embedding",
-                                     "configuration": {
-                                         "truncate": {
-                                             "default_value": null,
-                                             "depends_on": [],
-                                             "display": "dropdown",
-                                             "label": "Truncate",
-                                             "options": [
-                                                 {
-                                                     "label": "NONE",
-                                                     "value": "NONE"
-                                                 },
-                                                 {
-                                                     "label": "START",
-                                                     "value": "START"
-                                                 },
-                                                 {
-                                                     "label": "END",
-                                                     "value": "END"
-                                                 }
-                                             ],
-                                             "order": 2,
-                                             "required": false,
-                                             "sensitive": false,
-                                             "tooltip": "Specifies how the API handles inputs longer than the maximum token length.",
-                                             "type": "str",
-                                             "ui_restrictions": [],
-                                             "validations": [],
-                                             "value": ""
-                                         },
-                                         "input_type": {
-                                             "default_value": null,
-                                             "depends_on": [],
-                                             "display": "dropdown",
-                                             "label": "Input Type",
-                                             "options": [
-                                                 {
-                                                     "label": "classification",
-                                                     "value": "classification"
-                                                 },
-                                                 {
-                                                     "label": "clusterning",
-                                                     "value": "clusterning"
-                                                 },
-                                                 {
-                                                     "label": "ingest",
-                                                     "value": "ingest"
-                                                 },
-                                                 {
-                                                     "label": "search",
-                                                     "value": "search"
-                                                 }
-                                             ],
-                                             "order": 1,
-                                             "required": false,
-                                             "sensitive": false,
-                                             "tooltip": "Specifies the type of input passed to the model.",
-                                             "type": "str",
-                                             "ui_restrictions": [],
-                                             "validations": [],
-                                             "value": ""
-                                         }
-                                     }
-                                 },
-                                 {
-                                     "task_type": "rerank",
-                                     "configuration": {
-                                         "top_n": {
-                                             "default_value": null,
-                                             "depends_on": [],
-                                             "display": "numeric",
-                                             "label": "Top N",
-                                             "order": 2,
-                                             "required": false,
-                                             "sensitive": false,
-                                             "tooltip": "The number of most relevant documents to return, defaults to the number of the documents.",
-                                             "type": "int",
-                                             "ui_restrictions": [],
-                                             "validations": [],
-                                             "value": null
-                                         },
-                                         "return_documents": {
-                                             "default_value": null,
-                                             "depends_on": [],
-                                             "display": "toggle",
-                                             "label": "Return Documents",
-                                             "order": 1,
-                                             "required": false,
-                                             "sensitive": false,
-                                             "tooltip": "Specify whether to return doc text within the results.",
-                                             "type": "bool",
-                                             "ui_restrictions": [],
-                                             "validations": [],
-                                             "value": false
-                                         }
-                                     }
-                                 },
-                                 {
-                                     "task_type": "completion",
-                                     "configuration": {}
-                                 }
-                            ],
-                            "configuration": {
-                                "api_key": {
-                                    "default_value": null,
-                                    "depends_on": [],
-                                    "display": "textbox",
-                                    "label": "API Key",
-                                    "order": 1,
-                                    "required": true,
-                                    "sensitive": true,
-                                    "tooltip": "API Key for the provider you're connecting to.",
-                                    "type": "str",
-                                    "ui_restrictions": [],
-                                    "validations": [],
-                                    "value": null
-                                },
-                                "rate_limit.requests_per_minute": {
-                                    "default_value": null,
-                                    "depends_on": [],
-                                    "display": "numeric",
-                                    "label": "Rate Limit",
-                                    "order": 6,
-                                    "required": false,
-                                    "sensitive": false,
-                                    "tooltip": "Minimize the number of rate limit errors.",
-                                    "type": "int",
-                                    "ui_restrictions": [],
-                                    "validations": [],
-                                    "value": null
-                                }
+            String content = XContentHelper.stripWhitespace("""
+                {
+                        "service": "cohere",
+                        "name": "Cohere",
+                        "task_types": ["text_embedding", "rerank", "completion"],
+                        "configurations": {
+                            "api_key": {
+                                "description": "API Key for the provider you're connecting to.",
+                                "label": "API Key",
+                                "required": true,
+                                "sensitive": true,
+                                "updatable": true,
+                                "type": "str"
+                            },
+                            "rate_limit.requests_per_minute": {
+                                "description": "Minimize the number of rate limit errors.",
+                                "label": "Rate Limit",
+                                "required": false,
+                                "sensitive": false,
+                                "updatable": false,
+                                "type": "int"
                             }
                         }
-                    """
-            );
+                    }
+                """);
             InferenceServiceConfiguration configuration = InferenceServiceConfiguration.fromXContentBytes(
                 new BytesArray(content),
                 XContentType.JSON
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests.java
index 1751e1c3be5e8..dd205b12408ba 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests.java
@@ -71,7 +71,7 @@ public void testFromMap_InvalidElserModelId() {
         assertThat(validationException.getMessage(), containsString(Strings.format("unknown ELSER model id [%s]", invalidModelId)));
     }
 
-    public void testToXContent_WritesAlLFields() throws IOException {
+    public void testToXContent_WritesAllFields() throws IOException {
         var modelId = ElserModels.ELSER_V1_MODEL;
         var maxInputTokens = 10;
         var serviceSettings = new ElasticInferenceServiceSparseEmbeddingsServiceSettings(modelId, maxInputTokens, null);
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java
index 6cdd58c4cd99f..8a826e99c3c04 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java
@@ -114,22 +114,6 @@ public void testParseRequestConfig_CreatesASparseEmbeddingsModel() throws IOExce
         }
     }
 
-    public void testParseRequestConfig_ThrowsUnsupportedModelType() throws IOException {
-        try (var service = createServiceWithMockSender()) {
-            var failureListener = getModelListenerForException(
-                ElasticsearchStatusException.class,
-                "The [elastic] service does not support task type [completion]"
-            );
-
-            service.parseRequestConfig(
-                "id",
-                TaskType.COMPLETION,
-                getRequestConfigMap(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL), Map.of(), Map.of()),
-                failureListener
-            );
-        }
-    }
-
     public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInConfig() throws IOException {
         try (var service = createServiceWithMockSender()) {
             var config = getRequestConfigMap(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL), Map.of(), Map.of());
@@ -496,55 +480,33 @@ public void testGetConfiguration() throws Exception {
         try (var service = createServiceWithMockSender()) {
             String content = XContentHelper.stripWhitespace("""
                 {
-                       "provider": "elastic",
-                       "task_types": [
-                            {
-                                "task_type": "sparse_embedding",
-                                "configuration": {}
-                            }
-                       ],
-                       "configuration": {
+                       "service": "elastic",
+                       "name": "Elastic",
+                       "task_types": ["sparse_embedding" , "completion"],
+                       "configurations": {
                            "rate_limit.requests_per_minute": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "numeric",
+                               "description": "Minimize the number of rate limit errors.",
                                "label": "Rate Limit",
-                               "order": 6,
                                "required": false,
                                "sensitive": false,
-                               "tooltip": "Minimize the number of rate limit errors.",
-                               "type": "int",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "int"
                            },
                            "model_id": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "textbox",
+                               "description": "The name of the model to use for the inference task.",
                                "label": "Model ID",
-                               "order": 2,
                                "required": true,
                                "sensitive": false,
-                               "tooltip": "The name of the model to use for the inference task.",
-                               "type": "str",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "str"
                            },
                            "max_input_tokens": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "numeric",
+                               "description": "Allows you to specify the maximum number of tokens per input.",
                                "label": "Maximum Input Tokens",
-                               "order": 3,
                                "required": false,
                                "sensitive": false,
-                               "tooltip": "Allows you to specify the maximum number of tokens per input.",
-                               "type": "int",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "int"
                            }
                        }
                    }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/completion/ElasticInferenceServiceCompletionModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/completion/ElasticInferenceServiceCompletionModelTests.java
new file mode 100644
index 0000000000000..07da96cb32273
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/completion/ElasticInferenceServiceCompletionModelTests.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elastic.completion;
+
+import org.elasticsearch.inference.EmptySecretSettings;
+import org.elasticsearch.inference.EmptyTaskSettings;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.inference.UnifiedCompletionRequest;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceComponents;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+
+import java.util.List;
+
+import static org.hamcrest.Matchers.is;
+
+public class ElasticInferenceServiceCompletionModelTests extends ESTestCase {
+
+    public void testOverridingModelId() {
+        var originalModel = new ElasticInferenceServiceCompletionModel(
+            "id",
+            TaskType.COMPLETION,
+            "elastic",
+            new ElasticInferenceServiceCompletionServiceSettings("model_id", new RateLimitSettings(100)),
+            EmptyTaskSettings.INSTANCE,
+            EmptySecretSettings.INSTANCE,
+            new ElasticInferenceServiceComponents("url")
+        );
+
+        var request = new UnifiedCompletionRequest(
+            List.of(new UnifiedCompletionRequest.Message(new UnifiedCompletionRequest.ContentString("message"), "user", null, null)),
+            "new_model_id",
+            null,
+            null,
+            null,
+            null,
+            null,
+            null
+        );
+
+        var overriddenModel = ElasticInferenceServiceCompletionModel.of(originalModel, request);
+
+        assertThat(overriddenModel.getServiceSettings().modelId(), is("new_model_id"));
+        assertThat(overriddenModel.getTaskType(), is(TaskType.COMPLETION));
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/completion/ElasticInferenceServiceCompletionServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/completion/ElasticInferenceServiceCompletionServiceSettingsTests.java
new file mode 100644
index 0000000000000..0f6386f670338
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/completion/ElasticInferenceServiceCompletionServiceSettingsTests.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elastic.completion;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.ServiceFields;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+
+public class ElasticInferenceServiceCompletionServiceSettingsTests extends AbstractWireSerializingTestCase<
+    ElasticInferenceServiceCompletionServiceSettings> {
+
+    @Override
+    protected Writeable.Reader<ElasticInferenceServiceCompletionServiceSettings> instanceReader() {
+        return ElasticInferenceServiceCompletionServiceSettings::new;
+    }
+
+    @Override
+    protected ElasticInferenceServiceCompletionServiceSettings createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected ElasticInferenceServiceCompletionServiceSettings mutateInstance(ElasticInferenceServiceCompletionServiceSettings instance)
+        throws IOException {
+        return randomValueOtherThan(instance, ElasticInferenceServiceCompletionServiceSettingsTests::createRandom);
+    }
+
+    public void testFromMap() {
+        var modelId = "model_id";
+
+        var serviceSettings = ElasticInferenceServiceCompletionServiceSettings.fromMap(
+            new HashMap<>(Map.of(ServiceFields.MODEL_ID, modelId)),
+            ConfigurationParseContext.REQUEST
+        );
+
+        assertThat(serviceSettings, is(new ElasticInferenceServiceCompletionServiceSettings(modelId, new RateLimitSettings(240L))));
+    }
+
+    public void testFromMap_MissingModelId_ThrowsException() {
+        ValidationException validationException = expectThrows(
+            ValidationException.class,
+            () -> ElasticInferenceServiceCompletionServiceSettings.fromMap(new HashMap<>(Map.of()), ConfigurationParseContext.REQUEST)
+        );
+
+        assertThat(validationException.getMessage(), containsString("does not contain the required setting [model_id]"));
+    }
+
+    public void testToXContent_WritesAllFields() throws IOException {
+        var modelId = "model_id";
+        var serviceSettings = new ElasticInferenceServiceCompletionServiceSettings(modelId, new RateLimitSettings(1000));
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        serviceSettings.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, is(Strings.format("""
+            {"model_id":"%s","rate_limit":{"requests_per_minute":1000}}""", modelId)));
+    }
+
+    public static ElasticInferenceServiceCompletionServiceSettings createRandom() {
+        return new ElasticInferenceServiceCompletionServiceSettings(randomAlphaOfLength(4), RateLimitSettingsTests.createRandom());
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java
index 21d7efbc7b03c..803151a5b3476 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java
@@ -1553,100 +1553,36 @@ public void testGetConfiguration() throws Exception {
         try (var service = createService(mock(Client.class))) {
             String content = XContentHelper.stripWhitespace("""
                 {
-                       "provider": "elasticsearch",
-                       "task_types": [
-                            {
-                                "task_type": "text_embedding",
-                                "configuration": {}
-                            },
-                            {
-                                "task_type": "sparse_embedding",
-                                "configuration": {}
-                            },
-                            {
-                                "task_type": "rerank",
-                                "configuration": {
-                                    "return_documents": {
-                                        "default_value": null,
-                                        "depends_on": [],
-                                        "display": "toggle",
-                                        "label": "Return Documents",
-                                        "order": 1,
-                                        "required": false,
-                                        "sensitive": false,
-                                        "tooltip": "Returns the document instead of only the index.",
-                                        "type": "bool",
-                                        "ui_restrictions": [],
-                                        "validations": [],
-                                        "value": true
-                                    }
-                                }
-                            }
-                       ],
-                       "configuration": {
+                       "service": "elasticsearch",
+                       "name": "Elasticsearch",
+                       "task_types": ["text_embedding", "sparse_embedding", "rerank"],
+                       "configurations": {
                            "num_allocations": {
                                "default_value": 1,
-                               "depends_on": [],
-                               "display": "numeric",
+                               "description": "The total number of allocations this model is assigned across machine learning nodes.",
                                "label": "Number Allocations",
-                               "order": 2,
                                "required": true,
                                "sensitive": false,
-                               "tooltip": "The total number of allocations this model is assigned across machine learning nodes.",
-                               "type": "int",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": true,
+                               "type": "int"
                            },
                            "num_threads": {
                                "default_value": 2,
-                               "depends_on": [],
-                               "display": "numeric",
+                               "description": "Sets the number of threads used by each model allocation during inference.",
                                "label": "Number Threads",
-                               "order": 3,
                                "required": true,
                                "sensitive": false,
-                               "tooltip": "Sets the number of threads used by each model allocation during inference.",
-                               "type": "int",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "int"
                            },
                            "model_id": {
                                "default_value": ".multilingual-e5-small",
-                               "depends_on": [],
-                               "display": "dropdown",
+                               "description": "The name of the model to use for the inference task.",
                                "label": "Model ID",
-                               "options": [
-                                   {
-                                       "label": ".elser_model_1",
-                                       "value": ".elser_model_1"
-                                   },
-                                   {
-                                       "label": ".elser_model_2",
-                                       "value": ".elser_model_2"
-                                   },
-                                   {
-                                       "label": ".elser_model_2_linux-x86_64",
-                                       "value": ".elser_model_2_linux-x86_64"
-                                   },
-                                   {
-                                       "label": ".multilingual-e5-small",
-                                       "value": ".multilingual-e5-small"
-                                   },
-                                   {
-                                       "label": ".multilingual-e5-small_linux-x86_64",
-                                       "value": ".multilingual-e5-small_linux-x86_64"
-                                   }
-                               ],
-                               "order": 1,
                                "required": true,
                                "sensitive": false,
-                               "tooltip": "The name of the model to use for the inference task.",
-                               "type": "str",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "str"
                            }
                        }
                    }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java
index 3e43bd7c77684..aa45666eb0fb1 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java
@@ -1123,59 +1123,33 @@ public void testGetConfiguration() throws Exception {
         try (var service = createGoogleAiStudioService()) {
             String content = XContentHelper.stripWhitespace("""
                 {
-                       "provider": "googleaistudio",
-                       "task_types": [
-                            {
-                                "task_type": "text_embedding",
-                                "configuration": {}
-                            },
-                            {
-                                "task_type": "completion",
-                                "configuration": {}
-                            }
-                       ],
-                       "configuration": {
+                       "service": "googleaistudio",
+                       "name": "Google AI Studio",
+                       "task_types": ["text_embedding", "completion"],
+                       "configurations": {
                            "api_key": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "textbox",
+                               "description": "API Key for the provider you're connecting to.",
                                "label": "API Key",
-                               "order": 1,
                                "required": true,
                                "sensitive": true,
-                               "tooltip": "API Key for the provider you're connecting to.",
-                               "type": "str",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": true,
+                               "type": "str"
                            },
                            "rate_limit.requests_per_minute": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "numeric",
+                               "description": "Minimize the number of rate limit errors.",
                                "label": "Rate Limit",
-                               "order": 6,
                                "required": false,
                                "sensitive": false,
-                               "tooltip": "Minimize the number of rate limit errors.",
-                               "type": "int",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "int"
                            },
                            "model_id": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "textbox",
+                               "description": "ID of the LLM you're using.",
                                "label": "Model ID",
-                               "order": 2,
                                "required": true,
                                "sensitive": false,
-                               "tooltip": "ID of the LLM you're using.",
-                               "type": "str",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "str"
                            }
                        }
                    }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiServiceTests.java
index 2aeba5fcbe209..555e9f0785fa2 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiServiceTests.java
@@ -869,149 +869,49 @@ public void testGetConfiguration() throws Exception {
             String content = XContentHelper.stripWhitespace(
                 """
                     {
-                           "provider": "googlevertexai",
-                           "task_types": [
-                                {
-                                    "task_type": "text_embedding",
-                                    "configuration": {
-                                         "input_type": {
-                                             "default_value": null,
-                                             "depends_on": [],
-                                             "display": "dropdown",
-                                             "label": "Input Type",
-                                             "options": [
-                                                 {
-                                                     "label": "classification",
-                                                     "value": "classification"
-                                                 },
-                                                 {
-                                                     "label": "clustering",
-                                                     "value": "clustering"
-                                                 },
-                                                 {
-                                                     "label": "ingest",
-                                                     "value": "ingest"
-                                                 },
-                                                 {
-                                                     "label": "search",
-                                                     "value": "search"
-                                                 }
-                                             ],
-                                             "order": 1,
-                                             "required": false,
-                                             "sensitive": false,
-                                             "tooltip": "Specifies the type of input passed to the model.",
-                                             "type": "str",
-                                             "ui_restrictions": [],
-                                             "validations": [],
-                                             "value": ""
-                                        },
-                                        "auto_truncate": {
-                                            "default_value": null,
-                                            "depends_on": [],
-                                            "display": "toggle",
-                                            "label": "Auto Truncate",
-                                            "order": 2,
-                                            "required": false,
-                                            "sensitive": false,
-                                            "tooltip": "Specifies if the API truncates inputs longer than the maximum token length automatically.",
-                                            "type": "bool",
-                                            "ui_restrictions": [],
-                                            "validations": [],
-                                            "value": false
-                                        }
-                                    }
-                                },
-                                {
-                                    "task_type": "rerank",
-                                    "configuration": {
-                                        "top_n": {
-                                            "default_value": null,
-                                            "depends_on": [],
-                                            "display": "toggle",
-                                            "label": "Top N",
-                                            "order": 1,
-                                            "required": false,
-                                            "sensitive": false,
-                                            "tooltip": "Specifies the number of the top n documents, which should be returned.",
-                                            "type": "bool",
-                                            "ui_restrictions": [],
-                                            "validations": [],
-                                            "value": false
-                                        }
-                                    }
-                                }
-                           ],
-                           "configuration": {
+                           "service": "googlevertexai",
+                           "name": "Google Vertex AI",
+                           "task_types": ["text_embedding", "rerank"],
+                           "configurations": {
                                "service_account_json": {
-                                   "default_value": null,
-                                   "depends_on": [],
-                                   "display": "textbox",
+                                   "description": "API Key for the provider you're connecting to.",
                                    "label": "Credentials JSON",
-                                   "order": 1,
                                    "required": true,
                                    "sensitive": true,
-                                   "tooltip": "API Key for the provider you're connecting to.",
-                                   "type": "str",
-                                   "ui_restrictions": [],
-                                   "validations": [],
-                                   "value": null
+                                   "updatable": true,
+                                   "type": "str"
                                },
                                "project_id": {
-                                   "default_value": null,
-                                   "depends_on": [],
-                                   "display": "textbox",
+                                   "description": "The GCP Project ID which has Vertex AI API(s) enabled. For more information on the URL, refer to the {geminiVertexAIDocs}.",
                                    "label": "GCP Project",
-                                   "order": 4,
                                    "required": true,
                                    "sensitive": false,
-                                   "tooltip": "The GCP Project ID which has Vertex AI API(s) enabled. For more information on the URL, refer to the {geminiVertexAIDocs}.",
-                                   "type": "str",
-                                   "ui_restrictions": [],
-                                   "validations": [],
-                                   "value": null
+                                   "updatable": false,
+                                   "type": "str"
                                },
                                "location": {
-                                   "default_value": null,
-                                   "depends_on": [],
-                                   "display": "textbox",
+                                   "description": "Please provide the GCP region where the Vertex AI API(s) is enabled. For more information, refer to the {geminiVertexAIDocs}.",
                                    "label": "GCP Region",
-                                   "order": 3,
                                    "required": true,
                                    "sensitive": false,
-                                   "tooltip": "Please provide the GCP region where the Vertex AI API(s) is enabled. For more information, refer to the {geminiVertexAIDocs}.",
-                                   "type": "str",
-                                   "ui_restrictions": [],
-                                   "validations": [],
-                                   "value": null
+                                   "updatable": false,
+                                   "type": "str"
                                },
                                "rate_limit.requests_per_minute": {
-                                   "default_value": null,
-                                   "depends_on": [],
-                                   "display": "numeric",
+                                   "description": "Minimize the number of rate limit errors.",
                                    "label": "Rate Limit",
-                                   "order": 6,
                                    "required": false,
                                    "sensitive": false,
-                                   "tooltip": "Minimize the number of rate limit errors.",
-                                   "type": "int",
-                                   "ui_restrictions": [],
-                                   "validations": [],
-                                   "value": null
+                                   "updatable": false,
+                                   "type": "int"
                                },
                                "model_id": {
-                                   "default_value": null,
-                                   "depends_on": [],
-                                   "display": "textbox",
+                                   "description": "ID of the LLM you're using.",
                                    "label": "Model ID",
-                                   "order": 2,
                                    "required": true,
                                    "sensitive": false,
-                                   "tooltip": "ID of the LLM you're using.",
-                                   "type": "str",
-                                   "ui_restrictions": [],
-                                   "validations": [],
-                                   "value": null
+                                   "updatable": false,
+                                   "type": "str"
                                }
                            }
                        }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceElserServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceElserServiceTests.java
index 12811b6be87b3..0b774badd56b6 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceElserServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceElserServiceTests.java
@@ -140,55 +140,33 @@ public void testGetConfiguration() throws Exception {
         ) {
             String content = XContentHelper.stripWhitespace("""
                 {
-                       "provider": "hugging_face_elser",
-                       "task_types": [
-                            {
-                                "task_type": "sparse_embedding",
-                                "configuration": {}
-                            }
-                       ],
-                       "configuration": {
+                       "service": "hugging_face_elser",
+                       "name": "Hugging Face ELSER",
+                       "task_types": ["sparse_embedding"],
+                       "configurations": {
                            "api_key": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "textbox",
+                               "description": "API Key for the provider you're connecting to.",
                                "label": "API Key",
-                               "order": 1,
                                "required": true,
                                "sensitive": true,
-                               "tooltip": "API Key for the provider you're connecting to.",
-                               "type": "str",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": true,
+                               "type": "str"
                            },
                            "rate_limit.requests_per_minute": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "numeric",
+                               "description": "Minimize the number of rate limit errors.",
                                "label": "Rate Limit",
-                               "order": 6,
                                "required": false,
                                "sensitive": false,
-                               "tooltip": "Minimize the number of rate limit errors.",
-                               "type": "int",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "int"
                            },
                            "url": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "textbox",
+                               "description": "The URL endpoint to use for the requests.",
                                "label": "URL",
-                               "order": 1,
                                "required": true,
                                "sensitive": false,
-                               "tooltip": "The URL endpoint to use for the requests.",
-                               "type": "str",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "str"
                            }
                        }
                    }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java
index 2d30fd5aba4c2..1399712450af1 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java
@@ -858,59 +858,34 @@ public void testGetConfiguration() throws Exception {
         try (var service = createHuggingFaceService()) {
             String content = XContentHelper.stripWhitespace("""
                 {
-                       "provider": "hugging_face",
-                       "task_types": [
-                            {
-                                "task_type": "text_embedding",
-                                "configuration": {}
-                            },
-                            {
-                                "task_type": "sparse_embedding",
-                                "configuration": {}
-                            }
-                       ],
-                       "configuration": {
+                       "service": "hugging_face",
+                       "name": "Hugging Face",
+                       "task_types": ["text_embedding", "sparse_embedding"],
+                       "configurations": {
                            "api_key": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "textbox",
+                               "description": "API Key for the provider you're connecting to.",
                                "label": "API Key",
-                               "order": 1,
                                "required": true,
                                "sensitive": true,
-                               "tooltip": "API Key for the provider you're connecting to.",
-                               "type": "str",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": true,
+                               "type": "str"
                            },
                            "rate_limit.requests_per_minute": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "numeric",
+                               "description": "Minimize the number of rate limit errors.",
                                "label": "Rate Limit",
-                               "order": 6,
                                "required": false,
                                "sensitive": false,
-                               "tooltip": "Minimize the number of rate limit errors.",
-                               "type": "int",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "int"
                            },
                            "url": {
                                "default_value": "https://api.openai.com/v1/embeddings",
-                               "depends_on": [],
-                               "display": "textbox",
+                               "description": "The URL endpoint to use for the requests.",
                                "label": "URL",
-                               "order": 1,
                                "required": true,
                                "sensitive": false,
-                               "tooltip": "The URL endpoint to use for the requests.",
-                               "type": "str",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": "https://api.openai.com/v1/embeddings"
+                               "updatable": false,
+                               "type": "str"
                            }
                        }
                    }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java
index 3d298823ea19f..6e744fffbff41 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java
@@ -975,83 +975,49 @@ public void testGetConfiguration() throws Exception {
         try (var service = createIbmWatsonxService()) {
             String content = XContentHelper.stripWhitespace("""
                 {
-                       "provider": "watsonxai",
-                       "task_types": [
-                            {
-                                "task_type": "text_embedding",
-                                "configuration": {}
-                            }
-                       ],
-                       "configuration": {
+                       "service": "watsonxai",
+                       "name": "IBM Watsonx",
+                       "task_types": ["text_embedding"],
+                       "configurations": {
                            "project_id": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "textbox",
+                               "description": "",
                                "label": "Project ID",
-                               "order": 2,
                                "required": true,
                                "sensitive": false,
-                               "tooltip": "",
-                               "type": "str",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "str"
                            },
                            "model_id": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "textbox",
+                               "description": "The name of the model to use for the inference task.",
                                "label": "Model ID",
-                               "order": 3,
                                "required": true,
                                "sensitive": false,
-                               "tooltip": "The name of the model to use for the inference task.",
-                               "type": "str",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "str"
                            },
                            "api_version": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "textbox",
+                               "description": "The IBM Watsonx API version ID to use.",
                                "label": "API Version",
-                               "order": 1,
                                "required": true,
                                "sensitive": false,
-                               "tooltip": "The IBM Watsonx API version ID to use.",
-                               "type": "str",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "str"
                            },
                            "max_input_tokens": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "numeric",
+                               "description": "Allows you to specify the maximum number of tokens per input.",
                                "label": "Maximum Input Tokens",
-                               "order": 5,
                                "required": false,
                                "sensitive": false,
-                               "tooltip": "Allows you to specify the maximum number of tokens per input.",
-                               "type": "int",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "int"
                            },
                            "url": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "textbox",
+                               "description": "",
                                "label": "URL",
-                               "order": 4,
                                "required": true,
                                "sensitive": false,
-                               "tooltip": "",
-                               "type": "str",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "str"
                            }
                        }
                    }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceSettingsTests.java
new file mode 100644
index 0000000000000..4729e9e059d93
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceSettingsTests.java
@@ -0,0 +1,174 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.ServiceFields;
+import org.elasticsearch.xpack.inference.services.ServiceUtils;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+
+public class JinaAIServiceSettingsTests extends AbstractWireSerializingTestCase<JinaAIServiceSettings> {
+
+    public static JinaAIServiceSettings createRandomWithNonNullUrl() {
+        return createRandom(randomAlphaOfLength(15));
+    }
+
+    /**
+     * The created settings can have a url set to null.
+     */
+    public static JinaAIServiceSettings createRandom() {
+        var url = randomBoolean() ? randomAlphaOfLength(15) : null;
+        return createRandom(url);
+    }
+
+    private static JinaAIServiceSettings createRandom(String url) {
+        var model = randomAlphaOfLength(15);
+
+        return new JinaAIServiceSettings(ServiceUtils.createOptionalUri(url), model, RateLimitSettingsTests.createRandom());
+    }
+
+    public void testFromMap() {
+        var url = "https://www.abc.com";
+        var model = "model";
+        var serviceSettings = JinaAIServiceSettings.fromMap(
+            new HashMap<>(Map.of(ServiceFields.URL, url, JinaAIServiceSettings.MODEL_ID, model)),
+            ConfigurationParseContext.REQUEST
+        );
+
+        MatcherAssert.assertThat(serviceSettings, is(new JinaAIServiceSettings(ServiceUtils.createUri(url), model, null)));
+    }
+
+    public void testFromMap_WithRateLimit() {
+        var url = "https://www.abc.com";
+        var model = "model";
+        var serviceSettings = JinaAIServiceSettings.fromMap(
+            new HashMap<>(
+                Map.of(
+                    ServiceFields.URL,
+                    url,
+                    JinaAIServiceSettings.MODEL_ID,
+                    model,
+                    RateLimitSettings.FIELD_NAME,
+                    new HashMap<>(Map.of(RateLimitSettings.REQUESTS_PER_MINUTE_FIELD, 3))
+                )
+            ),
+            ConfigurationParseContext.REQUEST
+        );
+
+        MatcherAssert.assertThat(
+            serviceSettings,
+            is(new JinaAIServiceSettings(ServiceUtils.createUri(url), model, new RateLimitSettings(3)))
+        );
+    }
+
+    public void testFromMap_WhenUsingModelId() {
+        var url = "https://www.abc.com";
+        var model = "model";
+        var serviceSettings = JinaAIServiceSettings.fromMap(
+            new HashMap<>(Map.of(ServiceFields.URL, url, JinaAIServiceSettings.MODEL_ID, model)),
+            ConfigurationParseContext.PERSISTENT
+        );
+
+        MatcherAssert.assertThat(serviceSettings, is(new JinaAIServiceSettings(ServiceUtils.createUri(url), model, null)));
+    }
+
+    public void testFromMap_MissingUrl_DoesNotThrowException() {
+        var serviceSettings = JinaAIServiceSettings.fromMap(
+            new HashMap<>(Map.of(JinaAIServiceSettings.MODEL_ID, "model")),
+            ConfigurationParseContext.PERSISTENT
+        );
+        assertNull(serviceSettings.uri());
+    }
+
+    public void testFromMap_EmptyUrl_ThrowsError() {
+        var thrownException = expectThrows(
+            ValidationException.class,
+            () -> JinaAIServiceSettings.fromMap(new HashMap<>(Map.of(ServiceFields.URL, "")), ConfigurationParseContext.PERSISTENT)
+        );
+
+        MatcherAssert.assertThat(
+            thrownException.getMessage(),
+            containsString(
+                Strings.format(
+                    "Validation Failed: 1: [service_settings] Invalid value empty string. [%s] must be a non-empty string;",
+                    ServiceFields.URL
+                )
+            )
+        );
+    }
+
+    public void testFromMap_InvalidUrl_ThrowsError() {
+        var url = "https://www.abc^.com";
+        var thrownException = expectThrows(
+            ValidationException.class,
+            () -> JinaAIServiceSettings.fromMap(new HashMap<>(Map.of(ServiceFields.URL, url)), ConfigurationParseContext.PERSISTENT)
+        );
+
+        MatcherAssert.assertThat(
+            thrownException.getMessage(),
+            containsString(
+                Strings.format("Validation Failed: 1: [service_settings] Invalid url [%s] received for field [%s]", url, ServiceFields.URL)
+            )
+        );
+    }
+
+    public void testXContent_WritesModelId() throws IOException {
+        var entity = new JinaAIServiceSettings((String) null, "model", new RateLimitSettings(1));
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, is("""
+            {"model_id":"model","rate_limit":{"requests_per_minute":1}}"""));
+    }
+
+    @Override
+    protected Writeable.Reader<JinaAIServiceSettings> instanceReader() {
+        return JinaAIServiceSettings::new;
+    }
+
+    @Override
+    protected JinaAIServiceSettings createTestInstance() {
+        return createRandomWithNonNullUrl();
+    }
+
+    @Override
+    protected JinaAIServiceSettings mutateInstance(JinaAIServiceSettings instance) throws IOException {
+        return randomValueOtherThan(instance, JinaAIServiceSettingsTests::createRandom);
+    }
+
+    public static Map<String, Object> getServiceSettingsMap(@Nullable String url, String model) {
+        var map = new HashMap<String, Object>();
+
+        if (url != null) {
+            map.put(ServiceFields.URL, url);
+        }
+
+        map.put(JinaAIServiceSettings.MODEL_ID, model);
+
+        return map;
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java
new file mode 100644
index 0000000000000..6e68bde6a1266
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java
@@ -0,0 +1,1919 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ *
+ * this file was contributed to by a generative AI
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai;
+
+import org.apache.http.HttpHeaders;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.ChunkedInference;
+import org.elasticsearch.inference.ChunkingSettings;
+import org.elasticsearch.inference.InferenceServiceConfiguration;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.Model;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.http.MockResponse;
+import org.elasticsearch.test.http.MockWebServer;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.inference.action.InferenceAction;
+import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat;
+import org.elasticsearch.xpack.inference.external.http.HttpClientManager;
+import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
+import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests;
+import org.elasticsearch.xpack.inference.external.http.sender.Sender;
+import org.elasticsearch.xpack.inference.logging.ThrottlerManager;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModel;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModelTests;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsServiceSettingsTests;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettings;
+import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettingsTests;
+import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankModelTests;
+import org.hamcrest.CoreMatchers;
+import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+import static org.elasticsearch.xpack.inference.Utils.getInvalidModel;
+import static org.elasticsearch.xpack.inference.Utils.getPersistedConfigMap;
+import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool;
+import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty;
+import static org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests.createRandomChunkingSettings;
+import static org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests.createRandomChunkingSettingsMap;
+import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
+import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl;
+import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat;
+import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings;
+import static org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettingsTests.getSecretSettingsMap;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class JinaAIServiceTests extends ESTestCase {
+    private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS);
+    private final MockWebServer webServer = new MockWebServer();
+    private ThreadPool threadPool;
+    private HttpClientManager clientManager;
+
+    @Before
+    public void init() throws Exception {
+        webServer.start();
+        threadPool = createThreadPool(inferenceUtilityPool());
+        clientManager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty(), mock(ThrottlerManager.class));
+    }
+
+    @After
+    public void shutdown() throws IOException {
+        clientManager.close();
+        terminate(threadPool);
+        webServer.close();
+    }
+
+    public void testParseRequestConfig_CreatesAJinaAIEmbeddingsModel() throws IOException {
+        try (var service = createJinaAIService()) {
+            ActionListener<Model> modelListener = ActionListener.wrap(model -> {
+                MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+                var embeddingsModel = (JinaAIEmbeddingsModel) model;
+                MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+                MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+                MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings(InputType.INGEST)));
+                MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret"));
+            }, e -> fail("Model parsing should have succeeded " + e.getMessage()));
+
+            service.parseRequestConfig(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                getRequestConfigMap(
+                    JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                    JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.INGEST),
+                    getSecretSettingsMap("secret")
+                ),
+                modelListener
+            );
+
+        }
+    }
+
+    public void testParseRequestConfig_CreatesAJinaAIEmbeddingsModelWhenChunkingSettingsProvided() throws IOException {
+        try (var service = createJinaAIService()) {
+            ActionListener<Model> modelListener = ActionListener.wrap(model -> {
+                MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+                var embeddingsModel = (JinaAIEmbeddingsModel) model;
+                MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+                MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+                MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings(InputType.INGEST)));
+                MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class));
+                assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class));
+                MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret"));
+            }, e -> fail("Model parsing should have succeeded " + e.getMessage()));
+
+            service.parseRequestConfig(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                getRequestConfigMap(
+                    JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                    JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.INGEST),
+                    createRandomChunkingSettingsMap(),
+                    getSecretSettingsMap("secret")
+                ),
+                modelListener
+            );
+
+        }
+    }
+
+    public void testParseRequestConfig_CreatesAJinaAIEmbeddingsModelWhenChunkingSettingsNotProvided() throws IOException {
+        try (var service = createJinaAIService()) {
+            ActionListener<Model> modelListener = ActionListener.wrap(model -> {
+                MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+                var embeddingsModel = (JinaAIEmbeddingsModel) model;
+                MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+                MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+                MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings(InputType.INGEST)));
+                MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class));
+                assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class));
+                MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret"));
+            }, e -> fail("Model parsing should have succeeded " + e.getMessage()));
+
+            service.parseRequestConfig(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                getRequestConfigMap(
+                    JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                    JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.INGEST),
+                    getSecretSettingsMap("secret")
+                ),
+                modelListener
+            );
+
+        }
+    }
+
+    public void testParseRequestConfig_OptionalTaskSettings() throws IOException {
+        try (var service = createJinaAIService()) {
+
+            ActionListener<Model> modelListener = ActionListener.wrap(model -> {
+                MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+                var embeddingsModel = (JinaAIEmbeddingsModel) model;
+                MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+                MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+                MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), equalTo(JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS));
+                MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret"));
+            }, e -> fail("Model parsing should have succeeded " + e.getMessage()));
+
+            service.parseRequestConfig(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                getRequestConfigMap(
+                    JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                    getSecretSettingsMap("secret")
+                ),
+                modelListener
+            );
+
+        }
+    }
+
+    public void testParseRequestConfig_ThrowsUnsupportedTaskType() throws IOException {
+        try (var service = createJinaAIService()) {
+            var failureListener = getModelListenerForException(
+                ElasticsearchStatusException.class,
+                "The [jinaai] service does not support task type [sparse_embedding]"
+            );
+
+            service.parseRequestConfig(
+                "id",
+                TaskType.SPARSE_EMBEDDING,
+                getRequestConfigMap(
+                    JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                    JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(),
+                    getSecretSettingsMap("secret")
+                ),
+                failureListener
+            );
+        }
+    }
+
+    private static ActionListener<Model> getModelListenerForException(Class<?> exceptionClass, String expectedMessage) {
+        return ActionListener.<Model>wrap((model) -> fail("Model parsing should have failed"), e -> {
+            MatcherAssert.assertThat(e, instanceOf(exceptionClass));
+            MatcherAssert.assertThat(e.getMessage(), is(expectedMessage));
+        });
+    }
+
+    public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInConfig() throws IOException {
+        try (var service = createJinaAIService()) {
+            var config = getRequestConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(),
+                getSecretSettingsMap("secret")
+            );
+            config.put("extra_key", "value");
+
+            var failureListener = getModelListenerForException(
+                ElasticsearchStatusException.class,
+                "Model configuration contains settings [{extra_key=value}] unknown to the [jinaai] service"
+            );
+            service.parseRequestConfig("id", TaskType.TEXT_EMBEDDING, config, failureListener);
+        }
+    }
+
+    public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInServiceSettingsMap() throws IOException {
+        try (var service = createJinaAIService()) {
+            var serviceSettings = JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model");
+            serviceSettings.put("extra_key", "value");
+
+            var config = getRequestConfigMap(
+                serviceSettings,
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null),
+                getSecretSettingsMap("secret")
+            );
+
+            var failureListener = getModelListenerForException(
+                ElasticsearchStatusException.class,
+                "Model configuration contains settings [{extra_key=value}] unknown to the [jinaai] service"
+            );
+            service.parseRequestConfig("id", TaskType.TEXT_EMBEDDING, config, failureListener);
+        }
+    }
+
+    public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInTaskSettingsMap() throws IOException {
+        try (var service = createJinaAIService()) {
+            var taskSettingsMap = JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.INGEST);
+            taskSettingsMap.put("extra_key", "value");
+
+            var config = getRequestConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                taskSettingsMap,
+                getSecretSettingsMap("secret")
+            );
+
+            var failureListener = getModelListenerForException(
+                ElasticsearchStatusException.class,
+                "Model configuration contains settings [{extra_key=value}] unknown to the [jinaai] service"
+            );
+            service.parseRequestConfig("id", TaskType.TEXT_EMBEDDING, config, failureListener);
+
+        }
+    }
+
+    public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInSecretSettingsMap() throws IOException {
+        try (var service = createJinaAIService()) {
+            var secretSettingsMap = getSecretSettingsMap("secret");
+            secretSettingsMap.put("extra_key", "value");
+
+            var config = getRequestConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(),
+                secretSettingsMap
+            );
+
+            var failureListener = getModelListenerForException(
+                ElasticsearchStatusException.class,
+                "Model configuration contains settings [{extra_key=value}] unknown to the [jinaai] service"
+            );
+            service.parseRequestConfig("id", TaskType.TEXT_EMBEDDING, config, failureListener);
+        }
+    }
+
+    public void testParseRequestConfig_CreatesAJinaAIEmbeddingsModelWithoutUrl() throws IOException {
+        try (var service = createJinaAIService()) {
+            var modelListener = ActionListener.<Model>wrap((model) -> {
+                MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+                var embeddingsModel = (JinaAIEmbeddingsModel) model;
+                assertNull(embeddingsModel.getServiceSettings().getCommonSettings().uri());
+                MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS));
+                MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret"));
+            }, (e) -> fail("Model parsing should have succeeded " + e.getMessage()));
+
+            service.parseRequestConfig(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                getRequestConfigMap(
+                    JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap(null, "model"),
+                    JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(),
+                    getSecretSettingsMap("secret")
+                ),
+                modelListener
+            );
+
+        }
+    }
+
+    public void testParsePersistedConfigWithSecrets_CreatesAJinaAIEmbeddingsModel() throws IOException {
+        try (var service = createJinaAIService()) {
+            var persistedConfig = getPersistedConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null),
+                getSecretSettingsMap("secret")
+            );
+
+            var model = service.parsePersistedConfigWithSecrets(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                persistedConfig.config(),
+                persistedConfig.secrets()
+            );
+
+            MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+            var embeddingsModel = (JinaAIEmbeddingsModel) model;
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+            MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings((InputType) null)));
+            MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret"));
+        }
+    }
+
+    public void testParsePersistedConfigWithSecrets_CreatesAJinaAIEmbeddingsModelWhenChunkingSettingsProvided() throws IOException {
+        try (var service = createJinaAIService()) {
+            var persistedConfig = getPersistedConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null),
+                createRandomChunkingSettingsMap(),
+                getSecretSettingsMap("secret")
+            );
+
+            var model = service.parsePersistedConfigWithSecrets(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                persistedConfig.config(),
+                persistedConfig.secrets()
+            );
+
+            MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+            var embeddingsModel = (JinaAIEmbeddingsModel) model;
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+            MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings((InputType) null)));
+            MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class));
+            MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret"));
+        }
+    }
+
+    public void testParsePersistedConfigWithSecrets_CreatesAJinaAIEmbeddingsModelWhenChunkingSettingsNotProvided() throws IOException {
+        try (var service = createJinaAIService()) {
+            var persistedConfig = getPersistedConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null),
+                getSecretSettingsMap("secret")
+            );
+
+            var model = service.parsePersistedConfigWithSecrets(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                persistedConfig.config(),
+                persistedConfig.secrets()
+            );
+
+            MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+            var embeddingsModel = (JinaAIEmbeddingsModel) model;
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+            MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings((InputType) null)));
+            MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class));
+            MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret"));
+        }
+    }
+
+    public void testParsePersistedConfigWithSecrets_ThrowsErrorTryingToParseInvalidModel() throws IOException {
+        try (var service = createJinaAIService()) {
+            var persistedConfig = getPersistedConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "oldmodel"),
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(),
+                getSecretSettingsMap("secret")
+            );
+
+            var thrownException = expectThrows(
+                ElasticsearchStatusException.class,
+                () -> service.parsePersistedConfigWithSecrets(
+                    "id",
+                    TaskType.SPARSE_EMBEDDING,
+                    persistedConfig.config(),
+                    persistedConfig.secrets()
+                )
+            );
+
+            MatcherAssert.assertThat(
+                thrownException.getMessage(),
+                is("Failed to parse stored model [id] for [jinaai] service, please delete and add the service again")
+            );
+        }
+    }
+
+    public void testParsePersistedConfigWithSecrets_CreatesAJinaAIEmbeddingsModelWithoutUrl() throws IOException {
+        try (var service = createJinaAIService()) {
+            var persistedConfig = getPersistedConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap(null, "model"),
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.INGEST),
+                getSecretSettingsMap("secret")
+            );
+
+            var model = service.parsePersistedConfigWithSecrets(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                persistedConfig.config(),
+                persistedConfig.secrets()
+            );
+
+            MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+            var embeddingsModel = (JinaAIEmbeddingsModel) model;
+            assertNull(embeddingsModel.getServiceSettings().getCommonSettings().uri());
+            MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings(InputType.INGEST)));
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+            MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret"));
+        }
+    }
+
+    public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExistsInConfig() throws IOException {
+        try (var service = createJinaAIService()) {
+            var persistedConfig = getPersistedConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.SEARCH),
+                getSecretSettingsMap("secret")
+            );
+            persistedConfig.config().put("extra_key", "value");
+
+            var model = service.parsePersistedConfigWithSecrets(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                persistedConfig.config(),
+                persistedConfig.secrets()
+            );
+
+            MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+            var embeddingsModel = (JinaAIEmbeddingsModel) model;
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+            MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings(InputType.SEARCH)));
+            MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret"));
+        }
+    }
+
+    public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExistsInSecretsSettings() throws IOException {
+        try (var service = createJinaAIService()) {
+            var secretSettingsMap = getSecretSettingsMap("secret");
+            secretSettingsMap.put("extra_key", "value");
+
+            var persistedConfig = getPersistedConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(),
+                secretSettingsMap
+            );
+
+            var model = service.parsePersistedConfigWithSecrets(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                persistedConfig.config(),
+                persistedConfig.secrets()
+            );
+
+            MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+            var embeddingsModel = (JinaAIEmbeddingsModel) model;
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+            MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS));
+            MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret"));
+        }
+    }
+
+    public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInSecrets() throws IOException {
+        try (var service = createJinaAIService()) {
+            var persistedConfig = getPersistedConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null),
+                getSecretSettingsMap("secret")
+            );
+            persistedConfig.secrets().put("extra_key", "value");
+
+            var model = service.parsePersistedConfigWithSecrets(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                persistedConfig.config(),
+                persistedConfig.secrets()
+            );
+
+            MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+            var embeddingsModel = (JinaAIEmbeddingsModel) model;
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+            MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings((InputType) null)));
+            MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret"));
+        }
+    }
+
+    public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInServiceSettings() throws IOException {
+        try (var service = createJinaAIService()) {
+            var serviceSettingsMap = JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model");
+            serviceSettingsMap.put("extra_key", "value");
+
+            var persistedConfig = getPersistedConfigMap(
+                serviceSettingsMap,
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(),
+                getSecretSettingsMap("secret")
+            );
+
+            var model = service.parsePersistedConfigWithSecrets(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                persistedConfig.config(),
+                persistedConfig.secrets()
+            );
+
+            MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+            var embeddingsModel = (JinaAIEmbeddingsModel) model;
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+            MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS));
+            MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret"));
+        }
+    }
+
+    public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInTaskSettings() throws IOException {
+        try (var service = createJinaAIService()) {
+            var taskSettingsMap = JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.SEARCH);
+            taskSettingsMap.put("extra_key", "value");
+
+            var persistedConfig = getPersistedConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                taskSettingsMap,
+                getSecretSettingsMap("secret")
+            );
+
+            var model = service.parsePersistedConfigWithSecrets(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                persistedConfig.config(),
+                persistedConfig.secrets()
+            );
+
+            MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+            var embeddingsModel = (JinaAIEmbeddingsModel) model;
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+            MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings(InputType.SEARCH)));
+            MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret"));
+        }
+    }
+
+    public void testParsePersistedConfig_CreatesAJinaAIEmbeddingsModel() throws IOException {
+        try (var service = createJinaAIService()) {
+            var persistedConfig = getPersistedConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null)
+            );
+
+            var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config());
+
+            MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+            var embeddingsModel = (JinaAIEmbeddingsModel) model;
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+            MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings((InputType) null)));
+            assertNull(embeddingsModel.getSecretSettings());
+        }
+    }
+
+    public void testParsePersistedConfig_CreatesAJinaAIEmbeddingsModelWhenChunkingSettingsProvided() throws IOException {
+        try (var service = createJinaAIService()) {
+            var persistedConfig = getPersistedConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null),
+                createRandomChunkingSettingsMap()
+            );
+
+            var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config());
+
+            MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+            var embeddingsModel = (JinaAIEmbeddingsModel) model;
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+            MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings((InputType) null)));
+            MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class));
+            assertNull(embeddingsModel.getSecretSettings());
+        }
+    }
+
+    public void testParsePersistedConfig_CreatesAJinaAIEmbeddingsModelWhenChunkingSettingsNotProvided() throws IOException {
+        try (var service = createJinaAIService()) {
+            var persistedConfig = getPersistedConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null)
+            );
+
+            var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config());
+
+            MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+            var embeddingsModel = (JinaAIEmbeddingsModel) model;
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+            MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings((InputType) null)));
+            MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class));
+            assertNull(embeddingsModel.getSecretSettings());
+        }
+    }
+
+    public void testParsePersistedConfig_ThrowsErrorTryingToParseInvalidModel() throws IOException {
+        try (var service = createJinaAIService()) {
+            var persistedConfig = getPersistedConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model_old"),
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty()
+            );
+
+            var thrownException = expectThrows(
+                ElasticsearchStatusException.class,
+                () -> service.parsePersistedConfig("id", TaskType.SPARSE_EMBEDDING, persistedConfig.config())
+            );
+
+            MatcherAssert.assertThat(
+                thrownException.getMessage(),
+                is("Failed to parse stored model [id] for [jinaai] service, please delete and add the service again")
+            );
+        }
+    }
+
+    public void testParsePersistedConfig_CreatesAJinaAIEmbeddingsModelWithoutUrl() throws IOException {
+        try (var service = createJinaAIService()) {
+            var persistedConfig = getPersistedConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap(null, "model"),
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null)
+            );
+
+            var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config());
+
+            MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+            var embeddingsModel = (JinaAIEmbeddingsModel) model;
+            assertNull(embeddingsModel.getServiceSettings().getCommonSettings().uri());
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+            MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings((InputType) null)));
+            assertNull(embeddingsModel.getSecretSettings());
+        }
+    }
+
+    public void testParsePersistedConfig_DoesNotThrowWhenAnExtraKeyExistsInConfig() throws IOException {
+        try (var service = createJinaAIService()) {
+            var persistedConfig = getPersistedConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty()
+            );
+            persistedConfig.config().put("extra_key", "value");
+
+            var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config());
+
+            MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+            var embeddingsModel = (JinaAIEmbeddingsModel) model;
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+            MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS));
+            assertNull(embeddingsModel.getSecretSettings());
+        }
+    }
+
+    public void testParsePersistedConfig_NotThrowWhenAnExtraKeyExistsInServiceSettings() throws IOException {
+        try (var service = createJinaAIService()) {
+            var serviceSettingsMap = JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model");
+            serviceSettingsMap.put("extra_key", "value");
+
+            var persistedConfig = getPersistedConfigMap(
+                serviceSettingsMap,
+                JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.SEARCH)
+            );
+
+            var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config());
+
+            MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+            var embeddingsModel = (JinaAIEmbeddingsModel) model;
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+            MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings(InputType.SEARCH)));
+            assertNull(embeddingsModel.getSecretSettings());
+        }
+    }
+
+    public void testParsePersistedConfig_NotThrowWhenAnExtraKeyExistsInTaskSettings() throws IOException {
+        try (var service = createJinaAIService()) {
+            var taskSettingsMap = JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.INGEST);
+            taskSettingsMap.put("extra_key", "value");
+
+            var persistedConfig = getPersistedConfigMap(
+                JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"),
+                taskSettingsMap
+            );
+
+            var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config());
+            MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class));
+
+            var embeddingsModel = (JinaAIEmbeddingsModel) model;
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url"));
+            MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model"));
+            MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings(InputType.INGEST)));
+            assertNull(embeddingsModel.getSecretSettings());
+        }
+    }
+
+    public void testInfer_ThrowsErrorWhenModelIsNotJinaAIModel() throws IOException {
+        var sender = mock(Sender.class);
+
+        var factory = mock(HttpRequestSender.Factory.class);
+        when(factory.createSender()).thenReturn(sender);
+
+        var mockModel = getInvalidModel("model_id", "service_name");
+
+        try (var service = new JinaAIService(factory, createWithEmptySettings(threadPool))) {
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            service.infer(
+                mockModel,
+                null,
+                List.of(""),
+                false,
+                new HashMap<>(),
+                InputType.INGEST,
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT));
+            MatcherAssert.assertThat(
+                thrownException.getMessage(),
+                is("The internal model was invalid, please delete the service [service_name] with id [model_id] and add it again.")
+            );
+
+            verify(factory, times(1)).createSender();
+            verify(sender, times(1)).start();
+        }
+
+        verify(sender, times(1)).close();
+        verifyNoMoreInteractions(factory);
+        verifyNoMoreInteractions(sender);
+    }
+
+    public void testCheckModelConfig_UpdatesDimensions() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) {
+
+            String responseJson = """
+                {
+                    "model": "jina-clip-v2",
+                    "object": "list",
+                    "usage": {
+                        "total_tokens": 5,
+                        "prompt_tokens": 5
+                    },
+                    "data": [
+                        {
+                            "object": "embedding",
+                            "index": 0,
+                            "embedding": [
+                                0.123,
+                                -0.123
+                            ]
+                        }
+                    ]
+                }
+                """;
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = JinaAIEmbeddingsModelTests.createModel(
+                getUrl(webServer),
+                "secret",
+                JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS,
+                10,
+                1,
+                "jina-clip-v2"
+            );
+            PlainActionFuture<Model> listener = new PlainActionFuture<>();
+            service.checkModelConfig(model, listener);
+            var result = listener.actionGet(TIMEOUT);
+
+            MatcherAssert.assertThat(
+                result,
+                // the dimension is set to 2 because there are 2 embeddings returned from the mock server
+                is(
+                    JinaAIEmbeddingsModelTests.createModel(
+                        getUrl(webServer),
+                        "secret",
+                        JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS,
+                        10,
+                        2,
+                        "jina-clip-v2"
+                    )
+                )
+            );
+        }
+    }
+
+    public void testCheckModelConfig_UpdatesSimilarityToDotProduct_WhenItIsNull() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) {
+
+            String responseJson = """
+                {
+                    "model": "jina-clip-v2",
+                    "object": "list",
+                    "usage": {
+                        "total_tokens": 5,
+                        "prompt_tokens": 5
+                    },
+                    "data": [
+                        {
+                            "object": "embedding",
+                            "index": 0,
+                            "embedding": [
+                                0.123,
+                                -0.123
+                            ]
+                        }
+                    ]
+                }
+                """;
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = JinaAIEmbeddingsModelTests.createModel(
+                getUrl(webServer),
+                "secret",
+                JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS,
+                10,
+                1,
+                "jina-clip-v2",
+                null
+            );
+            PlainActionFuture<Model> listener = new PlainActionFuture<>();
+            service.checkModelConfig(model, listener);
+            var result = listener.actionGet(TIMEOUT);
+
+            MatcherAssert.assertThat(
+                result,
+                // the dimension is set to 2 because there are 2 embeddings returned from the mock server
+                is(
+                    JinaAIEmbeddingsModelTests.createModel(
+                        getUrl(webServer),
+                        "secret",
+                        JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS,
+                        10,
+                        2,
+                        "jina-clip-v2",
+                        SimilarityMeasure.DOT_PRODUCT
+                    )
+                )
+            );
+        }
+    }
+
+    public void testCheckModelConfig_DoesNotUpdateSimilarity_WhenItIsSpecifiedAsCosine() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) {
+
+            String responseJson = """
+                {
+                    "model": "jina-clip-v2",
+                    "object": "list",
+                    "usage": {
+                        "total_tokens": 5,
+                        "prompt_tokens": 5
+                    },
+                    "data": [
+                        {
+                            "object": "embedding",
+                            "index": 0,
+                            "embedding": [
+                                0.123,
+                                -0.123
+                            ]
+                        }
+                    ]
+                }
+                """;
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = JinaAIEmbeddingsModelTests.createModel(
+                getUrl(webServer),
+                "secret",
+                JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS,
+                10,
+                1,
+                "jina-clip-v2",
+                SimilarityMeasure.COSINE
+            );
+            PlainActionFuture<Model> listener = new PlainActionFuture<>();
+            service.checkModelConfig(model, listener);
+            var result = listener.actionGet(TIMEOUT);
+
+            MatcherAssert.assertThat(
+                result,
+                // the dimension is set to 2 because there are 2 embeddings returned from the mock server
+                is(
+                    JinaAIEmbeddingsModelTests.createModel(
+                        getUrl(webServer),
+                        "secret",
+                        JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS,
+                        10,
+                        2,
+                        "jina-clip-v2",
+                        SimilarityMeasure.COSINE
+                    )
+                )
+            );
+        }
+    }
+
+    public void testUpdateModelWithEmbeddingDetails_NullSimilarityInOriginalModel() throws IOException {
+        testUpdateModelWithEmbeddingDetails_Successful(null);
+    }
+
+    public void testUpdateModelWithEmbeddingDetails_NonNullSimilarityInOriginalModel() throws IOException {
+        testUpdateModelWithEmbeddingDetails_Successful(randomFrom(SimilarityMeasure.values()));
+    }
+
+    private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure similarityMeasure) throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) {
+            var embeddingSize = randomNonNegativeInt();
+            var model = JinaAIEmbeddingsModelTests.createModel(
+                randomAlphaOfLength(10),
+                randomAlphaOfLength(10),
+                JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS,
+                randomNonNegativeInt(),
+                randomNonNegativeInt(),
+                randomAlphaOfLength(10),
+                similarityMeasure
+            );
+
+            Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize);
+
+            SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null ? JinaAIService.defaultSimilarity() : similarityMeasure;
+            assertEquals(expectedSimilarityMeasure, updatedModel.getServiceSettings().similarity());
+            assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue());
+        }
+    }
+
+    public void testInfer_Embedding_UnauthorisedResponse() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) {
+
+            String responseJson = """
+                {
+                    "detail": "Unauthorized"
+                }
+                """;
+            webServer.enqueue(new MockResponse().setResponseCode(401).setBody(responseJson));
+
+            var model = JinaAIEmbeddingsModelTests.createModel(
+                getUrl(webServer),
+                "secret",
+                JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS,
+                1024,
+                1024,
+                "model",
+                null
+            );
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            service.infer(
+                model,
+                null,
+                List.of("abc"),
+                false,
+                new HashMap<>(),
+                InputType.INGEST,
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var error = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT));
+            MatcherAssert.assertThat(error.getMessage(), containsString("Received an authentication error status code for request"));
+            MatcherAssert.assertThat(error.getMessage(), containsString("Error message: [Unauthorized]"));
+            MatcherAssert.assertThat(webServer.requests(), hasSize(1));
+        }
+    }
+
+    public void testInfer_Rerank_UnauthorisedResponse() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) {
+
+            String responseJson = """
+                {
+                    "detail": "Unauthorized"
+                }
+                """;
+            webServer.enqueue(new MockResponse().setResponseCode(401).setBody(responseJson));
+
+            var model = JinaAIRerankModelTests.createModel(getUrl(webServer), "model", 1024, false);
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            service.infer(
+                model,
+                "query",
+                List.of("candidate1", "candidate2"),
+                false,
+                new HashMap<>(),
+                null,
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var error = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT));
+            MatcherAssert.assertThat(error.getMessage(), containsString("Received an authentication error status code for request"));
+            MatcherAssert.assertThat(error.getMessage(), containsString("Error message: [Unauthorized]"));
+            MatcherAssert.assertThat(webServer.requests(), hasSize(1));
+        }
+    }
+
+    public void testInfer_Embedding_Get_Response_Ingest() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) {
+
+            String responseJson = """
+                {
+                    "model": "jina-clip-v2",
+                    "object": "list",
+                    "usage": {
+                        "total_tokens": 5,
+                        "prompt_tokens": 5
+                    },
+                    "data": [
+                        {
+                            "object": "embedding",
+                            "index": 0,
+                            "embedding": [
+                                0.123,
+                                -0.123
+                            ]
+                        }
+                    ]
+                }
+                """;
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = JinaAIEmbeddingsModelTests.createModel(
+                getUrl(webServer),
+                "secret",
+                JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS,
+                1024,
+                1024,
+                "jina-clip-v2",
+                null
+            );
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            service.infer(
+                model,
+                null,
+                List.of("abc"),
+                false,
+                new HashMap<>(),
+                InputType.INGEST,
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var result = listener.actionGet(TIMEOUT);
+
+            assertEquals(buildExpectationFloat(List.of(new float[] { 0.123F, -0.123F })), result.asMap());
+
+            MatcherAssert.assertThat(webServer.requests(), hasSize(1));
+            assertNull(webServer.requests().get(0).getUri().getQuery());
+            MatcherAssert.assertThat(
+                webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE),
+                equalTo(XContentType.JSON.mediaType())
+            );
+            MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret"));
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"), "model", "jina-clip-v2", "task", "retrieval.passage")));
+        }
+    }
+
+    public void testInfer_Embedding_Get_Response_Search() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) {
+
+            String responseJson = """
+                {
+                    "model": "jina-clip-v2",
+                    "object": "list",
+                    "usage": {
+                        "total_tokens": 5,
+                        "prompt_tokens": 5
+                    },
+                    "data": [
+                        {
+                            "object": "embedding",
+                            "index": 0,
+                            "embedding": [
+                                0.123,
+                                -0.123
+                            ]
+                        }
+                    ]
+                }
+                """;
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = JinaAIEmbeddingsModelTests.createModel(
+                getUrl(webServer),
+                "secret",
+                JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS,
+                1024,
+                1024,
+                "jina-clip-v2",
+                null
+            );
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            service.infer(
+                model,
+                null,
+                List.of("abc"),
+                false,
+                new HashMap<>(),
+                InputType.SEARCH,
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var result = listener.actionGet(TIMEOUT);
+
+            assertEquals(buildExpectationFloat(List.of(new float[] { 0.123F, -0.123F })), result.asMap());
+
+            MatcherAssert.assertThat(webServer.requests(), hasSize(1));
+            assertNull(webServer.requests().get(0).getUri().getQuery());
+            MatcherAssert.assertThat(
+                webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE),
+                equalTo(XContentType.JSON.mediaType())
+            );
+            MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret"));
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"), "model", "jina-clip-v2", "task", "retrieval.query")));
+        }
+    }
+
+    public void testInfer_Embedding_Get_Response_clustering() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) {
+
+            String responseJson = """
+                {"model":"jina-clip-v2","object":"list","usage":{"total_tokens":5,"prompt_tokens":5},
+                "data":[{"object":"embedding","index":0,"embedding":[0.123, -0.123]}]}
+                """;
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = JinaAIEmbeddingsModelTests.createModel(
+                getUrl(webServer),
+                "secret",
+                JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS,
+                1024,
+                1024,
+                "jina-clip-v2",
+                null
+            );
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            service.infer(
+                model,
+                null,
+                List.of("abc"),
+                false,
+                new HashMap<>(),
+                InputType.CLUSTERING,
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var result = listener.actionGet(TIMEOUT);
+
+            assertEquals(buildExpectationFloat(List.of(new float[] { 0.123F, -0.123F })), result.asMap());
+
+            MatcherAssert.assertThat(webServer.requests(), hasSize(1));
+            assertNull(webServer.requests().get(0).getUri().getQuery());
+            MatcherAssert.assertThat(
+                webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE),
+                equalTo(XContentType.JSON.mediaType())
+            );
+            MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret"));
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"), "model", "jina-clip-v2", "task", "separation")));
+        }
+    }
+
+    public void testInfer_Embedding_Get_Response_NullInputType() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) {
+
+            String responseJson = """
+                {
+                    "model": "jina-clip-v2",
+                    "object": "list",
+                    "usage": {
+                        "total_tokens": 5,
+                        "prompt_tokens": 5
+                    },
+                    "data": [
+                        {
+                            "object": "embedding",
+                            "index": 0,
+                            "embedding": [
+                                0.123,
+                                -0.123
+                            ]
+                        }
+                    ]
+                }
+                """;
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = JinaAIEmbeddingsModelTests.createModel(
+                getUrl(webServer),
+                "secret",
+                JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS,
+                1024,
+                1024,
+                "jina-clip-v2",
+                null
+            );
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            service.infer(model, null, List.of("abc"), false, new HashMap<>(), null, InferenceAction.Request.DEFAULT_TIMEOUT, listener);
+
+            var result = listener.actionGet(TIMEOUT);
+
+            assertEquals(buildExpectationFloat(List.of(new float[] { 0.123F, -0.123F })), result.asMap());
+
+            MatcherAssert.assertThat(webServer.requests(), hasSize(1));
+            assertNull(webServer.requests().get(0).getUri().getQuery());
+            MatcherAssert.assertThat(
+                webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE),
+                equalTo(XContentType.JSON.mediaType())
+            );
+            MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret"));
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"), "model", "jina-clip-v2")));
+        }
+    }
+
+    public void testInfer_Rerank_Get_Response_NoReturnDocuments_NoTopN() throws IOException {
+        String responseJson = """
+            {
+                "model": "model",
+                "results": [
+                    {
+                        "index": 2,
+                        "relevance_score": 0.98005307
+                    },
+                    {
+                        "index": 1,
+                        "relevance_score": 0.27904198
+                    },
+                    {
+                        "index": 0,
+                        "relevance_score": 0.10194652
+                    }
+                ],
+                "usage": {
+                    "total_tokens": 15
+                }
+            }
+            """;
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) {
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+            var model = JinaAIRerankModelTests.createModel(getUrl(webServer), "secret", "model", null, false);
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            service.infer(
+                model,
+                "query",
+                List.of("candidate1", "candidate2", "candidate3"),
+                false,
+                new HashMap<>(),
+                null,
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var result = listener.actionGet(TIMEOUT);
+            var resultAsMap = result.asMap();
+            assertThat(
+                resultAsMap,
+                is(
+                    Map.of(
+                        "rerank",
+                        List.of(
+                            Map.of("ranked_doc", Map.of("index", 2, "relevance_score", 0.98005307F)),
+                            Map.of("ranked_doc", Map.of("index", 1, "relevance_score", 0.27904198F)),
+                            Map.of("ranked_doc", Map.of("index", 0, "relevance_score", 0.10194652F))
+                        )
+                    )
+                )
+            );
+
+            MatcherAssert.assertThat(webServer.requests(), hasSize(1));
+            MatcherAssert.assertThat(
+                webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE),
+                equalTo(XContentType.JSON.mediaType())
+            );
+            MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret"));
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            MatcherAssert.assertThat(
+                requestMap,
+                is(
+                    Map.of(
+                        "query",
+                        "query",
+                        "documents",
+                        List.of("candidate1", "candidate2", "candidate3"),
+                        "model",
+                        "model",
+                        "return_documents",
+                        false
+                    )
+                )
+            );
+
+        }
+    }
+
+    public void testInfer_Rerank_Get_Response_NoReturnDocuments_TopN() throws IOException {
+        String responseJson = """
+            {
+                "model": "model",
+                "results": [
+                    {
+                        "index": 2,
+                        "relevance_score": 0.98005307
+                    },
+                    {
+                        "index": 1,
+                        "relevance_score": 0.27904198
+                    },
+                    {
+                        "index": 0,
+                        "relevance_score": 0.10194652
+                    }
+                ],
+                "usage": {
+                    "total_tokens": 15
+                }
+            }
+            """;
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) {
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+            var model = JinaAIRerankModelTests.createModel(getUrl(webServer), "secret", "model", 3, false);
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            service.infer(
+                model,
+                "query",
+                List.of("candidate1", "candidate2", "candidate3", "candidate4"),
+                false,
+                new HashMap<>(),
+                null,
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var result = listener.actionGet(TIMEOUT);
+            var resultAsMap = result.asMap();
+            assertThat(
+                resultAsMap,
+                is(
+                    Map.of(
+                        "rerank",
+                        List.of(
+                            Map.of("ranked_doc", Map.of("index", 2, "relevance_score", 0.98005307F)),
+                            Map.of("ranked_doc", Map.of("index", 1, "relevance_score", 0.27904198F)),
+                            Map.of("ranked_doc", Map.of("index", 0, "relevance_score", 0.10194652F))
+                        )
+                    )
+                )
+            );
+
+            MatcherAssert.assertThat(webServer.requests(), hasSize(1));
+            MatcherAssert.assertThat(
+                webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE),
+                equalTo(XContentType.JSON.mediaType())
+            );
+            MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret"));
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            MatcherAssert.assertThat(
+                requestMap,
+                is(
+                    Map.of(
+                        "query",
+                        "query",
+                        "documents",
+                        List.of("candidate1", "candidate2", "candidate3", "candidate4"),
+                        "model",
+                        "model",
+                        "return_documents",
+                        false,
+                        "top_n",
+                        3
+                    )
+                )
+            );
+
+        }
+
+    }
+
+    public void testInfer_Rerank_Get_Response_ReturnDocumentsNull_NoTopN() throws IOException {
+        String responseJson = """
+            {
+                "model": "model",
+                "results": [
+                    {
+                        "index": 2,
+                        "relevance_score": 0.98005307,
+                        "document": {
+                            "text": "candidate3"
+                        }
+                    },
+                    {
+                        "index": 1,
+                        "relevance_score": 0.27904198,
+                        "document": {
+                            "text": "candidate2"
+                        }
+                    },
+                    {
+                        "index": 0,
+                        "relevance_score": 0.10194652,
+                        "document": {
+                            "text": "candidate1"
+                        }
+                    }
+                ],
+                "usage": {
+                    "total_tokens": 15
+                }
+            }
+            """;
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) {
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+            var model = JinaAIRerankModelTests.createModel(getUrl(webServer), "secret", "model", null, null);
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            service.infer(
+                model,
+                "query",
+                List.of("candidate1", "candidate2", "candidate3"),
+                false,
+                new HashMap<>(),
+                null,
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var result = listener.actionGet(TIMEOUT);
+            var resultAsMap = result.asMap();
+            assertThat(
+                resultAsMap,
+                is(
+                    Map.of(
+                        "rerank",
+                        List.of(
+                            Map.of("ranked_doc", Map.of("text", "candidate3", "index", 2, "relevance_score", 0.98005307F)),
+                            Map.of("ranked_doc", Map.of("text", "candidate2", "index", 1, "relevance_score", 0.27904198F)),
+                            Map.of("ranked_doc", Map.of("text", "candidate1", "index", 0, "relevance_score", 0.10194652F))
+                        )
+                    )
+                )
+            );
+            MatcherAssert.assertThat(webServer.requests(), hasSize(1));
+            MatcherAssert.assertThat(
+                webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE),
+                equalTo(XContentType.JSON.mediaType())
+            );
+            MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret"));
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            MatcherAssert.assertThat(
+                requestMap,
+                is(Map.of("query", "query", "documents", List.of("candidate1", "candidate2", "candidate3"), "model", "model"))
+            );
+
+        }
+
+    }
+
+    public void testInfer_Rerank_Get_Response_ReturnDocuments_TopN() throws IOException {
+        String responseJson = """
+            {
+                "model": "model",
+                "results": [
+                    {
+                        "index": 2,
+                        "relevance_score": 0.98005307,
+                        "document": {
+                            "text": "candidate3"
+                        }
+                    },
+                    {
+                        "index": 1,
+                        "relevance_score": 0.27904198,
+                        "document": {
+                            "text": "candidate2"
+                        }
+                    },
+                    {
+                        "index": 0,
+                        "relevance_score": 0.10194652,
+                        "document": {
+                            "text": "candidate1"
+                        }
+                    }
+                ],
+                "usage": {
+                    "total_tokens": 15
+                }
+            }
+            """;
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) {
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+            var model = JinaAIRerankModelTests.createModel(getUrl(webServer), "secret", "model", 3, true);
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            service.infer(
+                model,
+                "query",
+                List.of("candidate1", "candidate2", "candidate3", "candidate4"),
+                false,
+                new HashMap<>(),
+                null,
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var result = listener.actionGet(TIMEOUT);
+            var resultAsMap = result.asMap();
+            assertThat(
+                resultAsMap,
+                is(
+                    Map.of(
+                        "rerank",
+                        List.of(
+                            Map.of("ranked_doc", Map.of("text", "candidate3", "index", 2, "relevance_score", 0.98005307F)),
+                            Map.of("ranked_doc", Map.of("text", "candidate2", "index", 1, "relevance_score", 0.27904198F)),
+                            Map.of("ranked_doc", Map.of("text", "candidate1", "index", 0, "relevance_score", 0.10194652F))
+                        )
+                    )
+                )
+            );
+            MatcherAssert.assertThat(webServer.requests(), hasSize(1));
+            MatcherAssert.assertThat(
+                webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE),
+                equalTo(XContentType.JSON.mediaType())
+            );
+            MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret"));
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            MatcherAssert.assertThat(
+                requestMap,
+                is(
+                    Map.of(
+                        "query",
+                        "query",
+                        "documents",
+                        List.of("candidate1", "candidate2", "candidate3", "candidate4"),
+                        "model",
+                        "model",
+                        "return_documents",
+                        true,
+                        "top_n",
+                        3
+                    )
+                )
+            );
+
+        }
+
+    }
+
+    public void testInfer_Embedding_DoesNotSetInputType_WhenNotPresentInTaskSettings_AndUnspecifiedIsPassedInRequest() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) {
+
+            String responseJson = """
+                {
+                    "model": "jina-clip-v2",
+                    "object": "list",
+                    "usage": {
+                        "total_tokens": 5,
+                        "prompt_tokens": 5
+                    },
+                    "data": [
+                        {
+                            "object": "embedding",
+                            "index": 0,
+                            "embedding": [
+                                0.123,
+                                -0.123
+                            ]
+                        }
+                    ]
+                }
+                """;
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = JinaAIEmbeddingsModelTests.createModel(
+                getUrl(webServer),
+                "secret",
+                new JinaAIEmbeddingsTaskSettings((InputType) null),
+                1024,
+                1024,
+                "jina-clip-v2",
+                null
+            );
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            service.infer(
+                model,
+                null,
+                List.of("abc"),
+                false,
+                new HashMap<>(),
+                InputType.UNSPECIFIED,
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var result = listener.actionGet(TIMEOUT);
+
+            MatcherAssert.assertThat(result.asMap(), Matchers.is(buildExpectationFloat(List.of(new float[] { 0.123F, -0.123F }))));
+            MatcherAssert.assertThat(webServer.requests(), hasSize(1));
+            assertNull(webServer.requests().get(0).getUri().getQuery());
+            MatcherAssert.assertThat(
+                webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE),
+                equalTo(XContentType.JSON.mediaType())
+            );
+            MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret"));
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"), "model", "jina-clip-v2")));
+        }
+    }
+
+    public void test_Embedding_ChunkedInfer_BatchesCallsChunkingSettingsSet() throws IOException {
+        var model = JinaAIEmbeddingsModelTests.createModel(
+            getUrl(webServer),
+            "secret",
+            new JinaAIEmbeddingsTaskSettings((InputType) null),
+            createRandomChunkingSettings(),
+            1024,
+            1024,
+            "jina-clip-v2"
+        );
+
+        test_Embedding_ChunkedInfer_BatchesCalls(model);
+    }
+
+    public void test_Embedding_ChunkedInfer_ChunkingSettingsNotSet() throws IOException {
+        var model = JinaAIEmbeddingsModelTests.createModel(
+            getUrl(webServer),
+            "secret",
+            new JinaAIEmbeddingsTaskSettings((InputType) null),
+            null,
+            1024,
+            1024,
+            "jina-clip-v2"
+        );
+
+        test_Embedding_ChunkedInfer_BatchesCalls(model);
+    }
+
+    private void test_Embedding_ChunkedInfer_BatchesCalls(JinaAIEmbeddingsModel model) throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) {
+
+            // Batching will call the service with 2 input
+            String responseJson = """
+                {
+                    "model": "jina-clip-v2",
+                    "object": "list",
+                    "usage": {
+                        "total_tokens": 5,
+                        "prompt_tokens": 5
+                    },
+                    "data": [
+                        {
+                            "object": "embedding",
+                            "index": 0,
+                            "embedding": [
+                                0.123,
+                                -0.123
+                            ]
+                        },
+                        {
+                            "object": "embedding",
+                            "index": 1,
+                            "embedding": [
+                                0.223,
+                                -0.223
+                            ]
+                        }
+                    ]
+                }
+                """;
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            PlainActionFuture<List<ChunkedInference>> listener = new PlainActionFuture<>();
+            // 2 input
+            service.chunkedInfer(
+                model,
+                null,
+                List.of("foo", "bar"),
+                new HashMap<>(),
+                InputType.UNSPECIFIED,
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var results = listener.actionGet(TIMEOUT);
+            assertThat(results, hasSize(2));
+            {
+                assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class));
+                var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(0);
+                assertThat(floatResult.chunks(), hasSize(1));
+                assertEquals("foo", floatResult.chunks().get(0).matchedText());
+                assertArrayEquals(new float[] { 0.123f, -0.123f }, floatResult.chunks().get(0).embedding(), 0.0f);
+            }
+            {
+                assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class));
+                var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(1);
+                assertThat(floatResult.chunks(), hasSize(1));
+                assertEquals("bar", floatResult.chunks().get(0).matchedText());
+                assertArrayEquals(new float[] { 0.223f, -0.223f }, floatResult.chunks().get(0).embedding(), 0.0f);
+            }
+
+            MatcherAssert.assertThat(webServer.requests(), hasSize(1));
+            assertNull(webServer.requests().get(0).getUri().getQuery());
+            MatcherAssert.assertThat(
+                webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE),
+                equalTo(XContentType.JSON.mediaType())
+            );
+            MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret"));
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("foo", "bar"), "model", "jina-clip-v2")));
+        }
+    }
+
+    public void testDefaultSimilarity() {
+        assertEquals(SimilarityMeasure.DOT_PRODUCT, JinaAIService.defaultSimilarity());
+    }
+
+    @SuppressWarnings("checkstyle:LineLength")
+    public void testGetConfiguration() throws Exception {
+        try (var service = createJinaAIService()) {
+            String content = XContentHelper.stripWhitespace("""
+                {
+                        "service": "jinaai",
+                        "name": "Jina AI",
+                        "task_types": ["text_embedding", "rerank"],
+                        "configurations": {
+                            "api_key": {
+                                "description": "API Key for the provider you're connecting to.",
+                                "label": "API Key",
+                                "required": true,
+                                "sensitive": true,
+                                "updatable": true,
+                                "type": "str"
+                            },
+                            "rate_limit.requests_per_minute": {
+                                "description": "Minimize the number of rate limit errors.",
+                                "label": "Rate Limit",
+                                "required": false,
+                                "sensitive": false,
+                                "updatable": false,
+                                "type": "int"
+                            }
+                        }
+                    }
+                """);
+            InferenceServiceConfiguration configuration = InferenceServiceConfiguration.fromXContentBytes(
+                new BytesArray(content),
+                XContentType.JSON
+            );
+            boolean humanReadable = true;
+            BytesReference originalBytes = toShuffledXContent(configuration, XContentType.JSON, ToXContent.EMPTY_PARAMS, humanReadable);
+            InferenceServiceConfiguration serviceConfiguration = service.getConfiguration();
+            assertToXContentEquivalent(
+                originalBytes,
+                toXContent(serviceConfiguration, XContentType.JSON, humanReadable),
+                XContentType.JSON
+            );
+        }
+    }
+
+    public void testDoesNotSupportsStreaming() throws IOException {
+        try (var service = new JinaAIService(mock(), createWithEmptySettings(mock()))) {
+            assertFalse(service.canStream(TaskType.COMPLETION));
+            assertFalse(service.canStream(TaskType.ANY));
+        }
+    }
+
+    private Map<String, Object> getRequestConfigMap(
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        Map<String, Object> chunkingSettings,
+        Map<String, Object> secretSettings
+    ) {
+        var requestConfigMap = getRequestConfigMap(serviceSettings, taskSettings, secretSettings);
+        requestConfigMap.put(ModelConfigurations.CHUNKING_SETTINGS, chunkingSettings);
+
+        return requestConfigMap;
+    }
+
+    private Map<String, Object> getRequestConfigMap(
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        Map<String, Object> secretSettings
+    ) {
+        var builtServiceSettings = new HashMap<>();
+        builtServiceSettings.putAll(serviceSettings);
+        builtServiceSettings.putAll(secretSettings);
+
+        return new HashMap<>(
+            Map.of(ModelConfigurations.SERVICE_SETTINGS, builtServiceSettings, ModelConfigurations.TASK_SETTINGS, taskSettings)
+        );
+    }
+
+    private Map<String, Object> getRequestConfigMap(Map<String, Object> serviceSettings, Map<String, Object> secretSettings) {
+        var builtServiceSettings = new HashMap<>();
+        builtServiceSettings.putAll(serviceSettings);
+        builtServiceSettings.putAll(secretSettings);
+
+        return new HashMap<>(Map.of(ModelConfigurations.SERVICE_SETTINGS, builtServiceSettings));
+    }
+
+    private JinaAIService createJinaAIService() {
+        return new JinaAIService(mock(HttpRequestSender.Factory.class), createWithEmptySettings(threadPool));
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModelTests.java
new file mode 100644
index 0000000000000..58455bb1f54ea
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModelTests.java
@@ -0,0 +1,168 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai.embeddings;
+
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.ChunkingSettings;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettings;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
+import org.hamcrest.MatcherAssert;
+
+import java.util.Map;
+
+import static org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap;
+import static org.hamcrest.Matchers.is;
+
+public class JinaAIEmbeddingsModelTests extends ESTestCase {
+
+    public void testOverrideWith_DoesNotOverrideAndModelRemainsEqual_WhenSettingsAreEmpty_AndInputTypeIsInvalid() {
+        var model = createModel("url", "api_key", null, null, "model");
+
+        var overriddenModel = JinaAIEmbeddingsModel.of(model, Map.of(), InputType.UNSPECIFIED);
+        MatcherAssert.assertThat(overriddenModel, is(model));
+    }
+
+    public void testOverrideWith_DoesNotOverrideAndModelRemainsEqual_WhenSettingsAreNull_AndInputTypeIsInvalid() {
+        var model = createModel("url", "api_key", null, null, "model");
+
+        var overriddenModel = JinaAIEmbeddingsModel.of(model, null, InputType.UNSPECIFIED);
+        MatcherAssert.assertThat(overriddenModel, is(model));
+    }
+
+    public void testOverrideWith_SetsInputTypeToIngest_WhenTheFieldIsNullInModelTaskSettings_AndNullInRequestTaskSettings() {
+        var model = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings((InputType) null), null, null, "model");
+
+        var overriddenModel = JinaAIEmbeddingsModel.of(model, getTaskSettingsMap(null), InputType.INGEST);
+        var expectedModel = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings(InputType.INGEST), null, null, "model");
+        MatcherAssert.assertThat(overriddenModel, is(expectedModel));
+    }
+
+    public void testOverrideWith_SetsInputType_FromRequest_IfValid_OverridingStoredTaskSettings() {
+        var model = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings(InputType.INGEST), null, null, "model");
+
+        var overriddenModel = JinaAIEmbeddingsModel.of(model, getTaskSettingsMap(null), InputType.SEARCH);
+        var expectedModel = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings(InputType.SEARCH), null, null, "model");
+        MatcherAssert.assertThat(overriddenModel, is(expectedModel));
+    }
+
+    public void testOverrideWith_SetsInputType_FromRequest_IfValid_OverridingRequestTaskSettings() {
+        var model = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings((InputType) null), null, null, "model");
+
+        var overriddenModel = JinaAIEmbeddingsModel.of(model, getTaskSettingsMap(InputType.INGEST), InputType.SEARCH);
+        var expectedModel = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings(InputType.SEARCH), null, null, "model");
+        MatcherAssert.assertThat(overriddenModel, is(expectedModel));
+    }
+
+    public void testOverrideWith_OverridesInputType_WithRequestTaskSettingsSearch_WhenRequestInputTypeIsInvalid() {
+        var model = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings(InputType.INGEST), null, null, "model");
+
+        var overriddenModel = JinaAIEmbeddingsModel.of(model, getTaskSettingsMap(InputType.SEARCH), InputType.UNSPECIFIED);
+        var expectedModel = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings(InputType.SEARCH), null, null, "model");
+        MatcherAssert.assertThat(overriddenModel, is(expectedModel));
+    }
+
+    public void testOverrideWith_DoesNotSetInputType_FromRequest_IfInputTypeIsInvalid() {
+        var model = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings((InputType) null), null, null, "model");
+
+        var overriddenModel = JinaAIEmbeddingsModel.of(model, getTaskSettingsMap(null), InputType.UNSPECIFIED);
+        var expectedModel = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings((InputType) null), null, null, "model");
+        MatcherAssert.assertThat(overriddenModel, is(expectedModel));
+    }
+
+    public void testOverrideWith_DoesNotSetInputType_WhenRequestTaskSettingsIsNull_AndRequestInputTypeIsInvalid() {
+        var model = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings(InputType.INGEST), null, null, "model");
+
+        var overriddenModel = JinaAIEmbeddingsModel.of(model, getTaskSettingsMap(null), InputType.UNSPECIFIED);
+        var expectedModel = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings(InputType.INGEST), null, null, "model");
+        MatcherAssert.assertThat(overriddenModel, is(expectedModel));
+    }
+
+    public static JinaAIEmbeddingsModel createModel(String url, String apiKey, @Nullable Integer tokenLimit, @Nullable String model) {
+        return createModel(url, apiKey, JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, tokenLimit, null, model);
+    }
+
+    public static JinaAIEmbeddingsModel createModel(
+        String url,
+        String apiKey,
+        @Nullable Integer tokenLimit,
+        @Nullable Integer dimensions,
+        String model
+    ) {
+        return createModel(url, apiKey, JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, tokenLimit, dimensions, model);
+    }
+
+    public static JinaAIEmbeddingsModel createModel(
+        String url,
+        String apiKey,
+        JinaAIEmbeddingsTaskSettings taskSettings,
+        ChunkingSettings chunkingSettings,
+        @Nullable Integer tokenLimit,
+        @Nullable Integer dimensions,
+        String model
+    ) {
+        return new JinaAIEmbeddingsModel(
+            "id",
+            "service",
+            new JinaAIEmbeddingsServiceSettings(
+                new JinaAIServiceSettings(url, model, null),
+                SimilarityMeasure.DOT_PRODUCT,
+                dimensions,
+                tokenLimit
+            ),
+            taskSettings,
+            chunkingSettings,
+            new DefaultSecretSettings(new SecureString(apiKey.toCharArray()))
+        );
+    }
+
+    public static JinaAIEmbeddingsModel createModel(
+        String url,
+        String apiKey,
+        JinaAIEmbeddingsTaskSettings taskSettings,
+        @Nullable Integer tokenLimit,
+        @Nullable Integer dimensions,
+        String model
+    ) {
+        return new JinaAIEmbeddingsModel(
+            "id",
+            "service",
+            new JinaAIEmbeddingsServiceSettings(
+                new JinaAIServiceSettings(url, model, null),
+                SimilarityMeasure.DOT_PRODUCT,
+                dimensions,
+                tokenLimit
+            ),
+            taskSettings,
+            null,
+            new DefaultSecretSettings(new SecureString(apiKey.toCharArray()))
+        );
+    }
+
+    public static JinaAIEmbeddingsModel createModel(
+        String url,
+        String apiKey,
+        JinaAIEmbeddingsTaskSettings taskSettings,
+        @Nullable Integer tokenLimit,
+        @Nullable Integer dimensions,
+        String model,
+        @Nullable SimilarityMeasure similarityMeasure
+    ) {
+        return new JinaAIEmbeddingsModel(
+            "id",
+            "service",
+            new JinaAIEmbeddingsServiceSettings(new JinaAIServiceSettings(url, model, null), similarityMeasure, dimensions, tokenLimit),
+            taskSettings,
+            null,
+            new DefaultSecretSettings(new SecureString(apiKey.toCharArray()))
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsServiceSettingsTests.java
new file mode 100644
index 0000000000000..6847d249a57a0
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsServiceSettingsTests.java
@@ -0,0 +1,187 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai.embeddings;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
+import org.elasticsearch.xpack.inference.InferenceNamedWriteablesProvider;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.ServiceFields;
+import org.elasticsearch.xpack.inference.services.ServiceUtils;
+import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettings;
+import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettingsTests;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.is;
+
+public class JinaAIEmbeddingsServiceSettingsTests extends AbstractWireSerializingTestCase<JinaAIEmbeddingsServiceSettings> {
+    public static JinaAIEmbeddingsServiceSettings createRandom() {
+        SimilarityMeasure similarityMeasure = null;
+        Integer dims = null;
+        similarityMeasure = SimilarityMeasure.DOT_PRODUCT;
+        dims = 1024;
+        Integer maxInputTokens = randomBoolean() ? null : randomIntBetween(128, 256);
+
+        var commonSettings = JinaAIServiceSettingsTests.createRandom();
+
+        return new JinaAIEmbeddingsServiceSettings(commonSettings, similarityMeasure, dims, maxInputTokens);
+    }
+
+    public void testFromMap() {
+        var url = "https://www.abc.com";
+        var similarity = SimilarityMeasure.DOT_PRODUCT.toString();
+        var dims = 1536;
+        var maxInputTokens = 512;
+        var model = "model";
+        var serviceSettings = JinaAIEmbeddingsServiceSettings.fromMap(
+            new HashMap<>(
+                Map.of(
+                    ServiceFields.URL,
+                    url,
+                    ServiceFields.SIMILARITY,
+                    similarity,
+                    ServiceFields.DIMENSIONS,
+                    dims,
+                    ServiceFields.MAX_INPUT_TOKENS,
+                    maxInputTokens,
+                    JinaAIServiceSettings.MODEL_ID,
+                    model
+                )
+            ),
+            ConfigurationParseContext.PERSISTENT
+        );
+
+        MatcherAssert.assertThat(
+            serviceSettings,
+            is(
+                new JinaAIEmbeddingsServiceSettings(
+                    new JinaAIServiceSettings(ServiceUtils.createUri(url), model, null),
+                    SimilarityMeasure.DOT_PRODUCT,
+                    dims,
+                    maxInputTokens
+                )
+            )
+        );
+    }
+
+    public void testFromMap_WithModelId() {
+        var url = "https://www.abc.com";
+        var similarity = SimilarityMeasure.DOT_PRODUCT.toString();
+        var dims = 1536;
+        var maxInputTokens = 512;
+        var model = "model";
+        var serviceSettings = JinaAIEmbeddingsServiceSettings.fromMap(
+            new HashMap<>(
+                Map.of(
+                    ServiceFields.URL,
+                    url,
+                    ServiceFields.SIMILARITY,
+                    similarity,
+                    ServiceFields.DIMENSIONS,
+                    dims,
+                    ServiceFields.MAX_INPUT_TOKENS,
+                    maxInputTokens,
+                    JinaAIServiceSettings.MODEL_ID,
+                    model
+                )
+            ),
+            ConfigurationParseContext.REQUEST
+        );
+
+        MatcherAssert.assertThat(
+            serviceSettings,
+            is(
+                new JinaAIEmbeddingsServiceSettings(
+                    new JinaAIServiceSettings(ServiceUtils.createUri(url), model, null),
+                    SimilarityMeasure.DOT_PRODUCT,
+                    dims,
+                    maxInputTokens
+                )
+            )
+        );
+    }
+
+    public void testFromMap_InvalidSimilarity_ThrowsError() {
+        var similarity = "by_size";
+        var thrownException = expectThrows(
+            ValidationException.class,
+            () -> JinaAIEmbeddingsServiceSettings.fromMap(
+                new HashMap<>(Map.of(JinaAIServiceSettings.MODEL_ID, "model", ServiceFields.SIMILARITY, similarity)),
+                ConfigurationParseContext.PERSISTENT
+            )
+        );
+
+        MatcherAssert.assertThat(
+            thrownException.getMessage(),
+            is(
+                "Validation Failed: 1: [service_settings] Invalid value [by_size] received. [similarity] "
+                    + "must be one of [cosine, dot_product, l2_norm];"
+            )
+        );
+    }
+
+    public void testToXContent_WritesAllValues() throws IOException {
+        var serviceSettings = new JinaAIEmbeddingsServiceSettings(
+            new JinaAIServiceSettings("url", "model", new RateLimitSettings(3)),
+            SimilarityMeasure.COSINE,
+            5,
+            10
+        );
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        serviceSettings.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+        assertThat(xContentResult, is("""
+            {"url":"url","model_id":"model",""" + """
+            "rate_limit":{"requests_per_minute":3},"similarity":"cosine","dimensions":5,"max_input_tokens":10}"""));
+    }
+
+    @Override
+    protected Writeable.Reader<JinaAIEmbeddingsServiceSettings> instanceReader() {
+        return JinaAIEmbeddingsServiceSettings::new;
+    }
+
+    @Override
+    protected JinaAIEmbeddingsServiceSettings createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected JinaAIEmbeddingsServiceSettings mutateInstance(JinaAIEmbeddingsServiceSettings instance) throws IOException {
+        return randomValueOtherThan(instance, JinaAIEmbeddingsServiceSettingsTests::createRandom);
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
+        entries.addAll(new MlInferenceNamedXContentProvider().getNamedWriteables());
+        entries.addAll(InferenceNamedWriteablesProvider.getNamedWriteables());
+        return new NamedWriteableRegistry(entries);
+    }
+
+    public static Map<String, Object> getServiceSettingsMap(@Nullable String url, String model) {
+        var map = new HashMap<>(JinaAIServiceSettingsTests.getServiceSettingsMap(url, model));
+        return map;
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsTaskSettingsTests.java
new file mode 100644
index 0000000000000..8535381b43adc
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsTaskSettingsTests.java
@@ -0,0 +1,193 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai.embeddings;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.inference.InputTypeTests.randomWithoutUnspecified;
+import static org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettings.VALID_REQUEST_VALUES;
+import static org.hamcrest.Matchers.is;
+
+public class JinaAIEmbeddingsTaskSettingsTests extends AbstractWireSerializingTestCase<JinaAIEmbeddingsTaskSettings> {
+
+    public static JinaAIEmbeddingsTaskSettings createRandom() {
+        var inputType = randomBoolean() ? randomWithoutUnspecified() : null;
+
+        return new JinaAIEmbeddingsTaskSettings(inputType);
+    }
+
+    public void testIsEmpty() {
+        var randomSettings = createRandom();
+        var stringRep = Strings.toString(randomSettings);
+        assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}"));
+    }
+
+    public void testUpdatedTaskSettings_NotUpdated_UseInitialSettings() {
+        var initialSettings = createRandom();
+        var newSettings = new JinaAIEmbeddingsTaskSettings((InputType) null);
+        Map<String, Object> newSettingsMap = new HashMap<>();
+        JinaAIEmbeddingsTaskSettings updatedSettings = (JinaAIEmbeddingsTaskSettings) initialSettings.updatedTaskSettings(
+            Collections.unmodifiableMap(newSettingsMap)
+        );
+        assertEquals(initialSettings.getInputType(), updatedSettings.getInputType());
+    }
+
+    public void testUpdatedTaskSettings_Updated_UseNewSettings() {
+        var initialSettings = createRandom();
+        var newSettings = new JinaAIEmbeddingsTaskSettings(randomWithoutUnspecified());
+        Map<String, Object> newSettingsMap = new HashMap<>();
+        newSettingsMap.put(JinaAIEmbeddingsTaskSettings.INPUT_TYPE, newSettings.getInputType().toString());
+        JinaAIEmbeddingsTaskSettings updatedSettings = (JinaAIEmbeddingsTaskSettings) initialSettings.updatedTaskSettings(
+            Collections.unmodifiableMap(newSettingsMap)
+        );
+        assertEquals(newSettings.getInputType(), updatedSettings.getInputType());
+    }
+
+    public void testFromMap_CreatesEmptySettings_WhenAllFieldsAreNull() {
+        MatcherAssert.assertThat(
+            JinaAIEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of())),
+            is(new JinaAIEmbeddingsTaskSettings((InputType) null))
+        );
+    }
+
+    public void testFromMap_CreatesEmptySettings_WhenMapIsNull() {
+        MatcherAssert.assertThat(JinaAIEmbeddingsTaskSettings.fromMap(null), is(new JinaAIEmbeddingsTaskSettings((InputType) null)));
+    }
+
+    public void testFromMap_CreatesSettings_WhenAllFieldsOfSettingsArePresent() {
+        MatcherAssert.assertThat(
+            JinaAIEmbeddingsTaskSettings.fromMap(
+                new HashMap<>(Map.of(JinaAIEmbeddingsTaskSettings.INPUT_TYPE, InputType.INGEST.toString()))
+            ),
+            is(new JinaAIEmbeddingsTaskSettings(InputType.INGEST))
+        );
+    }
+
+    public void testFromMap_ReturnsFailure_WhenInputTypeIsInvalid() {
+        var exception = expectThrows(
+            ValidationException.class,
+            () -> JinaAIEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(JinaAIEmbeddingsTaskSettings.INPUT_TYPE, "abc")))
+        );
+
+        MatcherAssert.assertThat(
+            exception.getMessage(),
+            is(
+                Strings.format(
+                    "Validation Failed: 1: [task_settings] Invalid value [abc] received. [input_type] must be one of [%s];",
+                    getValidValuesSortedAndCombined(VALID_REQUEST_VALUES)
+                )
+            )
+        );
+    }
+
+    public void testFromMap_ReturnsFailure_WhenInputTypeIsUnspecified() {
+        var exception = expectThrows(
+            ValidationException.class,
+            () -> JinaAIEmbeddingsTaskSettings.fromMap(
+                new HashMap<>(Map.of(JinaAIEmbeddingsTaskSettings.INPUT_TYPE, InputType.UNSPECIFIED.toString()))
+            )
+        );
+
+        MatcherAssert.assertThat(
+            exception.getMessage(),
+            is(
+                Strings.format(
+                    "Validation Failed: 1: [task_settings] Invalid value [unspecified] received. [input_type] must be one of [%s];",
+                    getValidValuesSortedAndCombined(VALID_REQUEST_VALUES)
+                )
+            )
+        );
+    }
+
+    private static <E extends Enum<E>> String getValidValuesSortedAndCombined(EnumSet<E> validValues) {
+        var validValuesAsStrings = validValues.stream().map(value -> value.toString().toLowerCase(Locale.ROOT)).toArray(String[]::new);
+        Arrays.sort(validValuesAsStrings);
+
+        return String.join(", ", validValuesAsStrings);
+    }
+
+    public void testXContent_ThrowsAssertionFailure_WhenInputTypeIsUnspecified() {
+        var thrownException = expectThrows(AssertionError.class, () -> new JinaAIEmbeddingsTaskSettings(InputType.UNSPECIFIED));
+        MatcherAssert.assertThat(thrownException.getMessage(), is("received invalid input type value [unspecified]"));
+    }
+
+    public void testOf_KeepsOriginalValuesWhenRequestSettingsAreNull_AndRequestInputTypeIsInvalid() {
+        var taskSettings = new JinaAIEmbeddingsTaskSettings(InputType.INGEST);
+        var overriddenTaskSettings = JinaAIEmbeddingsTaskSettings.of(
+            taskSettings,
+            JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS,
+            InputType.UNSPECIFIED
+        );
+        MatcherAssert.assertThat(overriddenTaskSettings, is(taskSettings));
+    }
+
+    public void testOf_UsesRequestTaskSettings() {
+        var taskSettings = new JinaAIEmbeddingsTaskSettings((InputType) null);
+        var overriddenTaskSettings = JinaAIEmbeddingsTaskSettings.of(
+            taskSettings,
+            new JinaAIEmbeddingsTaskSettings(InputType.INGEST),
+            InputType.UNSPECIFIED
+        );
+
+        MatcherAssert.assertThat(overriddenTaskSettings, is(new JinaAIEmbeddingsTaskSettings(InputType.INGEST)));
+    }
+
+    public void testOf_UsesRequestTaskSettings_AndRequestInputType() {
+        var taskSettings = new JinaAIEmbeddingsTaskSettings(InputType.SEARCH);
+        var overriddenTaskSettings = JinaAIEmbeddingsTaskSettings.of(
+            taskSettings,
+            new JinaAIEmbeddingsTaskSettings((InputType) null),
+            InputType.INGEST
+        );
+
+        MatcherAssert.assertThat(overriddenTaskSettings, is(new JinaAIEmbeddingsTaskSettings(InputType.INGEST)));
+    }
+
+    @Override
+    protected Writeable.Reader<JinaAIEmbeddingsTaskSettings> instanceReader() {
+        return JinaAIEmbeddingsTaskSettings::new;
+    }
+
+    @Override
+    protected JinaAIEmbeddingsTaskSettings createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected JinaAIEmbeddingsTaskSettings mutateInstance(JinaAIEmbeddingsTaskSettings instance) throws IOException {
+        return randomValueOtherThan(instance, JinaAIEmbeddingsTaskSettingsTests::createRandom);
+    }
+
+    public static Map<String, Object> getTaskSettingsMapEmpty() {
+        return new HashMap<>();
+    }
+
+    public static Map<String, Object> getTaskSettingsMap(@Nullable InputType inputType) {
+        var map = new HashMap<String, Object>();
+
+        if (inputType != null) {
+            map.put(JinaAIEmbeddingsTaskSettings.INPUT_TYPE, inputType.toString());
+        }
+
+        return map;
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModelTests.java
new file mode 100644
index 0000000000000..d6b3df5fd3717
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModelTests.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai.rerank;
+
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettings;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
+
+public class JinaAIRerankModelTests {
+
+    public static JinaAIRerankModel createModel(String apiKey, String modelId, @Nullable Integer topN) {
+        return new JinaAIRerankModel(
+            "id",
+            "service",
+            new JinaAIRerankServiceSettings(new JinaAIServiceSettings(ESTestCase.randomAlphaOfLength(10), modelId, null)),
+            new JinaAIRerankTaskSettings(topN, null),
+            new DefaultSecretSettings(new SecureString(apiKey.toCharArray()))
+        );
+    }
+
+    public static JinaAIRerankModel createModel(String modelId, @Nullable Integer topN) {
+        return new JinaAIRerankModel(
+            "id",
+            "service",
+            new JinaAIRerankServiceSettings(new JinaAIServiceSettings(ESTestCase.randomAlphaOfLength(10), modelId, null)),
+            new JinaAIRerankTaskSettings(topN, null),
+            new DefaultSecretSettings(ESTestCase.randomSecureStringOfLength(8))
+        );
+    }
+
+    public static JinaAIRerankModel createModel(String modelId, @Nullable Integer topN, Boolean returnDocuments) {
+        return new JinaAIRerankModel(
+            "id",
+            "service",
+            new JinaAIRerankServiceSettings(new JinaAIServiceSettings(ESTestCase.randomAlphaOfLength(10), modelId, null)),
+            new JinaAIRerankTaskSettings(topN, returnDocuments),
+            new DefaultSecretSettings(ESTestCase.randomSecureStringOfLength(8))
+        );
+    }
+
+    public static JinaAIRerankModel createModel(String url, String modelId, @Nullable Integer topN, Boolean returnDocuments) {
+        return new JinaAIRerankModel(
+            "id",
+            "service",
+            new JinaAIRerankServiceSettings(new JinaAIServiceSettings(url, modelId, null)),
+            new JinaAIRerankTaskSettings(topN, returnDocuments),
+            new DefaultSecretSettings(ESTestCase.randomSecureStringOfLength(8))
+        );
+    }
+
+    public static JinaAIRerankModel createModel(
+        String url,
+        String apiKey,
+        String modelId,
+        @Nullable Integer topN,
+        Boolean returnDocuments
+    ) {
+        return new JinaAIRerankModel(
+            "id",
+            "service",
+            new JinaAIRerankServiceSettings(new JinaAIServiceSettings(url, modelId, null)),
+            new JinaAIRerankTaskSettings(topN, returnDocuments),
+            new DefaultSecretSettings(new SecureString(apiKey.toCharArray()))
+        );
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankServiceSettingsTests.java
new file mode 100644
index 0000000000000..47f67bd8cefb8
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankServiceSettingsTests.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai.rerank;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase;
+import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettings;
+import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettingsTests;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.inference.MatchersUtils.equalToIgnoringWhitespaceInJsonString;
+
+public class JinaAIRerankServiceSettingsTests extends AbstractBWCWireSerializationTestCase<JinaAIRerankServiceSettings> {
+    public static JinaAIRerankServiceSettings createRandom() {
+        return new JinaAIRerankServiceSettings(
+            new JinaAIServiceSettings(
+                randomFrom(new String[] { null, Strings.format("http://%s.com", randomAlphaOfLength(8)) }),
+                randomAlphaOfLength(10),
+                RateLimitSettingsTests.createRandom()
+            )
+        );
+    }
+
+    public void testToXContent_WritesAllValues() throws IOException {
+        var url = "http://www.abc.com";
+        var model = "model";
+
+        var serviceSettings = new JinaAIRerankServiceSettings(new JinaAIServiceSettings(url, model, null));
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        serviceSettings.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString("""
+            {
+                "url":"http://www.abc.com",
+                "model_id":"model",
+                "rate_limit": {
+                    "requests_per_minute": 2000
+                }
+            }
+            """));
+    }
+
+    @Override
+    protected Writeable.Reader<JinaAIRerankServiceSettings> instanceReader() {
+        return JinaAIRerankServiceSettings::new;
+    }
+
+    @Override
+    protected JinaAIRerankServiceSettings createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected JinaAIRerankServiceSettings mutateInstance(JinaAIRerankServiceSettings instance) throws IOException {
+        return randomValueOtherThan(instance, JinaAIRerankServiceSettingsTests::createRandom);
+    }
+
+    @Override
+    protected JinaAIRerankServiceSettings mutateInstanceForVersion(JinaAIRerankServiceSettings instance, TransportVersion version) {
+        return instance;
+    }
+
+    public static Map<String, Object> getServiceSettingsMap(@Nullable String url, @Nullable String model) {
+        return new HashMap<>(JinaAIServiceSettingsTests.getServiceSettingsMap(url, model));
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankTaskSettingsTests.java
new file mode 100644
index 0000000000000..fa70248d01513
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankTaskSettingsTests.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.jinaai.rerank;
+
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class JinaAIRerankTaskSettingsTests extends AbstractWireSerializingTestCase<JinaAIRerankTaskSettings> {
+
+    public static JinaAIRerankTaskSettings createRandom() {
+        var returnDocuments = randomBoolean() ? randomBoolean() : null;
+        var topNDocsOnly = randomBoolean() ? randomIntBetween(1, 10) : null;
+
+        return new JinaAIRerankTaskSettings(topNDocsOnly, returnDocuments);
+    }
+
+    public void testFromMap_WithValidValues_ReturnsSettings() {
+        Map<String, Object> taskMap = Map.of(JinaAIRerankTaskSettings.RETURN_DOCUMENTS, true, JinaAIRerankTaskSettings.TOP_N_DOCS_ONLY, 5);
+        var settings = JinaAIRerankTaskSettings.fromMap(new HashMap<>(taskMap));
+        assertTrue(settings.getReturnDocuments());
+        assertEquals(5, settings.getTopNDocumentsOnly().intValue());
+    }
+
+    public void testFromMap_WithNullValues_ReturnsSettingsWithNulls() {
+        var settings = JinaAIRerankTaskSettings.fromMap(Map.of());
+        assertNull(settings.getReturnDocuments());
+        assertNull(settings.getTopNDocumentsOnly());
+    }
+
+    public void testFromMap_WithInvalidReturnDocuments_ThrowsValidationException() {
+        Map<String, Object> taskMap = Map.of(
+            JinaAIRerankTaskSettings.RETURN_DOCUMENTS,
+            "invalid",
+            JinaAIRerankTaskSettings.TOP_N_DOCS_ONLY,
+            5
+        );
+        var thrownException = expectThrows(ValidationException.class, () -> JinaAIRerankTaskSettings.fromMap(new HashMap<>(taskMap)));
+        assertThat(thrownException.getMessage(), containsString("field [return_documents] is not of the expected type"));
+    }
+
+    public void testFromMap_WithInvalidTopNDocsOnly_ThrowsValidationException() {
+        Map<String, Object> taskMap = Map.of(
+            JinaAIRerankTaskSettings.RETURN_DOCUMENTS,
+            true,
+            JinaAIRerankTaskSettings.TOP_N_DOCS_ONLY,
+            "invalid"
+        );
+        var thrownException = expectThrows(ValidationException.class, () -> JinaAIRerankTaskSettings.fromMap(new HashMap<>(taskMap)));
+        assertThat(thrownException.getMessage(), containsString("field [top_n] is not of the expected type"));
+    }
+
+    public void testUpdatedTaskSettings_WithEmptyMap_ReturnsSameSettings() {
+        var initialSettings = new JinaAIRerankTaskSettings(5, true);
+        JinaAIRerankTaskSettings updatedSettings = (JinaAIRerankTaskSettings) initialSettings.updatedTaskSettings(Map.of());
+        assertEquals(initialSettings, updatedSettings);
+    }
+
+    public void testUpdatedTaskSettings_WithNewReturnDocuments_ReturnsUpdatedSettings() {
+        var initialSettings = new JinaAIRerankTaskSettings(5, true);
+        Map<String, Object> newSettings = Map.of(JinaAIRerankTaskSettings.RETURN_DOCUMENTS, false);
+        JinaAIRerankTaskSettings updatedSettings = (JinaAIRerankTaskSettings) initialSettings.updatedTaskSettings(newSettings);
+        assertFalse(updatedSettings.getReturnDocuments());
+        assertEquals(initialSettings.getTopNDocumentsOnly(), updatedSettings.getTopNDocumentsOnly());
+    }
+
+    public void testUpdatedTaskSettings_WithNewTopNDocsOnly_ReturnsUpdatedSettings() {
+        var initialSettings = new JinaAIRerankTaskSettings(5, true);
+        Map<String, Object> newSettings = Map.of(JinaAIRerankTaskSettings.TOP_N_DOCS_ONLY, 7);
+        JinaAIRerankTaskSettings updatedSettings = (JinaAIRerankTaskSettings) initialSettings.updatedTaskSettings(newSettings);
+        assertEquals(7, updatedSettings.getTopNDocumentsOnly().intValue());
+        assertEquals(initialSettings.getReturnDocuments(), updatedSettings.getReturnDocuments());
+    }
+
+    public void testUpdatedTaskSettings_WithMultipleNewValues_ReturnsUpdatedSettings() {
+        var initialSettings = new JinaAIRerankTaskSettings(5, true);
+        Map<String, Object> newSettings = Map.of(
+            JinaAIRerankTaskSettings.RETURN_DOCUMENTS,
+            false,
+            JinaAIRerankTaskSettings.TOP_N_DOCS_ONLY,
+            7
+        );
+        JinaAIRerankTaskSettings updatedSettings = (JinaAIRerankTaskSettings) initialSettings.updatedTaskSettings(newSettings);
+        assertFalse(updatedSettings.getReturnDocuments());
+        assertEquals(7, updatedSettings.getTopNDocumentsOnly().intValue());
+    }
+
+    @Override
+    protected Writeable.Reader<JinaAIRerankTaskSettings> instanceReader() {
+        return JinaAIRerankTaskSettings::new;
+    }
+
+    @Override
+    protected JinaAIRerankTaskSettings createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected JinaAIRerankTaskSettings mutateInstance(JinaAIRerankTaskSettings instance) throws IOException {
+        return randomValueOtherThan(instance, JinaAIRerankTaskSettingsTests::createRandom);
+    }
+
+    public static Map<String, Object> getTaskSettingsMapEmpty() {
+        return new HashMap<>();
+    }
+
+    public static Map<String, Object> getTaskSettingsMap(@Nullable Integer topNDocumentsOnly, Boolean returnDocuments) {
+        var map = new HashMap<String, Object>();
+
+        if (topNDocumentsOnly != null) {
+            map.put(JinaAIRerankTaskSettings.TOP_N_DOCS_ONLY, topNDocumentsOnly.toString());
+        }
+
+        if (returnDocuments != null) {
+            map.put(JinaAIRerankTaskSettings.RETURN_DOCUMENTS, returnDocuments.toString());
+        }
+
+        return map;
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java
index f164afe604b43..395e29240dfd4 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java
@@ -748,69 +748,41 @@ public void testGetConfiguration() throws Exception {
         try (var service = createService()) {
             String content = XContentHelper.stripWhitespace("""
                 {
-                       "provider": "mistral",
-                       "task_types": [
-                            {
-                                "task_type": "text_embedding",
-                                "configuration": {}
-                            }
-                       ],
-                       "configuration": {
+                       "service": "mistral",
+                       "name": "Mistral",
+                       "task_types": ["text_embedding"],
+                       "configurations": {
                            "api_key": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "textbox",
+                               "description": "API Key for the provider you're connecting to.",
                                "label": "API Key",
-                               "order": 1,
                                "required": true,
                                "sensitive": true,
-                               "tooltip": "API Key for the provider you're connecting to.",
-                               "type": "str",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": true,
+                               "type": "str"
                            },
                            "model": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "textbox",
+                               "description": "Refer to the Mistral models documentation for the list of available text embedding models.",
                                "label": "Model",
-                               "order": 2,
                                "required": true,
                                "sensitive": false,
-                               "tooltip": "Refer to the Mistral models documentation for the list of available text embedding models.",
-                               "type": "str",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "str"
                            },
                            "rate_limit.requests_per_minute": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "numeric",
+                               "description": "Minimize the number of rate limit errors.",
                                "label": "Rate Limit",
-                               "order": 6,
                                "required": false,
                                "sensitive": false,
-                               "tooltip": "Minimize the number of rate limit errors.",
-                               "type": "int",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "int"
                            },
                            "max_input_tokens": {
-                               "default_value": null,
-                               "depends_on": [],
-                               "display": "numeric",
+                               "description": "Allows you to specify the maximum number of tokens per input.",
                                "label": "Maximum Input Tokens",
-                               "order": 3,
                                "required": false,
                                "sensitive": false,
-                               "tooltip": "Allows you to specify the maximum number of tokens per input.",
-                               "type": "int",
-                               "ui_restrictions": [],
-                               "validations": [],
-                               "value": null
+                               "updatable": false,
+                               "type": "int"
                            }
                        }
                    }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java
index b595862775053..678c4528a3f41 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java
@@ -967,9 +967,7 @@ public void testUnifiedCompletionInfer() throws Exception {
             service.unifiedCompletionInfer(
                 model,
                 UnifiedCompletionRequest.of(
-                    List.of(
-                        new UnifiedCompletionRequest.Message(new UnifiedCompletionRequest.ContentString("hello"), "user", null, null, null)
-                    )
+                    List.of(new UnifiedCompletionRequest.Message(new UnifiedCompletionRequest.ContentString("hello"), "user", null, null))
                 ),
                 InferenceAction.Request.DEFAULT_TIMEOUT,
                 listener
@@ -1661,117 +1659,50 @@ public void testGetConfiguration() throws Exception {
             String content = XContentHelper.stripWhitespace(
                 """
                     {
-                            "provider": "openai",
-                            "task_types": [
-                                 {
-                                     "task_type": "text_embedding",
-                                     "configuration": {
-                                         "user": {
-                                             "default_value": null,
-                                             "depends_on": [],
-                                             "display": "textbox",
-                                             "label": "User",
-                                             "order": 1,
-                                             "required": false,
-                                             "sensitive": false,
-                                             "tooltip": "Specifies the user issuing the request.",
-                                             "type": "str",
-                                             "ui_restrictions": [],
-                                             "validations": [],
-                                             "value": ""
-                                         }
-                                     }
-                                 },
-                                 {
-                                     "task_type": "completion",
-                                     "configuration": {
-                                         "user": {
-                                             "default_value": null,
-                                             "depends_on": [],
-                                             "display": "textbox",
-                                             "label": "User",
-                                             "order": 1,
-                                             "required": false,
-                                             "sensitive": false,
-                                             "tooltip": "Specifies the user issuing the request.",
-                                             "type": "str",
-                                             "ui_restrictions": [],
-                                             "validations": [],
-                                             "value": ""
-                                         }
-                                     }
-                                 }
-                            ],
-                            "configuration": {
+                            "service": "openai",
+                            "name": "OpenAI",
+                            "task_types": ["text_embedding", "completion"],
+                            "configurations": {
                                 "api_key": {
-                                    "default_value": null,
-                                    "depends_on": [],
-                                    "display": "textbox",
+                                    "description": "The OpenAI API authentication key. For more details about generating OpenAI API keys, refer to the https://platform.openai.com/account/api-keys.",
                                     "label": "API Key",
-                                    "order": 1,
                                     "required": true,
                                     "sensitive": true,
-                                    "tooltip": "The OpenAI API authentication key. For more details about generating OpenAI API keys, refer to the https://platform.openai.com/account/api-keys.",
-                                    "type": "str",
-                                    "ui_restrictions": [],
-                                    "validations": [],
-                                    "value": null
+                                    "updatable": true,
+                                    "type": "str"
                                 },
                                 "organization_id": {
-                                    "default_value": null,
-                                    "depends_on": [],
-                                    "display": "textbox",
+                                    "description": "The unique identifier of your organization.",
                                     "label": "Organization ID",
-                                    "order": 3,
                                     "required": false,
                                     "sensitive": false,
-                                    "tooltip": "The unique identifier of your organization.",
-                                    "type": "str",
-                                    "ui_restrictions": [],
-                                    "validations": [],
-                                    "value": null
+                                    "updatable": false,
+                                    "type": "str"
                                 },
                                 "rate_limit.requests_per_minute": {
-                                    "default_value": null,
-                                    "depends_on": [],
-                                    "display": "numeric",
+                                    "description": "Default number of requests allowed per minute. For text_embedding is 3000. For completion is 500.",
                                     "label": "Rate Limit",
-                                    "order": 6,
                                     "required": false,
                                     "sensitive": false,
-                                    "tooltip": "Default number of requests allowed per minute. For text_embedding is 3000. For completion is 500.",
-                                    "type": "int",
-                                    "ui_restrictions": [],
-                                    "validations": [],
-                                    "value": null
+                                    "updatable": false,
+                                    "type": "int"
                                 },
                                 "model_id": {
-                                    "default_value": null,
-                                    "depends_on": [],
-                                    "display": "textbox",
+                                    "description": "The name of the model to use for the inference task.",
                                     "label": "Model ID",
-                                    "order": 2,
                                     "required": true,
                                     "sensitive": false,
-                                    "tooltip": "The name of the model to use for the inference task.",
-                                    "type": "str",
-                                    "ui_restrictions": [],
-                                    "validations": [],
-                                    "value": null
+                                    "updatable": false,
+                                    "type": "str"
                                 },
                                 "url": {
                                     "default_value": "https://api.openai.com/v1/chat/completions",
-                                    "depends_on": [],
-                                    "display": "textbox",
+                                    "description": "The OpenAI API endpoint URL. For more information on the URL, refer to the https://platform.openai.com/docs/api-reference.",
                                     "label": "URL",
-                                    "order": 4,
                                     "required": true,
                                     "sensitive": false,
-                                    "tooltip": "The OpenAI API endpoint URL. For more information on the URL, refer to the https://platform.openai.com/docs/api-reference.",
-                                    "type": "str",
-                                    "ui_restrictions": [],
-                                    "validations": [],
-                                    "value": null
+                                    "updatable": false,
+                                    "type": "str"
                                 }
                             }
                         }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModelTests.java
index e7ac4cf879e92..2a5415f45c6d9 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModelTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModelTests.java
@@ -51,7 +51,7 @@ public void testOverrideWith_NullMap() {
     public void testOverrideWith_UnifiedCompletionRequest_OverridesModelId() {
         var model = createChatCompletionModel("url", "org", "api_key", "model_name", "user");
         var request = new UnifiedCompletionRequest(
-            List.of(new UnifiedCompletionRequest.Message(new UnifiedCompletionRequest.ContentString("hello"), "role", null, null, null)),
+            List.of(new UnifiedCompletionRequest.Message(new UnifiedCompletionRequest.ContentString("hello"), "role", null, null)),
             "different_model",
             null,
             null,
@@ -70,7 +70,7 @@ public void testOverrideWith_UnifiedCompletionRequest_OverridesModelId() {
     public void testOverrideWith_UnifiedCompletionRequest_UsesModelFields_WhenRequestDoesNotOverride() {
         var model = createChatCompletionModel("url", "org", "api_key", "model_name", "user");
         var request = new UnifiedCompletionRequest(
-            List.of(new UnifiedCompletionRequest.Message(new UnifiedCompletionRequest.ContentString("hello"), "role", null, null, null)),
+            List.of(new UnifiedCompletionRequest.Message(new UnifiedCompletionRequest.ContentString("hello"), "role", null, null)),
             null, // not overriding model
             null,
             null,
diff --git a/x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/sample-doc-legacy.json.gz b/x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/sample-doc-legacy.json.gz
new file mode 100644
index 0000000000000..8e6a6dd0a72a5
Binary files /dev/null and b/x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/sample-doc-legacy.json.gz differ
diff --git a/x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/sample-doc.json.gz b/x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/sample-doc.json.gz
index 881524e46e186..d49625f096987 100644
Binary files a/x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/sample-doc.json.gz and b/x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/sample-doc.json.gz differ
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml
index 882f1df03e926..5b1f1b9509d9d 100644
--- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml
@@ -7,6 +7,11 @@ setup:
       indices.create:
         index: test-index
         body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: false
           mappings:
             properties:
               sparse_field:
@@ -18,7 +23,6 @@ setup:
 
 ---
 "Indexes sparse vector document":
-
   # Checks mapping is not updated until first doc arrives
   - do:
       indices.get_mapping:
@@ -33,25 +37,28 @@ setup:
         index: test-index
         id: doc_1
         body:
-          sparse_field:
-            text: "these are not the droids you're looking for. He's free to go around"
+          sparse_field: "these are not the droids you're looking for. He's free to go around"
+          _inference_fields.sparse_field:
             inference:
               inference_id: sparse-inference-id
               model_settings:
                 task_type: sparse_embedding
               chunks:
-                - text: "these are not the droids you're looking for"
-                  embeddings:
-                    feature_0: 1.0
-                    feature_1: 2.0
-                    feature_2: 3.0
-                    feature_3: 4.0
-                - text: "He's free to go around"
-                  embeddings:
-                    feature_4: 0.1
-                    feature_5: 0.2
-                    feature_6: 0.3
-                    feature_7: 0.4
+                sparse_field:
+                  - start_offset: 0
+                    end_offset: 44
+                    embeddings:
+                      feature_0: 1.0
+                      feature_1: 2.0
+                      feature_2: 3.0
+                      feature_3: 4.0
+                  - start_offset: 44
+                    end_offset: 67
+                    embeddings:
+                      feature_4: 0.1
+                      feature_5: 0.2
+                      feature_6: 0.3
+                      feature_7: 0.4
 
   # Checks mapping is updated when first doc arrives
   - do:
@@ -65,7 +72,6 @@ setup:
 
 ---
 "Field caps with sparse embedding":
-
   - requires:
       cluster_features: "gte_v8.16.0"
       reason: field_caps support for semantic_text added in 8.16.0
@@ -95,25 +101,28 @@ setup:
         index: test-index
         id: doc_1
         body:
-          sparse_field:
-            text: "these are not the droids you're looking for. He's free to go around"
+          sparse_field: "these are not the droids you're looking for. He's free to go around"
+          _inference_fields.sparse_field:
             inference:
               inference_id: sparse-inference-id
               model_settings:
                 task_type: sparse_embedding
               chunks:
-                - text: "these are not the droids you're looking for"
-                  embeddings:
-                    feature_0: 1.0
-                    feature_1: 2.0
-                    feature_2: 3.0
-                    feature_3: 4.0
-                - text: "He's free to go around"
-                  embeddings:
-                    feature_4: 0.1
-                    feature_5: 0.2
-                    feature_6: 0.3
-                    feature_7: 0.4
+                sparse_field:
+                  - start_offset: 0
+                    end_offset: 44
+                    embeddings:
+                      feature_0: 1.0
+                      feature_1: 2.0
+                      feature_2: 3.0
+                      feature_3: 4.0
+                  - start_offset: 44
+                    end_offset: 67
+                    embeddings:
+                      feature_4: 0.1
+                      feature_5: 0.2
+                      feature_6: 0.3
+                      feature_7: 0.4
         refresh: true
 
   - do:
@@ -125,8 +134,8 @@ setup:
   - match: { indices: [ "test-index" ] }
   - exists: fields.sparse_field
   - exists: fields.dense_field
-  - match: { fields.sparse_field.semantic_text.searchable: true }
-  - match: { fields.dense_field.semantic_text.searchable: true }
+  - match: { fields.sparse_field.text.searchable: true }
+  - match: { fields.dense_field.text.searchable: true }
 
   - do:
       field_caps:
@@ -137,11 +146,10 @@ setup:
   - match: { indices: [ "test-index" ] }
   - exists: fields.sparse_field
   - not_exists: fields.dense_field
-  - match: { fields.sparse_field.semantic_text.searchable: true }
+  - match: { fields.sparse_field.text.searchable: true }
 
 ---
 "Indexes dense vector document":
-
   # Checks mapping is not updated until first doc arrives
   - do:
       indices.get_mapping:
@@ -156,8 +164,8 @@ setup:
         index: test-index
         id: doc_2
         body:
-          dense_field:
-            text: "these are not the droids you're looking for. He's free to go around"
+          dense_field: "these are not the droids you're looking for. He's free to go around"
+          _inference_fields.dense_field:
             inference:
               inference_id: dense-inference-id
               model_settings:
@@ -166,10 +174,13 @@ setup:
                 similarity: cosine
                 element_type: float
               chunks:
-                - text: "these are not the droids you're looking for"
-                  embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
-                - text: "He's free to go around"
-                  embeddings: [0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896]
+                dense_field:
+                  - start_offset: 0
+                    end_offset: 44
+                    embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
+                  - start_offset: 44
+                    end_offset: 67
+                    embeddings: [0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896]
 
   # Checks mapping is updated when first doc arrives
   - do:
@@ -183,7 +194,6 @@ setup:
 
 ---
 "Field caps with text embedding":
-
   - requires:
       cluster_features: "gte_v8.16.0"
       reason: field_caps support for semantic_text added in 8.16.0
@@ -213,8 +223,8 @@ setup:
         index: test-index
         id: doc_2
         body:
-          dense_field:
-            text: "these are not the droids you're looking for. He's free to go around"
+          dense_field: "these are not the droids you're looking for. He's free to go around"
+          _inference_fields.dense_field:
             inference:
               inference_id: dense-inference-id
               model_settings:
@@ -223,10 +233,13 @@ setup:
                 similarity: cosine
                 element_type: float
               chunks:
-                - text: "these are not the droids you're looking for"
-                  embeddings: [ 0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416 ]
-                - text: "He's free to go around"
-                  embeddings: [ 0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896 ]
+                dense_field:
+                  - start_offset: 0
+                    end_offset: 44
+                    embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
+                  - start_offset: 44
+                    end_offset: 67
+                    embeddings: [0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896]
         refresh: true
 
   - do:
@@ -238,8 +251,8 @@ setup:
   - match: { indices: [ "test-index" ] }
   - exists: fields.sparse_field
   - exists: fields.dense_field
-  - match: { fields.sparse_field.semantic_text.searchable: true }
-  - match: { fields.dense_field.semantic_text.searchable: true }
+  - match: { fields.sparse_field.text.searchable: true }
+  - match: { fields.dense_field.text.searchable: true }
 
   - do:
       field_caps:
@@ -250,7 +263,7 @@ setup:
   - match: { indices: [ "test-index" ] }
   - not_exists: fields.sparse_field
   - exists: fields.dense_field
-  - match: { fields.dense_field.semantic_text.searchable: true }
+  - match: { fields.dense_field.text.searchable: true }
 
 ---
 "Can't be used as a multifield":
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml
new file mode 100644
index 0000000000000..8da30a8581a2d
--- /dev/null
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml
@@ -0,0 +1,254 @@
+setup:
+  - requires:
+      cluster_features: "gte_v8.15.0"
+      reason: semantic_text introduced in 8.15.0
+
+  - do:
+      indices.create:
+        index: test-index
+        body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: true
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+              dense_field:
+                type: semantic_text
+                inference_id: dense-inference-id
+
+---
+"Indexes sparse vector document":
+  # Checks mapping is not updated until first doc arrives
+  - do:
+      indices.get_mapping:
+        index: test-index
+
+  - match: { "test-index.mappings.properties.sparse_field.type": semantic_text }
+  - match: { "test-index.mappings.properties.sparse_field.inference_id": sparse-inference-id }
+  - length: { "test-index.mappings.properties.sparse_field": 2 }
+
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field:
+            text: "these are not the droids you're looking for. He's free to go around"
+            inference:
+              inference_id: sparse-inference-id
+              model_settings:
+                task_type: sparse_embedding
+              chunks:
+                - text: "these are not the droids you're looking for"
+                  embeddings:
+                    feature_0: 1.0
+                    feature_1: 2.0
+                    feature_2: 3.0
+                    feature_3: 4.0
+                - text: "He's free to go around"
+                  embeddings:
+                    feature_4: 0.1
+                    feature_5: 0.2
+                    feature_6: 0.3
+                    feature_7: 0.4
+
+  # Checks mapping is updated when first doc arrives
+  - do:
+      indices.get_mapping:
+        index: test-index
+
+  - match: { "test-index.mappings.properties.sparse_field.type": semantic_text }
+  - match: { "test-index.mappings.properties.sparse_field.inference_id": sparse-inference-id }
+  - match: { "test-index.mappings.properties.sparse_field.model_settings.task_type": sparse_embedding }
+  - length: { "test-index.mappings.properties.sparse_field": 3 }
+
+---
+"Field caps with sparse embedding":
+  - requires:
+      cluster_features: "gte_v8.16.0"
+      reason: field_caps support for semantic_text added in 8.16.0
+
+  - do:
+      field_caps:
+        include_empty_fields: true
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - exists: fields.sparse_field
+  - exists: fields.dense_field
+
+  - do:
+      field_caps:
+        include_empty_fields: false
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - not_exists: fields.sparse_field
+  - not_exists: fields.dense_field
+
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field:
+            text: "these are not the droids you're looking for. He's free to go around"
+            inference:
+              inference_id: sparse-inference-id
+              model_settings:
+                task_type: sparse_embedding
+              chunks:
+                - text: "these are not the droids you're looking for"
+                  embeddings:
+                    feature_0: 1.0
+                    feature_1: 2.0
+                    feature_2: 3.0
+                    feature_3: 4.0
+                - text: "He's free to go around"
+                  embeddings:
+                    feature_4: 0.1
+                    feature_5: 0.2
+                    feature_6: 0.3
+                    feature_7: 0.4
+        refresh: true
+
+  - do:
+      field_caps:
+        include_empty_fields: true
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - exists: fields.sparse_field
+  - exists: fields.dense_field
+  - match: { fields.sparse_field.text.searchable: true }
+  - match: { fields.dense_field.text.searchable: true }
+
+  - do:
+      field_caps:
+        include_empty_fields: false
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - exists: fields.sparse_field
+  - not_exists: fields.dense_field
+  - match: { fields.sparse_field.text.searchable: true }
+
+---
+"Indexes dense vector document":
+  # Checks mapping is not updated until first doc arrives
+  - do:
+      indices.get_mapping:
+        index: test-index
+
+  - match: { "test-index.mappings.properties.dense_field.type": semantic_text }
+  - match: { "test-index.mappings.properties.dense_field.inference_id": dense-inference-id }
+  - length: { "test-index.mappings.properties.dense_field": 2 }
+
+  - do:
+      index:
+        index: test-index
+        id: doc_2
+        body:
+          dense_field:
+            text: "these are not the droids you're looking for. He's free to go around"
+            inference:
+              inference_id: dense-inference-id
+              model_settings:
+                task_type: text_embedding
+                dimensions: 4
+                similarity: cosine
+                element_type: float
+              chunks:
+                - text: "these are not the droids you're looking for"
+                  embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
+                - text: "He's free to go around"
+                  embeddings: [0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896]
+
+  # Checks mapping is updated when first doc arrives
+  - do:
+      indices.get_mapping:
+        index: test-index
+
+  - match: { "test-index.mappings.properties.dense_field.type": semantic_text }
+  - match: { "test-index.mappings.properties.dense_field.inference_id": dense-inference-id }
+  - match: { "test-index.mappings.properties.dense_field.model_settings.task_type": text_embedding }
+  - length: { "test-index.mappings.properties.dense_field": 3 }
+
+---
+"Field caps with text embedding":
+  - requires:
+      cluster_features: "gte_v8.16.0"
+      reason: field_caps support for semantic_text added in 8.16.0
+
+  - do:
+      field_caps:
+        include_empty_fields: true
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - exists: fields.sparse_field
+  - exists: fields.dense_field
+
+  - do:
+      field_caps:
+        include_empty_fields: false
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - not_exists: fields.sparse_field
+  - not_exists: fields.dense_field
+
+  - do:
+      index:
+        index: test-index
+        id: doc_2
+        body:
+          dense_field:
+            text: "these are not the droids you're looking for. He's free to go around"
+            inference:
+              inference_id: dense-inference-id
+              model_settings:
+                task_type: text_embedding
+                dimensions: 4
+                similarity: cosine
+                element_type: float
+              chunks:
+                - text: "these are not the droids you're looking for"
+                  embeddings: [ 0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416 ]
+                - text: "He's free to go around"
+                  embeddings: [ 0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896 ]
+        refresh: true
+
+  - do:
+      field_caps:
+        include_empty_fields: true
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - exists: fields.sparse_field
+  - exists: fields.dense_field
+  - match: { fields.sparse_field.text.searchable: true }
+  - match: { fields.dense_field.text.searchable: true }
+
+  - do:
+      field_caps:
+        include_empty_fields: false
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - not_exists: fields.sparse_field
+  - exists: fields.dense_field
+  - match: { fields.dense_field.text.searchable: true }
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/20_semantic_text_field_mapping_incompatible_field_mapping.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/20_semantic_text_field_mapping_incompatible_field_mapping.yml
index 3d46c3b23d7e3..39f3a1641c18c 100644
--- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/20_semantic_text_field_mapping_incompatible_field_mapping.yml
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/20_semantic_text_field_mapping_incompatible_field_mapping.yml
@@ -7,6 +7,11 @@ setup:
       indices.create:
         index: test-index
         body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: false
           mappings:
             properties:
               sparse_field:
@@ -22,27 +27,30 @@ setup:
         index: test-index
         id: doc_1
         body:
-          sparse_field:
-            text: "these are not the droids you're looking for. He's free to go around"
+          sparse_field: "these are not the droids you're looking for. He's free to go around"
+          _inference_fields.sparse_field:
             inference:
               inference_id: sparse-inference-id
               model_settings:
                 task_type: sparse_embedding
               chunks:
-                - text: "these are not the droids you're looking for"
-                  embeddings:
-                    feature_0: 1.0
-                    feature_1: 2.0
-                    feature_2: 3.0
-                    feature_3: 4.0
-                - text: "He's free to go around"
-                  embeddings:
-                    feature_4: 0.1
-                    feature_5: 0.2
-                    feature_6: 0.3
-                    feature_7: 0.4
-          dense_field:
-            text: "these are not the droids you're looking for. He's free to go around"
+                sparse_field:
+                  - start_offset: 0
+                    end_offset: 44
+                    embeddings:
+                      feature_0: 1.0
+                      feature_1: 2.0
+                      feature_2: 3.0
+                      feature_3: 4.0
+                  - start_offset: 44
+                    end_offset: 67
+                    embeddings:
+                      feature_4: 0.1
+                      feature_5: 0.2
+                      feature_6: 0.3
+                      feature_7: 0.4
+          dense_field: "these are not the droids you're looking for. He's free to go around"
+          _inference_fields.dense_field:
             inference:
               inference_id: dense-inference-id
               model_settings:
@@ -51,23 +59,25 @@ setup:
                 similarity: cosine
                 element_type: float
               chunks:
-                - text: "these are not the droids you're looking for"
-                  embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
-                - text: "He's free to go around"
-                  embeddings: [0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896]
+                dense_field:
+                  - start_offset: 0
+                    end_offset: 44
+                    embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
+                  - start_offset: 44
+                    end_offset: 67
+                    embeddings: [0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896]
 
 
 ---
 "Fails for non-compatible dimensions":
-
   - do:
       catch: /Incompatible model settings for field \[dense_field\].+/
       index:
         index: test-index
         id: doc_2
         body:
-          dense_field:
-            text: "other text"
+          dense_field: "other text"
+          _inference_fields.dense_field:
             inference:
               inference_id: dense-inference-id
               model_settings:
@@ -76,20 +86,21 @@ setup:
                 similarity: cosine
                 element_type: float
               chunks:
-                - text: "other text"
-                  embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416, 0.053438711911439896]
+                dense_field:
+                  - start_offset: 0
+                    end_offset: 10
+                    embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416, 0.053438711911439896]
 
 ---
 "Fails for non-compatible inference id":
-
   - do:
       catch: /The configured inference_id \[a-different-inference-id\] for field \[dense_field\] doesn't match the inference_id \[dense-inference-id\].+/
       index:
         index: test-index
         id: doc_2
         body:
-          dense_field:
-            text: "other text"
+          dense_field: "other text"
+          _inference_fields.dense_field:
             inference:
               inference_id: a-different-inference-id
               model_settings:
@@ -98,20 +109,21 @@ setup:
                 similarity: cosine
                 element_type: float
               chunks:
-                - text: "other text"
-                  embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
+                dense_field:
+                  - start_offset: 0
+                    end_offset: 10
+                    embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
 
 ---
 "Fails for non-compatible similarity":
-
   - do:
       catch: /Incompatible model settings for field \[dense_field\].+/
       index:
         index: test-index
         id: doc_2
         body:
-          dense_field:
-            text: "other text"
+          dense_field: "other text"
+          _inference_fields.dense_field:
             inference:
               inference_id: dense-inference-id
               model_settings:
@@ -120,20 +132,21 @@ setup:
                 similarity: dot_product
                 element_type: float
               chunks:
-                - text: "other text"
-                  embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
+                dense_field:
+                  - start_offset: 0
+                    end_offset: 10
+                    embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
 
 ---
 "Fails for non-compatible element type":
-
   - do:
       catch: /Incompatible model settings for field \[dense_field\].+/
       index:
         index: test-index
         id: doc_2
         body:
-          dense_field:
-            text: "other text"
+          dense_field: "other text"
+          _inference_fields.dense_field:
             inference:
               inference_id: dense-inference-id
               model_settings:
@@ -142,43 +155,45 @@ setup:
                 similarity: cosine
                 element_type: byte
               chunks:
-                - text: "other text"
-                  embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
+                dense_field:
+                  - start_offset: 0
+                    end_offset: 10
+                    embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
 
 ---
 "Fails for non-compatible task type for dense vectors":
-
   - do:
       catch: /Incompatible model settings for field \[dense_field\].+/
       index:
         index: test-index
         id: doc_2
         body:
-          dense_field:
-            text: "other text"
+          dense_field: "other text"
+          _inference_fields.dense_field:
             inference:
               inference_id: dense-inference-id
               model_settings:
                 task_type: sparse_embedding
               chunks:
-                - text: "these are not the droids you're looking for"
-                  embeddings:
-                    feature_0: 1.0
-                    feature_1: 2.0
-                    feature_2: 3.0
-                    feature_3: 4.0
+                dense_field:
+                  - start_offset: 0
+                    end_offset: 10
+                    embeddings:
+                      feature_0: 1.0
+                      feature_1: 2.0
+                      feature_2: 3.0
+                      feature_3: 4.0
 
 ---
 "Fails for non-compatible task type for sparse vectors":
-
   - do:
       catch: /Incompatible model settings for field \[sparse_field\].+/
       index:
         index: test-index
         id: doc_2
         body:
-          sparse_field:
-            text: "these are not the droids you're looking for. He's free to go around"
+          sparse_field: "these are not the droids you're looking for. He's free to go around"
+          _inference_fields.sparse_field:
             inference:
               inference_id: sparse-inference-id
               model_settings:
@@ -187,20 +202,21 @@ setup:
                 similarity: cosine
                 element_type: float
               chunks:
-                - text: "these are not the droids you're looking for"
-                  embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
+                sparse_field:
+                  - start_offset: 0
+                    end_offset: 44
+                    embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
 
 ---
 "Fails for missing dense vector inference results in chunks":
-
   - do:
-      catch: /failed to parse field \[dense_field\] of type \[semantic_text\]/
+      catch: /failed to parse field \[_inference_fields\] of type \[_inference_fields\]/
       index:
         index: test-index
         id: doc_2
         body:
-          dense_field:
-            text: "these are not the droids you're looking for. He's free to go around"
+          dense_field: "these are not the droids you're looking for. He's free to go around"
+          _inference_fields.dense_field:
             inference:
               inference_id: dense-inference-id
               model_settings:
@@ -209,37 +225,39 @@ setup:
                 similarity: cosine
                 element_type: float
               chunks:
-                - text: "these are not the droids you're looking for"
+                dense_field:
+                  - start_offset: 0
+                    end_offset: 44
 
 ---
 "Fails for missing sparse vector inference results in chunks":
-
   - do:
-      catch: /failed to parse field \[sparse_field\] of type \[semantic_text\]/
+      catch: /failed to parse field \[_inference_fields\] of type \[_inference_fields\]/
       index:
         index: test-index
         id: doc_2
         body:
-          sparse_field:
-            text: "these are not the droids you're looking for. He's free to go around"
+          sparse_field: "these are not the droids you're looking for. He's free to go around"
+          _inference_fields.sparse_field:
             inference:
               inference_id: sparse-inference-id
               model_settings:
                 task_type: sparse_embedding
               chunks:
-                - text: "these are not the droids you're looking for"
+                sparse_field:
+                  - start_offset: 0
+                    end_offset: 44
 
 ---
-"Fails for missing text in chunks":
-
+"Fails for missing offsets in chunks":
   - do:
-      catch: /failed to parse field \[dense_field\] of type \[semantic_text\]/
+      catch: /failed to parse field \[_inference_fields\] of type \[_inference_fields\]/
       index:
         index: test-index
         id: doc_2
         body:
-          dense_field:
-            text: "these are not the droids you're looking for. He's free to go around"
+          dense_field: "these are not the droids you're looking for. He's free to go around"
+          _inference_fields.dense_field:
             inference:
               inference_id: dense-inference-id
               model_settings:
@@ -248,5 +266,6 @@ setup:
                 similarity: cosine
                 element_type: float
               chunks:
-                - embeddings: [ 0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416 ]
+                dense_field:
+                  - embeddings: [ 0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416 ]
 
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/20_semantic_text_field_mapping_incompatible_field_mapping_bwc.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/20_semantic_text_field_mapping_incompatible_field_mapping_bwc.yml
new file mode 100644
index 0000000000000..ffdb2a24c3dcf
--- /dev/null
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/20_semantic_text_field_mapping_incompatible_field_mapping_bwc.yml
@@ -0,0 +1,248 @@
+setup:
+  - requires:
+      cluster_features: "gte_v8.15.0"
+      reason: semantic_text introduced in 8.15.0
+
+  - do:
+      indices.create:
+        index: test-index
+        body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: true
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+              dense_field:
+                type: semantic_text
+                inference_id: dense-inference-id
+
+  # Indexes a doc with inference results to update mappings
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field:
+            text: "these are not the droids you're looking for. He's free to go around"
+            inference:
+              inference_id: sparse-inference-id
+              model_settings:
+                task_type: sparse_embedding
+              chunks:
+                - text: "these are not the droids you're looking for"
+                  embeddings:
+                    feature_0: 1.0
+                    feature_1: 2.0
+                    feature_2: 3.0
+                    feature_3: 4.0
+                - text: "He's free to go around"
+                  embeddings:
+                    feature_4: 0.1
+                    feature_5: 0.2
+                    feature_6: 0.3
+                    feature_7: 0.4
+          dense_field:
+            text: "these are not the droids you're looking for. He's free to go around"
+            inference:
+              inference_id: dense-inference-id
+              model_settings:
+                task_type: text_embedding
+                dimensions: 4
+                similarity: cosine
+                element_type: float
+              chunks:
+                - text: "these are not the droids you're looking for"
+                  embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
+                - text: "He's free to go around"
+                  embeddings: [0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896]
+
+
+---
+"Fails for non-compatible dimensions":
+  - do:
+      catch: /Incompatible model settings for field \[dense_field\].+/
+      index:
+        index: test-index
+        id: doc_2
+        body:
+          dense_field:
+            text: "other text"
+            inference:
+              inference_id: dense-inference-id
+              model_settings:
+                task_type: text_embedding
+                dimensions: 5
+                similarity: cosine
+                element_type: float
+              chunks:
+                - text: "other text"
+                  embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416, 0.053438711911439896]
+
+---
+"Fails for non-compatible inference id":
+  - do:
+      catch: /The configured inference_id \[a-different-inference-id\] for field \[dense_field\] doesn't match the inference_id \[dense-inference-id\].+/
+      index:
+        index: test-index
+        id: doc_2
+        body:
+          dense_field:
+            text: "other text"
+            inference:
+              inference_id: a-different-inference-id
+              model_settings:
+                task_type: text_embedding
+                dimensions: 4
+                similarity: cosine
+                element_type: float
+              chunks:
+                - text: "other text"
+                  embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
+
+---
+"Fails for non-compatible similarity":
+  - do:
+      catch: /Incompatible model settings for field \[dense_field\].+/
+      index:
+        index: test-index
+        id: doc_2
+        body:
+          dense_field:
+            text: "other text"
+            inference:
+              inference_id: dense-inference-id
+              model_settings:
+                task_type: text_embedding
+                dimensions: 4
+                similarity: dot_product
+                element_type: float
+              chunks:
+                - text: "other text"
+                  embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
+
+---
+"Fails for non-compatible element type":
+  - do:
+      catch: /Incompatible model settings for field \[dense_field\].+/
+      index:
+        index: test-index
+        id: doc_2
+        body:
+          dense_field:
+            text: "other text"
+            inference:
+              inference_id: dense-inference-id
+              model_settings:
+                task_type: text_embedding
+                dimensions: 4
+                similarity: cosine
+                element_type: byte
+              chunks:
+                - text: "other text"
+                  embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
+
+---
+"Fails for non-compatible task type for dense vectors":
+  - do:
+      catch: /Incompatible model settings for field \[dense_field\].+/
+      index:
+        index: test-index
+        id: doc_2
+        body:
+          dense_field:
+            text: "other text"
+            inference:
+              inference_id: dense-inference-id
+              model_settings:
+                task_type: sparse_embedding
+              chunks:
+                - text: "these are not the droids you're looking for"
+                  embeddings:
+                    feature_0: 1.0
+                    feature_1: 2.0
+                    feature_2: 3.0
+                    feature_3: 4.0
+
+---
+"Fails for non-compatible task type for sparse vectors":
+  - do:
+      catch: /Incompatible model settings for field \[sparse_field\].+/
+      index:
+        index: test-index
+        id: doc_2
+        body:
+          sparse_field:
+            text: "these are not the droids you're looking for. He's free to go around"
+            inference:
+              inference_id: sparse-inference-id
+              model_settings:
+                task_type: text_embedding
+                dimensions: 4
+                similarity: cosine
+                element_type: float
+              chunks:
+                - text: "these are not the droids you're looking for"
+                  embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
+
+---
+"Fails for missing dense vector inference results in chunks":
+  - do:
+      catch: /failed to parse field \[dense_field\] of type \[semantic_text\]/
+      index:
+        index: test-index
+        id: doc_2
+        body:
+          dense_field:
+            text: "these are not the droids you're looking for. He's free to go around"
+            inference:
+              inference_id: dense-inference-id
+              model_settings:
+                task_type: text_embedding
+                dimensions: 4
+                similarity: cosine
+                element_type: float
+              chunks:
+                - text: "these are not the droids you're looking for"
+
+---
+"Fails for missing sparse vector inference results in chunks":
+  - do:
+      catch: /failed to parse field \[sparse_field\] of type \[semantic_text\]/
+      index:
+        index: test-index
+        id: doc_2
+        body:
+          sparse_field:
+            text: "these are not the droids you're looking for. He's free to go around"
+            inference:
+              inference_id: sparse-inference-id
+              model_settings:
+                task_type: sparse_embedding
+              chunks:
+                - text: "these are not the droids you're looking for"
+
+---
+"Fails for missing text in chunks":
+  - do:
+      catch: /failed to parse field \[dense_field\] of type \[semantic_text\]/
+      index:
+        index: test-index
+        id: doc_2
+        body:
+          dense_field:
+            text: "these are not the droids you're looking for. He's free to go around"
+            inference:
+              inference_id: dense-inference-id
+              model_settings:
+                task_type: text_embedding
+                dimensions: 4
+                similarity: cosine
+                element_type: float
+              chunks:
+                - embeddings: [ 0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416 ]
+
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml
index 534e4831c4a0a..1adf0d58de8e3 100644
--- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml
@@ -39,6 +39,11 @@ setup:
       indices.create:
         index: test-index
         body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: false
           mappings:
             properties:
               sparse_field:
@@ -60,19 +65,30 @@ setup:
           sparse_field: "inference test"
           dense_field: "another inference test"
           non_inference_field: "non inference test"
+        refresh: true
 
   - do:
-      get:
+      search:
         index: test-index
-        id: doc_1
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
 
-  - match: { _source.sparse_field.text: "inference test" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
-  - match: { _source.dense_field.text: "another inference test" }
-  - exists: _source.dense_field.inference.chunks.0.embeddings
-  - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
-  - match: { _source.non_inference_field: "non inference test" }
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 14 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 22 }
 
 ---
 "Calculates sparse embedding and text embedding results for new documents with integer value":
@@ -83,18 +99,30 @@ setup:
         body:
           sparse_field: 75
           dense_field: 100
+        refresh: true
 
   - do:
-      get:
+      search:
         index: test-index
-        id: doc_1
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
 
-  - match: { _source.sparse_field.text: "75" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.sparse_field.inference.chunks.0.text: "75" }
-  - match: { _source.dense_field.text: "100" }
-  - exists: _source.dense_field.inference.chunks.0.embeddings
-  - match: { _source.dense_field.inference.chunks.0.text: "100" }
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 2 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 3 }
 
 ---
 "Calculates sparse embedding and text embedding results for new documents with boolean value":
@@ -105,18 +133,30 @@ setup:
         body:
           sparse_field: true
           dense_field: false
+        refresh: true
 
   - do:
-      get:
+      search:
         index: test-index
-        id: doc_1
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
 
-  - match: { _source.sparse_field.text: "true" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.sparse_field.inference.chunks.0.text: "true" }
-  - match: { _source.dense_field.text: "false" }
-  - exists: _source.dense_field.inference.chunks.0.embeddings
-  - match: { _source.dense_field.inference.chunks.0.text: "false" }
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 4 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 5 }
 
 ---
 "Calculates sparse embedding and text embedding results for new documents with a collection of mixed data types":
@@ -127,39 +167,48 @@ setup:
         body:
           sparse_field: [false, 75, "inference test", 13.49]
           dense_field: [true, 49.99, "another inference test", 5654]
+        refresh: true
 
   - do:
-      get:
+      search:
         index: test-index
-        id: doc_1
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
 
-  - length: { _source.sparse_field.text: 4 }
-  - match: { _source.sparse_field.text.0: "false" }
-  - match: { _source.sparse_field.text.1: "75" }
-  - match: { _source.sparse_field.text.2: "inference test" }
-  - match: { _source.sparse_field.text.3: "13.49" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - exists: _source.sparse_field.inference.chunks.1.embeddings
-  - exists: _source.sparse_field.inference.chunks.2.embeddings
-  - exists: _source.sparse_field.inference.chunks.3.embeddings
-  - match: { _source.sparse_field.inference.chunks.0.text: "false" }
-  - match: { _source.sparse_field.inference.chunks.1.text: "75" }
-  - match: { _source.sparse_field.inference.chunks.2.text: "inference test" }
-  - match: { _source.sparse_field.inference.chunks.3.text: "13.49" }
-
-  - length: { _source.dense_field.text: 4 }
-  - match: { _source.dense_field.text.0: "true" }
-  - match: { _source.dense_field.text.1: "49.99" }
-  - match: { _source.dense_field.text.2: "another inference test" }
-  - match: { _source.dense_field.text.3: "5654" }
-  - exists: _source.dense_field.inference.chunks.0.embeddings
-  - exists: _source.dense_field.inference.chunks.1.embeddings
-  - exists: _source.dense_field.inference.chunks.2.embeddings
-  - exists: _source.dense_field.inference.chunks.3.embeddings
-  - match: { _source.dense_field.inference.chunks.0.text: "true" }
-  - match: { _source.dense_field.inference.chunks.1.text: "49.99" }
-  - match: { _source.dense_field.inference.chunks.2.text: "another inference test" }
-  - match: { _source.dense_field.inference.chunks.3.text: "5654" }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 4 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 5 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.1.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.1.start_offset: 6 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.1.end_offset: 8 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.2.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.2.start_offset: 9 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.2.end_offset: 23 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.3.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.3.start_offset: 24 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.3.end_offset: 29 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 4 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 4 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.1.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.1.start_offset: 5 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.1.end_offset: 10 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.2.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.2.start_offset: 11 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.2.end_offset: 33 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.3.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.3.start_offset: 34 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.3.end_offset: 38 }
 
 ---
 "Inference fields do not create new mappings":
@@ -197,6 +246,7 @@ setup:
       search:
         index: test-index
         body:
+          fields: [ _inference_fields ]
           query:
             nested:
               path: sparse_field.inference.chunks
@@ -208,37 +258,24 @@ setup:
 
   - match: { hits.total.value: 2 }
   - match: { hits.total.relation: eq }
-  - length: { hits.hits.0._source.sparse_field.inference.chunks: 2 }
-  - length: { hits.hits.1._source.sparse_field.inference.chunks: 2 }
-
-  # Search with inner hits
-  - do:
-      search:
-        _source: false
-        index: test-index
-        body:
-          query:
-            nested:
-              path: sparse_field.inference.chunks
-              inner_hits:
-                _source: false
-                fields: [ sparse_field.inference.chunks.text ]
-              query:
-                sparse_vector:
-                  field: sparse_field.inference.chunks.embeddings
-                  inference_id: sparse-inference-id
-                  query: "you know, for testing"
-
-  - match: { hits.total.value: 2 }
-  - match: { hits.total.relation: eq }
-  - match: { hits.hits.0.inner_hits.sparse_field\.inference\.chunks.hits.total.value: 2 }
-  - match: { hits.hits.0.inner_hits.sparse_field\.inference\.chunks.hits.total.relation: eq }
-
-  - length: { hits.hits.0.inner_hits.sparse_field\.inference\.chunks.hits.hits.0.fields.sparse_field\.inference\.chunks.0.text: 1 }
-  - length: { hits.hits.0.inner_hits.sparse_field\.inference\.chunks.hits.hits.1.fields.sparse_field\.inference\.chunks.0.text: 1 }
-  - length: { hits.hits.1.inner_hits.sparse_field\.inference\.chunks.hits.hits.0.fields.sparse_field\.inference\.chunks.0.text: 1 }
-  - length: { hits.hits.1.inner_hits.sparse_field\.inference\.chunks.hits.hits.1.fields.sparse_field\.inference\.chunks.0.text: 1 }
 
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 2 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 21 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.1.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.1.start_offset: 22 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.1.end_offset: 37 }
+
+  - length: { hits.hits.1._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.1._source._inference_fields.sparse_field.inference.chunks.sparse_field: 2 }
+  - exists: hits.hits.1._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.1._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.1._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 15 }
+  - exists: hits.hits.1._source._inference_fields.sparse_field.inference.chunks.sparse_field.1.embeddings
+  - match: { hits.hits.1._source._inference_fields.sparse_field.inference.chunks.sparse_field.1.start_offset: 16 }
+  - match: { hits.hits.1._source._inference_fields.sparse_field.inference.chunks.sparse_field.1.end_offset: 35 }
 
 ---
 "Dense vector results are indexed as nested chunks and searchable":
@@ -256,6 +293,7 @@ setup:
       search:
         index: test-index
         body:
+          fields: [ _inference_fields ]
           query:
             nested:
               path: dense_field.inference.chunks
@@ -269,38 +307,24 @@ setup:
 
   - match: { hits.total.value: 2 }
   - match: { hits.total.relation: eq }
-  - length: { hits.hits.0._source.dense_field.inference.chunks: 2 }
-  - length: { hits.hits.1._source.dense_field.inference.chunks: 2 }
-
-  # Search with inner hits
-  - do:
-      search:
-        _source: false
-        index: test-index
-        body:
-          query:
-            nested:
-              path: dense_field.inference.chunks
-              inner_hits:
-                _source: false
-                fields: [ dense_field.inference.chunks.text ]
-              query:
-                knn:
-                  field: dense_field.inference.chunks.embeddings
-                  query_vector_builder:
-                    text_embedding:
-                      model_id: dense-inference-id
-                      model_text: "you know, for testing"
-
-  - match: { hits.total.value: 2 }
-  - match: { hits.total.relation: eq }
-  - match: { hits.hits.0.inner_hits.dense_field\.inference\.chunks.hits.total.value: 2 }
-  - match: { hits.hits.0.inner_hits.dense_field\.inference\.chunks.hits.total.relation: eq }
 
-  - length: { hits.hits.0.inner_hits.dense_field\.inference\.chunks.hits.hits.0.fields.dense_field\.inference\.chunks.0.text: 1 }
-  - length: { hits.hits.0.inner_hits.dense_field\.inference\.chunks.hits.hits.1.fields.dense_field\.inference\.chunks.0.text: 1 }
-  - length: { hits.hits.1.inner_hits.dense_field\.inference\.chunks.hits.hits.0.fields.dense_field\.inference\.chunks.0.text: 1 }
-  - length: { hits.hits.1.inner_hits.dense_field\.inference\.chunks.hits.hits.1.fields.dense_field\.inference\.chunks.0.text: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 2 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 21 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.1.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.1.start_offset: 22 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.1.end_offset: 37 }
+
+  - length: { hits.hits.1._source._inference_fields.dense_field.inference.chunks: 1 }
+  - length: { hits.hits.1._source._inference_fields.dense_field.inference.chunks.dense_field: 2 }
+  - exists: hits.hits.1._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings
+  - match: { hits.hits.1._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.1._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 15 }
+  - exists: hits.hits.1._source._inference_fields.dense_field.inference.chunks.dense_field.1.embeddings
+  - match: { hits.hits.1._source._inference_fields.dense_field.inference.chunks.dense_field.1.start_offset: 16 }
+  - match: { hits.hits.1._source._inference_fields.dense_field.inference.chunks.dense_field.1.end_offset: 35 }
 
 ---
 "Reindex works for semantic_text fields":
@@ -312,22 +336,17 @@ setup:
           sparse_field: "inference test"
           dense_field: "another inference test"
           non_inference_field: "non inference test"
-
-  - do:
-      get:
-        index: test-index
-        id: doc_1
-
-  - set: { _source.sparse_field.inference.chunks.0.embeddings: sparse_field_embedding }
-  - set: { _source.dense_field.inference.chunks.0.embeddings: dense_field_embedding }
-
-  - do:
-      indices.refresh: { }
+        refresh: true
 
   - do:
       indices.create:
         index: destination-index
         body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: false
           mappings:
             properties:
               sparse_field:
@@ -347,19 +366,42 @@ setup:
             index: test-index
           dest:
             index: destination-index
+
+  - do:
+      indices.refresh: { }
+
   - do:
       get:
         index: destination-index
         id: doc_1
 
-  - match: { _source.sparse_field.text: "inference test" }
-  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
-  - match: { _source.sparse_field.inference.chunks.0.embeddings: $sparse_field_embedding }
-  - match: { _source.dense_field.text: "another inference test" }
-  - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
-  - match: { _source.dense_field.inference.chunks.0.embeddings: $dense_field_embedding }
+  - match: { _source.sparse_field: "inference test" }
+  - match: { _source.dense_field: "another inference test" }
   - match: { _source.non_inference_field: "non inference test" }
 
+  - do:
+      search:
+        index: destination-index
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 14 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 22 }
+
 ---
 "Fails for non-existent inference":
   - do:
@@ -399,6 +441,11 @@ setup:
       indices.create:
         index: test-copy-to-index
         body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: false
           mappings:
             properties:
               sparse_field:
@@ -419,20 +466,32 @@ setup:
           source_field: "copy_to inference test"
           sparse_field: "inference test"
           another_source_field: "another copy_to inference test"
+        refresh: true
 
   - do:
-      get:
+      search:
         index: test-copy-to-index
-        id: doc_1
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
 
-  - match: { _source.sparse_field.text: "inference test" }
-  - length: { _source.sparse_field.inference.chunks: 3 }
-  - match: { _source.sparse_field.inference.chunks.0.text: "another copy_to inference test" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.sparse_field.inference.chunks.1.text: "copy_to inference test" }
-  - exists: _source.sparse_field.inference.chunks.1.embeddings
-  - match: { _source.sparse_field.inference.chunks.2.text: "inference test" }
-  - exists: _source.sparse_field.inference.chunks.2.embeddings
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 3 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 14 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.source_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.source_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.source_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.source_field.0.end_offset: 22 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.another_source_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.another_source_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.another_source_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.another_source_field.0.end_offset: 30 }
 
 ---
 "Calculates embeddings for bulk operations - index":
@@ -441,6 +500,7 @@ setup:
         body:
           - '{"index": {"_index": "test-index", "_id": "doc_1"}}'
           - '{"sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test"}'
+        refresh: true
 
   - match: { errors: false }
   - match: { items.0.index.result: "created" }
@@ -450,21 +510,44 @@ setup:
         index: test-index
         id: doc_1
 
-  - match: { _source.sparse_field.text: "inference test" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
-  - match: { _source.dense_field.text: "another inference test" }
-  - exists: _source.dense_field.inference.chunks.0.embeddings
-  - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
+  - match: { _source.sparse_field: "inference test" }
+  - match: { _source.dense_field: "another inference test" }
   - match: { _source.non_inference_field: "non inference test" }
 
+  - do:
+      search:
+        index: test-index
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 14 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 22 }
+
 ---
 "Update by query picks up new semantic_text fields":
-
   - do:
       indices.create:
         index: mapping-update-index
         body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: false
           mappings:
             dynamic: false
             properties:
@@ -502,22 +585,43 @@ setup:
 
   - match: { updated: 1 }
 
+  - do:
+      indices.refresh: { }
+
   - do:
       get:
         index: mapping-update-index
         id: doc_1
 
-  - match: { _source.sparse_field.text: "inference test" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
-  - match: { _source.dense_field.text: "another inference test" }
-  - exists: _source.dense_field.inference.chunks.0.embeddings
-  - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
+  - match: { _source.sparse_field: "inference test" }
+  - match: { _source.dense_field: "another inference test" }
   - match: { _source.non_inference_field: "non inference test" }
 
+  - do:
+      search:
+        index: mapping-update-index
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 14 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 22 }
+
 ---
 "Update by query works for scripts":
-
   - do:
       index:
         index: test-index
@@ -528,8 +632,26 @@ setup:
           dense_field: "another inference test"
           non_inference_field: "non inference test"
 
+  - do:
+      search:
+        index: test-index
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 1 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 14 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 22 }
+
   - do:
       update_by_query:
+        refresh: true
         wait_for_completion: true
         index: test-index
         body: { "script": "ctx._source.sparse_field = 'updated inference test'; ctx._source.dense_field = 'another updated inference test'" }
@@ -541,12 +663,27 @@ setup:
         index: test-index
         id: doc_1
 
-  - match: { _source.sparse_field.text: "updated inference test" }
-  - match: { _source.sparse_field.inference.chunks.0.text: "updated inference test" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.dense_field.text: "another updated inference test" }
-  - match: { _source.dense_field.inference.chunks.0.text: "another updated inference test" }
-  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field: "updated inference test" }
+  - match: { _source.dense_field: "another updated inference test" }
+
+  - do:
+      search:
+        index: test-index
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 1 }
+  # We can't directly check that the embeddings are different since there isn't a "does not match" assertion in the
+  # YAML test framework. Check that the start and end offsets change as expected as a proxy.
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 22 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 30 }
 
 ---
 "Calculates embeddings using the default ELSER 2 endpoint":
@@ -562,6 +699,11 @@ setup:
       indices.create:
         index: test-elser-2-default-index
         body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: false
           mappings:
             properties:
               sparse_field:
@@ -573,15 +715,24 @@ setup:
         id: doc_1
         body:
           sparse_field: "inference test"
+        refresh: true
 
   - do:
-      get:
+      search:
         index: test-elser-2-default-index
-        id: doc_1
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
 
-  - match: { _source.sparse_field.text: "inference test" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 14 }
 
 ---
 "Can be used inside an object field":
@@ -593,6 +744,11 @@ setup:
       indices.create:
         index: test-in-object-index
         body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: false
           mappings:
             properties:
               level_1:
@@ -612,18 +768,38 @@ setup:
           level_1:
             sparse_field: "inference test"
             dense_field: "another inference test"
+        refresh: true
 
   - do:
       get:
         index: test-in-object-index
         id: doc_1
 
-  - match: { _source.level_1.sparse_field.text: "inference test" }
-  - exists: _source.level_1.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.level_1.sparse_field.inference.chunks.0.text: "inference test" }
-  - match: { _source.level_1.dense_field.text: "another inference test" }
-  - exists: _source.level_1.dense_field.inference.chunks.0.embeddings
-  - match: { _source.level_1.dense_field.inference.chunks.0.text: "another inference test" }
+  - match: { _source.level_1.sparse_field: "inference test" }
+  - match: { _source.level_1.dense_field: "another inference test" }
+
+  - do:
+      search:
+        index: test-in-object-index
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.level_1\\.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.level_1\\.sparse_field.inference.chunks.level_1\\.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.level_1\\.sparse_field.inference.chunks.level_1\\.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.level_1\\.sparse_field.inference.chunks.level_1\\.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.level_1\\.sparse_field.inference.chunks.level_1\\.sparse_field.0.end_offset: 14 }
+
+  - length: { hits.hits.0._source._inference_fields.level_1\\.dense_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.level_1\\.dense_field.inference.chunks.level_1\\.dense_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.level_1\\.dense_field.inference.chunks.level_1\\.dense_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.level_1\\.dense_field.inference.chunks.level_1\\.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.level_1\\.dense_field.inference.chunks.level_1\\.dense_field.0.end_offset: 22 }
 
 ---
 "Deletes on bulk operation":
@@ -652,8 +828,8 @@ setup:
 
   - match: { hits.total.value: 2 }
   - match: { hits.total.relation: eq }
-  - match: { hits.hits.0._source.dense_field.text: ["you know, for testing", "now with chunks"] }
-  - match: { hits.hits.1._source.dense_field.text: ["some more tests", "that include chunks"] }
+  - match: { hits.hits.0._source.dense_field: ["you know, for testing", "now with chunks"] }
+  - match: { hits.hits.1._source.dense_field: ["some more tests", "that include chunks"] }
 
   - do:
       bulk:
@@ -675,4 +851,4 @@ setup:
 
   - match: { hits.total.value: 1 }
   - match: { hits.total.relation: eq }
-  - match: { hits.hits.0._source.dense_field.text: "updated text" }
+  - match: { hits.hits.0._source.dense_field: "updated text" }
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference_bwc.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference_bwc.yml
new file mode 100644
index 0000000000000..07341273151bc
--- /dev/null
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference_bwc.yml
@@ -0,0 +1,652 @@
+setup:
+  - requires:
+      cluster_features: "gte_v8.15.0"
+      reason: semantic_text introduced in 8.15.0
+
+  - do:
+      inference.put:
+        task_type: sparse_embedding
+        inference_id: sparse-inference-id
+        body: >
+          {
+            "service": "test_service",
+            "service_settings": {
+              "model": "my_model",
+              "api_key": "abc64"
+            },
+            "task_settings": {
+            }
+          }
+
+  - do:
+      inference.put:
+        task_type: text_embedding
+        inference_id: dense-inference-id
+        body: >
+          {
+            "service": "text_embedding_test_service",
+            "service_settings": {
+              "model": "my_model",
+              "dimensions": 10,
+              "similarity": "cosine",
+              "api_key": "abc64"
+            },
+            "task_settings": {
+            }
+          }
+
+  - do:
+      indices.create:
+        index: test-index
+        body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: true
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+              dense_field:
+                type: semantic_text
+                inference_id: dense-inference-id
+              non_inference_field:
+                type: text
+
+---
+"Calculates sparse embedding and text embedding results for new documents":
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field: "inference test"
+          dense_field: "another inference test"
+          non_inference_field: "non inference test"
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "inference test" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
+  - match: { _source.dense_field.text: "another inference test" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
+  - match: { _source.non_inference_field: "non inference test" }
+
+---
+"Calculates sparse embedding and text embedding results for new documents with integer value":
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field: 75
+          dense_field: 100
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "75" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "75" }
+  - match: { _source.dense_field.text: "100" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.inference.chunks.0.text: "100" }
+
+---
+"Calculates sparse embedding and text embedding results for new documents with boolean value":
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field: true
+          dense_field: false
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "true" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "true" }
+  - match: { _source.dense_field.text: "false" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.inference.chunks.0.text: "false" }
+
+---
+"Calculates sparse embedding and text embedding results for new documents with a collection of mixed data types":
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field: [false, 75, "inference test", 13.49]
+          dense_field: [true, 49.99, "another inference test", 5654]
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - length: { _source.sparse_field.text: 4 }
+  - match: { _source.sparse_field.text.0: "false" }
+  - match: { _source.sparse_field.text.1: "75" }
+  - match: { _source.sparse_field.text.2: "inference test" }
+  - match: { _source.sparse_field.text.3: "13.49" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - exists: _source.sparse_field.inference.chunks.1.embeddings
+  - exists: _source.sparse_field.inference.chunks.2.embeddings
+  - exists: _source.sparse_field.inference.chunks.3.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "false" }
+  - match: { _source.sparse_field.inference.chunks.1.text: "75" }
+  - match: { _source.sparse_field.inference.chunks.2.text: "inference test" }
+  - match: { _source.sparse_field.inference.chunks.3.text: "13.49" }
+
+  - length: { _source.dense_field.text: 4 }
+  - match: { _source.dense_field.text.0: "true" }
+  - match: { _source.dense_field.text.1: "49.99" }
+  - match: { _source.dense_field.text.2: "another inference test" }
+  - match: { _source.dense_field.text.3: "5654" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - exists: _source.dense_field.inference.chunks.1.embeddings
+  - exists: _source.dense_field.inference.chunks.2.embeddings
+  - exists: _source.dense_field.inference.chunks.3.embeddings
+  - match: { _source.dense_field.inference.chunks.0.text: "true" }
+  - match: { _source.dense_field.inference.chunks.1.text: "49.99" }
+  - match: { _source.dense_field.inference.chunks.2.text: "another inference test" }
+  - match: { _source.dense_field.inference.chunks.3.text: "5654" }
+
+---
+"Sparse vector results are indexed as nested chunks and searchable":
+  - do:
+      bulk:
+        index: test-index
+        refresh: true
+        body: |
+          {"index":{}}
+          {"sparse_field": ["you know, for testing", "now with chunks"]}
+          {"index":{}}
+          {"sparse_field": ["some more tests", "that include chunks"]}
+
+  - do:
+      search:
+        index: test-index
+        body:
+          query:
+            nested:
+              path: sparse_field.inference.chunks
+              query:
+                sparse_vector:
+                  field: sparse_field.inference.chunks.embeddings
+                  inference_id: sparse-inference-id
+                  query: "you know, for testing"
+
+  - match: { hits.total.value: 2 }
+  - match: { hits.total.relation: eq }
+  - length: { hits.hits.0._source.sparse_field.inference.chunks: 2 }
+  - length: { hits.hits.1._source.sparse_field.inference.chunks: 2 }
+
+  # Search with inner hits
+  - do:
+      search:
+        _source: false
+        index: test-index
+        body:
+          query:
+            nested:
+              path: sparse_field.inference.chunks
+              inner_hits:
+                _source: false
+                fields: [ sparse_field.inference.chunks.text ]
+              query:
+                sparse_vector:
+                  field: sparse_field.inference.chunks.embeddings
+                  inference_id: sparse-inference-id
+                  query: "you know, for testing"
+
+  - match: { hits.total.value: 2 }
+  - match: { hits.total.relation: eq }
+  - match: { hits.hits.0.inner_hits.sparse_field\.inference\.chunks.hits.total.value: 2 }
+  - match: { hits.hits.0.inner_hits.sparse_field\.inference\.chunks.hits.total.relation: eq }
+
+  - length: { hits.hits.0.inner_hits.sparse_field\.inference\.chunks.hits.hits.0.fields.sparse_field\.inference\.chunks.0.text: 1 }
+  - length: { hits.hits.0.inner_hits.sparse_field\.inference\.chunks.hits.hits.1.fields.sparse_field\.inference\.chunks.0.text: 1 }
+  - length: { hits.hits.1.inner_hits.sparse_field\.inference\.chunks.hits.hits.0.fields.sparse_field\.inference\.chunks.0.text: 1 }
+  - length: { hits.hits.1.inner_hits.sparse_field\.inference\.chunks.hits.hits.1.fields.sparse_field\.inference\.chunks.0.text: 1 }
+
+---
+"Dense vector results are indexed as nested chunks and searchable":
+  - do:
+      bulk:
+        index: test-index
+        refresh: true
+        body: |
+          {"index":{}}
+          {"dense_field": ["you know, for testing", "now with chunks"]}
+          {"index":{}}
+          {"dense_field": ["some more tests", "that include chunks"]}
+
+  - do:
+      search:
+        index: test-index
+        body:
+          query:
+            nested:
+              path: dense_field.inference.chunks
+              query:
+                knn:
+                  field: dense_field.inference.chunks.embeddings
+                  query_vector_builder:
+                    text_embedding:
+                      model_id: dense-inference-id
+                      model_text: "you know, for testing"
+
+  - match: { hits.total.value: 2 }
+  - match: { hits.total.relation: eq }
+  - length: { hits.hits.0._source.dense_field.inference.chunks: 2 }
+  - length: { hits.hits.1._source.dense_field.inference.chunks: 2 }
+
+  # Search with inner hits
+  - do:
+      search:
+        _source: false
+        index: test-index
+        body:
+          query:
+            nested:
+              path: dense_field.inference.chunks
+              inner_hits:
+                _source: false
+                fields: [ dense_field.inference.chunks.text ]
+              query:
+                knn:
+                  field: dense_field.inference.chunks.embeddings
+                  query_vector_builder:
+                    text_embedding:
+                      model_id: dense-inference-id
+                      model_text: "you know, for testing"
+
+  - match: { hits.total.value: 2 }
+  - match: { hits.total.relation: eq }
+  - match: { hits.hits.0.inner_hits.dense_field\.inference\.chunks.hits.total.value: 2 }
+  - match: { hits.hits.0.inner_hits.dense_field\.inference\.chunks.hits.total.relation: eq }
+
+  - length: { hits.hits.0.inner_hits.dense_field\.inference\.chunks.hits.hits.0.fields.dense_field\.inference\.chunks.0.text: 1 }
+  - length: { hits.hits.0.inner_hits.dense_field\.inference\.chunks.hits.hits.1.fields.dense_field\.inference\.chunks.0.text: 1 }
+  - length: { hits.hits.1.inner_hits.dense_field\.inference\.chunks.hits.hits.0.fields.dense_field\.inference\.chunks.0.text: 1 }
+  - length: { hits.hits.1.inner_hits.dense_field\.inference\.chunks.hits.hits.1.fields.dense_field\.inference\.chunks.0.text: 1 }
+
+---
+"Reindex works for semantic_text fields":
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field: "inference test"
+          dense_field: "another inference test"
+          non_inference_field: "non inference test"
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - set: { _source.sparse_field.inference.chunks.0.embeddings: sparse_field_embedding }
+  - set: { _source.dense_field.inference.chunks.0.embeddings: dense_field_embedding }
+
+  - do:
+      indices.refresh: { }
+
+  - do:
+      indices.create:
+        index: destination-index
+        body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: true
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+              dense_field:
+                type: semantic_text
+                inference_id: dense-inference-id
+              non_inference_field:
+                type: text
+
+  - do:
+      reindex:
+        wait_for_completion: true
+        body:
+          source:
+            index: test-index
+          dest:
+            index: destination-index
+  - do:
+      get:
+        index: destination-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "inference test" }
+  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
+  - match: { _source.sparse_field.inference.chunks.0.embeddings: $sparse_field_embedding }
+  - match: { _source.dense_field.text: "another inference test" }
+  - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
+  - match: { _source.dense_field.inference.chunks.0.embeddings: $dense_field_embedding }
+  - match: { _source.non_inference_field: "non inference test" }
+
+---
+"semantic_text copy_to calculates embeddings for source fields":
+  - do:
+      indices.create:
+        index: test-copy-to-index
+        body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: true
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+              source_field:
+                type: text
+                copy_to: sparse_field
+              another_source_field:
+                type: text
+                copy_to: sparse_field
+
+  - do:
+      index:
+        index: test-copy-to-index
+        id: doc_1
+        body:
+          source_field: "copy_to inference test"
+          sparse_field: "inference test"
+          another_source_field: "another copy_to inference test"
+
+  - do:
+      get:
+        index: test-copy-to-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "inference test" }
+  - length: { _source.sparse_field.inference.chunks: 3 }
+  - match: { _source.sparse_field.inference.chunks.0.text: "another copy_to inference test" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field.inference.chunks.1.text: "copy_to inference test" }
+  - exists: _source.sparse_field.inference.chunks.1.embeddings
+  - match: { _source.sparse_field.inference.chunks.2.text: "inference test" }
+  - exists: _source.sparse_field.inference.chunks.2.embeddings
+
+---
+"Calculates embeddings for bulk operations - index":
+  - do:
+      bulk:
+        body:
+          - '{"index": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test"}'
+
+  - match: { errors: false }
+  - match: { items.0.index.result: "created" }
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "inference test" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
+  - match: { _source.dense_field.text: "another inference test" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
+  - match: { _source.non_inference_field: "non inference test" }
+
+---
+"Update by query picks up new semantic_text fields":
+  - do:
+      indices.create:
+        index: mapping-update-index
+        body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: true
+          mappings:
+            dynamic: false
+            properties:
+              non_inference_field:
+                type: text
+
+  - do:
+      index:
+        index: mapping-update-index
+        id: doc_1
+        refresh: true
+        body:
+          sparse_field: "inference test"
+          dense_field: "another inference test"
+          non_inference_field: "non inference test"
+
+  - do:
+      indices.put_mapping:
+        index: mapping-update-index
+        body:
+          properties:
+            sparse_field:
+              type: semantic_text
+              inference_id: sparse-inference-id
+            dense_field:
+              type: semantic_text
+              inference_id: dense-inference-id
+            non_inference_field:
+              type: text
+
+  - do:
+      update_by_query:
+        wait_for_completion: true
+        index: mapping-update-index
+
+  - match: { updated: 1 }
+
+  - do:
+      get:
+        index: mapping-update-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "inference test" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
+  - match: { _source.dense_field.text: "another inference test" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
+  - match: { _source.non_inference_field: "non inference test" }
+
+---
+"Update by query works for scripts":
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        refresh: true
+        body:
+          sparse_field: "inference test"
+          dense_field: "another inference test"
+          non_inference_field: "non inference test"
+
+  - do:
+      update_by_query:
+        wait_for_completion: true
+        index: test-index
+        body: { "script": "ctx._source.sparse_field = 'updated inference test'; ctx._source.dense_field = 'another updated inference test'" }
+
+  - match: { updated: 1 }
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "updated inference test" }
+  - match: { _source.sparse_field.inference.chunks.0.text: "updated inference test" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.text: "another updated inference test" }
+  - match: { _source.dense_field.inference.chunks.0.text: "another updated inference test" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+
+---
+"Calculates embeddings using the default ELSER 2 endpoint":
+  - requires:
+      reason: "default ELSER 2 inference ID is enabled via a capability"
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: GET
+          path: /_inference
+          capabilities: [default_elser_2]
+
+  - do:
+      indices.create:
+        index: test-elser-2-default-index
+        body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: true
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+
+  - do:
+      index:
+        index: test-elser-2-default-index
+        id: doc_1
+        body:
+          sparse_field: "inference test"
+
+  - do:
+      get:
+        index: test-elser-2-default-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "inference test" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
+
+---
+"Can be used inside an object field":
+  - requires:
+      cluster_features: "semantic_text.in_object_field_fix"
+      reason: object field fix added in 8.16.0 & 8.15.4
+
+  - do:
+      indices.create:
+        index: test-in-object-index
+        body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: true
+          mappings:
+            properties:
+              level_1:
+                properties:
+                  sparse_field:
+                    type: semantic_text
+                    inference_id: sparse-inference-id
+                  dense_field:
+                    type: semantic_text
+                    inference_id: dense-inference-id
+
+  - do:
+      index:
+        index: test-in-object-index
+        id: doc_1
+        body:
+          level_1:
+            sparse_field: "inference test"
+            dense_field: "another inference test"
+
+  - do:
+      get:
+        index: test-in-object-index
+        id: doc_1
+
+  - match: { _source.level_1.sparse_field.text: "inference test" }
+  - exists: _source.level_1.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.level_1.sparse_field.inference.chunks.0.text: "inference test" }
+  - match: { _source.level_1.dense_field.text: "another inference test" }
+  - exists: _source.level_1.dense_field.inference.chunks.0.embeddings
+  - match: { _source.level_1.dense_field.inference.chunks.0.text: "another inference test" }
+
+---
+"Deletes on bulk operation":
+  - requires:
+      cluster_features: semantic_text.delete_fix
+      reason: Delete operations are properly applied when subsequent operations include a semantic text field.
+
+  - do:
+      bulk:
+        index: test-index
+        refresh: true
+        body: |
+          {"index":{"_id": "1"}}
+          {"dense_field": ["you know, for testing", "now with chunks"]}
+          {"index":{"_id": "2"}}
+          {"dense_field": ["some more tests", "that include chunks"]}
+
+  - do:
+      search:
+        index: test-index
+        body:
+          query:
+            semantic:
+              field: dense_field
+              query: "you know, for testing"
+
+  - match: { hits.total.value: 2 }
+  - match: { hits.total.relation: eq }
+  - match: { hits.hits.0._source.dense_field.text: ["you know, for testing", "now with chunks"] }
+  - match: { hits.hits.1._source.dense_field.text: ["some more tests", "that include chunks"] }
+
+  - do:
+      bulk:
+        index: test-index
+        refresh: true
+        body: |
+          {"delete":{ "_id": "2"}}
+          {"update":{"_id": "1"}}
+          {"doc":{"dense_field": "updated text"}}
+
+  - do:
+      search:
+        index: test-index
+        body:
+          query:
+            semantic:
+              field: dense_field
+              query: "you know, for testing"
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+  - match: { hits.hits.0._source.dense_field.text: "updated text" }
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/40_semantic_text_query.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/40_semantic_text_query.yml
index 3d3790d879ef1..71db83e4667df 100644
--- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/40_semantic_text_query.yml
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/40_semantic_text_query.yml
@@ -120,7 +120,6 @@ setup:
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._id: "doc_1" }
   - close_to: { hits.hits.0._score: { value: 3.7837332e17, error: 1e10 } }
-  - length: { hits.hits.0._source.inference_field.inference.chunks: 2 }
 
 ---
 "Numeric query using a sparse embedding model":
@@ -146,7 +145,6 @@ setup:
 
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._id: "doc_1" }
-  - length: { hits.hits.0._source.inference_field.inference.chunks: 2 }
 
 ---
 "Boolean query using a sparse embedding model":
@@ -172,14 +170,9 @@ setup:
 
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._id: "doc_1" }
-  - length: { hits.hits.0._source.inference_field.inference.chunks: 1 }
 
 ---
 "Query using a sparse embedding model via a search inference ID":
-  - requires:
-      cluster_features: "semantic_text.search_inference_id"
-      reason: search_inference_id introduced in 8.16.0
-
   - skip:
       features: [ "headers", "close_to" ]
 
@@ -217,7 +210,6 @@ setup:
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._id: "doc_1" }
   - close_to: { hits.hits.0._score: { value: 3.7837332e17, error: 1e10 } }
-  - length: { hits.hits.0._source.inference_field.inference.chunks: 2 }
 
 ---
 "Query using a dense embedding model":
@@ -248,7 +240,6 @@ setup:
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._id: "doc_1" }
   - close_to: { hits.hits.0._score: { value: 1.0, error: 0.0001 } }
-  - length: { hits.hits.0._source.inference_field.inference.chunks: 2 }
 
 ---
 "Numeric query using a dense embedding model":
@@ -274,7 +265,6 @@ setup:
 
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._id: "doc_1" }
-  - length: { hits.hits.0._source.inference_field.inference.chunks: 2 }
 
 ---
 "Boolean query using a dense embedding model":
@@ -300,7 +290,6 @@ setup:
 
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._id: "doc_1" }
-  - length: { hits.hits.0._source.inference_field.inference.chunks: 1 }
 
 ---
 "Query using a dense embedding model that uses byte embeddings":
@@ -361,14 +350,9 @@ setup:
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._id: "doc_1" }
   - close_to: { hits.hits.0._score: { value: 1.0, error: 0.0001 } }
-  - length: { hits.hits.0._source.inference_field.inference.chunks: 2 }
 
 ---
 "Query using a dense embedding model via a search inference ID":
-  - requires:
-      cluster_features: "semantic_text.search_inference_id"
-      reason: search_inference_id introduced in 8.16.0
-
   - skip:
       features: [ "headers", "close_to" ]
 
@@ -406,7 +390,6 @@ setup:
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._id: "doc_1" }
   - close_to: { hits.hits.0._score: { value: 1.0, error: 0.0001 } }
-  - length: { hits.hits.0._source.inference_field.inference.chunks: 2 }
 
 ---
 "Apply boost and query name":
@@ -439,7 +422,6 @@ setup:
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._id: "doc_1" }
   - close_to: { hits.hits.0._score: { value: 3.783733e19, error: 1e13 } }
-  - length: { hits.hits.0._source.inference_field.inference.chunks: 2 }
   - match: { hits.hits.0.matched_queries: [ "i-like-naming-my-queries" ] }
 
 ---
@@ -476,7 +458,6 @@ setup:
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._id: "doc_1" }
   - close_to: { hits.hits.0._score: { value: 3.7837332e17, error: 1e10 } }
-  - length: { hits.hits.0._source.inference_field.inference.chunks: 2 }
 
 ---
 "Query the wrong field type":
@@ -748,10 +729,6 @@ setup:
 
 ---
 "Query a field with a search inference ID that uses the wrong task type":
-  - requires:
-      cluster_features: "semantic_text.search_inference_id"
-      reason: search_inference_id introduced in 8.16.0
-
   - do:
       indices.put_mapping:
         index: test-sparse-index
@@ -789,10 +766,6 @@ setup:
 
 ---
 "Query a field with a search inference ID that uses the wrong dimension count":
-  - requires:
-      cluster_features: "semantic_text.search_inference_id"
-      reason: search_inference_id introduced in 8.16.0
-
   - do:
       inference.put:
         task_type: text_embedding
@@ -846,10 +819,6 @@ setup:
 
 ---
 "Query a field with an invalid search inference ID":
-  - requires:
-      cluster_features: "semantic_text.search_inference_id"
-      reason: search_inference_id introduced in 8.16.0
-
   - do:
       indices.put_mapping:
         index: test-dense-index
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/45_semantic_text_match.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/45_semantic_text_match.yml
index cdbf73d31a272..28093ba49e6cc 100644
--- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/45_semantic_text_match.yml
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/45_semantic_text_match.yml
@@ -210,8 +210,6 @@ setup:
                 query: "inference test"
 
   - match: { hits.total.value: 2 }
-  - match: { hits.hits.0._id: "doc_1" }
-  - match: { hits.hits.1._id: "doc_2" }
 
   # Test querying multiple indices that either use the same inference ID or combine semantic_text with lexical search
   - do:
@@ -246,9 +244,6 @@ setup:
                 query: "inference test"
 
   - match: { hits.total.value: 3 }
-  - match: { hits.hits.0._id: "doc_1" }
-  - match: { hits.hits.1._id: "doc_3" }
-  - match: { hits.hits.2._id: "doc_2" }
 
 ---
 "Query a field that has no indexed inference results":
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/46_semantic_text_sparse_vector.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/46_semantic_text_sparse_vector.yml
new file mode 100644
index 0000000000000..f1cff512fd209
--- /dev/null
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/46_semantic_text_sparse_vector.yml
@@ -0,0 +1,249 @@
+setup:
+  - requires:
+      cluster_features: "search.semantic_sparse_vector_query_rewrite_interception_supported"
+      reason: semantic_text sparse_vector support introduced in 8.18.0
+
+  - do:
+      inference.put:
+        task_type: sparse_embedding
+        inference_id: sparse-inference-id
+        body: >
+          {
+            "service": "test_service",
+            "service_settings": {
+              "model": "my_model",
+              "api_key": "abc64"
+            },
+            "task_settings": {
+            }
+          }
+
+  - do:
+      inference.put:
+        task_type: sparse_embedding
+        inference_id: sparse-inference-id-2
+        body: >
+          {
+            "service": "test_service",
+            "service_settings": {
+              "model": "my_model",
+              "api_key": "abc64"
+            },
+            "task_settings": {
+            }
+          }
+
+  - do:
+      indices.create:
+        index: test-semantic-text-index
+        body:
+          mappings:
+            properties:
+              inference_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+
+  - do:
+      indices.create:
+        index: test-semantic-text-index-2
+        body:
+          mappings:
+            properties:
+              inference_field:
+                type: semantic_text
+                inference_id: sparse-inference-id-2
+
+  - do:
+      indices.create:
+        index: test-sparse-vector-index
+        body:
+          mappings:
+            properties:
+              inference_field:
+                type: sparse_vector
+
+  - do:
+      index:
+        index: test-semantic-text-index
+        id: doc_1
+        body:
+          inference_field: [ "inference test", "another inference test" ]
+        refresh: true
+
+  - do:
+      index:
+        index: test-semantic-text-index-2
+        id: doc_3
+        body:
+          inference_field: [ "inference test", "another inference test" ]
+        refresh: true
+
+  - do:
+      index:
+        index: test-sparse-vector-index
+        id: doc_2
+        body:
+          inference_field: { "feature_0": 1, "feature_1": 2, "feature_2": 3, "feature_3": 4, "feature_4": 5 }
+        refresh: true
+
+---
+"Nested sparse_vector queries using the old format on semantic_text embeddings and inference still work":
+  - skip:
+      features: [ "headers" ]
+
+  - do:
+      headers:
+        # Force JSON content type so that we use a parser that interprets the floating-point score as a double
+        Content-Type: application/json
+      search:
+        index: test-semantic-text-index
+        body:
+          query:
+            nested:
+              path: inference_field.inference.chunks
+              query:
+                sparse_vector:
+                  field: inference_field.inference.chunks.embeddings
+                  inference_id: sparse-inference-id
+                  query: test
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "doc_1" }
+
+---
+"Nested sparse_vector queries using the old format on semantic_text embeddings and query vectors still work":
+  - skip:
+      features: [ "headers" ]
+
+  - do:
+      headers:
+        # Force JSON content type so that we use a parser that interprets the floating-point score as a double
+        Content-Type: application/json
+      search:
+        index: test-semantic-text-index
+        body:
+          query:
+            nested:
+              path: inference_field.inference.chunks
+              query:
+                sparse_vector:
+                  field: inference_field.inference.chunks.embeddings
+                  query_vector: { "feature_0": 1, "feature_1": 2, "feature_2": 3, "feature_3": 4, "feature_4": 5 }
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "doc_1" }
+
+---
+"sparse_vector query against semantic_text field using a specified inference ID":
+
+  - do:
+      search:
+        index: test-semantic-text-index
+        body:
+          query:
+            sparse_vector:
+              field: inference_field
+              inference_id: sparse-inference-id
+              query: "inference test"
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "doc_1" }
+
+---
+"sparse_vector query against semantic_text field using inference ID configured in semantic_text field":
+
+  - do:
+      search:
+        index: test-semantic-text-index
+        body:
+          query:
+            sparse_vector:
+              field: inference_field
+              query: "inference test"
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "doc_1" }
+
+---
+"sparse_vector query against semantic_text field using query vectors":
+
+  - do:
+      search:
+        index: test-semantic-text-index
+        body:
+          query:
+            sparse_vector:
+              field: inference_field
+              query_vector: { "feature_0": 1, "feature_1": 2, "feature_2": 3, "feature_3": 4, "feature_4": 5 }
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "doc_1" }
+
+---
+"sparse_vector query against combined sparse_vector and semantic_text fields using inference":
+
+  - do:
+      search:
+        index:
+          - test-semantic-text-index
+          - test-sparse-vector-index
+        body:
+          query:
+            sparse_vector:
+              field: inference_field
+              inference_id: sparse-inference-id
+              query: "inference test"
+
+  - match: { hits.total.value: 2 }
+
+---
+"sparse_vector query against combined sparse_vector and semantic_text fields still requires inference ID":
+
+  - do:
+      catch: bad_request
+      search:
+        index:
+          - test-semantic-text-index
+          - test-sparse-vector-index
+        body:
+          query:
+            sparse_vector:
+              field: inference_field
+              query: "inference test"
+
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "inference_id required to perform vector search on query string" }
+
+---
+"sparse_vector query against combined sparse_vector and semantic_text fields using query vectors":
+
+  - do:
+      search:
+        index:
+          - test-semantic-text-index
+          - test-sparse-vector-index
+        body:
+          query:
+            sparse_vector:
+              field: inference_field
+              query_vector: { "feature_0": 1, "feature_1": 2, "feature_2": 3, "feature_3": 4, "feature_4": 5 }
+
+  - match: { hits.total.value: 2 }
+
+
+---
+"sparse_vector query against multiple semantic_text fields with multiple inference IDs specified in semantic_text fields":
+
+  - do:
+      search:
+        index:
+          - test-semantic-text-index
+          - test-semantic-text-index-2
+        body:
+          query:
+            sparse_vector:
+              field: inference_field
+              query: "inference test"
+
+  - match: { hits.total.value: 2 }
+
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/47_semantic_text_knn.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/47_semantic_text_knn.yml
new file mode 100644
index 0000000000000..dec4e127e501c
--- /dev/null
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/47_semantic_text_knn.yml
@@ -0,0 +1,367 @@
+setup:
+  - requires:
+      cluster_features: "search.semantic_knn_vector_query_rewrite_interception_supported"
+      reason: semantic_text knn support introduced in 8.18.0
+
+  - do:
+      inference.put:
+        task_type: text_embedding
+        inference_id: dense-inference-id
+        body: >
+          {
+            "service": "text_embedding_test_service",
+            "service_settings": {
+              "model": "my_model",
+              "dimensions": 10,
+              "similarity": "cosine",
+              "api_key": "abc64"
+            },
+            "task_settings": {
+            }
+          }
+
+  - do:
+      inference.put:
+        task_type: text_embedding
+        inference_id: dense-inference-id-2
+        body: >
+          {
+            "service": "text_embedding_test_service",
+            "service_settings": {
+              "model": "my_model",
+              "dimensions": 5,
+              "similarity": "cosine",
+              "api_key": "abc64"
+            },
+            "task_settings": {
+            }
+          }
+
+  - do:
+      indices.create:
+        index: test-semantic-text-index
+        body:
+          mappings:
+            properties:
+              inference_field:
+                type: semantic_text
+                inference_id: dense-inference-id
+
+  - do:
+      indices.create:
+        index: test-semantic-text-index-2
+        body:
+          mappings:
+            properties:
+              inference_field:
+                type: semantic_text
+                inference_id: dense-inference-id-2
+
+  - do:
+      indices.create:
+        index: test-dense-vector-index
+        body:
+          mappings:
+            properties:
+              inference_field:
+                type: dense_vector
+                dims: 10
+                similarity: cosine
+
+  - do:
+      indices.create:
+        index: test-incompatible-dense-vector-index
+        body:
+          mappings:
+            properties:
+              inference_field:
+                type: dense_vector
+                dims: 3
+                similarity: cosine
+
+  - do:
+      index:
+        index: test-semantic-text-index
+        id: doc_1
+        body:
+          inference_field: [ "inference test", "another inference test" ]
+        refresh: true
+
+  - do:
+      index:
+        index: test-semantic-text-index-2
+        id: doc_2
+        body:
+          inference_field: [ "inference test", "another inference test" ]
+        refresh: true
+
+  - do:
+      index:
+        index: test-dense-vector-index
+        id: doc_3
+        body:
+          inference_field: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
+        refresh: true
+
+  - do:
+      index:
+        index: test-incompatible-dense-vector-index
+        id: doc_4
+        body:
+          inference_field: [ 1, 2, 3 ]
+        refresh: true
+
+---
+"Nested knn queries using the old format on semantic_text embeddings and inference still work":
+  - do:
+      search:
+        index: test-semantic-text-index
+        body:
+          query:
+            nested:
+              path: inference_field.inference.chunks
+              query:
+                knn:
+                  field: inference_field.inference.chunks.embeddings
+                  k: 10
+                  num_candidates: 100
+                  query_vector_builder:
+                    text_embedding:
+                      model_id: dense-inference-id
+                      model_text: test
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "doc_1" }
+
+---
+"Nested knn queries using the old format on semantic_text embeddings and query vectors still work":
+  - do:
+      search:
+        index: test-semantic-text-index
+        body:
+          query:
+            nested:
+              path: inference_field.inference.chunks
+              query:
+                knn:
+                  field: inference_field.inference.chunks.embeddings
+                  k: 10
+                  num_candidates: 100
+                  query_vector: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "doc_1" }
+
+---
+"knn query against semantic_text field using a specified inference ID":
+
+  - do:
+      search:
+        index: test-semantic-text-index
+        body:
+          query:
+            knn:
+              field: inference_field
+              k: 10
+              num_candidates: 100
+              query_vector_builder:
+                text_embedding:
+                  model_id: dense-inference-id
+                  model_text: test
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "doc_1" }
+
+---
+"knn query against semantic_text field using inference ID configured in semantic_text field":
+
+  - do:
+      search:
+        index: test-semantic-text-index
+        body:
+          query:
+            knn:
+              field: inference_field
+              k: 10
+              num_candidates: 100
+              query_vector_builder:
+                text_embedding:
+                  model_text: test
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "doc_1" }
+
+---
+"knn query against semantic_text field using query vectors":
+
+  - do:
+      search:
+        index: test-semantic-text-index
+        body:
+          query:
+            knn:
+              field: inference_field
+              k: 10
+              num_candidates: 100
+              query_vector: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "doc_1" }
+
+---
+"knn query against combined dense_vector and semantic_text fields using inference":
+
+  - do:
+      search:
+        index:
+          - test-semantic-text-index
+          - test-dense-vector-index
+        body:
+          query:
+            knn:
+              field: inference_field
+              k: 10
+              num_candidates: 100
+              query_vector_builder:
+                text_embedding:
+                  model_id: dense-inference-id
+                  model_text: test
+
+  - match: { hits.total.value: 2 }
+
+---
+"knn query against combined dense_vector and semantic_text fields still requires inference ID":
+
+  - do:
+      catch: bad_request
+      search:
+        index:
+          - test-semantic-text-index
+          - test-dense-vector-index
+        body:
+          query:
+            knn:
+              field: inference_field
+              k: 10
+              num_candidates: 100
+              query_vector_builder:
+                text_embedding:
+                  model_text: test
+
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "[model_id] must not be null." }
+
+---
+"knn query against combined dense_vector and semantic_text fields using query vectors":
+
+  - do:
+      search:
+        index:
+          - test-semantic-text-index
+          - test-dense-vector-index
+        body:
+          query:
+            knn:
+              field: inference_field
+              k: 10
+              num_candidates: 100
+              query_vector: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
+
+  - match: { hits.total.value: 2 }
+
+
+---
+"knn query against incompatible dense_vector and semantic_text fields using query vectors returns the matching semantic vectors and failures for incompatible dims":
+
+  - do:
+      search:
+        index:
+          - test-semantic-text-index
+          - test-incompatible-dense-vector-index
+        body:
+          query:
+            knn:
+              field: inference_field
+              k: 10
+              num_candidates: 100
+              query_vector: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "doc_1" }
+  - match: { _shards.failures.0.index: "test-incompatible-dense-vector-index" }
+  - match: { _shards.failures.0.reason.reason: "failed to create query: The query vector has a different number of dimensions [10] than the document vectors [3]." }
+
+---
+"knn query against multiple semantic_text fields with multiple inference IDs specified in semantic_text fields":
+
+  - do:
+      search:
+        index:
+          - test-semantic-text-index
+          - test-semantic-text-index-2
+        body:
+          query:
+            knn:
+              field: inference_field
+              k: 10
+              num_candidates: 100
+              query_vector_builder:
+                text_embedding:
+                  model_text: test
+
+  - match: { hits.total.value: 2 }
+
+
+---
+"knn query against multiple semantic_text fields with multiple inference IDs specified in semantic_text fields with smaller k returns k for each index":
+
+  - do:
+      index:
+        index: test-semantic-text-index
+        id: doc_4
+        body:
+          inference_field: [ "inference test", "another inference test" ]
+        refresh: true
+
+  - do:
+      index:
+        index: test-semantic-text-index
+        id: doc_5
+        body:
+          inference_field: [ "inference test", "another inference test" ]
+        refresh: true
+
+  - do:
+      index:
+        index: test-semantic-text-index-2
+        id: doc_6
+        body:
+          inference_field: [ "inference test", "another inference test" ]
+        refresh: true
+
+  - do:
+      index:
+        index: test-semantic-text-index-2
+        id: doc_7
+        body:
+          inference_field: [ "inference test", "another inference test" ]
+        refresh: true
+
+  - do:
+      search:
+        index:
+          - test-semantic-text-index
+          - test-semantic-text-index-2
+        body:
+          query:
+            knn:
+              field: inference_field
+              k: 2
+              num_candidates: 100
+              query_vector_builder:
+                text_embedding:
+                  model_text: test
+
+  - match: { hits.total.value: 4 }
+
+
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update.yml
index 294761608ee81..27c405f6c23bf 100644
--- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update.yml
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update.yml
@@ -39,6 +39,11 @@ setup:
       indices.create:
         index: test-index
         body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: false
           mappings:
             properties:
               sparse_field:
@@ -52,6 +57,9 @@ setup:
 
 ---
 "Updating non semantic_text fields does not recalculate embeddings":
+  - skip:
+      features: [ "headers" ]
+
   - do:
       index:
         index: test-index
@@ -60,14 +68,23 @@ setup:
           sparse_field: "inference test"
           dense_field: "another inference test"
           non_inference_field: "non inference test"
+        refresh: true
 
   - do:
-      get:
+      headers:
+        # Force JSON content type so that we use a parser that interprets the embeddings as doubles
+        Content-Type: application/json
+      search:
         index: test-index
-        id: doc_1
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
 
-  - set: { _source.sparse_field.inference.chunks.0.embeddings: sparse_field_embedding }
-  - set: { _source.dense_field.inference.chunks.0.embeddings: dense_field_embedding }
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+  - set: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings: sparse_field_embedding }
+  - set: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings: dense_field_embedding }
 
   - do:
       update:
@@ -76,19 +93,23 @@ setup:
         body:
           doc:
             non_inference_field: "another non inference test"
+        refresh: true
 
   - do:
-      get:
+      headers:
+        # Force JSON content type so that we use a parser that interprets the embeddings as doubles
+        Content-Type: application/json
+      search:
         index: test-index
-        id: doc_1
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
 
-  - match:  { _source.sparse_field.text: "inference test" }
-  - match:  { _source.sparse_field.inference.chunks.0.text: "inference test" }
-  - match:  { _source.sparse_field.inference.chunks.0.embeddings: $sparse_field_embedding }
-  - match:  { _source.dense_field.text: "another inference test" }
-  - match:  { _source.dense_field.inference.chunks.0.text: "another inference test" }
-  - match:  { _source.dense_field.inference.chunks.0.embeddings: $dense_field_embedding }
-  - match:  { _source.non_inference_field: "another non inference test" }
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings: $sparse_field_embedding }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings: $dense_field_embedding }
 
 ---
 "Updating semantic_text fields recalculates embeddings":
@@ -100,17 +121,22 @@ setup:
           sparse_field: "inference test"
           dense_field: "another inference test"
           non_inference_field: "non inference test"
+        refresh: true
 
   - do:
-      get:
+      search:
         index: test-index
-        id: doc_1
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
 
-  - match:  { _source.sparse_field.text: "inference test" }
-  - match:  { _source.sparse_field.inference.chunks.0.text: "inference test" }
-  - match:  { _source.dense_field.text: "another inference test" }
-  - match:  { _source.dense_field.inference.chunks.0.text: "another inference test" }
-  - match:  { _source.non_inference_field: "non inference test" }
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 14 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 22 }
 
   - do:
       bulk:
@@ -118,17 +144,24 @@ setup:
         body:
           - '{"update": {"_id": "doc_1"}}'
           - '{"doc":{"sparse_field": "I am a test", "dense_field": "I am a teapot"}}'
+        refresh: true
 
   - do:
-      get:
+      search:
         index: test-index
-        id: doc_1
-
-  - match:  { _source.sparse_field.text: "I am a test" }
-  - match:  { _source.sparse_field.inference.chunks.0.text: "I am a test" }
-  - match:  { _source.dense_field.text: "I am a teapot" }
-  - match:  { _source.dense_field.inference.chunks.0.text: "I am a teapot" }
-  - match:  { _source.non_inference_field: "non inference test" }
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
+
+  # We can't directly check that the embeddings are different since there isn't a "does not match" assertion in the
+  # YAML test framework. Check that the start and end offsets change as expected as a proxy.
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 11 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 13 }
 
   - do:
       update:
@@ -138,17 +171,22 @@ setup:
           doc:
             sparse_field: "updated inference test"
             dense_field: "another updated inference test"
+        refresh: true
 
   - do:
-      get:
+      search:
         index: test-index
-        id: doc_1
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
 
-  - match:  { _source.sparse_field.text: "updated inference test" }
-  - match:  { _source.sparse_field.inference.chunks.0.text: "updated inference test" }
-  - match:  { _source.dense_field.text: "another updated inference test" }
-  - match:  { _source.dense_field.inference.chunks.0.text: "another updated inference test" }
-  - match:  { _source.non_inference_field: "non inference test" }
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 22 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 30 }
 
   - do:
       bulk:
@@ -156,17 +194,22 @@ setup:
         body:
           - '{"update": {"_id": "doc_1"}}'
           - '{"doc":{"sparse_field": "bulk inference test", "dense_field": "bulk updated inference test"}}'
+        refresh: true
 
   - do:
-      get:
+      search:
         index: test-index
-        id: doc_1
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
 
-  - match:  { _source.sparse_field.text: "bulk inference test" }
-  - match:  { _source.sparse_field.inference.chunks.0.text: "bulk inference test" }
-  - match:  { _source.dense_field.text: "bulk updated inference test" }
-  - match:  { _source.dense_field.inference.chunks.0.text: "bulk updated inference test" }
-  - match:  { _source.non_inference_field: "non inference test" }
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 19 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 27 }
 
 ---
 "Update logic handles source fields in object fields":
@@ -174,6 +217,11 @@ setup:
       indices.create:
         index: test-copy-to-index
         body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: false
           mappings:
             properties:
               sparse_field:
@@ -198,18 +246,22 @@ setup:
         body:
           sparse_field: "sparse data 1"
           dense_field: "dense data 1"
+        refresh: true
 
   - do:
-      get:
+      search:
         index: test-copy-to-index
-        id: doc_1
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
 
-  - match: { _source.sparse_field.text: "sparse data 1" }
-  - length: { _source.sparse_field.inference.chunks: 1 }
-  - match: { _source.sparse_field.inference.chunks.0.text: "sparse data 1" }
-  - match: { _source.dense_field.text: "dense data 1" }
-  - length: { _source.dense_field.inference.chunks: 1 }
-  - match: { _source.dense_field.inference.chunks.0.text: "dense data 1" }
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field
 
   - do:
       bulk:
@@ -217,28 +269,38 @@ setup:
         body:
           - '{"update": {"_id": "doc_1"}}'
           - >
-           {
-             "doc": {
-               "sparse_field": "sparse data 1",
-               "object_source.sparse_field": "sparse data 2",
-               "dense_field": "dense data 1",
-               "object_source.dense_field": "dense data 2"
-             }
-           }
+            {
+              "doc": {
+                "object_source.sparse_field": "sparse data two",
+                "object_source.dense_field": "dense data two"
+              }
+            }
+        refresh: true
 
   - do:
-      get:
+      search:
         index: test-copy-to-index
-        id: doc_1
-
-  - match: { _source.sparse_field.text: "sparse data 1" }
-  - length: { _source.sparse_field.inference.chunks: 2 }
-  - match: { _source.sparse_field.inference.chunks.0.text: "sparse data 2" }
-  - match: { _source.sparse_field.inference.chunks.1.text: "sparse data 1" }
-  - match: { _source.dense_field.text: "dense data 1" }
-  - length: { _source.dense_field.inference.chunks: 2 }
-  - match: { _source.dense_field.inference.chunks.0.text: "dense data 1" }
-  - match: { _source.dense_field.inference.chunks.1.text: "dense data 2" }
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 2 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.object_source\\.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.object_source\\.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.object_source\\.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.object_source\\.sparse_field.0.end_offset: 15 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 2 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.object_source\\.dense_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.object_source\\.dense_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.object_source\\.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.object_source\\.dense_field.0.end_offset: 14 }
 
   - do:
       update:
@@ -247,25 +309,35 @@ setup:
         body:
           doc:
             {
-              "sparse_field": "sparse data 1",
-              "object_source.sparse_field": "sparse data 3",
-              "dense_field": "dense data 1",
-              "object_source.dense_field": "dense data 3"
+              "object_source.sparse_field": "sparse data three",
+              "object_source.dense_field": "dense data three"
             }
+        refresh: true
 
   - do:
-      get:
+      search:
         index: test-copy-to-index
-        id: doc_1
-
-  - match: { _source.sparse_field.text: "sparse data 1" }
-  - length: { _source.sparse_field.inference.chunks: 2 }
-  - match: { _source.sparse_field.inference.chunks.0.text: "sparse data 3" }
-  - match: { _source.sparse_field.inference.chunks.1.text: "sparse data 1" }
-  - match: { _source.dense_field.text: "dense data 1" }
-  - length: { _source.dense_field.inference.chunks: 2 }
-  - match: { _source.dense_field.inference.chunks.0.text: "dense data 1" }
-  - match: { _source.dense_field.inference.chunks.1.text: "dense data 3" }
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 2 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.object_source\\.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.object_source\\.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.object_source\\.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.object_source\\.sparse_field.0.end_offset: 17 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 2 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.object_source\\.dense_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.object_source\\.dense_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.object_source\\.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.object_source\\.dense_field.0.end_offset: 16 }
 
 ---
 "Updates fail when the updated value is invalid":
@@ -278,17 +350,6 @@ setup:
           dense_field: "another inference test"
           non_inference_field: "non inference test"
 
-  - do:
-      get:
-        index: test-index
-        id: doc_1
-
-  - match: { _source.sparse_field.text: "inference test" }
-  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
-  - match: { _source.dense_field.text: "another inference test" }
-  - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
-  - match: { _source.non_inference_field: "non inference test" }
-
   - do:
       bulk:
         index: test-index
@@ -317,6 +378,11 @@ setup:
       indices.create:
         index: test-copy-to-index
         body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: false
           mappings:
             properties:
               sparse_field:
@@ -338,42 +404,68 @@ setup:
         id: doc_1
         body:
           sparse_field: "sparse data 1"
-          sparse_source_field: "sparse data 2"
+          sparse_source_field: "sparse data two"
           dense_field: "dense data 1"
-          dense_source_field: "dense data 2"
+          dense_source_field: "dense data two"
+        refresh: true
 
   - do:
-      get:
+      search:
         index: test-copy-to-index
-        id: doc_1
-
-  - length: { _source.sparse_field.inference.chunks: 2 }
-  - match: { _source.sparse_field.inference.chunks.1.text: "sparse data 2" }
-  - exists: _source.sparse_field.inference.chunks.1.embeddings
-  - length: { _source.dense_field.inference.chunks: 2 }
-  - match: { _source.dense_field.inference.chunks.1.text: "dense data 2" }
-  - exists: _source.dense_field.inference.chunks.1.embeddings
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 2 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.end_offset: 15 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 2 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.end_offset: 14 }
 
   - do:
       update:
         index: test-copy-to-index
         id: doc_1
         body:
-          doc: { "sparse_source_field": "sparse data 3", "dense_source_field": "dense data 3" }
+          doc: { "sparse_source_field": "sparse data three", "dense_source_field": "dense data three" }
+        refresh: true
 
   - do:
-      get:
+      search:
         index: test-copy-to-index
-        id: doc_1
-
-  - match: { _source.sparse_field.text: "sparse data 1" }
-  - length: { _source.sparse_field.inference.chunks: 2 }
-  - match: { _source.sparse_field.inference.chunks.1.text: "sparse data 3" }
-  - exists: _source.sparse_field.inference.chunks.1.embeddings
-  - match: { _source.dense_field.text: "dense data 1" }
-  - length: { _source.dense_field.inference.chunks: 2 }
-  - match: { _source.dense_field.inference.chunks.1.text: "dense data 3" }
-  - exists: _source.dense_field.inference.chunks.1.embeddings
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 2 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.end_offset: 17 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 2 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.end_offset: 16 }
 
 ---
 "Partial updates work when using the update API and the semantic_text field's original value is null":
@@ -381,6 +473,11 @@ setup:
       indices.create:
         index: test-copy-to-index
         body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: false
           mappings:
             properties:
               sparse_field:
@@ -402,83 +499,142 @@ setup:
         index: test-copy-to-index
         id: doc_1
         body:
-          sparse_source_field: "sparse data 2"
-          dense_source_field: "dense data 2"
+          sparse_source_field: "sparse data two"
+          dense_source_field: "dense data two"
+        refresh: true
 
   - do:
-      get:
+      search:
         index: test-copy-to-index
-        id: doc_1
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.end_offset: 15 }
 
-  - match: { _source.sparse_field.text: null }
-  - length: { _source.sparse_field.inference.chunks: 1 }
-  - match: { _source.sparse_field.inference.chunks.0.text: "sparse data 2" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.dense_field.text: null }
-  - length: { _source.dense_field.inference.chunks: 1 }
-  - match: { _source.dense_field.inference.chunks.0.text: "dense data 2" }
-  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.end_offset: 14 }
 
   - do:
       update:
         index: test-copy-to-index
         id: doc_1
         body:
-          doc: { "sparse_source_field": "sparse data 3", "dense_source_field": "dense data 3" }
+          doc: { "sparse_source_field": "sparse data three", "dense_source_field": "dense data three" }
+        refresh: true
 
   - do:
-      get:
+      search:
         index: test-copy-to-index
-        id: doc_1
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: {}
 
-  - match: { _source.sparse_field.text: null }
-  - length: { _source.sparse_field.inference.chunks: 1 }
-  - match: { _source.sparse_field.inference.chunks.0.text: "sparse data 3" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.dense_field.text: null }
-  - length: { _source.dense_field.inference.chunks: 1 }
-  - match: { _source.dense_field.inference.chunks.0.text: "dense data 3" }
-  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.end_offset: 17 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.end_offset: 16 }
 
 ---
-"Updates with script are not allowed":
+"Updates with script via bulk API are not allowed":
   - do:
       bulk:
         index: test-index
         body:
           - '{"index": {"_id": "doc_1"}}'
           - '{"doc":{"sparse_field": "I am a test", "dense_field": "I am a teapot"}}'
+        refresh: true
+
+  - match: { errors: false }
 
   - do:
       bulk:
         index: test-index
         body:
           - '{"update": {"_id": "doc_1"}}'
-          - '{"script": "ctx._source.new_field = \"hello\"", "scripted_upsert": true}'
+          - '{"script": "ctx._source.sparse_field = \"sparse data two\"", "scripted_upsert": true}'
+        refresh: true
 
   - match: { errors: true }
   - match: { items.0.update.status: 400 }
   - match: { items.0.update.error.reason: "Cannot apply update with a script on indices that contain [semantic_text] field(s)" }
 
+---
+"Updates with script works when using the update API":
+  - do:
+      bulk:
+        index: test-index
+        body:
+          - '{"index": {"_id": "doc_1"}}'
+          - '{"doc":{"sparse_field": "I am a test", "dense_field": "I am a teapot"}}'
+        refresh: true
+
+  - match: { errors: false }
+
   - do:
-      catch: bad_request
       update:
         index: test-index
         id: doc_1
         body:
           script:
-            source: "ctx._source.new_field = \"hello\""
+            source: "ctx._source.sparse_field = \"sparse data two\"; ctx._source.dense_field = \"dense data two\""
             lang: "painless"
+        refresh: true
 
-  - match: { error.type: "status_exception" }
-  - match: { error.reason: "Cannot apply update with a script on indices that contain inference field(s)" }
+  - do:
+      search:
+        index: test-index
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: { }
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 15 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 14 }
 
 ---
-"semantic_text copy_to needs values for every source field for bulk updates":
+"semantic_text copy_to does not need values for every source field for bulk updates":
   - do:
       indices.create:
         index: test-copy-to-index
         body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: false
           mappings:
             properties:
               sparse_field:
@@ -491,7 +647,6 @@ setup:
                 type: text
                 copy_to: sparse_field
 
-  # Not every source field needed on creation
   - do:
       index:
         index: test-copy-to-index
@@ -500,15 +655,35 @@ setup:
           source_field: "a single source field provided"
           sparse_field: "inference test"
 
-  # Every source field needed on bulk updates
   - do:
       bulk:
         body:
           - '{"update": {"_index": "test-copy-to-index", "_id": "doc_1"}}'
-          - '{"doc": {"source_field": "a single source field is kept as provided via bulk", "sparse_field": "updated inference test" }}'
+          - '{"doc": {"source_field": "updated source field value"}}'
+        refresh: true
 
-  - match: { items.0.update.status: 400 }
-  - match: { items.0.update.error.reason: "Field [another_source_field] must be specified on an update request to calculate inference for field [sparse_field]" }
+  - match: { errors: false }
+
+  - do:
+      search:
+        index: test-copy-to-index
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: { }
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 2 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 14 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.source_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.source_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.source_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.source_field.0.end_offset: 26 }
 
 ---
 "Calculates embeddings for bulk operations - update":
@@ -526,22 +701,33 @@ setup:
         body:
           - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
           - '{"doc": { "sparse_field": "updated inference test", "dense_field": "another updated inference test", "non_inference_field": "updated non inference test" }}'
+        refresh: true
 
   - match: { errors: false }
   - match: { items.0.update.result: "updated" }
 
   - do:
-      get:
+      search:
         index: test-index
-        id: doc_1
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: { }
 
-  - match: { _source.sparse_field.text: "updated inference test" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.sparse_field.inference.chunks.0.text: "updated inference test" }
-  - match: { _source.dense_field.text: "another updated inference test" }
-  - exists: _source.dense_field.inference.chunks.0.embeddings
-  - match: { _source.dense_field.inference.chunks.0.text: "another updated inference test" }
-  - match: { _source.non_inference_field: "updated non inference test" }
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 22 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 30 }
 
   # Script update not supported
   - do:
@@ -572,51 +758,69 @@ setup:
         body:
           - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
           - '{"doc": { "sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test" }, "doc_as_upsert": true}'
+        refresh: true
 
   - match: { errors: false }
   - match: { items.0.update.result: "created" }
 
   - do:
-      get:
+      search:
         index: test-index
-        id: doc_1
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: { }
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
 
-  - match: { _source.sparse_field.text: "inference test" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
-  - match: { _source.dense_field.text: "another inference test" }
-  - exists: _source.dense_field.inference.chunks.0.embeddings
-  - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
-  - match: { _source.non_inference_field: "non inference test" }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 14 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 22 }
 
   - do:
       bulk:
         body:
           - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
           - '{"doc": { "sparse_field": "updated inference test", "dense_field": "another updated inference test", "non_inference_field": "updated non inference test" }, "doc_as_upsert": true}'
+        refresh: true
 
   - match: { errors: false }
   - match: { items.0.update.result: "updated" }
 
   - do:
-      get:
+      search:
         index: test-index
-        id: doc_1
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: { }
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
 
-  - match: { _source.sparse_field.text: "updated inference test" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.sparse_field.inference.chunks.0.text: "updated inference test" }
-  - match: { _source.dense_field.text: "another updated inference test" }
-  - exists: _source.dense_field.inference.chunks.0.embeddings
-  - match: { _source.dense_field.inference.chunks.0.text: "another updated inference test" }
-  - match: { _source.non_inference_field: "updated non inference test" }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 22 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 30 }
 
 ---
 "Bypass inference on bulk update operation":
-  - requires:
-      cluster_features: semantic_text.single_field_update_fix
-      reason: Standalone semantic text fields are now optional in a bulk update operation
-
   # Update as upsert
   - do:
       bulk:
@@ -632,37 +836,193 @@ setup:
         body:
           - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
           - '{"doc": { "non_inference_field": "another value" }, "doc_as_upsert": true}'
+        refresh: true
 
   - match: { errors: false }
   - match: { items.0.update.result: "updated" }
 
   - do:
-      get:
+      search:
         index: test-index
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: { }
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 14 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 22 }
+
+  - match: { hits.hits.0._source.sparse_field: "inference test" }
+  - match: { hits.hits.0._source.dense_field: "another inference test" }
+  - match: { hits.hits.0._source.non_inference_field: "another value" }
+
+---
+"Explicit nulls clear inference results on bulk update operation":
+  - requires:
+      cluster_features: "semantic_text.inference_metadata_fields.explicit_null_fixes"
+      reason: Fixes explicit null handling when using the _inference_fields metafield
+
+  - skip:
+      features: [ "headers" ]
+
+  - do:
+      indices.create:
+        index: test-copy-to-index
+        body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: false
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+              sparse_source_field:
+                type: text
+                copy_to: sparse_field
+              dense_field:
+                type: semantic_text
+                inference_id: dense-inference-id
+              dense_source_field:
+                type: text
+                copy_to: dense_field
+              non_inference_field:
+                type: text
+
+  - do:
+      index:
+        index: test-copy-to-index
         id: doc_1
+        body:
+          sparse_field: "inference test"
+          sparse_source_field: "sparse source test"
+          dense_field: "another inference test"
+          dense_source_field: "dense source test"
+          non_inference_field: "non inference test"
+        refresh: true
 
-  - match: { _source.sparse_field.text: "inference test" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
-  - match: { _source.dense_field.text: "another inference test" }
-  - exists: _source.dense_field.inference.chunks.0.embeddings
-  - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
-  - match: { _source.non_inference_field: "another value" }
+  - do:
+      headers:
+        # Force JSON content type so that we use a parser that interprets the embeddings as doubles
+        Content-Type: application/json
+      search:
+        index: test-copy-to-index
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: { }
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 2 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 14 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.embeddings
+  - set: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.embeddings: sparse_source_field_embeddings }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.end_offset: 18 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 2 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 22 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field: 1 }
+  - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.embeddings
+  - set: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.embeddings: dense_source_field_embeddings }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.end_offset: 17 }
+
+  - match: { hits.hits.0._source.sparse_field: "inference test" }
+  - match: { hits.hits.0._source.sparse_source_field: "sparse source test" }
+  - match: { hits.hits.0._source.dense_field: "another inference test" }
+  - match: { hits.hits.0._source.dense_source_field: "dense source test" }
+  - match: { hits.hits.0._source.non_inference_field: "non inference test" }
 
   - do:
       bulk:
         body:
-          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"update": {"_index": "test-copy-to-index", "_id": "doc_1"}}'
           - '{"doc": { "sparse_field": null, "dense_field": null, "non_inference_field": "updated value" }, "doc_as_upsert": true}'
+        refresh: true
 
   - match: { errors: false }
   - match: { items.0.update.result: "updated" }
 
   - do:
-      get:
-        index: test-index
-        id: doc_1
+      headers:
+        # Force JSON content type so that we use a parser that interprets the embeddings as doubles
+        Content-Type: application/json
+      search:
+        index: test-copy-to-index
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: { }
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field: 1 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.embeddings: $sparse_source_field_embeddings }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.end_offset: 18 }
+
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 }
+  - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field: 1 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.embeddings: $dense_source_field_embeddings }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.start_offset: 0 }
+  - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.end_offset: 17 }
+
+  - not_exists: hits.hits.0._source.sparse_field
+  - match: { hits.hits.0._source.sparse_source_field: "sparse source test" }
+  - not_exists: hits.hits.0._source.dense_field
+  - match: { hits.hits.0._source.dense_source_field: "dense source test" }
+  - match: { hits.hits.0._source.non_inference_field: "updated value" }
 
-  - match: { _source.sparse_field: null }
-  - match: { _source.dense_field: null }
-  - match: { _source.non_inference_field: "updated value" }
+  - do:
+      bulk:
+        body:
+          - '{"update": {"_index": "test-copy-to-index", "_id": "doc_1"}}'
+          - '{"doc": { "sparse_source_field": null, "dense_source_field": null, "non_inference_field": "another value" }, "doc_as_upsert": true}'
+        refresh: true
+
+  - match: { errors: false }
+  - match: { items.0.update.result: "updated" }
+
+  - do:
+      search:
+        index: test-copy-to-index
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: { }
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.total.relation: eq }
+
+  - not_exists: hits.hits.0._source._inference_fields
+  - not_exists: hits.hits.0._source.sparse_field
+  - not_exists: hits.hits.0._source.sparse_source_field
+  - not_exists: hits.hits.0._source.dense_field
+  - not_exists: hits.hits.0._source.dense_source_field
+  - match: { hits.hits.0._source.non_inference_field: "another value" }
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update_bwc.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update_bwc.yml
new file mode 100644
index 0000000000000..912cdb5a85d35
--- /dev/null
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update_bwc.yml
@@ -0,0 +1,676 @@
+setup:
+  - requires:
+      cluster_features: "gte_v8.15.0"
+      reason: semantic_text introduced in 8.15.0
+
+  - do:
+      inference.put:
+        task_type: sparse_embedding
+        inference_id: sparse-inference-id
+        body: >
+          {
+            "service": "test_service",
+            "service_settings": {
+              "model": "my_model",
+              "api_key": "abc64"
+            },
+            "task_settings": {
+            }
+          }
+
+  - do:
+      inference.put:
+        task_type: text_embedding
+        inference_id: dense-inference-id
+        body: >
+          {
+            "service": "text_embedding_test_service",
+            "service_settings": {
+              "model": "my_model",
+              "dimensions": 10,
+              "similarity": "cosine",
+              "api_key": "abc64"
+            },
+            "task_settings": {
+            }
+          }
+
+  - do:
+      indices.create:
+        index: test-index
+        body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: true
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+              dense_field:
+                type: semantic_text
+                inference_id: dense-inference-id
+              non_inference_field:
+                type: text
+
+---
+"Updating non semantic_text fields does not recalculate embeddings":
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field: "inference test"
+          dense_field: "another inference test"
+          non_inference_field: "non inference test"
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - set: { _source.sparse_field.inference.chunks.0.embeddings: sparse_field_embedding }
+  - set: { _source.dense_field.inference.chunks.0.embeddings: dense_field_embedding }
+
+  - do:
+      update:
+        index: test-index
+        id: doc_1
+        body:
+          doc:
+            non_inference_field: "another non inference test"
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match:  { _source.sparse_field.text: "inference test" }
+  - match:  { _source.sparse_field.inference.chunks.0.text: "inference test" }
+  - match:  { _source.sparse_field.inference.chunks.0.embeddings: $sparse_field_embedding }
+  - match:  { _source.dense_field.text: "another inference test" }
+  - match:  { _source.dense_field.inference.chunks.0.text: "another inference test" }
+  - match:  { _source.dense_field.inference.chunks.0.embeddings: $dense_field_embedding }
+  - match:  { _source.non_inference_field: "another non inference test" }
+
+---
+"Updating semantic_text fields recalculates embeddings":
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field: "inference test"
+          dense_field: "another inference test"
+          non_inference_field: "non inference test"
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match:  { _source.sparse_field.text: "inference test" }
+  - match:  { _source.sparse_field.inference.chunks.0.text: "inference test" }
+  - match:  { _source.dense_field.text: "another inference test" }
+  - match:  { _source.dense_field.inference.chunks.0.text: "another inference test" }
+  - match:  { _source.non_inference_field: "non inference test" }
+
+  - do:
+      bulk:
+        index: test-index
+        body:
+          - '{"update": {"_id": "doc_1"}}'
+          - '{"doc":{"sparse_field": "I am a test", "dense_field": "I am a teapot"}}'
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match:  { _source.sparse_field.text: "I am a test" }
+  - match:  { _source.sparse_field.inference.chunks.0.text: "I am a test" }
+  - match:  { _source.dense_field.text: "I am a teapot" }
+  - match:  { _source.dense_field.inference.chunks.0.text: "I am a teapot" }
+  - match:  { _source.non_inference_field: "non inference test" }
+
+  - do:
+      update:
+        index: test-index
+        id: doc_1
+        body:
+          doc:
+            sparse_field: "updated inference test"
+            dense_field: "another updated inference test"
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match:  { _source.sparse_field.text: "updated inference test" }
+  - match:  { _source.sparse_field.inference.chunks.0.text: "updated inference test" }
+  - match:  { _source.dense_field.text: "another updated inference test" }
+  - match:  { _source.dense_field.inference.chunks.0.text: "another updated inference test" }
+  - match:  { _source.non_inference_field: "non inference test" }
+
+  - do:
+      bulk:
+        index: test-index
+        body:
+          - '{"update": {"_id": "doc_1"}}'
+          - '{"doc":{"sparse_field": "bulk inference test", "dense_field": "bulk updated inference test"}}'
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match:  { _source.sparse_field.text: "bulk inference test" }
+  - match:  { _source.sparse_field.inference.chunks.0.text: "bulk inference test" }
+  - match:  { _source.dense_field.text: "bulk updated inference test" }
+  - match:  { _source.dense_field.inference.chunks.0.text: "bulk updated inference test" }
+  - match:  { _source.non_inference_field: "non inference test" }
+
+---
+"Update logic handles source fields in object fields":
+  - do:
+      indices.create:
+        index: test-copy-to-index
+        body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: true
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+              dense_field:
+                type: semantic_text
+                inference_id: dense-inference-id
+              object_source:
+                properties:
+                  sparse_field:
+                    type: text
+                    copy_to: sparse_field
+                  dense_field:
+                    type: text
+                    copy_to: dense_field
+
+  - do:
+      index:
+        index: test-copy-to-index
+        id: doc_1
+        body:
+          sparse_field: "sparse data 1"
+          dense_field: "dense data 1"
+
+  - do:
+      get:
+        index: test-copy-to-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "sparse data 1" }
+  - length: { _source.sparse_field.inference.chunks: 1 }
+  - match: { _source.sparse_field.inference.chunks.0.text: "sparse data 1" }
+  - match: { _source.dense_field.text: "dense data 1" }
+  - length: { _source.dense_field.inference.chunks: 1 }
+  - match: { _source.dense_field.inference.chunks.0.text: "dense data 1" }
+
+  - do:
+      bulk:
+        index: test-copy-to-index
+        body:
+          - '{"update": {"_id": "doc_1"}}'
+          - >
+            {
+              "doc": {
+                "sparse_field": "sparse data 1",
+                "object_source.sparse_field": "sparse data 2",
+                "dense_field": "dense data 1",
+                "object_source.dense_field": "dense data 2"
+              }
+            }
+
+  - do:
+      get:
+        index: test-copy-to-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "sparse data 1" }
+  - length: { _source.sparse_field.inference.chunks: 2 }
+  - match: { _source.sparse_field.inference.chunks.0.text: "sparse data 2" }
+  - match: { _source.sparse_field.inference.chunks.1.text: "sparse data 1" }
+  - match: { _source.dense_field.text: "dense data 1" }
+  - length: { _source.dense_field.inference.chunks: 2 }
+  - match: { _source.dense_field.inference.chunks.0.text: "dense data 1" }
+  - match: { _source.dense_field.inference.chunks.1.text: "dense data 2" }
+
+  - do:
+      update:
+        index: test-copy-to-index
+        id: doc_1
+        body:
+          doc:
+            {
+              "sparse_field": "sparse data 1",
+              "object_source.sparse_field": "sparse data 3",
+              "dense_field": "dense data 1",
+              "object_source.dense_field": "dense data 3"
+            }
+
+  - do:
+      get:
+        index: test-copy-to-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "sparse data 1" }
+  - length: { _source.sparse_field.inference.chunks: 2 }
+  - match: { _source.sparse_field.inference.chunks.0.text: "sparse data 3" }
+  - match: { _source.sparse_field.inference.chunks.1.text: "sparse data 1" }
+  - match: { _source.dense_field.text: "dense data 1" }
+  - length: { _source.dense_field.inference.chunks: 2 }
+  - match: { _source.dense_field.inference.chunks.0.text: "dense data 1" }
+  - match: { _source.dense_field.inference.chunks.1.text: "dense data 3" }
+
+---
+"Partial updates work when using the update API":
+  - do:
+      indices.create:
+        index: test-copy-to-index
+        body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: true
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+              sparse_source_field:
+                type: text
+                copy_to: sparse_field
+              dense_field:
+                type: semantic_text
+                inference_id: dense-inference-id
+              dense_source_field:
+                type: text
+                copy_to: dense_field
+
+  - do:
+      index:
+        index: test-copy-to-index
+        id: doc_1
+        body:
+          sparse_field: "sparse data 1"
+          sparse_source_field: "sparse data 2"
+          dense_field: "dense data 1"
+          dense_source_field: "dense data 2"
+
+  - do:
+      get:
+        index: test-copy-to-index
+        id: doc_1
+
+  - length: { _source.sparse_field.inference.chunks: 2 }
+  - match: { _source.sparse_field.inference.chunks.1.text: "sparse data 2" }
+  - exists: _source.sparse_field.inference.chunks.1.embeddings
+  - length: { _source.dense_field.inference.chunks: 2 }
+  - match: { _source.dense_field.inference.chunks.1.text: "dense data 2" }
+  - exists: _source.dense_field.inference.chunks.1.embeddings
+
+  - do:
+      update:
+        index: test-copy-to-index
+        id: doc_1
+        body:
+          doc: { "sparse_source_field": "sparse data 3", "dense_source_field": "dense data 3" }
+
+  - do:
+      get:
+        index: test-copy-to-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "sparse data 1" }
+  - length: { _source.sparse_field.inference.chunks: 2 }
+  - match: { _source.sparse_field.inference.chunks.1.text: "sparse data 3" }
+  - exists: _source.sparse_field.inference.chunks.1.embeddings
+  - match: { _source.dense_field.text: "dense data 1" }
+  - length: { _source.dense_field.inference.chunks: 2 }
+  - match: { _source.dense_field.inference.chunks.1.text: "dense data 3" }
+  - exists: _source.dense_field.inference.chunks.1.embeddings
+
+---
+"Partial updates work when using the update API and the semantic_text field's original value is null":
+  - do:
+      indices.create:
+        index: test-copy-to-index
+        body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: true
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+              sparse_source_field:
+                type: text
+                copy_to: sparse_field
+              dense_field:
+                type: semantic_text
+                inference_id: dense-inference-id
+              dense_source_field:
+                type: text
+                copy_to: dense_field
+
+  # Don't set sparse_field or dense_field so their original value is null
+  - do:
+      index:
+        index: test-copy-to-index
+        id: doc_1
+        body:
+          sparse_source_field: "sparse data 2"
+          dense_source_field: "dense data 2"
+
+  - do:
+      get:
+        index: test-copy-to-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: null }
+  - length: { _source.sparse_field.inference.chunks: 1 }
+  - match: { _source.sparse_field.inference.chunks.0.text: "sparse data 2" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.text: null }
+  - length: { _source.dense_field.inference.chunks: 1 }
+  - match: { _source.dense_field.inference.chunks.0.text: "dense data 2" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+
+  - do:
+      update:
+        index: test-copy-to-index
+        id: doc_1
+        body:
+          doc: { "sparse_source_field": "sparse data 3", "dense_source_field": "dense data 3" }
+
+  - do:
+      get:
+        index: test-copy-to-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: null }
+  - length: { _source.sparse_field.inference.chunks: 1 }
+  - match: { _source.sparse_field.inference.chunks.0.text: "sparse data 3" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.text: null }
+  - length: { _source.dense_field.inference.chunks: 1 }
+  - match: { _source.dense_field.inference.chunks.0.text: "dense data 3" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+
+---
+"Updates with script are not allowed":
+  - do:
+      bulk:
+        index: test-index
+        body:
+          - '{"index": {"_id": "doc_1"}}'
+          - '{"doc":{"sparse_field": "I am a test", "dense_field": "I am a teapot"}}'
+
+  - match: { errors: false }
+
+  - do:
+      bulk:
+        index: test-index
+        body:
+          - '{"update": {"_id": "doc_1"}}'
+          - '{"script": "ctx._source.new_field = \"hello\"", "scripted_upsert": true}'
+
+  - match: { errors: true }
+  - match: { items.0.update.status: 400 }
+  - match: { items.0.update.error.reason: "Cannot apply update with a script on indices that contain [semantic_text] field(s)" }
+
+  - do:
+      catch: bad_request
+      update:
+        index: test-index
+        id: doc_1
+        body:
+          script:
+            source: "ctx._source.new_field = \"hello\""
+            lang: "painless"
+
+  - match: { error.type: "status_exception" }
+  - match: { error.reason: "Cannot apply update with a script on indices that contain inference field(s)" }
+
+---
+"semantic_text copy_to needs values for every source field for bulk updates":
+  - do:
+      indices.create:
+        index: test-copy-to-index
+        body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: true
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+              source_field:
+                type: text
+                copy_to: sparse_field
+              another_source_field:
+                type: text
+                copy_to: sparse_field
+
+  # Not every source field needed on creation
+  - do:
+      index:
+        index: test-copy-to-index
+        id: doc_1
+        body:
+          source_field: "a single source field provided"
+          sparse_field: "inference test"
+
+  # Every source field needed on bulk updates
+  - do:
+      bulk:
+        body:
+          - '{"update": {"_index": "test-copy-to-index", "_id": "doc_1"}}'
+          - '{"doc": {"source_field": "a single source field is kept as provided via bulk", "sparse_field": "updated inference test" }}'
+
+  - match: { items.0.update.status: 400 }
+  - match: { items.0.update.error.reason: "Field [another_source_field] must be specified on an update request to calculate inference for field [sparse_field]" }
+
+---
+"Calculates embeddings for bulk operations - update":
+  - do:
+      bulk:
+        body:
+          - '{"index": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test"}'
+
+  - match: { errors: false }
+  - match: { items.0.index.result: "created" }
+
+  - do:
+      bulk:
+        body:
+          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"doc": { "sparse_field": "updated inference test", "dense_field": "another updated inference test", "non_inference_field": "updated non inference test" }}'
+
+  - match: { errors: false }
+  - match: { items.0.update.result: "updated" }
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "updated inference test" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "updated inference test" }
+  - match: { _source.dense_field.text: "another updated inference test" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.inference.chunks.0.text: "another updated inference test" }
+  - match: { _source.non_inference_field: "updated non inference test" }
+
+  # Script update not supported
+  - do:
+      bulk:
+        body:
+          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"script": {"source": {"ctx.sparse_field": "updated inference test"}}}'
+
+  - match: { errors: true }
+  - match: { items.0.update.status: 400 }
+  - match: { items.0.update.error.reason: "Cannot apply update with a script on indices that contain [semantic_text] field(s)" }
+
+---
+"Calculates embeddings for bulk operations - upsert":
+  # Initial update fails
+  - do:
+      bulk:
+        body:
+          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"doc": { "sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test" }}'
+
+  - match: { errors: true }
+  - match: { items.0.update.status: 404 }
+
+  # Update as upsert
+  - do:
+      bulk:
+        body:
+          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"doc": { "sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test" }, "doc_as_upsert": true}'
+
+  - match: { errors: false }
+  - match: { items.0.update.result: "created" }
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "inference test" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
+  - match: { _source.dense_field.text: "another inference test" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
+  - match: { _source.non_inference_field: "non inference test" }
+
+  - do:
+      bulk:
+        body:
+          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"doc": { "sparse_field": "updated inference test", "dense_field": "another updated inference test", "non_inference_field": "updated non inference test" }, "doc_as_upsert": true}'
+
+  - match: { errors: false }
+  - match: { items.0.update.result: "updated" }
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "updated inference test" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "updated inference test" }
+  - match: { _source.dense_field.text: "another updated inference test" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.inference.chunks.0.text: "another updated inference test" }
+  - match: { _source.non_inference_field: "updated non inference test" }
+
+---
+"Bypass inference on bulk update operation":
+  - requires:
+      cluster_features: semantic_text.single_field_update_fix
+      reason: Standalone semantic text fields are now optional in a bulk update operation
+
+  # Update as upsert
+  - do:
+      bulk:
+        body:
+          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"doc": { "sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test" }, "doc_as_upsert": true}'
+
+  - match: { errors: false }
+  - match: { items.0.update.result: "created" }
+
+  - do:
+      bulk:
+        body:
+          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"doc": { "non_inference_field": "another value" }, "doc_as_upsert": true}'
+
+  - match: { errors: false }
+  - match: { items.0.update.result: "updated" }
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "inference test" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
+  - match: { _source.dense_field.text: "another inference test" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
+  - match: { _source.non_inference_field: "another value" }
+
+---
+"Explicit nulls clear inference results on bulk update operation":
+  # Update as upsert
+  - do:
+      bulk:
+        body:
+          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"doc": { "sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test" }, "doc_as_upsert": true}'
+
+  - match: { errors: false }
+  - match: { items.0.update.result: "created" }
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "inference test" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
+  - match: { _source.dense_field.text: "another inference test" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
+  - match: { _source.non_inference_field: "non inference test" }
+
+  - do:
+      bulk:
+        body:
+          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"doc": { "sparse_field": null, "dense_field": null, "non_inference_field": "updated value" }, "doc_as_upsert": true}'
+
+  - match: { errors: false }
+  - match: { items.0.update.result: "updated" }
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match: { _source.sparse_field: null }
+  - match: { _source.dense_field: null }
+  - match: { _source.non_inference_field: "updated value" }
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml
index 2980d42d22c3e..88569daaa6070 100644
--- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml
@@ -4,8 +4,6 @@ setup:
         - close_to
         - contains
   - requires:
-      cluster_features: "text_similarity_reranker_retriever_supported"
-      reason: semantic reranking introduced in 8.15.0
       test_runner_features: "close_to"
 
   - do:
@@ -216,3 +214,82 @@ setup:
   - close_to: { hits.hits.0._explanation.value: { value: 0.4, error: 0.000001 } }
   - match: {hits.hits.0._explanation.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[text\\].*/" }
   - match: {hits.hits.0._explanation.details.0.description: "/weight.*science.*/" }
+
+---
+"text similarity reranker properly handles aliases":
+  - requires:
+      cluster_features: "text_similarity_reranker_alias_handling_fix"
+      reason: Test for alias handling fix
+
+  # Create an empty index that will have an earlier shard index than the index with the desired result when referenced
+  # via the alias
+  - do:
+      indices.create:
+        index: first-test-index
+        body:
+          mappings:
+            properties:
+              text:
+                type: text
+              topic:
+                type: keyword
+              subtopic:
+                type: keyword
+
+  - do:
+      indices.create:
+        index: second-test-index
+        body:
+          settings:
+            number_of_shards: 2
+            number_of_replicas: 0
+          mappings:
+            properties:
+              text:
+                type: text
+              topic:
+                type: keyword
+              subtopic:
+                type: keyword
+
+  - do:
+      indices.put_alias:
+        index: first-test-index
+        name: test-alias
+
+  - do:
+      indices.put_alias:
+        index: second-test-index
+        name: test-alias
+
+  - do:
+      index:
+        index: second-test-index
+        id: doc_1
+        body:
+          text: "As seen from Earth, a solar eclipse happens when the Moon is directly between the Earth and the Sun."
+          topic: [ "science" ]
+          subtopic: [ "technology" ]
+        refresh: true
+
+  - do:
+      search:
+        index: test-alias
+        body:
+          track_total_hits: true
+          retriever:
+            text_similarity_reranker:
+              retriever:
+                standard:
+                  query:
+                    term:
+                      topic: "science"
+              rank_window_size: 10
+              inference_id: my-rerank-model
+              inference_text: "How often does the moon hide the sun?"
+              field: text
+          size: 10
+
+  - match: { hits.total.value: 1 }
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._id: "doc_1" }
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/90_semantic_text_highlighter.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/90_semantic_text_highlighter.yml
index 25cd1b5aec48a..ca87c97fc3acd 100644
--- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/90_semantic_text_highlighter.yml
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/90_semantic_text_highlighter.yml
@@ -39,6 +39,8 @@ setup:
       indices.create:
         index: test-sparse-index
         body:
+          settings:
+            index.mapping.semantic_text.use_legacy_format: false
           mappings:
             properties:
               body:
@@ -49,6 +51,8 @@ setup:
       indices.create:
         index: test-dense-index
         body:
+          settings:
+            index.mapping.semantic_text.use_legacy_format: false
           mappings:
             properties:
               body:
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/90_semantic_text_highlighter_bwc.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/90_semantic_text_highlighter_bwc.yml
new file mode 100644
index 0000000000000..3b2c8ba4f8082
--- /dev/null
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/90_semantic_text_highlighter_bwc.yml
@@ -0,0 +1,246 @@
+setup:
+  - requires:
+      cluster_features: "semantic_text.highlighter"
+      reason: a new highlighter for semantic text field
+
+  - do:
+      inference.put:
+        task_type: sparse_embedding
+        inference_id: sparse-inference-id
+        body: >
+          {
+            "service": "test_service",
+            "service_settings": {
+              "model": "my_model",
+              "api_key": "abc64"
+            },
+            "task_settings": {
+            }
+          }
+
+  - do:
+      inference.put:
+        task_type: text_embedding
+        inference_id: dense-inference-id
+        body: >
+          {
+            "service": "text_embedding_test_service",
+            "service_settings": {
+              "model": "my_model",
+              "dimensions": 10,
+              "api_key": "abc64",
+              "similarity": "COSINE"
+            },
+            "task_settings": {
+            }
+          }
+
+  - do:
+      indices.create:
+        index: test-sparse-index
+        body:
+          settings:
+            index.mapping.semantic_text.use_legacy_format: true
+          mappings:
+            properties:
+              body:
+                type: semantic_text
+                inference_id: sparse-inference-id
+
+  - do:
+      indices.create:
+        index: test-dense-index
+        body:
+          settings:
+            index.mapping.semantic_text.use_legacy_format: true
+          mappings:
+            properties:
+              body:
+                type: semantic_text
+                inference_id: dense-inference-id
+
+---
+"Highlighting using a sparse embedding model":
+  - do:
+      index:
+        index: test-sparse-index
+        id: doc_1
+        body:
+          body: ["ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides.", "You Know, for Search!"]
+        refresh: true
+
+  - match: { result: created }
+
+  - do:
+      search:
+        index: test-sparse-index
+        body:
+          query:
+            semantic:
+              field: "body"
+              query: "What is Elasticsearch?"
+          highlight:
+            fields:
+              body:
+                type: "semantic"
+                number_of_fragments: 1
+
+  - match:  { hits.total.value: 1 }
+  - match:  { hits.hits.0._id: "doc_1" }
+  - length: { hits.hits.0.highlight.body: 1 }
+  - match:  { hits.hits.0.highlight.body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." }
+
+  - do:
+      search:
+        index: test-sparse-index
+        body:
+          query:
+            semantic:
+              field: "body"
+              query: "What is Elasticsearch?"
+          highlight:
+            fields:
+              body:
+                type: "semantic"
+                number_of_fragments: 2
+
+  - match:  { hits.total.value: 1 }
+  - match:  { hits.hits.0._id: "doc_1" }
+  - length: { hits.hits.0.highlight.body: 2 }
+  - match:  { hits.hits.0.highlight.body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." }
+  - match:  { hits.hits.0.highlight.body.1: "You Know, for Search!" }
+
+  - do:
+      search:
+        index: test-sparse-index
+        body:
+          query:
+            semantic:
+              field: "body"
+              query: "What is Elasticsearch?"
+          highlight:
+            fields:
+              body:
+                type: "semantic"
+                order: "score"
+                number_of_fragments: 1
+
+  - match:  { hits.total.value: 1 }
+  - match:  { hits.hits.0._id: "doc_1" }
+  - length: { hits.hits.0.highlight.body: 1 }
+  - match:  { hits.hits.0.highlight.body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." }
+
+  - do:
+      search:
+        index: test-sparse-index
+        body:
+          query:
+            semantic:
+              field: "body"
+              query: "What is Elasticsearch?"
+          highlight:
+            fields:
+              body:
+                type: "semantic"
+                order: "score"
+                number_of_fragments: 2
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "doc_1" }
+  - length: { hits.hits.0.highlight.body: 2 }
+  - match: { hits.hits.0.highlight.body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." }
+  - match: { hits.hits.0.highlight.body.1: "You Know, for Search!" }
+
+---
+"Highlighting using a dense embedding model":
+  - do:
+      index:
+        index: test-dense-index
+        id: doc_1
+        body:
+          body: ["ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides.", "You Know, for Search!"]
+        refresh: true
+
+  - match: { result: created }
+
+  - do:
+      search:
+        index: test-dense-index
+        body:
+          query:
+            semantic:
+              field: "body"
+              query: "What is Elasticsearch?"
+          highlight:
+            fields:
+              body:
+                type: "semantic"
+                number_of_fragments: 1
+
+  - match:  { hits.total.value: 1 }
+  - match:  { hits.hits.0._id: "doc_1" }
+  - length: { hits.hits.0.highlight.body: 1 }
+  - match:  { hits.hits.0.highlight.body.0: "You Know, for Search!" }
+
+  - do:
+      search:
+        index: test-dense-index
+        body:
+          query:
+            semantic:
+              field: "body"
+              query: "What is Elasticsearch?"
+          highlight:
+            fields:
+              body:
+                type: "semantic"
+                number_of_fragments: 2
+
+  - match:  { hits.total.value: 1 }
+  - match:  { hits.hits.0._id: "doc_1" }
+  - length: { hits.hits.0.highlight.body: 2 }
+  - match:  { hits.hits.0.highlight.body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." }
+  - match:  { hits.hits.0.highlight.body.1: "You Know, for Search!" }
+
+  - do:
+      search:
+        index: test-dense-index
+        body:
+          query:
+            semantic:
+              field: "body"
+              query: "What is Elasticsearch?"
+          highlight:
+            fields:
+              body:
+                type: "semantic"
+                order: "score"
+                number_of_fragments: 1
+
+  - match:  { hits.total.value: 1 }
+  - match:  { hits.hits.0._id: "doc_1" }
+  - length: { hits.hits.0.highlight.body: 1 }
+  - match:  { hits.hits.0.highlight.body.0: "You Know, for Search!" }
+
+  - do:
+      search:
+        index: test-dense-index
+        body:
+          query:
+            semantic:
+              field: "body"
+              query: "What is Elasticsearch?"
+          highlight:
+            fields:
+              body:
+                type: "semantic"
+                order: "score"
+                number_of_fragments: 2
+
+  - match:  { hits.total.value: 1 }
+  - match:  { hits.hits.0._id: "doc_1" }
+  - length: { hits.hits.0.highlight.body: 2 }
+  - match:  { hits.hits.0.highlight.body.0: "You Know, for Search!" }
+  - match:  { hits.hits.0.highlight.body.1: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." }
+
+
diff --git a/x-pack/plugin/kql/build.gradle b/x-pack/plugin/kql/build.gradle
index 76a4bd5aff777..3d27906b0bc8f 100644
--- a/x-pack/plugin/kql/build.gradle
+++ b/x-pack/plugin/kql/build.gradle
@@ -13,9 +13,9 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.publish'
 
 esplugin {
-  name 'x-pack-kql'
-  description 'Elasticsearch Expanded Pack Plugin - KQL query'
-  classname 'org.elasticsearch.xpack.kql.KqlPlugin'
+  name = 'x-pack-kql'
+  description = 'Elasticsearch Expanded Pack Plugin - KQL query'
+  classname ='org.elasticsearch.xpack.kql.KqlPlugin'
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java
index 217513bd2c0da..e31db37953d43 100644
--- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java
+++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.kql;
 
-import org.elasticsearch.Build;
 import org.elasticsearch.plugins.ExtensiblePlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
@@ -18,10 +17,6 @@
 public class KqlPlugin extends Plugin implements SearchPlugin, ExtensiblePlugin {
     @Override
     public List<QuerySpec<?>> getQueries() {
-        if (Build.current().isSnapshot()) {
-            return List.of(new SearchPlugin.QuerySpec<>(KqlQueryBuilder.NAME, KqlQueryBuilder::new, KqlQueryBuilder::fromXContent));
-        }
-
-        return List.of();
+        return List.of(new SearchPlugin.QuerySpec<>(KqlQueryBuilder.NAME, KqlQueryBuilder::new, KqlQueryBuilder::fromXContent));
     }
 }
diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java
index e2817665d8f79..a77e1453e66ca 100644
--- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java
+++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java
@@ -97,7 +97,7 @@ public static KqlQueryBuilder fromXContent(XContentParser parser) {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.KQL_QUERY_ADDED;
+        return TransportVersions.KQL_QUERY_TECH_PREVIEW;
     }
 
     public String queryString() {
diff --git a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java
index 7323f7d6d1a4e..2643e95df8946 100644
--- a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java
+++ b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.kql.query;
 
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Build;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -22,7 +21,6 @@
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.xpack.kql.KqlPlugin;
 import org.hamcrest.Matchers;
-import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -36,12 +34,6 @@
 import static org.hamcrest.Matchers.nullValue;
 
 public class KqlQueryBuilderTests extends AbstractQueryTestCase<KqlQueryBuilder> {
-    @BeforeClass
-    protected static void ensureSnapshotBuild() {
-        assumeTrue("requires snapshot builds", Build.current().isSnapshot());
-    }
-
-    @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return List.of(KqlPlugin.class);
     }
diff --git a/x-pack/plugin/logsdb/build.gradle b/x-pack/plugin/logsdb/build.gradle
index 1aef69e0e3fac..bef07258a8e33 100644
--- a/x-pack/plugin/logsdb/build.gradle
+++ b/x-pack/plugin/logsdb/build.gradle
@@ -8,13 +8,14 @@
 evaluationDependsOn(xpackModule('core'))
 
 apply plugin: 'elasticsearch.internal-es-plugin'
+apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
 esplugin {
-  name 'logsdb'
-  description 'A plugin for logsdb related functionality'
-  classname 'org.elasticsearch.xpack.logsdb.LogsDBPlugin'
+  name = 'logsdb'
+  description = 'A plugin for logsdb related functionality'
+  classname ='org.elasticsearch.xpack.logsdb.LogsDBPlugin'
   extendedPlugins = ['x-pack-core']
 }
 base {
@@ -29,7 +30,9 @@ restResources {
 
 dependencies {
   compileOnly project(path: xpackModule('core'))
+  testImplementation project(':modules:data-streams')
   testImplementation(testArtifact(project(xpackModule('core'))))
+  internalClusterTestImplementation(testArtifact(project(xpackModule('core'))))
 }
 
 tasks.named("javaRestTest").configure {
diff --git a/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java b/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java
index 381c83ceee289..4a9d13bc642d7 100644
--- a/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java
+++ b/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java
@@ -9,7 +9,9 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
@@ -203,4 +205,59 @@ public void testLogsdbOverrideDefaultModeForLogsIndex() throws IOException {
         assertEquals("logsdb", settings.get("index.mode"));
         assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode"));
     }
+
+    public void testLogsdbRouteOnSortFields() throws IOException {
+        Request request = new Request("PUT", "/_cluster/settings");
+        request.setJsonEntity("{ \"transient\": { \"cluster.logsdb.enabled\": true } }");
+        assertOK(client().performRequest(request));
+
+        request = new Request("POST", "/_index_template/1");
+        request.setJsonEntity("""
+            {
+                "index_patterns": ["my-log-*"],
+                "data_stream": {
+                },
+                "template": {
+                    "settings":{
+                        "index": {
+                            "mode": "logsdb",
+                            "sort.field": [ "host.name", "message", "@timestamp" ],
+                            "logsdb.route_on_sort_fields": true
+                        }
+                    },
+                    "mappings": {
+                        "properties": {
+                            "@timestamp" : {
+                                "type": "date"
+                            },
+                            "host.name": {
+                                "type": "keyword"
+                            },
+                            "message": {
+                                "type": "keyword"
+                            }
+                        }
+                    }
+                }
+            }
+            """);
+        assertOK(client().performRequest(request));
+
+        request = new Request("POST", "/my-log-foo/_doc");
+        request.setJsonEntity("""
+            {
+                "@timestamp": "2020-01-01T00:00:00.000Z",
+                "host.name": "foo",
+                "message": "bar"
+            }
+            """);
+        assertOK(client().performRequest(request));
+
+        String index = DataStream.getDefaultBackingIndexName("my-log-foo", 1);
+        var settings = (Map<?, ?>) ((Map<?, ?>) getIndexSettings(index).get(index)).get("settings");
+        assertEquals("logsdb", settings.get("index.mode"));
+        assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode"));
+        assertEquals("true", settings.get(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey()));
+        assertEquals(List.of("host.name", "message"), settings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey()));
+    }
 }
diff --git a/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsIndexingIT.java b/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsIndexingIT.java
new file mode 100644
index 0000000000000..1ab49f91376f1
--- /dev/null
+++ b/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsIndexingIT.java
@@ -0,0 +1,213 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb;
+
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.Template;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.common.time.FormatNames;
+import org.elasticsearch.datastreams.DataStreamsPlugin;
+import org.elasticsearch.license.LicenseSettings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.XPackPlugin;
+
+import java.time.Instant;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class LogsIndexingIT extends ESSingleNodeTestCase {
+
+    public static final String MAPPING_TEMPLATE = """
+        {
+          "_doc":{
+            "properties": {
+              "@timestamp" : {
+                "type": "date"
+              },
+              "message": {
+                "type": "keyword"
+              },
+              "k8s": {
+                "properties": {
+                  "pod": {
+                    "properties": {
+                      "uid": {
+                        "type": "keyword"
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }""";
+
+    private static final String DOC = """
+        {
+            "@timestamp": "$time",
+            "message": "$pod",
+            "k8s": {
+                "pod": {
+                    "name": "dog",
+                    "uid":"$uuid",
+                    "ip": "10.10.55.3",
+                    "network": {
+                        "tx": 1434595272,
+                        "rx": 530605511
+                    }
+                }
+            }
+        }
+        """;
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return List.of(InternalSettingsPlugin.class, XPackPlugin.class, LogsDBPlugin.class, DataStreamsPlugin.class);
+    }
+
+    @Override
+    protected Settings nodeSettings() {
+        return Settings.builder()
+            .put(super.nodeSettings())
+            .put("cluster.logsdb.enabled", "true")
+            .put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial")
+            .build();
+    }
+
+    public void testStandard() throws Exception {
+        String dataStreamName = "k8s";
+        var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id");
+        putTemplateRequest.indexTemplate(
+            ComposableIndexTemplate.builder()
+                .indexPatterns(List.of(dataStreamName + "*"))
+                .template(
+                    new Template(
+                        indexSettings(4, 0).put("index.mode", "logsdb").put("index.sort.field", "message,k8s.pod.uid,@timestamp").build(),
+                        new CompressedXContent(MAPPING_TEMPLATE),
+                        null
+                    )
+                )
+                .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
+                .build()
+        );
+        client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest).actionGet();
+        checkIndexSearchAndRetrieval(dataStreamName, false);
+    }
+
+    public void testRouteOnSortFields() throws Exception {
+        String dataStreamName = "k8s";
+        var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id");
+        putTemplateRequest.indexTemplate(
+            ComposableIndexTemplate.builder()
+                .indexPatterns(List.of(dataStreamName + "*"))
+                .template(
+                    new Template(
+                        indexSettings(4, 0).put("index.mode", "logsdb")
+                            .put("index.sort.field", "message,k8s.pod.uid,@timestamp")
+                            .put("index.logsdb.route_on_sort_fields", true)
+                            .build(),
+                        new CompressedXContent(MAPPING_TEMPLATE),
+                        null
+                    )
+                )
+                .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
+                .build()
+        );
+        client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest).actionGet();
+        checkIndexSearchAndRetrieval(dataStreamName, true);
+    }
+
+    private void checkIndexSearchAndRetrieval(String dataStreamName, boolean routeOnSortFields) throws Exception {
+        String[] uuis = {
+            UUID.randomUUID().toString(),
+            UUID.randomUUID().toString(),
+            UUID.randomUUID().toString(),
+            UUID.randomUUID().toString() };
+        int numBulkRequests = randomIntBetween(128, 1024);
+        int numDocsPerBulk = randomIntBetween(16, 256);
+        String indexName = null;
+        {
+            Instant time = Instant.now();
+            for (int i = 0; i < numBulkRequests; i++) {
+                BulkRequest bulkRequest = new BulkRequest(dataStreamName);
+                for (int j = 0; j < numDocsPerBulk; j++) {
+                    var indexRequest = new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE);
+                    indexRequest.source(
+                        DOC.replace("$time", formatInstant(time))
+                            .replace("$uuid", uuis[j % uuis.length])
+                            .replace("$pod", "pod-" + randomIntBetween(0, 10)),
+                        XContentType.JSON
+                    );
+                    bulkRequest.add(indexRequest);
+                    time = time.plusMillis(1);
+                }
+                var bulkResponse = client().bulk(bulkRequest).actionGet();
+                assertThat(bulkResponse.hasFailures(), is(false));
+                indexName = bulkResponse.getItems()[0].getIndex();
+            }
+            client().admin().indices().refresh(new RefreshRequest(dataStreamName)).actionGet();
+        }
+
+        // Verify settings.
+        final GetSettingsResponse getSettingsResponse = indicesAdmin().getSettings(
+            new GetSettingsRequest().indices(indexName).includeDefaults(false)
+        ).actionGet();
+        final Settings settings = getSettingsResponse.getIndexToSettings().get(indexName);
+        assertEquals("message,k8s.pod.uid,@timestamp", settings.get("index.sort.field"));
+        if (routeOnSortFields) {
+            assertEquals("[message, k8s.pod.uid]", settings.get("index.routing_path"));
+            assertEquals("true", settings.get("index.logsdb.route_on_sort_fields"));
+        } else {
+            assertNull(settings.get("index.routing_path"));
+            assertNull(settings.get("index.logsdb.route_on_sort_fields"));
+        }
+
+        // Check the search api can synthesize _id
+        final String idxName = indexName;
+        var searchRequest = new SearchRequest(dataStreamName);
+        searchRequest.source().trackTotalHits(true);
+        assertResponse(client().search(searchRequest), searchResponse -> {
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo((long) numBulkRequests * numDocsPerBulk));
+
+            for (int i = 0; i < searchResponse.getHits().getHits().length; i++) {
+                String id = searchResponse.getHits().getHits()[i].getId();
+                assertThat(id, notNullValue());
+
+                // Check that the _id is gettable:
+                var getResponse = client().get(new GetRequest(idxName).id(id)).actionGet();
+                assertThat(getResponse.isExists(), is(true));
+                assertThat(getResponse.getId(), equalTo(id));
+            }
+        });
+    }
+
+    static String formatInstant(Instant instant) {
+        return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant);
+    }
+
+}
diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java
index ef9480681f559..bd8093c0a01c1 100644
--- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java
@@ -9,9 +9,11 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.FormatNames;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -225,4 +227,58 @@ static String formatInstant(Instant instant) {
         return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant);
     }
 
+    public void testLogsdbRouteOnSortFields() throws IOException {
+        Request request = new Request("PUT", "/_cluster/settings");
+        request.setJsonEntity("{ \"transient\": { \"cluster.logsdb.enabled\": true } }");
+        assertOK(client().performRequest(request));
+
+        request = new Request("POST", "/_index_template/1");
+        request.setJsonEntity("""
+            {
+                "index_patterns": ["my-log-*"],
+                "data_stream": {
+                },
+                "template": {
+                    "settings":{
+                        "index": {
+                            "mode": "logsdb",
+                            "sort.field": [ "host.name", "message", "@timestamp" ],
+                            "logsdb.route_on_sort_fields": true
+                        }
+                    },
+                    "mappings": {
+                        "properties": {
+                            "@timestamp" : {
+                                "type": "date"
+                            },
+                            "host.name": {
+                                "type": "keyword"
+                            },
+                            "message": {
+                                "type": "keyword"
+                            }
+                        }
+                    }
+                }
+            }
+            """);
+        assertOK(client().performRequest(request));
+
+        request = new Request("POST", "/my-log-foo/_doc");
+        request.setJsonEntity("""
+            {
+                "@timestamp": "2020-01-01T00:00:00.000Z",
+                "host.name": "foo",
+                "message": "bar"
+            }
+            """);
+        assertOK(client().performRequest(request));
+
+        String index = DataStream.getDefaultBackingIndexName("my-log-foo", 1);
+        var settings = (Map<?, ?>) ((Map<?, ?>) getIndexSettings(index).get(index)).get("settings");
+        assertEquals("logsdb", settings.get("index.mode"));
+        assertNull(settings.get("index.mapping.source.mode"));
+        assertEquals("true", settings.get(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey()));
+        assertEquals(List.of("host.name", "message"), settings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey()));
+    }
 }
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java
index a8085f3d50a82..266847209f495 100644
--- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java
+++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java
@@ -67,12 +67,11 @@ public Collection<?> createComponents(PluginServices services) {
 
     @Override
     public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) {
-        if (DiscoveryNode.isStateless(settings) == false) {
-            logsdbIndexModeSettingsProvider.init(
-                parameters.mapperServiceFactory(),
-                () -> parameters.clusterService().state().nodes().getMinSupportedIndexVersion()
-            );
-        }
+        logsdbIndexModeSettingsProvider.init(
+            parameters.mapperServiceFactory(),
+            () -> parameters.clusterService().state().nodes().getMinSupportedIndexVersion(),
+            DiscoveryNode.isStateless(settings) == false
+        );
         return List.of(logsdbIndexModeSettingsProvider);
     }
 
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java
index f4fa2a29d79a0..19f5039a1075d 100644
--- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java
+++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.monitor.metrics.IndexModeStatsActionType;
@@ -23,24 +23,19 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.XPackFeatures;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
 import org.elasticsearch.xpack.core.application.LogsDBFeatureSetUsage;
 
-import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING;
-
 public class LogsDBUsageTransportAction extends XPackUsageFeatureTransportAction {
     private final ClusterService clusterService;
-    private final FeatureService featureService;
     private final Client client;
 
     @Inject
     public LogsDBUsageTransportAction(
         TransportService transportService,
         ClusterService clusterService,
-        FeatureService featureService,
         ThreadPool threadPool,
         ActionFilters actionFilters,
         Client client,
@@ -55,7 +50,6 @@ public LogsDBUsageTransportAction(
             indexNameExpressionResolver
         );
         this.clusterService = clusterService;
-        this.featureService = featureService;
         this.client = client;
     }
 
@@ -71,38 +65,30 @@ protected void masterOperation(
         for (IndexMetadata indexMetadata : state.metadata()) {
             if (indexMetadata.getIndexMode() == IndexMode.LOGSDB) {
                 numIndices++;
-                if (INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC) {
+                if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC) {
                     numIndicesWithSyntheticSources++;
                 }
             }
         }
         final boolean enabled = LogsDBPlugin.CLUSTER_LOGSDB_ENABLED.get(clusterService.getSettings());
         final boolean hasCustomCutoffDate = System.getProperty(SyntheticSourceLicenseService.CUTOFF_DATE_SYS_PROP_NAME) != null;
-        if (featureService.clusterHasFeature(state, XPackFeatures.LOGSDB_TELMETRY_STATS)) {
-            final DiscoveryNode[] nodes = state.nodes().getDataNodes().values().toArray(DiscoveryNode[]::new);
-            final var statsRequest = new IndexModeStatsActionType.StatsRequest(nodes);
-            final int finalNumIndices = numIndices;
-            final int finalNumIndicesWithSyntheticSources = numIndicesWithSyntheticSources;
-            client.execute(IndexModeStatsActionType.TYPE, statsRequest, listener.map(statsResponse -> {
-                final var indexStats = statsResponse.stats().get(IndexMode.LOGSDB);
-                return new XPackUsageFeatureResponse(
-                    new LogsDBFeatureSetUsage(
-                        true,
-                        enabled,
-                        finalNumIndices,
-                        finalNumIndicesWithSyntheticSources,
-                        indexStats.numDocs(),
-                        indexStats.numBytes(),
-                        hasCustomCutoffDate
-                    )
-                );
-            }));
-        } else {
-            listener.onResponse(
-                new XPackUsageFeatureResponse(
-                    new LogsDBFeatureSetUsage(true, enabled, numIndices, numIndicesWithSyntheticSources, 0L, 0L, hasCustomCutoffDate)
+        final DiscoveryNode[] nodes = state.nodes().getDataNodes().values().toArray(DiscoveryNode[]::new);
+        final var statsRequest = new IndexModeStatsActionType.StatsRequest(nodes);
+        final int finalNumIndices = numIndices;
+        final int finalNumIndicesWithSyntheticSources = numIndicesWithSyntheticSources;
+        client.execute(IndexModeStatsActionType.TYPE, statsRequest, listener.map(statsResponse -> {
+            final var indexStats = statsResponse.stats().get(IndexMode.LOGSDB);
+            return new XPackUsageFeatureResponse(
+                new LogsDBFeatureSetUsage(
+                    true,
+                    enabled,
+                    finalNumIndices,
+                    finalNumIndicesWithSyntheticSources,
+                    indexStats.numDocs(),
+                    indexStats.numBytes(),
+                    hasCustomCutoffDate
                 )
             );
-        }
+        }));
     }
 }
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java
index 977b0e1c57578..633dd6123c088 100644
--- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java
+++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java
@@ -21,12 +21,19 @@
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexSortConfig;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 
 import java.io.IOException;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 import java.util.function.Supplier;
@@ -41,6 +48,7 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider {
     private final SyntheticSourceLicenseService syntheticSourceLicenseService;
     private final SetOnce<CheckedFunction<IndexMetadata, MapperService, IOException>> mapperServiceFactory = new SetOnce<>();
     private final SetOnce<Supplier<IndexVersion>> createdIndexVersion = new SetOnce<>();
+    private final SetOnce<Boolean> supportFallbackToStoredSource = new SetOnce<>();
 
     private volatile boolean isLogsdbEnabled;
 
@@ -53,13 +61,14 @@ void updateClusterIndexModeLogsdbEnabled(boolean isLogsdbEnabled) {
         this.isLogsdbEnabled = isLogsdbEnabled;
     }
 
-    void init(CheckedFunction<IndexMetadata, MapperService, IOException> factory, Supplier<IndexVersion> indexVersion) {
-        mapperServiceFactory.set(factory);
-        createdIndexVersion.set(indexVersion);
-    }
-
-    private boolean supportFallbackToStoredSource() {
-        return mapperServiceFactory.get() != null;
+    void init(
+        CheckedFunction<IndexMetadata, MapperService, IOException> factory,
+        Supplier<IndexVersion> indexVersion,
+        boolean supportFallbackToStoredSource
+    ) {
+        this.mapperServiceFactory.set(factory);
+        this.createdIndexVersion.set(indexVersion);
+        this.supportFallbackToStoredSource.set(supportFallbackToStoredSource);
     }
 
     @Override
@@ -79,17 +88,22 @@ public Settings getAdditionalIndexSettings(
         final List<CompressedXContent> combinedTemplateMappings
     ) {
         Settings.Builder settingsBuilder = null;
+        boolean isLogsDB = templateIndexMode == IndexMode.LOGSDB;
+
+        // Inject logsdb index mode, based on the logs pattern.
         if (isLogsdbEnabled
             && dataStreamName != null
             && resolveIndexMode(settings.get(IndexSettings.MODE.getKey())) == null
             && matchesLogsPattern(dataStreamName)) {
             settingsBuilder = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName());
-            if (supportFallbackToStoredSource()) {
-                settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()).put(settings).build();
-            }
+            settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()).put(settings).build();
+            isLogsDB = true;
         }
 
-        if (supportFallbackToStoredSource()) {
+        MappingHints mappingHints = getMappingHints(indexName, templateIndexMode, settings, combinedTemplateMappings);
+
+        // Inject stored source mode if synthetic source if not available per licence.
+        if (mappingHints.hasSyntheticSourceUsage && supportFallbackToStoredSource.get()) {
             // This index name is used when validating component and index templates, we should skip this check in that case.
             // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
             boolean isTemplateValidation = "validate-index-name".equals(indexName);
@@ -98,30 +112,85 @@ && matchesLogsPattern(dataStreamName)) {
                 indexName,
                 dataStreamName
             );
-            if (newIndexHasSyntheticSourceUsage(indexName, templateIndexMode, settings, combinedTemplateMappings)
-                && syntheticSourceLicenseService.fallbackToStoredSource(
-                    isTemplateValidation,
-                    legacyLicensedUsageOfSyntheticSourceAllowed
-                )) {
+            if (syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation, legacyLicensedUsageOfSyntheticSourceAllowed)) {
                 LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName);
                 if (settingsBuilder == null) {
                     settingsBuilder = Settings.builder();
                 }
-                settingsBuilder.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString());
+                settingsBuilder.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString());
             }
         }
+
+        if (isLogsDB) {
+            // Inject sorting on [host.name], in addition to [@timestamp].
+            if (mappingHints.sortOnHostName) {
+                if (settingsBuilder == null) {
+                    settingsBuilder = Settings.builder();
+                }
+                if (mappingHints.addHostNameField) {
+                    // Inject keyword field [host.name] too.
+                    settingsBuilder.put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true);
+                }
+                settingsBuilder.put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true);
+            }
+
+            // Inject routing path matching sort fields.
+            if (settings.getAsBoolean(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), false)) {
+                List<String> sortFields = new ArrayList<>(settings.getAsList(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey()));
+                sortFields.removeIf(s -> s.equals(DataStreamTimestampFieldMapper.DEFAULT_PATH));
+                if (sortFields.size() < 2) {
+                    throw new IllegalStateException(
+                        String.format(
+                            Locale.ROOT,
+                            "data stream [%s] in logsdb mode and with [%s] index setting has only %d sort fields "
+                                + "(excluding timestamp), needs at least 2",
+                            dataStreamName,
+                            IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(),
+                            sortFields.size()
+                        )
+                    );
+                }
+                if (settings.hasValue(IndexMetadata.INDEX_ROUTING_PATH.getKey())) {
+                    List<String> routingPaths = settings.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey());
+                    if (routingPaths.equals(sortFields) == false) {
+                        throw new IllegalStateException(
+                            String.format(
+                                Locale.ROOT,
+                                "data stream [%s] in logsdb mode and with [%s] index setting has mismatching sort "
+                                    + "and routing fields, [index.routing_path:%s], [index.sort.fields:%s]",
+                                dataStreamName,
+                                IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(),
+                                routingPaths,
+                                sortFields
+                            )
+                        );
+                    }
+                } else {
+                    if (settingsBuilder == null) {
+                        settingsBuilder = Settings.builder();
+                    }
+                    settingsBuilder.putList(INDEX_ROUTING_PATH.getKey(), sortFields);
+                }
+            }
+        }
+
         return settingsBuilder == null ? Settings.EMPTY : settingsBuilder.build();
+
+    }
+
+    record MappingHints(boolean hasSyntheticSourceUsage, boolean sortOnHostName, boolean addHostNameField) {
+        static MappingHints EMPTY = new MappingHints(false, false, false);
     }
 
     private static boolean matchesLogsPattern(final String name) {
         return Regex.simpleMatch(LOGS_PATTERN, name);
     }
 
-    private IndexMode resolveIndexMode(final String mode) {
+    private static IndexMode resolveIndexMode(final String mode) {
         return mode != null ? Enum.valueOf(IndexMode.class, mode.toUpperCase(Locale.ROOT)) : null;
     }
 
-    boolean newIndexHasSyntheticSourceUsage(
+    MappingHints getMappingHints(
         String indexName,
         IndexMode templateIndexMode,
         Settings indexTemplateAndCreateRequestSettings,
@@ -130,13 +199,14 @@ boolean newIndexHasSyntheticSourceUsage(
         if ("validate-index-name".equals(indexName)) {
             // This index name is used when validating component and index templates, we should skip this check in that case.
             // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
-            return false;
+            return MappingHints.EMPTY;
         }
 
         try {
             var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, templateIndexMode, indexTemplateAndCreateRequestSettings);
             var indexMode = tmpIndexMetadata.getIndexMode();
-            if (SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(tmpIndexMetadata.getSettings())
+            boolean hasSyntheticSourceUsage = false;
+            if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(tmpIndexMetadata.getSettings())
                 || indexMode == IndexMode.LOGSDB
                 || indexMode == IndexMode.TIME_SERIES) {
                 // In case when index mode is tsdb or logsdb and only _source.mode mapping attribute is specified, then the default
@@ -144,11 +214,19 @@ boolean newIndexHasSyntheticSourceUsage(
                 // then configuring the index.mapping.source.mode setting to stored has no effect. Additionally _source.mode can't be set
                 // to disabled, because that isn't allowed with logsdb/tsdb. In other words setting index.mapping.source.mode setting to
                 // stored when _source.mode mapping attribute is stored is fine as it has no effect, but avoids creating MapperService.
-                var sourceMode = SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(tmpIndexMetadata.getSettings());
-                return sourceMode == SourceFieldMapper.Mode.SYNTHETIC;
+                var sourceMode = IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(tmpIndexMetadata.getSettings());
+                hasSyntheticSourceUsage = sourceMode == SourceFieldMapper.Mode.SYNTHETIC;
+                if (IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(indexTemplateAndCreateRequestSettings).isEmpty() == false) {
+                    // Custom sort config, no point for further checks on [host.name] field.
+                    return new MappingHints(hasSyntheticSourceUsage, false, false);
+                }
+                if (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(indexTemplateAndCreateRequestSettings)
+                    && IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(indexTemplateAndCreateRequestSettings)) {
+                    // Settings for adding and sorting on [host.name] are already set, propagate them.
+                    return new MappingHints(hasSyntheticSourceUsage, true, true);
+                }
             }
 
-            // TODO: remove this when _source.mode attribute has been removed:
             try (var mapperService = mapperServiceFactory.get().apply(tmpIndexMetadata)) {
                 // combinedTemplateMappings can be null when creating system indices
                 // combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping.
@@ -156,14 +234,25 @@ boolean newIndexHasSyntheticSourceUsage(
                     combinedTemplateMappings = List.of(new CompressedXContent("{}"));
                 }
                 mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE);
-                return mapperService.documentMapper().sourceMapper().isSynthetic();
+                Mapper hostName = mapperService.mappingLookup().getMapper("host.name");
+                hasSyntheticSourceUsage = hasSyntheticSourceUsage || mapperService.documentMapper().sourceMapper().isSynthetic();
+                boolean addHostNameField = IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(indexTemplateAndCreateRequestSettings)
+                    || (hostName == null
+                        && mapperService.mappingLookup().objectMappers().get("host.name") == null
+                        && (mapperService.mappingLookup().getMapper("host") == null
+                            || mapperService.mappingLookup().getMapping().getRoot().subobjects() == ObjectMapper.Subobjects.DISABLED));
+                boolean sortOnHostName = IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(indexTemplateAndCreateRequestSettings)
+                    || addHostNameField
+                    || ((hostName instanceof NumberFieldMapper nfm && nfm.fieldType().hasDocValues())
+                        || (hostName instanceof KeywordFieldMapper kfm && kfm.fieldType().hasDocValues()));
+                return new MappingHints(hasSyntheticSourceUsage, sortOnHostName, addHostNameField);
             }
         } catch (AssertionError | Exception e) {
             // In case invalid mappings or setting are provided, then mapper service creation can fail.
             // In that case it is ok to return false here. The index creation will fail anyway later, so no need to fallback to stored
             // source.
             LOGGER.info(() -> Strings.format("unable to create mapper service for index [%s]", indexName), e);
-            return false;
+            return MappingHints.EMPTY;
         }
     }
 
diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java
index f8f307b572f33..b2a533f6b76c9 100644
--- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java
+++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java
@@ -9,7 +9,7 @@
 
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.license.AbstractLicensesIntegrationTestCase;
 import org.elasticsearch.license.GetFeatureUsageRequest;
 import org.elasticsearch.license.GetFeatureUsageResponse;
@@ -104,11 +104,11 @@ public void testSyntheticSourceUsageTracksBothLegacyAndRegularFeature() throws E
     }
 
     private void createIndexWithSyntheticSourceAndAssertExpectedType(String indexName, String expectedType) {
-        var settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic").build();
+        var settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic").build();
         createIndex(indexName, settings);
         var response = admin().indices().getSettings(new GetSettingsRequest().indices(indexName)).actionGet();
         assertThat(
-            response.getIndexToSettings().get(indexName).get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()),
+            response.getIndexToSettings().get(indexName).get(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()),
             equalTo(expectedType)
         );
     }
diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java
index de4f0960f50e7..77319a881f1e5 100644
--- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java
+++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -18,6 +19,7 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexSortConfig;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
@@ -36,6 +38,7 @@
 
 import static org.elasticsearch.common.settings.Settings.builder;
 import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseServiceTests.createEnterpriseLicense;
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
@@ -43,6 +46,7 @@
 
 public class LogsdbIndexModeSettingsProviderTests extends ESTestCase {
 
+    private static final String DATA_STREAM_NAME = "logs-app1";
     public static final String DEFAULT_MAPPING = """
         {
             "_doc": {
@@ -78,7 +82,7 @@ public void setup() throws Exception {
         syntheticSourceLicenseService.setLicenseService(mockLicenseService);
     }
 
-    LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled) {
+    private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled) {
         newMapperServiceCounter.set(0);
         var provider = new LogsdbIndexModeSettingsProvider(
             syntheticSourceLicenseService,
@@ -87,11 +91,37 @@ LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabl
         provider.init(im -> {
             newMapperServiceCounter.incrementAndGet();
             return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName());
-        }, IndexVersion::current);
+        }, IndexVersion::current, true);
         return provider;
     }
 
-    public void testLogsDbDisabled() throws IOException {
+    private Settings generateLogsdbSettings(Settings settings) throws IOException {
+        return generateLogsdbSettings(settings, null);
+    }
+
+    private Settings generateLogsdbSettings(Settings settings, String mapping) throws IOException {
+        Metadata metadata = Metadata.EMPTY_METADATA;
+        var provider = new LogsdbIndexModeSettingsProvider(
+            syntheticSourceLicenseService,
+            Settings.builder().put("cluster.logsdb.enabled", true).build()
+        );
+        provider.init(im -> {
+            newMapperServiceCounter.incrementAndGet();
+            return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName());
+        }, IndexVersion::current, true);
+        var result = provider.getAdditionalIndexSettings(
+            DataStream.getDefaultBackingIndexName(DATA_STREAM_NAME, 0),
+            DATA_STREAM_NAME,
+            IndexMode.LOGSDB,
+            metadata,
+            Instant.now(),
+            settings,
+            mapping == null ? List.of() : List.of(new CompressedXContent(mapping))
+        );
+        return builder().put(result).build();
+    }
+
+    public void testDisabled() throws IOException {
         final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider(
             syntheticSourceLicenseService,
             Settings.builder().put("cluster.logsdb.enabled", false).build()
@@ -385,7 +415,7 @@ private void assertIndexMode(final Settings settings, final String expectedIndex
     }
 
     public void testNewIndexHasSyntheticSourceUsage() throws IOException {
-        String dataStreamName = "logs-app1";
+        String dataStreamName = DATA_STREAM_NAME;
         String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0);
         Settings settings = Settings.EMPTY;
         LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false);
@@ -404,7 +434,8 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException {
                     }
                 }
                 """;
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping)));
+            boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping)))
+                .hasSyntheticSourceUsage();
             assertTrue(result);
             assertThat(newMapperServiceCounter.get(), equalTo(1));
             assertWarnings(SourceFieldMapper.DEPRECATION_WARNING);
@@ -440,7 +471,8 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException {
                     }
                     """;
             }
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping)));
+            boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping)))
+                .hasSyntheticSourceUsage();
             assertFalse(result);
             assertThat(newMapperServiceCounter.get(), equalTo(2));
             if (withSourceMode) {
@@ -467,12 +499,13 @@ public void testValidateIndexName() throws IOException {
             """;
         Settings settings = Settings.EMPTY;
         LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false);
-        boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping)));
+        boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping)))
+            .hasSyntheticSourceUsage();
         assertFalse(result);
     }
 
     public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException {
-        String dataStreamName = "logs-app1";
+        String dataStreamName = DATA_STREAM_NAME;
         String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0);
         String mapping = """
             {
@@ -488,35 +521,32 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException
         LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false);
         {
             Settings settings = Settings.builder().put("index.mode", "logsdb").build();
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping)));
+            boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping)))
+                .hasSyntheticSourceUsage();
             assertTrue(result);
-            assertThat(newMapperServiceCounter.get(), equalTo(0));
+            assertThat(newMapperServiceCounter.get(), equalTo(1));
         }
         {
             Settings settings = Settings.builder().put("index.mode", "logsdb").build();
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of());
+            boolean result = provider.getMappingHints(indexName, null, settings, List.of()).hasSyntheticSourceUsage();
             assertTrue(result);
-            assertThat(newMapperServiceCounter.get(), equalTo(0));
+            assertThat(newMapperServiceCounter.get(), equalTo(2));
         }
         {
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, Settings.EMPTY, List.of());
+            boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of()).hasSyntheticSourceUsage();
             assertFalse(result);
-            assertThat(newMapperServiceCounter.get(), equalTo(1));
+            assertThat(newMapperServiceCounter.get(), equalTo(3));
         }
         {
-            boolean result = provider.newIndexHasSyntheticSourceUsage(
-                indexName,
-                null,
-                Settings.EMPTY,
-                List.of(new CompressedXContent(mapping))
-            );
+            boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(mapping)))
+                .hasSyntheticSourceUsage();
             assertFalse(result);
-            assertThat(newMapperServiceCounter.get(), equalTo(2));
+            assertThat(newMapperServiceCounter.get(), equalTo(4));
         }
     }
 
     public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException {
-        String dataStreamName = "logs-app1";
+        String dataStreamName = DATA_STREAM_NAME;
         String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0);
         String mapping = """
             {
@@ -533,31 +563,28 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException {
         LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false);
         {
             Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build();
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping)));
+            boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping)))
+                .hasSyntheticSourceUsage();
             assertTrue(result);
         }
         {
             Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build();
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of());
+            boolean result = provider.getMappingHints(indexName, null, settings, List.of()).hasSyntheticSourceUsage();
             assertTrue(result);
         }
         {
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, Settings.EMPTY, List.of());
+            boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of()).hasSyntheticSourceUsage();
             assertFalse(result);
         }
         {
-            boolean result = provider.newIndexHasSyntheticSourceUsage(
-                indexName,
-                null,
-                Settings.EMPTY,
-                List.of(new CompressedXContent(mapping))
-            );
+            boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(mapping)))
+                .hasSyntheticSourceUsage();
             assertFalse(result);
         }
     }
 
     public void testNewIndexHasSyntheticSourceUsage_invalidSettings() throws IOException {
-        String dataStreamName = "logs-app1";
+        String dataStreamName = DATA_STREAM_NAME;
         String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0);
         Settings settings = Settings.builder().put("index.soft_deletes.enabled", false).build();
         LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false);
@@ -576,7 +603,8 @@ public void testNewIndexHasSyntheticSourceUsage_invalidSettings() throws IOExcep
                     }
                 }
                 """;
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping)));
+            boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping)))
+                .hasSyntheticSourceUsage();
             assertFalse(result);
             assertThat(newMapperServiceCounter.get(), equalTo(1));
         }
@@ -592,14 +620,15 @@ public void testNewIndexHasSyntheticSourceUsage_invalidSettings() throws IOExcep
                     }
                 }
                 """;
-            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping)));
+            boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping)))
+                .hasSyntheticSourceUsage();
             assertFalse(result);
             assertThat(newMapperServiceCounter.get(), equalTo(2));
         }
     }
 
     public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws IOException {
-        String dataStreamName = "logs-app1";
+        String dataStreamName = DATA_STREAM_NAME;
         Metadata.Builder mb = Metadata.builder(
             DataStreamTestHelper.getClusterStateWithDataStreams(
                 List.of(Tuple.tuple(dataStreamName, 1)),
@@ -611,7 +640,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws
         );
         Metadata metadata = mb.build();
         LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false);
-        Settings settings = builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
+        Settings settings = builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
             .build();
 
         Settings result = provider.getAdditionalIndexSettings(
@@ -624,7 +653,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws
             List.of()
         );
         assertThat(result.size(), equalTo(0));
-        assertThat(newMapperServiceCounter.get(), equalTo(0));
+        assertThat(newMapperServiceCounter.get(), equalTo(1));
 
         syntheticSourceLicenseService.setSyntheticSourceFallback(true);
         result = provider.getAdditionalIndexSettings(
@@ -637,8 +666,8 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws
             List.of()
         );
         assertThat(result.size(), equalTo(1));
-        assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result));
-        assertThat(newMapperServiceCounter.get(), equalTo(0));
+        assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result));
+        assertThat(newMapperServiceCounter.get(), equalTo(2));
 
         result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 2),
@@ -650,8 +679,8 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws
             List.of()
         );
         assertThat(result.size(), equalTo(1));
-        assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result));
-        assertThat(newMapperServiceCounter.get(), equalTo(0));
+        assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result));
+        assertThat(newMapperServiceCounter.get(), equalTo(3));
 
         result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 2),
@@ -662,9 +691,11 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws
             settings,
             List.of()
         );
-        assertThat(result.size(), equalTo(1));
-        assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result));
-        assertThat(newMapperServiceCounter.get(), equalTo(0));
+        assertThat(result.size(), equalTo(3));
+        assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result));
+        assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result));
+        assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result));
+        assertThat(newMapperServiceCounter.get(), equalTo(4));
     }
 
     public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch() throws IOException {
@@ -672,7 +703,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch(
         LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(true);
         final Settings settings = Settings.EMPTY;
 
-        String dataStreamName = "logs-app1";
+        String dataStreamName = DATA_STREAM_NAME;
         Metadata.Builder mb = Metadata.builder(
             DataStreamTestHelper.getClusterStateWithDataStreams(
                 List.of(Tuple.tuple(dataStreamName, 1)),
@@ -715,9 +746,11 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch(
             settings,
             List.of()
         );
-        assertThat(result.size(), equalTo(2));
-        assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result));
+        assertThat(result.size(), equalTo(4));
+        assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result));
         assertEquals(IndexMode.LOGSDB, IndexSettings.MODE.get(result));
+        assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result));
+        assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result));
 
         result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 2),
@@ -731,4 +764,292 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch(
         assertThat(result.size(), equalTo(0));
     }
 
+    public void testRoutingPathOnSortFields() throws Exception {
+        var settings = Settings.builder()
+            .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message")
+            .put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), true)
+            .build();
+        Settings result = generateLogsdbSettings(settings);
+        assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), contains("host", "message"));
+    }
+
+    public void testRoutingPathOnSortFieldsFilterTimestamp() throws Exception {
+        var settings = Settings.builder()
+            .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message,@timestamp")
+            .put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), true)
+            .build();
+        Settings result = generateLogsdbSettings(settings);
+        assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), contains("host", "message"));
+    }
+
+    public void testRoutingPathOnSortSingleField() throws Exception {
+        var settings = Settings.builder()
+            .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host")
+            .put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), true)
+            .build();
+        Exception e = expectThrows(IllegalStateException.class, () -> generateLogsdbSettings(settings));
+        assertThat(
+            e.getMessage(),
+            equalTo(
+                "data stream ["
+                    + DATA_STREAM_NAME
+                    + "] in logsdb mode and with [index.logsdb.route_on_sort_fields] index setting has only 1 sort fields "
+                    + "(excluding timestamp), needs at least 2"
+            )
+        );
+    }
+
+    public void testExplicitRoutingPathMatchesSortFields() throws Exception {
+        var settings = Settings.builder()
+            .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB)
+            .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message,@timestamp")
+            .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "host,message")
+            .put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), true)
+            .build();
+        Settings result = generateLogsdbSettings(settings);
+        assertTrue(result.isEmpty());
+    }
+
+    public void testExplicitRoutingPathDoesNotMatchSortFields() throws Exception {
+        var settings = Settings.builder()
+            .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message,@timestamp")
+            .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "host,message,foo")
+            .put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), true)
+            .build();
+        Exception e = expectThrows(IllegalStateException.class, () -> generateLogsdbSettings(settings));
+        assertThat(
+            e.getMessage(),
+            equalTo(
+                "data stream ["
+                    + DATA_STREAM_NAME
+                    + "] in logsdb mode and with [index."
+                    + "logsdb.route_on_sort_fields] index setting has mismatching sort "
+                    + "and routing fields, [index.routing_path:[host, message, foo]], [index.sort.fields:[host, message]]"
+            )
+        );
+    }
+
+    public void testSortAndHostNamePropagateValue() throws Exception {
+        var settings = Settings.builder()
+            .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB)
+            .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true)
+            .put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true)
+            .build();
+        Settings result = generateLogsdbSettings(settings);
+        assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result));
+        assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result));
+        assertEquals(0, newMapperServiceCounter.get());
+    }
+
+    public void testSortAndHostNameWithCustomSortConfig() throws Exception {
+        var settings = Settings.builder()
+            .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB)
+            .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "foo,bar")
+            .build();
+        Settings result = generateLogsdbSettings(settings);
+        assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result));
+        assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result));
+        assertEquals(0, newMapperServiceCounter.get());
+    }
+
+    public void testSortAndHostNameKeyword() throws Exception {
+        var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build();
+        var mappings = """
+            {
+                "_doc": {
+                    "properties": {
+                        "@timestamp": {
+                            "type": "date"
+                        },
+                        "host.name": {
+                            "type": "keyword"
+                        }
+                    }
+                }
+            }
+            """;
+        Settings result = generateLogsdbSettings(settings, mappings);
+        assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result));
+        assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result));
+        assertEquals(1, newMapperServiceCounter.get());
+    }
+
+    public void testSortAndHostNameKeywordNoDocvalues() throws Exception {
+        var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build();
+        var mappings = """
+            {
+                "_doc": {
+                    "properties": {
+                        "@timestamp": {
+                            "type": "date"
+                        },
+                        "host.name": {
+                            "type": "keyword",
+                            "doc_values": false
+                        }
+                    }
+                }
+            }
+            """;
+        Settings result = generateLogsdbSettings(settings, mappings);
+        assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result));
+        assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result));
+        assertEquals(1, newMapperServiceCounter.get());
+    }
+
+    public void testSortAndHostNameInteger() throws Exception {
+        var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build();
+        var mappings = """
+            {
+                "_doc": {
+                    "properties": {
+                        "@timestamp": {
+                            "type": "date"
+                        },
+                        "host.name": {
+                            "type": "integer"
+                        }
+                    }
+                }
+            }
+            """;
+        Settings result = generateLogsdbSettings(settings, mappings);
+        assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result));
+        assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result));
+        assertEquals(1, newMapperServiceCounter.get());
+    }
+
+    public void testSortAndHostNameIntegerNoDocvalues() throws Exception {
+        var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build();
+        var mappings = """
+            {
+                "_doc": {
+                    "properties": {
+                        "@timestamp": {
+                            "type": "date"
+                        },
+                        "host.name": {
+                            "type": "integer",
+                            "doc_values": false
+                        }
+                    }
+                }
+            }
+            """;
+        Settings result = generateLogsdbSettings(settings, mappings);
+        assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result));
+        assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result));
+        assertEquals(1, newMapperServiceCounter.get());
+    }
+
+    public void testSortAndHostNameBoolean() throws Exception {
+        var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build();
+        var mappings = """
+            {
+                "_doc": {
+                    "properties": {
+                        "@timestamp": {
+                            "type": "date"
+                        },
+                        "host.name": {
+                            "type": "boolean"
+                        }
+                    }
+                }
+            }
+            """;
+        Settings result = generateLogsdbSettings(settings, mappings);
+        assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result));
+        assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result));
+        assertEquals(1, newMapperServiceCounter.get());
+    }
+
+    public void testSortAndHostObject() throws Exception {
+        var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build();
+        var mappings = """
+            {
+                "_doc": {
+                    "properties": {
+                        "@timestamp": {
+                            "type": "date"
+                        },
+                        "host": {
+                            "type": "object"
+                        }
+                    }
+                }
+            }
+            """;
+        Settings result = generateLogsdbSettings(settings, mappings);
+        assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result));
+        assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result));
+        assertEquals(1, newMapperServiceCounter.get());
+    }
+
+    public void testSortAndHostField() throws Exception {
+        var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build();
+        var mappings = """
+            {
+                "_doc": {
+                    "properties": {
+                        "@timestamp": {
+                            "type": "date"
+                        },
+                        "host": {
+                            "type": "keyword"
+                        }
+                    }
+                }
+            }
+            """;
+        Settings result = generateLogsdbSettings(settings, mappings);
+        assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result));
+        assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result));
+        assertEquals(1, newMapperServiceCounter.get());
+    }
+
+    public void testSortAndHostFieldSubobjectsFalse() throws Exception {
+        var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build();
+        var mappings = """
+            {
+                "_doc": {
+                    "subobjects": false,
+                    "properties": {
+                        "@timestamp": {
+                            "type": "date"
+                        },
+                        "host": {
+                            "type": "keyword"
+                        }
+                    }
+                }
+            }
+            """;
+        Settings result = generateLogsdbSettings(settings, mappings);
+        assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result));
+        assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result));
+        assertEquals(1, newMapperServiceCounter.get());
+    }
+
+    public void testSortAndHostNameObject() throws Exception {
+        var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build();
+        var mappings = """
+            {
+                "_doc": {
+                    "properties": {
+                        "@timestamp": {
+                            "type": "date"
+                        },
+                        "host.name.sub": {
+                            "type": "keyword"
+                        }
+                    }
+                }
+            }
+            """;
+        Settings result = generateLogsdbSettings(settings, mappings);
+        assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result));
+        assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result));
+        assertEquals(1, newMapperServiceCounter.get());
+    }
 }
diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java
index 8a4adf18b3e67..7fa2f11880f40 100644
--- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java
+++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java
@@ -10,9 +10,9 @@
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.MapperTestUtils;
-import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.license.License;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.license.XPackLicenseState;
@@ -52,21 +52,22 @@ public void setup() throws Exception {
         provider = new LogsdbIndexModeSettingsProvider(syntheticSourceLicenseService, Settings.EMPTY);
         provider.init(
             im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()),
-            IndexVersion::current
+            IndexVersion::current,
+            true
         );
     }
 
     public void testGetAdditionalIndexSettingsDefault() {
-        Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build();
+        Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build();
         String dataStreamName = "metrics-my-app";
         String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0);
         var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, null, null, null, settings, List.of());
         assertThat(result.size(), equalTo(1));
-        assertThat(result.get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED"));
+        assertThat(result.get(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED"));
     }
 
     public void testGetAdditionalIndexSettingsApm() throws IOException {
-        Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build();
+        Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build();
         String dataStreamName = "metrics-apm.app.test";
         String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0);
         var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, null, null, null, settings, List.of());
@@ -74,7 +75,7 @@ public void testGetAdditionalIndexSettingsApm() throws IOException {
     }
 
     public void testGetAdditionalIndexSettingsProfiling() throws IOException {
-        Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build();
+        Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build();
         for (String dataStreamName : new String[] { "profiling-metrics", "profiling-events" }) {
             String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0);
             var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, null, null, null, settings, List.of());
@@ -88,7 +89,7 @@ public void testGetAdditionalIndexSettingsProfiling() throws IOException {
     }
 
     public void testGetAdditionalIndexSettingsTsdb() throws IOException {
-        Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build();
+        Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build();
         String dataStreamName = "metrics-my-app";
         String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0);
         var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, IndexMode.TIME_SERIES, null, null, settings, List.of());
@@ -113,14 +114,15 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception
         provider = new LogsdbIndexModeSettingsProvider(syntheticSourceLicenseService, Settings.EMPTY);
         provider.init(
             im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()),
-            IndexVersion::current
+            IndexVersion::current,
+            true
         );
 
-        Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build();
+        Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build();
         String dataStreamName = "metrics-my-app";
         String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0);
         var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, IndexMode.TIME_SERIES, null, null, settings, List.of());
         assertThat(result.size(), equalTo(1));
-        assertThat(result.get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED"));
+        assertThat(result.get(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED"));
     }
 }
diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml
index 2f320c2cad966..b802c8958f421 100644
--- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml
+++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml
@@ -2,9 +2,6 @@
 create logsdb data stream with host.name as keyword:
   - requires:
       test_runner_features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       cluster.put_component_template:
@@ -38,9 +35,6 @@ create logsdb data stream with host.name as keyword:
 create logsdb data stream with host.name as keyword and timestamp as date:
   - requires:
       test_runner_features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       cluster.put_component_template:
@@ -73,12 +67,44 @@ create logsdb data stream with host.name as keyword and timestamp as date:
   - is_true: acknowledged
 
 ---
-create logsdb data stream with host as keyword:
+create logsdb data stream with host.name as integer and timestamp as date:
   - requires:
       test_runner_features: [ "allowed_warnings" ]
+
+  - do:
+      cluster.put_component_template:
+        name: "logsdb-mappings"
+        body:
+          template:
+            settings:
+              mode: "logsdb"
+            mappings:
+              properties:
+                host.name:
+                  type: "integer"
+                "@timestamp":
+                  type: "date"
+
+  - do:
+      indices.put_index_template:
+        name: "logsdb-index-template"
+        body:
+          index_patterns: ["logsdb"]
+          data_stream: {}
+          composed_of: ["logsdb-mappings"]
+      allowed_warnings:
+        - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation"
+
+  - do:
+      indices.create_data_stream:
+        name: "logsdb"
+
+  - is_true: acknowledged
+
+---
+create logsdb data stream with host as keyword:
   - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
+      test_runner_features: [ "allowed_warnings" ]
 
   - do:
       cluster.put_component_template:
@@ -103,20 +129,15 @@ create logsdb data stream with host as keyword:
         - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation"
 
   - do:
-      catch: bad_request
       indices.create_data_stream:
         name: "logsdb"
 
-  - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: can't merge a non object mapping [host] with an object mapping" }
+  - is_true: acknowledged
 
 ---
 create logsdb data stream with host as text and multi fields:
   - requires:
       test_runner_features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       cluster.put_component_template:
@@ -148,20 +169,15 @@ create logsdb data stream with host as text and multi fields:
         - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation"
 
   - do:
-      catch: bad_request
       indices.create_data_stream:
         name: "logsdb"
 
-  - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: can't merge a non object mapping [host] with an object mapping" }
+  - is_true: acknowledged
 
 ---
 create logsdb data stream with host as text:
   - requires:
       test_runner_features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: ["mapper.keyword_normalizer_synthetic_source"]
-      reason: "Support for normalizer on keyword fields"
 
   - do:
       cluster.put_component_template:
@@ -189,20 +205,15 @@ create logsdb data stream with host as text:
         - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation"
 
   - do:
-      catch: bad_request
       indices.create_data_stream:
         name: "logsdb"
 
-  - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: can't merge a non object mapping [host] with an object mapping" }
+  - is_true: acknowledged
 
 ---
 create logsdb data stream with host as text and name as double:
   - requires:
       test_runner_features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: ["mapper.keyword_normalizer_synthetic_source"]
-      reason: "Support for normalizer on keyword fields"
 
   - do:
       cluster.put_component_template:
@@ -233,20 +244,15 @@ create logsdb data stream with host as text and name as double:
         - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation"
 
   - do:
-      catch: bad_request
       indices.create_data_stream:
         name: "logsdb"
 
-  - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: can't merge a non object mapping [host] with an object mapping" }
+  - is_true: acknowledged
 
 ---
 create logsdb data stream with timestamp object mapping:
   - requires:
       test_runner_features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: ["mapper.keyword_normalizer_synthetic_source"]
-      reason: "Support for normalizer on keyword fields"
 
   - do:
       cluster.put_component_template:
@@ -285,9 +291,6 @@ create logsdb data stream with timestamp object mapping:
 create logsdb data stream with custom sorting without host.name:
   - skip:
       features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       allowed_warnings:
@@ -335,9 +338,6 @@ create logsdb data stream with custom sorting without host.name:
 create logsdb data stream with custom sorting and host object:
   - skip:
       features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       allowed_warnings:
@@ -388,15 +388,12 @@ create logsdb data stream with custom sorting and host object:
   - match: { .$backing_index.mappings.properties.host.properties.ip.type: ip }
   - match: { .$backing_index.mappings.properties.host.properties.hostname.type: keyword }
   - match: { .$backing_index.mappings.properties.host.properties.region.type: keyword }
-  - match: { .$backing_index.mappings.properties.host.properties.name.type: integer } # Overrides LogsDB injected
+  - match: { .$backing_index.mappings.properties.host.properties.name.type: integer }
 
 ---
 create logsdb data stream with custom sorting and dynamically mapped host.name:
   - skip:
       features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       allowed_warnings:
@@ -459,9 +456,6 @@ create logsdb data stream with custom sorting and dynamically mapped host.name:
 create logsdb data stream with custom sorting and host.name object:
   - skip:
       features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       allowed_warnings:
@@ -515,9 +509,6 @@ create logsdb data stream with custom sorting and host.name object:
 create logsdb data stream with default sorting on malformed host.name:
   - skip:
       features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       allowed_warnings:
@@ -579,9 +570,6 @@ create logsdb data stream with default sorting on malformed host.name:
 create logsdb data stream with custom sorting and host.name date field:
   - skip:
       features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       allowed_warnings:
@@ -632,9 +620,6 @@ create logsdb data stream with custom sorting and host.name date field:
 create logsdb data stream with custom sorting and missing host.name field mapping:
   - skip:
       features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       allowed_warnings:
@@ -647,8 +632,8 @@ create logsdb data stream with custom sorting and missing host.name field mappin
           template:
             settings:
               index:
-                sort.field: [ host.name, host.hostname ]
-                sort.order: [ desc, desc ]
+                sort.field: [ host.hostname ]
+                sort.order: [ desc ]
                 mode: logsdb
             mappings:
               properties:
@@ -677,16 +662,12 @@ create logsdb data stream with custom sorting and missing host.name field mappin
 
   - match: { .$backing_index.mappings.properties.@timestamp.type: date }
   - match: { .$backing_index.mappings.properties.host.properties.hostname.type: keyword }
-  - match: { .$backing_index.mappings.properties.host.properties.name.type: keyword }
-  - match: { .$backing_index.mappings.properties.host.properties.name.ignore_above: 1024 }
+  - match: { .$backing_index.mappings.properties.host.properties.name: null }
 
 ---
 create logsdb data stream with custom sorting and host.name field without doc values:
   - skip:
       features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       allowed_warnings:
@@ -726,9 +707,6 @@ create logsdb data stream with custom sorting and host.name field without doc va
 create logsdb data stream with incompatible ignore_above on host.name:
   - skip:
       features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       allowed_warnings:
@@ -774,9 +752,6 @@ create logsdb data stream with incompatible ignore_above on host.name:
 create logsdb data stream with no sorting and host.name as text:
   - skip:
       features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       allowed_warnings:
@@ -797,20 +772,26 @@ create logsdb data stream with no sorting and host.name as text:
   - is_true: acknowledged
 
   - do:
-      catch: bad_request
       indices.create_data_stream:
         name: logsdb-non-keyword
+  - is_true: acknowledged
 
-  - match: { error.type: "illegal_argument_exception" }
-  - match: { error.reason: "docvalues not found for index sort field:[host.name]" }
+  - do:
+      indices.get_data_stream:
+        name: logsdb-non-keyword
+
+  - set: { data_streams.0.indices.0.index_name: backing_index }
+  - do:
+      indices.get_mapping:
+        index: $backing_index
+
+  - match: { .$backing_index.mappings.properties.@timestamp.type: date }
+  - match: { .$backing_index.mappings.properties.host.properties.name.type: text }
 
 ---
 create logsdb data stream without index sorting and ignore_above on host.name:
   - skip:
       features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       allowed_warnings:
@@ -854,9 +835,6 @@ create logsdb data stream without index sorting and ignore_above on host.name:
 create logsdb data stream with host.name as alias and sorting on it:
   - skip:
       features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       allowed_warnings:
@@ -892,9 +870,6 @@ create logsdb data stream with host.name as alias and sorting on it:
 create logsdb data stream with multi-fields on host.name:
   - skip:
       features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       allowed_warnings:
@@ -939,9 +914,6 @@ create logsdb data stream with multi-fields on host.name:
 create logsdb data stream with multi-fields on host.name and no sorting:
   - skip:
       features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       allowed_warnings:
@@ -963,20 +935,26 @@ create logsdb data stream with multi-fields on host.name and no sorting:
           data_stream: {}
 
   - do:
-      catch: bad_request
       indices.create_data_stream:
         name: logsdb-no-sort-multi-fields
+  - is_true: acknowledged
 
-  - match: { error.type: "illegal_argument_exception" }
-  - match: { error.reason: "docvalues not found for index sort field:[host.name]" }
+  - do:
+      indices.get_data_stream:
+        name: logsdb-no-sort-multi-fields
+
+  - set: { data_streams.0.indices.0.index_name: backing_index }
+  - do:
+      indices.get_mapping:
+        index: $backing_index
+
+  - match: { .$backing_index.mappings.properties.@timestamp.type: date }
+  - match: { .$backing_index.mappings.properties.host.properties.name.type: text }
 
 ---
 create logsdb data stream with custom empty sorting:
   - skip:
       features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       allowed_warnings:
@@ -1021,9 +999,6 @@ create logsdb data stream with custom empty sorting:
 create logsdb data stream with custom sorting on timestamp:
   - skip:
       features: [ "allowed_warnings" ]
-  - requires:
-      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
-      reason: support for normalizer on keyword fields
 
   - do:
       allowed_warnings:
diff --git a/x-pack/plugin/logstash/build.gradle b/x-pack/plugin/logstash/build.gradle
index 8e47ebaebdf8d..37bd972629aa2 100644
--- a/x-pack/plugin/logstash/build.gradle
+++ b/x-pack/plugin/logstash/build.gradle
@@ -2,9 +2,9 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 esplugin {
-  name 'x-pack-logstash'
-  description 'Elasticsearch Expanded Pack Plugin - Logstash'
-  classname 'org.elasticsearch.xpack.logstash.Logstash'
+  name = 'x-pack-logstash'
+  description = 'Elasticsearch Expanded Pack Plugin - Logstash'
+  classname ='org.elasticsearch.xpack.logstash.Logstash'
   extendedPlugins = ['x-pack-core']
 }
 
diff --git a/x-pack/plugin/mapper-aggregate-metric/build.gradle b/x-pack/plugin/mapper-aggregate-metric/build.gradle
index 2a7841929b21d..cb7dad88ac61a 100644
--- a/x-pack/plugin/mapper-aggregate-metric/build.gradle
+++ b/x-pack/plugin/mapper-aggregate-metric/build.gradle
@@ -10,9 +10,9 @@ evaluationDependsOn(xpackModule('core'))
 apply plugin: 'elasticsearch.internal-es-plugin'
 
 esplugin {
-  name 'x-pack-aggregate-metric'
-  description 'Module for the aggregate_metric_double field type, which allows pre-aggregated fields to be stored as a single field'
-  classname 'org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin'
+  name = 'x-pack-aggregate-metric'
+  description = 'Module for the aggregate_metric_double field type, which allows pre-aggregated fields to be stored as a single field'
+  classname ='org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin'
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/mapper-constant-keyword/build.gradle b/x-pack/plugin/mapper-constant-keyword/build.gradle
index c1e0eb61b611b..1650016949661 100644
--- a/x-pack/plugin/mapper-constant-keyword/build.gradle
+++ b/x-pack/plugin/mapper-constant-keyword/build.gradle
@@ -9,9 +9,9 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
 esplugin {
-  name 'constant-keyword'
-  description 'Module for the constant-keyword field type, which is a specialization of keyword for the case when all documents have the same value.'
-  classname 'org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin'
+  name = 'constant-keyword'
+  description = 'Module for the constant-keyword field type, which is a specialization of keyword for the case when all documents have the same value.'
+  classname ='org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin'
   extendedPlugins = ['x-pack-core', 'lang-painless']
 }
 base {
diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java
index fa5d9428bb0c6..fbc89fd563cc7 100644
--- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java
+++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.constantkeyword.mapper;
 
+import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.MatchAllDocsQuery;
@@ -41,6 +42,7 @@
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.SortedNumericDocValuesSyntheticFieldLoader;
 import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.ValueFetcher;
 import org.elasticsearch.index.query.QueryRewriteContext;
@@ -339,6 +341,12 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
                     + "]"
             );
         }
+
+        if (context.mappingLookup().isSourceSynthetic()) {
+            // Remember which documents had value in source so that it can be correctly
+            // reconstructed in synthetic source
+            context.doc().add(new SortedNumericDocValuesField(fieldType().name(), 1));
+        }
     }
 
     @Override
@@ -348,20 +356,19 @@ protected String contentType() {
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport() {
-        String value = fieldType().value();
+        String const_value = fieldType().value();
 
-        if (value == null) {
+        if (const_value == null) {
             return new SyntheticSourceSupport.Native(SourceLoader.SyntheticFieldLoader.NOTHING);
         }
 
-        /*
-        If there was no value in the document, synthetic source should not have the value too.
-        This is consistent with stored source behavior and is important for scenarios
-        like reindexing into an index that has a different value of this value in the mapping.
+        var loader = new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), false) {
+            @Override
+            protected void writeValue(XContentBuilder b, long ignored) throws IOException {
+                b.value(const_value);
+            }
+        };
 
-        In order to do that we use fallback logic which implements exactly such logic (_source only contains value
-        if it was in the original document).
-         */
-        return new SyntheticSourceSupport.Fallback();
+        return new SyntheticSourceSupport.Native(loader);
     }
 }
diff --git a/x-pack/plugin/mapper-constant-keyword/src/yamlRestTest/resources/rest-api-spec/test/20_synthetic_source.yml b/x-pack/plugin/mapper-constant-keyword/src/yamlRestTest/resources/rest-api-spec/test/20_synthetic_source.yml
index 012b1006b8d20..7ea1937aa39e9 100644
--- a/x-pack/plugin/mapper-constant-keyword/src/yamlRestTest/resources/rest-api-spec/test/20_synthetic_source.yml
+++ b/x-pack/plugin/mapper-constant-keyword/src/yamlRestTest/resources/rest-api-spec/test/20_synthetic_source.yml
@@ -59,3 +59,103 @@ constant_keyword:
       hits.hits.0._source:
         kwd: foo
         const_kwd: bar
+
+---
+constant_keyword update value:
+  - requires:
+      cluster_features: [ "mapper.constant_keyword.synthetic_source_write_fix" ]
+      reason: "Behavior fix"
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mapping.source.mode: synthetic
+          mappings:
+            properties:
+              const_kwd:
+                type: constant_keyword
+              kwd:
+                type: keyword
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        refresh: true
+        body:
+          kwd: foo
+
+  - do:
+      index:
+        index:   test
+        id:      2
+        refresh: true
+        body:
+          const_kwd: bar
+
+  - do:
+      index:
+        index:   test
+        id:      3
+        refresh: true
+        body:
+          kwd: foo
+
+  - do:
+      index:
+        index:   test
+        id:      4
+        refresh: true
+        body:
+          const_kwd: bar
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            ids:
+              values: [1]
+
+  - match:
+      hits.hits.0._source:
+        kwd: foo
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            ids:
+              values: [2]
+
+  - match:
+      hits.hits.0._source:
+        const_kwd: bar
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            ids:
+              values: [3]
+
+  - match:
+      hits.hits.0._source:
+        kwd: foo
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            ids:
+              values: [4]
+
+  - match:
+      hits.hits.0._source:
+        const_kwd: bar
diff --git a/x-pack/plugin/mapper-counted-keyword/build.gradle b/x-pack/plugin/mapper-counted-keyword/build.gradle
index f2e7152406962..b69a599628d2c 100644
--- a/x-pack/plugin/mapper-counted-keyword/build.gradle
+++ b/x-pack/plugin/mapper-counted-keyword/build.gradle
@@ -8,9 +8,9 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 
 esplugin {
-  name 'counted-keyword'
-  description 'Module for the counted-keyword field type, which allows to consider duplicates in an array of values of that type.'
-  classname 'org.elasticsearch.xpack.countedkeyword.CountedKeywordMapperPlugin'
+  name = 'counted-keyword'
+  description = 'Module for the counted-keyword field type, which allows to consider duplicates in an array of values of that type.'
+  classname ='org.elasticsearch.xpack.countedkeyword.CountedKeywordMapperPlugin'
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/mapper-unsigned-long/build.gradle b/x-pack/plugin/mapper-unsigned-long/build.gradle
index 17a4f8a03fa57..71afd2e9082f3 100644
--- a/x-pack/plugin/mapper-unsigned-long/build.gradle
+++ b/x-pack/plugin/mapper-unsigned-long/build.gradle
@@ -14,9 +14,9 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.yaml-rest-compat-test'
 
 esplugin {
-  name 'unsigned-long'
-  description 'Module for the unsigned long field type'
-  classname 'org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin'
+  name = 'unsigned-long'
+  description = 'Module for the unsigned long field type'
+  classname ='org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin'
   extendedPlugins = ['x-pack-core', 'lang-painless']
 }
 base {
diff --git a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/80_synthetic_source.yml b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/80_synthetic_source.yml
index 22f69e30650fd..eb7c1df91de60 100644
--- a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/80_synthetic_source.yml
+++ b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/80_synthetic_source.yml
@@ -5,10 +5,6 @@ setup:
 
 ---
 synthetic source:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"]
-      reason: requires synthetic source support
-
   - do:
       indices.create:
         index: synthetic_source_test
@@ -51,10 +47,6 @@ synthetic source:
 
 ---
 synthetic source with copy_to:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"]
-      reason: requires copy_to support in synthetic source
-
   - do:
       indices.create:
         index: synthetic_source_test
@@ -111,10 +103,6 @@ synthetic source with copy_to:
 
 ---
 synthetic source with disabled doc_values:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"]
-      reason: requires disabled doc_values support in synthetic source
-
   - do:
       indices.create:
         index: synthetic_source_test
diff --git a/x-pack/plugin/mapper-version/build.gradle b/x-pack/plugin/mapper-version/build.gradle
index bf78c61523e39..4ee97c2d1a46e 100644
--- a/x-pack/plugin/mapper-version/build.gradle
+++ b/x-pack/plugin/mapper-version/build.gradle
@@ -13,9 +13,9 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  name 'mapper-version'
-  description 'A plugin for a field type to store software versions'
-  classname 'org.elasticsearch.xpack.versionfield.VersionFieldPlugin'
+  name = 'mapper-version'
+  description = 'A plugin for a field type to store software versions'
+  classname = 'org.elasticsearch.xpack.versionfield.VersionFieldPlugin'
   extendedPlugins = ['x-pack-core', 'lang-painless']
 }
 base {
diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java
index 6bf2917c601ac..0bdb5fc0101a5 100644
--- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java
+++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java
@@ -403,7 +403,7 @@ public Iterator<Mapper> iterator() {
         return concat;
     }
 
-    public static DocValueFormat VERSION_DOCVALUE = new DocValueFormat() {
+    public static final DocValueFormat VERSION_DOCVALUE = new DocValueFormat() {
 
         @Override
         public String getWriteableName() {
diff --git a/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/40_synthetic_source.yml b/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/40_synthetic_source.yml
index 3095a19fa29d0..0989a92f2ca01 100644
--- a/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/40_synthetic_source.yml
+++ b/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/40_synthetic_source.yml
@@ -69,10 +69,6 @@ script values:
 
 ---
 synthetic source with copy_to:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"]
-      reason: requires copy_to support in synthetic source
-
   - do:
       indices.create:
         index: synthetic_source_test
diff --git a/x-pack/plugin/migrate/build.gradle b/x-pack/plugin/migrate/build.gradle
index 87ea7a07ab414..283362a637e78 100644
--- a/x-pack/plugin/migrate/build.gradle
+++ b/x-pack/plugin/migrate/build.gradle
@@ -2,12 +2,12 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  name 'x-pack-migrate'
-  description 'Elasticsearch Expanded Pack Plugin - Index and Data Stream Migration'
-  classname 'org.elasticsearch.xpack.migrate.MigratePlugin'
+  name = 'x-pack-migrate'
+  description = 'Elasticsearch Expanded Pack Plugin - Index and Data Stream Migration'
+  classname = 'org.elasticsearch.xpack.migrate.MigratePlugin'
   extendedPlugins = ['x-pack-core']
-  hasNativeController false
-  requiresKeystore true
+  hasNativeController =false
+  requiresKeystore =true
 }
 base {
   archivesName = 'x-pack-migrate'
diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java
new file mode 100644
index 0000000000000..f1b14658c6b8e
--- /dev/null
+++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java
@@ -0,0 +1,251 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.migrate.action;
+
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.datastreams.DataStreamsPlugin;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.reindex.ReindexPlugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.xcontent.json.JsonXContent;
+import org.elasticsearch.xpack.migrate.MigratePlugin;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.REINDEX_DATA_STREAM_FEATURE_FLAG;
+
+public class CreateIndexFromSourceActionIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(MigratePlugin.class, ReindexPlugin.class, MockTransportService.TestPlugin.class, DataStreamsPlugin.class);
+    }
+
+    public void testDestIndexCreated() throws Exception {
+        assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
+
+        var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get();
+
+        // create from source
+        var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        assertAcked(
+            client().execute(CreateIndexFromSourceAction.INSTANCE, new CreateIndexFromSourceAction.Request(sourceIndex, destIndex))
+        );
+
+        try {
+            indicesAdmin().getIndex(new GetIndexRequest().indices(destIndex)).actionGet();
+        } catch (IndexNotFoundException e) {
+            fail();
+        }
+    }
+
+    public void testSettingsCopiedFromSource() throws Exception {
+        assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
+
+        // start with a static setting
+        var numShards = randomIntBetween(1, 10);
+        var staticSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build();
+        var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        indicesAdmin().create(new CreateIndexRequest(sourceIndex, staticSettings)).get();
+
+        // update with a dynamic setting
+        var numReplicas = randomIntBetween(0, 10);
+        var dynamicSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas).build();
+        indicesAdmin().updateSettings(new UpdateSettingsRequest(dynamicSettings, sourceIndex)).actionGet();
+
+        // create from source
+        var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        assertAcked(
+            client().execute(CreateIndexFromSourceAction.INSTANCE, new CreateIndexFromSourceAction.Request(sourceIndex, destIndex))
+        );
+
+        // assert both static and dynamic settings set on dest index
+        var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet();
+        assertEquals(numReplicas, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS)));
+        assertEquals(numShards, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS)));
+    }
+
+    public void testMappingsCopiedFromSource() {
+        assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
+
+        var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        String mapping = """
+            {
+              "_doc":{
+                "dynamic":"strict",
+                "properties":{
+                  "foo1":{
+                    "type":"text"
+                  }
+                }
+              }
+            }
+            """;
+        indicesAdmin().create(new CreateIndexRequest(sourceIndex).mapping(mapping)).actionGet();
+
+        // create from source
+        var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        assertAcked(
+            client().execute(CreateIndexFromSourceAction.INSTANCE, new CreateIndexFromSourceAction.Request(sourceIndex, destIndex))
+        );
+
+        var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(sourceIndex, destIndex))
+            .actionGet();
+        Map<String, MappingMetadata> mappings = mappingsResponse.mappings();
+        var destMappings = mappings.get(destIndex).sourceAsMap();
+        var sourceMappings = mappings.get(sourceIndex).sourceAsMap();
+
+        assertEquals(sourceMappings, destMappings);
+        // sanity check specific value from dest mapping
+        assertEquals("text", XContentMapValues.extractValue("properties.foo1.type", destMappings));
+    }
+
+    public void testSettingsOverridden() throws Exception {
+        assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
+
+        var numShardsSource = randomIntBetween(1, 10);
+        var numReplicasSource = randomIntBetween(0, 10);
+        var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        var sourceSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShardsSource)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicasSource)
+            .build();
+        indicesAdmin().create(new CreateIndexRequest(sourceIndex, sourceSettings)).get();
+
+        boolean overrideNumShards = randomBoolean();
+        Settings settingsOverride = overrideNumShards
+            ? Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShardsSource + 1).build()
+            : Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicasSource + 1).build();
+
+        // create from source
+        var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        assertAcked(
+            client().execute(
+                CreateIndexFromSourceAction.INSTANCE,
+                new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, settingsOverride, Map.of())
+            )
+        );
+
+        // assert settings overridden
+        int expectedShards = overrideNumShards ? numShardsSource + 1 : numShardsSource;
+        int expectedReplicas = overrideNumShards ? numReplicasSource : numReplicasSource + 1;
+        var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet();
+        assertEquals(expectedShards, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS)));
+        assertEquals(expectedReplicas, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS)));
+    }
+
+    public void testSettingsNullOverride() throws Exception {
+        assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
+
+        var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        var sourceSettings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_WRITE, true).build();
+        indicesAdmin().create(new CreateIndexRequest(sourceIndex, sourceSettings)).get();
+
+        Settings settingsOverride = Settings.builder().putNull(IndexMetadata.SETTING_BLOCKS_WRITE).build();
+
+        // create from source
+        var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        assertAcked(
+            client().execute(
+                CreateIndexFromSourceAction.INSTANCE,
+                new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, settingsOverride, Map.of())
+            )
+        );
+
+        // assert settings overridden
+        var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet();
+        assertNull(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_BLOCKS_WRITE));
+    }
+
+    public void testMappingsOverridden() {
+        assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
+
+        var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        String sourceMapping = """
+            {
+              "_doc":{
+                "dynamic":"strict",
+                "properties":{
+                  "foo1":{
+                    "type":"text"
+                  },
+                  "foo2":{
+                    "type":"boolean"
+                  }
+                }
+              }
+            }
+            """;
+        indicesAdmin().create(new CreateIndexRequest(sourceIndex).mapping(sourceMapping)).actionGet();
+
+        String mappingOverrideStr = """
+            {
+              "_doc":{
+                "dynamic":"strict",
+                "properties":{
+                  "foo1":{
+                    "type":"integer"
+                  },
+                  "foo3": {
+                    "type":"keyword"
+                  }
+                }
+              }
+            }
+            """;
+        var mappingOverride = XContentHelper.convertToMap(JsonXContent.jsonXContent, mappingOverrideStr, false);
+
+        // create from source
+        var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        assertAcked(
+            client().execute(
+                CreateIndexFromSourceAction.INSTANCE,
+                new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, Settings.EMPTY, mappingOverride)
+            )
+        );
+
+        var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(destIndex)).actionGet();
+        Map<String, MappingMetadata> mappings = mappingsResponse.mappings();
+        var destMappings = mappings.get(destIndex).sourceAsMap();
+
+        String expectedMappingStr = """
+            {
+                "dynamic":"strict",
+                "properties":{
+                  "foo1":{
+                    "type":"integer"
+                  },
+                  "foo2": {
+                    "type":"boolean"
+                  },
+                  "foo3": {
+                    "type":"keyword"
+                  }
+                }
+            }
+            """;
+        var expectedMapping = XContentHelper.convertToMap(JsonXContent.jsonXContent, expectedMappingStr, false);
+        assertEquals(expectedMapping, destMappings);
+    }
+}
diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportActionIT.java
index 6e24e644cb2af..b32a6efb854d7 100644
--- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportActionIT.java
+++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportActionIT.java
@@ -29,7 +29,7 @@
 import org.elasticsearch.xpack.migrate.MigratePlugin;
 import org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.ReindexDataStreamRequest;
 import org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.ReindexDataStreamResponse;
-import org.elasticsearch.xpack.migrate.task.ReindexDataStreamStatus;
+import org.elasticsearch.xpack.migrate.task.ReindexDataStreamEnrichedStatus;
 import org.elasticsearch.xpack.migrate.task.ReindexDataStreamTask;
 
 import java.util.Collection;
@@ -37,6 +37,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -100,7 +101,7 @@ public void testAlreadyUpToDateDataStream() throws Exception {
         assertThat(task.getStatus().complete(), equalTo(true));
         assertNull(task.getStatus().exception());
         assertThat(task.getStatus().pending(), equalTo(0));
-        assertThat(task.getStatus().inProgress(), equalTo(0));
+        assertThat(task.getStatus().inProgress(), equalTo(Set.of()));
         assertThat(task.getStatus().errors().size(), equalTo(0));
 
         assertBusy(() -> {
@@ -108,12 +109,12 @@ public void testAlreadyUpToDateDataStream() throws Exception {
                 new ActionType<GetMigrationReindexStatusAction.Response>(GetMigrationReindexStatusAction.NAME),
                 new GetMigrationReindexStatusAction.Request(dataStreamName)
             ).actionGet();
-            ReindexDataStreamStatus status = (ReindexDataStreamStatus) statusResponse.getTask().getTask().status();
+            ReindexDataStreamEnrichedStatus status = statusResponse.getEnrichedStatus();
             assertThat(status.complete(), equalTo(true));
             assertThat(status.errors(), equalTo(List.of()));
             assertThat(status.exception(), equalTo(null));
             assertThat(status.pending(), equalTo(0));
-            assertThat(status.inProgress(), equalTo(0));
+            assertThat(status.inProgress().size(), equalTo(0));
             assertThat(status.totalIndices(), equalTo(backingIndexCount));
             assertThat(status.totalIndicesToBeUpgraded(), equalTo(0));
         });
diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java
similarity index 70%
rename from x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexIT.java
rename to x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java
index e492f035da866..b310d332fdd5c 100644
--- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexIT.java
+++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.migrate.action;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
@@ -15,6 +16,8 @@
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
+import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteIndexTemplateAction;
 import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
@@ -53,7 +56,7 @@
 import static org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.REINDEX_DATA_STREAM_FEATURE_FLAG;
 import static org.hamcrest.Matchers.equalTo;
 
-public class ReindexDatastreamIndexIT extends ESIntegTestCase {
+public class ReindexDatastreamIndexTransportActionIT extends ESIntegTestCase {
 
     private static final String MAPPING = """
         {
@@ -94,7 +97,7 @@ public void testDestIndexDeletedIfExists() throws Exception {
         assertHitCount(prepareSearch(destIndex).setSize(0), 0);
     }
 
-    public void testDestIndexNameSet() throws Exception {
+    public void testDestIndexNameSet_noDotPrefix() throws Exception {
         assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
 
         var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
@@ -108,6 +111,20 @@ public void testDestIndexNameSet() throws Exception {
         assertEquals(expectedDestIndexName, response.getDestIndex());
     }
 
+    public void testDestIndexNameSet_withDotPrefix() throws Exception {
+        assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
+
+        var sourceIndex = "." + randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get();
+
+        // call reindex
+        var response = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex))
+            .actionGet();
+
+        var expectedDestIndexName = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex);
+        assertEquals(expectedDestIndexName, response.getDestIndex());
+    }
+
     public void testDestIndexContainsDocs() throws Exception {
         assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
 
@@ -126,12 +143,14 @@ public void testDestIndexContainsDocs() throws Exception {
         assertHitCount(prepareSearch(response.getDestIndex()).setSize(0), numDocs);
     }
 
-    public void testSetSourceToReadOnly() throws Exception {
+    public void testSetSourceToBlockWrites() throws Exception {
         assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
 
+        var settings = randomBoolean() ? Settings.builder().put(IndexMetadata.SETTING_BLOCKS_WRITE, true).build() : Settings.EMPTY;
+
         // empty source index
         var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
-        indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get();
+        indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)).get();
 
         // call reindex
         client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)).actionGet();
@@ -153,7 +172,11 @@ public void testSettingsAddedBeforeReindex() throws Exception {
 
         // update with a dynamic setting
         var numReplicas = randomIntBetween(0, 10);
-        var dynamicSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas).build();
+        var refreshInterval = randomIntBetween(1, 100) + "s";
+        var dynamicSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas)
+            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), refreshInterval)
+            .build();
         indicesAdmin().updateSettings(new UpdateSettingsRequest(dynamicSettings, sourceIndex)).actionGet();
 
         // call reindex
@@ -165,6 +188,7 @@ public void testSettingsAddedBeforeReindex() throws Exception {
         var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet();
         assertEquals(numReplicas, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS)));
         assertEquals(numShards, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS)));
+        assertEquals(refreshInterval, settingsResponse.getSetting(destIndex, IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()));
     }
 
     public void testMappingsAddedToDestIndex() throws Exception {
@@ -190,7 +214,8 @@ public void testMappingsAddedToDestIndex() throws Exception {
             .actionGet()
             .getDestIndex();
 
-        var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet();
+        var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(sourceIndex, destIndex))
+            .actionGet();
         Map<String, MappingMetadata> mappings = mappingsResponse.mappings();
         var destMappings = mappings.get(destIndex).sourceAsMap();
         var sourceMappings = mappings.get(sourceIndex).sourceAsMap();
@@ -200,16 +225,46 @@ public void testMappingsAddedToDestIndex() throws Exception {
         assertEquals("text", XContentMapValues.extractValue("properties.foo1.type", destMappings));
     }
 
-    public void testReadOnlyAddedBack() {
+    public void testFailIfMetadataBlockSet() {
+        assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
+
+        var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        var settings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_METADATA, true).build();
+        indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)).actionGet();
+
+        try {
+            client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)).actionGet();
+        } catch (ElasticsearchException e) {
+            assertTrue(e.getMessage().contains("Cannot reindex index") || e.getCause().getMessage().equals("Cannot reindex index"));
+        }
+
+        cleanupMetadataBlocks(sourceIndex);
+    }
+
+    public void testFailIfReadBlockSet() {
+        assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
+
+        var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        var settings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_READ, true).build();
+        indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)).actionGet();
+
+        try {
+            client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)).actionGet();
+        } catch (ElasticsearchException e) {
+            assertTrue(e.getMessage().contains("Cannot reindex index") || e.getCause().getMessage().equals("Cannot reindex index"));
+        }
+
+        cleanupMetadataBlocks(sourceIndex);
+    }
+
+    public void testReadOnlyBlocksNotAddedBack() {
         assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
 
-        // Create source index with read-only and/or block-writes
         var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
-        boolean isReadOnly = randomBoolean();
-        boolean isBlockWrites = randomBoolean();
         var settings = Settings.builder()
-            .put(IndexMetadata.SETTING_READ_ONLY, isReadOnly)
-            .put(IndexMetadata.SETTING_BLOCKS_WRITE, isBlockWrites)
+            .put(IndexMetadata.SETTING_READ_ONLY, randomBoolean())
+            .put(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE, randomBoolean())
+            .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean())
             .build();
         indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)).actionGet();
 
@@ -218,13 +273,45 @@ public void testReadOnlyAddedBack() {
             .actionGet()
             .getDestIndex();
 
-        // assert read-only settings added back to dest index
         var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet();
-        assertEquals(isReadOnly, Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY)));
-        assertEquals(isBlockWrites, Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_BLOCKS_WRITE)));
+        assertFalse(Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY)));
+        assertFalse(Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE)));
+        assertFalse(Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_BLOCKS_WRITE)));
 
-        removeReadOnly(sourceIndex);
-        removeReadOnly(destIndex);
+        cleanupMetadataBlocks(sourceIndex);
+        cleanupMetadataBlocks(destIndex);
+    }
+
+    public void testUpdateSettingsDefaultsRestored() {
+        assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
+
+        // ESIntegTestCase creates a template random_index_template which contains a value for number_of_replicas.
+        // Since this test checks the behavior of default settings, there cannot be a value for number_of_replicas,
+        // so we delete the template within this method. This has no effect on other tests which will still
+        // have the template created during their setup.
+        assertAcked(
+            indicesAdmin().execute(TransportDeleteIndexTemplateAction.TYPE, new DeleteIndexTemplateRequest("random_index_template"))
+        );
+
+        var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        assertAcked(indicesAdmin().create(new CreateIndexRequest(sourceIndex)));
+
+        // call reindex
+        var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex))
+            .actionGet()
+            .getDestIndex();
+
+        var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(sourceIndex, destIndex)).actionGet();
+        var destSettings = settingsResponse.getIndexToSettings().get(destIndex);
+
+        assertEquals(
+            IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getDefault(destSettings),
+            IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(destSettings)
+        );
+        assertEquals(
+            IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getDefault(destSettings),
+            IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.get(destSettings)
+        );
     }
 
     public void testSettingsAndMappingsFromTemplate() throws IOException {
@@ -264,7 +351,8 @@ public void testSettingsAndMappingsFromTemplate() throws IOException {
 
         // verify mappings from templates copied to dest index
         {
-            var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet();
+            var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(sourceIndex, destIndex))
+                .actionGet();
             var destMappings = mappingsResponse.mappings().get(destIndex).sourceAsMap();
             var sourceMappings = mappingsResponse.mappings().get(sourceIndex).sourceAsMap();
             assertEquals(sourceMappings, destMappings);
@@ -371,10 +459,11 @@ public void testTsdbStartEndSet() throws Exception {
     // TODO check other IndexMetadata fields that need to be fixed after the fact
     // TODO what happens if don't have necessary perms for a given index?
 
-    private static void removeReadOnly(String index) {
+    private static void cleanupMetadataBlocks(String index) {
         var settings = Settings.builder()
-            .put(IndexMetadata.SETTING_READ_ONLY, false)
-            .put(IndexMetadata.SETTING_BLOCKS_WRITE, false)
+            .putNull(IndexMetadata.SETTING_READ_ONLY)
+            .putNull(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE)
+            .putNull(IndexMetadata.SETTING_BLOCKS_METADATA)
             .build();
         assertAcked(indicesAdmin().updateSettings(new UpdateSettingsRequest(settings, index)).actionGet());
     }
@@ -405,4 +494,32 @@ private static String getIndexUUID(String index) {
             .get(index)
             .get(IndexMetadata.SETTING_INDEX_UUID);
     }
+
+    public void testGenerateDestIndexName_noDotPrefix() {
+        String sourceIndex = "sourceindex";
+        String expectedDestIndex = "migrated-sourceindex";
+        String actualDestIndex = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex);
+        assertEquals(expectedDestIndex, actualDestIndex);
+    }
+
+    public void testGenerateDestIndexName_withDotPrefix() {
+        String sourceIndex = ".sourceindex";
+        String expectedDestIndex = ".migrated-sourceindex";
+        String actualDestIndex = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex);
+        assertEquals(expectedDestIndex, actualDestIndex);
+    }
+
+    public void testGenerateDestIndexName_withHyphen() {
+        String sourceIndex = "source-index";
+        String expectedDestIndex = "migrated-source-index";
+        String actualDestIndex = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex);
+        assertEquals(expectedDestIndex, actualDestIndex);
+    }
+
+    public void testGenerateDestIndexName_withUnderscore() {
+        String sourceIndex = "source_index";
+        String expectedDestIndex = "migrated-source_index";
+        String actualDestIndex = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex);
+        assertEquals(expectedDestIndex, actualDestIndex);
+    }
 }
diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java
index f42d05727b9fd..55ec4065be8c4 100644
--- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java
+++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.settings.SettingsModule;
@@ -34,6 +35,8 @@
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xpack.migrate.action.CancelReindexDataStreamAction;
 import org.elasticsearch.xpack.migrate.action.CancelReindexDataStreamTransportAction;
+import org.elasticsearch.xpack.migrate.action.CreateIndexFromSourceAction;
+import org.elasticsearch.xpack.migrate.action.CreateIndexFromSourceTransportAction;
 import org.elasticsearch.xpack.migrate.action.GetMigrationReindexStatusAction;
 import org.elasticsearch.xpack.migrate.action.GetMigrationReindexStatusTransportAction;
 import org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction;
@@ -41,6 +44,7 @@
 import org.elasticsearch.xpack.migrate.action.ReindexDataStreamIndexTransportAction;
 import org.elasticsearch.xpack.migrate.action.ReindexDataStreamTransportAction;
 import org.elasticsearch.xpack.migrate.rest.RestCancelReindexDataStreamAction;
+import org.elasticsearch.xpack.migrate.rest.RestCreateIndexFromSourceAction;
 import org.elasticsearch.xpack.migrate.rest.RestGetMigrationReindexStatusAction;
 import org.elasticsearch.xpack.migrate.rest.RestMigrationReindexAction;
 import org.elasticsearch.xpack.migrate.task.ReindexDataStreamPersistentTaskExecutor;
@@ -55,6 +59,7 @@
 import java.util.function.Supplier;
 
 import static org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.REINDEX_DATA_STREAM_FEATURE_FLAG;
+import static org.elasticsearch.xpack.migrate.task.ReindexDataStreamPersistentTaskExecutor.MAX_CONCURRENT_INDICES_REINDEXED_PER_DATA_STREAM_SETTING;
 
 public class MigratePlugin extends Plugin implements ActionPlugin, PersistentTaskPlugin {
 
@@ -75,6 +80,7 @@ public List<RestHandler> getRestHandlers(
             handlers.add(new RestMigrationReindexAction());
             handlers.add(new RestGetMigrationReindexStatusAction());
             handlers.add(new RestCancelReindexDataStreamAction());
+            handlers.add(new RestCreateIndexFromSourceAction());
         }
         return handlers;
     }
@@ -87,6 +93,7 @@ public List<RestHandler> getRestHandlers(
             actions.add(new ActionHandler<>(GetMigrationReindexStatusAction.INSTANCE, GetMigrationReindexStatusTransportAction.class));
             actions.add(new ActionHandler<>(CancelReindexDataStreamAction.INSTANCE, CancelReindexDataStreamTransportAction.class));
             actions.add(new ActionHandler<>(ReindexDataStreamIndexAction.INSTANCE, ReindexDataStreamIndexTransportAction.class));
+            actions.add(new ActionHandler<>(CreateIndexFromSourceAction.INSTANCE, CreateIndexFromSourceTransportAction.class));
         }
         return actions;
     }
@@ -148,4 +155,11 @@ public List<PersistentTasksExecutor<?>> getPersistentTasksExecutor(
             return List.of();
         }
     }
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        List<Setting<?>> pluginSettings = new ArrayList<>();
+        pluginSettings.add(MAX_CONCURRENT_INDICES_REINDEXED_PER_DATA_STREAM_SETTING);
+        return pluginSettings;
+    }
 }
diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java
new file mode 100644
index 0000000000000..b2c6a3ea4b779
--- /dev/null
+++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java
@@ -0,0 +1,169 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.migrate.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xcontent.ObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+public class CreateIndexFromSourceAction extends ActionType<AcknowledgedResponse> {
+
+    public static final String NAME = "indices:admin/index/create_from_source";
+
+    public static final ActionType<AcknowledgedResponse> INSTANCE = new CreateIndexFromSourceAction();
+
+    private CreateIndexFromSourceAction() {
+        super(NAME);
+    }
+
+    public static class Request extends ActionRequest implements IndicesRequest, ToXContent {
+        private final String sourceIndex;
+        private final String destIndex;
+        private Settings settingsOverride = Settings.EMPTY;
+        private Map<String, Object> mappingsOverride = Map.of();
+        private static final ParseField SETTINGS_OVERRIDE_FIELD = new ParseField("settings_override");
+        private static final ParseField MAPPINGS_OVERRIDE_FIELD = new ParseField("mappings_override");
+        private static final ObjectParser<Request, Void> PARSER = new ObjectParser<>("create_index_from_source_request");
+
+        static {
+            PARSER.declareField(
+                (parser, request, context) -> request.settingsOverride(Settings.fromXContent(parser)),
+                SETTINGS_OVERRIDE_FIELD,
+                ObjectParser.ValueType.OBJECT
+            );
+
+            PARSER.declareField(
+                (parser, request, context) -> request.mappingsOverride(Map.of("_doc", parser.map())),
+                MAPPINGS_OVERRIDE_FIELD,
+                ObjectParser.ValueType.OBJECT
+            );
+        }
+
+        public Request(String sourceIndex, String destIndex) {
+            this(sourceIndex, destIndex, Settings.EMPTY, Map.of());
+        }
+
+        public Request(String sourceIndex, String destIndex, Settings settingsOverride, Map<String, Object> mappingsOverride) {
+            Objects.requireNonNull(settingsOverride);
+            Objects.requireNonNull(mappingsOverride);
+            this.sourceIndex = sourceIndex;
+            this.destIndex = destIndex;
+            this.settingsOverride = settingsOverride;
+            this.mappingsOverride = mappingsOverride;
+        }
+
+        @SuppressWarnings("unchecked")
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            this.sourceIndex = in.readString();
+            this.destIndex = in.readString();
+            this.settingsOverride = Settings.readSettingsFromStream(in);
+            this.mappingsOverride = (Map<String, Object>) in.readGenericValue();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(sourceIndex);
+            out.writeString(destIndex);
+            settingsOverride.writeTo(out);
+            out.writeGenericValue(mappingsOverride);
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        public String sourceIndex() {
+            return sourceIndex;
+        }
+
+        public String destIndex() {
+            return destIndex;
+        }
+
+        public Settings settingsOverride() {
+            return settingsOverride;
+        }
+
+        public Map<String, Object> mappingsOverride() {
+            return mappingsOverride;
+        }
+
+        public void settingsOverride(Settings settingsOverride) {
+            this.settingsOverride = settingsOverride;
+        }
+
+        public void mappingsOverride(Map<String, Object> mappingsOverride) {
+            this.mappingsOverride = mappingsOverride;
+        }
+
+        public void fromXContent(XContentParser parser) throws IOException {
+            PARSER.parse(parser, this, null);
+        }
+
+        /*
+         * This only exists for the sake of testing the xcontent parser
+         */
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+            if (mappingsOverride.containsKey("_doc")) {
+                builder.field(MAPPINGS_OVERRIDE_FIELD.getPreferredName(), mappingsOverride.get("_doc"));
+            }
+
+            if (settingsOverride.isEmpty() == false) {
+                builder.startObject(SETTINGS_OVERRIDE_FIELD.getPreferredName());
+                settingsOverride.toXContent(builder, params);
+                builder.endObject();
+            }
+
+            return builder;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Request request = (Request) o;
+            return Objects.equals(sourceIndex, request.sourceIndex)
+                && Objects.equals(destIndex, request.destIndex)
+                && Objects.equals(settingsOverride, request.settingsOverride)
+                && Objects.equals(mappingsOverride, request.mappingsOverride);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(sourceIndex, destIndex, settingsOverride, mappingsOverride);
+        }
+
+        @Override
+        public String[] indices() {
+            return new String[] { sourceIndex };
+        }
+
+        @Override
+        public IndicesOptions indicesOptions() {
+            return IndicesOptions.strictSingleIndexNoExpandForbidClosed();
+        }
+    }
+}
diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceTransportAction.java
new file mode 100644
index 0000000000000..09f9d48b62558
--- /dev/null
+++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceTransportAction.java
@@ -0,0 +1,128 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.migrate.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+public class CreateIndexFromSourceTransportAction extends HandledTransportAction<
+    CreateIndexFromSourceAction.Request,
+    AcknowledgedResponse> {
+    private static final Logger logger = LogManager.getLogger(CreateIndexFromSourceTransportAction.class);
+
+    private final ClusterService clusterService;
+    private final Client client;
+    private final IndexScopedSettings indexScopedSettings;
+
+    @Inject
+    public CreateIndexFromSourceTransportAction(
+        TransportService transportService,
+        ClusterService clusterService,
+        ActionFilters actionFilters,
+        Client client,
+        IndexScopedSettings indexScopedSettings
+    ) {
+        super(
+            CreateIndexFromSourceAction.NAME,
+            false,
+            transportService,
+            actionFilters,
+            CreateIndexFromSourceAction.Request::new,
+            transportService.getThreadPool().executor(ThreadPool.Names.GENERIC)
+        );
+        this.clusterService = clusterService;
+        this.client = client;
+        this.indexScopedSettings = indexScopedSettings;
+    }
+
+    @Override
+    protected void doExecute(Task task, CreateIndexFromSourceAction.Request request, ActionListener<AcknowledgedResponse> listener) {
+
+        IndexMetadata sourceIndex = clusterService.state().getMetadata().index(request.sourceIndex());
+
+        if (sourceIndex == null) {
+            listener.onFailure(new IndexNotFoundException(request.sourceIndex()));
+            return;
+        }
+
+        logger.debug("Creating destination index [{}] for source index [{}]", request.destIndex(), request.sourceIndex());
+
+        Settings settings = Settings.builder()
+            // add source settings
+            .put(filterSettings(sourceIndex))
+            // add override settings from request
+            .put(request.settingsOverride())
+            .build();
+
+        Map<String, Object> mergeMappings;
+        try {
+            mergeMappings = mergeMappings(sourceIndex.mapping(), request.mappingsOverride());
+        } catch (IOException e) {
+            listener.onFailure(e);
+            return;
+        }
+
+        var createIndexRequest = new CreateIndexRequest(request.destIndex()).settings(settings);
+        if (mergeMappings.isEmpty() == false) {
+            createIndexRequest.mapping(mergeMappings);
+        }
+        createIndexRequest.setParentTask(new TaskId(clusterService.localNode().getId(), task.getId()));
+
+        client.admin().indices().create(createIndexRequest, listener.map(response -> response));
+    }
+
+    private static Map<String, Object> toMap(@Nullable MappingMetadata sourceMapping) {
+        return Optional.ofNullable(sourceMapping)
+            .map(MappingMetadata::source)
+            .map(CompressedXContent::uncompressed)
+            .map(s -> XContentHelper.convertToMap(s, true, XContentType.JSON).v2())
+            .orElse(Map.of());
+    }
+
+    private static Map<String, Object> mergeMappings(@Nullable MappingMetadata sourceMapping, Map<String, Object> mappingAddition)
+        throws IOException {
+        Map<String, Object> combinedMappingMap = new HashMap<>(toMap(sourceMapping));
+        XContentHelper.update(combinedMappingMap, mappingAddition, true);
+        return combinedMappingMap;
+    }
+
+    // Filter source index settings to subset of settings that can be included during reindex.
+    // Similar to the settings filtering done when reindexing for upgrade in Kibana
+    // https://github.com/elastic/kibana/blob/8a8363f02cc990732eb9cbb60cd388643a336bed/x-pack
+    // /plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts#L155
+    private Settings filterSettings(IndexMetadata sourceIndex) {
+        return MetadataCreateIndexService.copySettingsFromSource(false, sourceIndex.getSettings(), indexScopedSettings, Settings.builder())
+            .build();
+    }
+}
diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/GetMigrationReindexStatusAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/GetMigrationReindexStatusAction.java
index 68ccaef4bf02c..bc084f3e0b5d6 100644
--- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/GetMigrationReindexStatusAction.java
+++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/GetMigrationReindexStatusAction.java
@@ -16,10 +16,9 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.tasks.TaskResult;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.migrate.task.ReindexDataStreamEnrichedStatus;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -36,46 +35,43 @@ public GetMigrationReindexStatusAction() {
     }
 
     public static class Response extends ActionResponse implements ToXContentObject {
-        private final TaskResult task;
+        private final ReindexDataStreamEnrichedStatus enrichedStatus;
 
-        public Response(TaskResult task) {
-            this.task = requireNonNull(task, "task is required");
+        public Response(ReindexDataStreamEnrichedStatus enrichedStatus) {
+            this.enrichedStatus = requireNonNull(enrichedStatus, "status is required");
         }
 
         public Response(StreamInput in) throws IOException {
             super(in);
-            task = in.readOptionalWriteable(TaskResult::new);
+            enrichedStatus = in.readOptionalWriteable(ReindexDataStreamEnrichedStatus::new);
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeOptionalWriteable(task);
+            out.writeOptionalWriteable(enrichedStatus);
         }
 
         /**
          * Get the actual result of the fetch.
          */
-        public TaskResult getTask() {
-            return task;
+        public ReindexDataStreamEnrichedStatus getEnrichedStatus() {
+            return enrichedStatus;
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            Task.Status status = task.getTask().status();
-            if (status != null) {
-                task.getTask().status().toXContent(builder, params);
-            }
+            enrichedStatus.toXContent(builder, params);
             return builder;
         }
 
         @Override
         public int hashCode() {
-            return Objects.hashCode(task);
+            return Objects.hashCode(enrichedStatus);
         }
 
         @Override
         public boolean equals(Object other) {
-            return other instanceof Response && task.equals(((Response) other).task);
+            return other instanceof Response && enrichedStatus.equals(((Response) other).enrichedStatus);
         }
 
         @Override
diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/GetMigrationReindexStatusTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/GetMigrationReindexStatusTransportAction.java
index ca81a03fc5630..64864491191e5 100644
--- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/GetMigrationReindexStatusTransportAction.java
+++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/GetMigrationReindexStatusTransportAction.java
@@ -11,40 +11,55 @@
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
+import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.AllocatedPersistentTask;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskInfo;
-import org.elasticsearch.tasks.TaskResult;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.migrate.action.GetMigrationReindexStatusAction.Request;
 import org.elasticsearch.xpack.migrate.action.GetMigrationReindexStatusAction.Response;
+import org.elasticsearch.xpack.migrate.task.ReindexDataStreamEnrichedStatus;
+import org.elasticsearch.xpack.migrate.task.ReindexDataStreamStatus;
 
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Stream;
 
 public class GetMigrationReindexStatusTransportAction extends HandledTransportAction<Request, Response> {
     private final ClusterService clusterService;
     private final TransportService transportService;
+    private final Client client;
 
     @Inject
     public GetMigrationReindexStatusTransportAction(
         ClusterService clusterService,
         TransportService transportService,
-        ActionFilters actionFilters
+        ActionFilters actionFilters,
+        Client client
     ) {
         super(GetMigrationReindexStatusAction.NAME, transportService, actionFilters, Request::new, EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.clusterService = clusterService;
         this.transportService = transportService;
+        this.client = client;
     }
 
     @Override
@@ -60,9 +75,9 @@ protected void doExecute(Task task, Request request, ActionListener<Response> li
         } else if (persistentTask.isAssigned()) {
             String nodeId = persistentTask.getExecutorNode();
             if (clusterService.localNode().getId().equals(nodeId)) {
-                getRunningTaskFromNode(persistentTaskId, listener);
+                fetchAndReportStatusForTaskOnThisNode(persistentTaskId, listener);
             } else {
-                runOnNodeWithTaskIfPossible(task, request, nodeId, listener);
+                fetchAndReportStatusForTaskOnRemoteNode(task, request, nodeId, listener);
             }
         } else {
             listener.onFailure(new ElasticsearchException("Persistent task with id [{}] is not assigned to a node", persistentTaskId));
@@ -82,7 +97,7 @@ private Task getRunningPersistentTaskFromTaskManager(String persistentTaskId) {
         return optionalTask.<Task>map(Map.Entry::getValue).orElse(null);
     }
 
-    void getRunningTaskFromNode(String persistentTaskId, ActionListener<Response> listener) {
+    void fetchAndReportStatusForTaskOnThisNode(String persistentTaskId, ActionListener<Response> listener) {
         Task runningTask = getRunningPersistentTaskFromTaskManager(persistentTaskId);
         if (runningTask == null) {
             listener.onFailure(
@@ -96,11 +111,97 @@ void getRunningTaskFromNode(String persistentTaskId, ActionListener<Response> li
             );
         } else {
             TaskInfo info = runningTask.taskInfo(clusterService.localNode().getId(), true);
-            listener.onResponse(new Response(new TaskResult(false, info)));
+            ReindexDataStreamStatus status = (ReindexDataStreamStatus) info.status();
+            Set<String> inProgressIndices = status.inProgress();
+            if (inProgressIndices.isEmpty()) {
+                // We have no reason to fetch index stats since there are no in progress indices
+                reportStatus(Map.of(), status, listener);
+            } else {
+                fetchInProgressStatsAndReportStatus(inProgressIndices, status, listener);
+            }
         }
     }
 
-    private void runOnNodeWithTaskIfPossible(Task thisTask, Request request, String nodeId, ActionListener<Response> listener) {
+    /*
+     * The status is enriched with the information from inProgressMap to create a new ReindexDataStreamEnrichedStatus, which is used in the
+     * response sent to the listener.
+     */
+    private void reportStatus(
+        Map<String, Tuple<Long, Long>> inProgressMap,
+        ReindexDataStreamStatus status,
+        ActionListener<Response> listener
+    ) {
+        ReindexDataStreamEnrichedStatus enrichedStatus = new ReindexDataStreamEnrichedStatus(
+            status.persistentTaskStartTime(),
+            status.totalIndices(),
+            status.totalIndicesToBeUpgraded(),
+            status.complete(),
+            status.exception(),
+            inProgressMap,
+            status.pending(),
+            status.errors()
+        );
+        listener.onResponse(new Response(enrichedStatus));
+    }
+
+    /*
+     * This method feches doc counts for all indices in inProgressIndices (and the indices they are being reindexed into). After
+     * successfully fetching those, reportStatus is called.
+     */
+    private void fetchInProgressStatsAndReportStatus(
+        Set<String> inProgressIndices,
+        ReindexDataStreamStatus status,
+        ActionListener<Response> listener
+    ) {
+        IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
+        String[] indices = inProgressIndices.stream()
+            .flatMap(index -> Stream.of(index, ReindexDataStreamIndexTransportAction.generateDestIndexName(index)))
+            .toList()
+            .toArray(new String[0]);
+        indicesStatsRequest.indices(indices);
+        /*
+         * It is possible that the destination index will not exist yet, so we want to ignore the fact that it is missing
+         */
+        indicesStatsRequest.indicesOptions(IndicesOptions.fromOptions(true, true, true, true));
+        client.execute(IndicesStatsAction.INSTANCE, indicesStatsRequest, new ActionListener<IndicesStatsResponse>() {
+            @Override
+            public void onResponse(IndicesStatsResponse indicesStatsResponse) {
+                Map<String, Tuple<Long, Long>> inProgressMap = new HashMap<>();
+                for (String index : inProgressIndices) {
+                    IndexStats sourceIndexStats = indicesStatsResponse.getIndex(index);
+                    final long totalDocsInIndex;
+                    if (sourceIndexStats == null) {
+                        totalDocsInIndex = 0;
+                    } else {
+                        DocsStats totalDocsStats = sourceIndexStats.getTotal().getDocs();
+                        totalDocsInIndex = totalDocsStats == null ? 0 : totalDocsStats.getCount();
+                    }
+                    IndexStats migratedIndexStats = indicesStatsResponse.getIndex(
+                        ReindexDataStreamIndexTransportAction.generateDestIndexName(index)
+                    );
+                    final long reindexedDocsInIndex;
+                    if (migratedIndexStats == null) {
+                        reindexedDocsInIndex = 0;
+                    } else {
+                        DocsStats reindexedDocsStats = migratedIndexStats.getTotal().getDocs();
+                        reindexedDocsInIndex = reindexedDocsStats == null ? 0 : reindexedDocsStats.getCount();
+                    }
+                    inProgressMap.put(index, Tuple.tuple(totalDocsInIndex, reindexedDocsInIndex));
+                }
+                reportStatus(inProgressMap, status, listener);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                listener.onFailure(e);
+            }
+        });
+    }
+
+    /*
+     * The task and its status exist on some other node, so this method forwards the request to that node.
+     */
+    private void fetchAndReportStatusForTaskOnRemoteNode(Task thisTask, Request request, String nodeId, ActionListener<Response> listener) {
         DiscoveryNode node = clusterService.state().nodes().get(nodeId);
         if (node == null) {
             listener.onFailure(
diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java
index b10bea9e54230..9e4cbb1082215 100644
--- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java
+++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java
@@ -13,14 +13,10 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.features.NodeFeature;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexVersion;
-import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
@@ -43,24 +39,10 @@ public class ReindexDataStreamAction extends ActionType<ReindexDataStreamAction.
     public static final ParseField SOURCE_FIELD = new ParseField("source");
     public static final ParseField INDEX_FIELD = new ParseField("index");
 
-    /*
-     * The version before which we do not support writes in the _next_ major version of Elasticsearch. For example, Elasticsearch 10.x will
-     * not support writing to indices created before version 9.0.0.
-     */
-    public static final IndexVersion MINIMUM_WRITEABLE_VERSION_AFTER_UPGRADE = IndexVersions.UPGRADE_TO_LUCENE_10_0_0;
-
     public ReindexDataStreamAction() {
         super(NAME);
     }
 
-    /*
-     * This predicate allows through only indices that were created with a previous lucene version, meaning that they need to be reindexed
-     * in order to be writable in the _next_ lucene version.
-     */
-    public static Predicate<Index> getOldIndexVersionPredicate(Metadata metadata) {
-        return index -> metadata.index(index).getCreationVersion().onOrBefore(MINIMUM_WRITEABLE_VERSION_AFTER_UPGRADE);
-    }
-
     public enum Mode {
         UPGRADE
     }
diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexAction.java
index 00c81fdc9fbc6..2e3fd1b76ed32 100644
--- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexAction.java
+++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexAction.java
@@ -41,10 +41,6 @@ public Request(StreamInput in) throws IOException {
             this.sourceIndex = in.readString();
         }
 
-        public static Request readFrom(StreamInput in) throws IOException {
-            return new Request(in);
-        }
-
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java
index 8863c45691c92..7bb440bc52a15 100644
--- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java
+++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java
@@ -10,9 +10,10 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockRequest;
+import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockResponse;
+import org.elasticsearch.action.admin.indices.readonly.TransportAddIndexBlockAction;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
@@ -22,43 +23,40 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.ReindexAction;
 import org.elasticsearch.index.reindex.ReindexRequest;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.deprecation.DeprecatedIndexPredicate;
 
 import java.util.Locale;
 import java.util.Map;
-import java.util.Set;
+
+import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.WRITE;
 
 public class ReindexDataStreamIndexTransportAction extends HandledTransportAction<
     ReindexDataStreamIndexAction.Request,
     ReindexDataStreamIndexAction.Response> {
 
     private static final Logger logger = LogManager.getLogger(ReindexDataStreamIndexTransportAction.class);
-
-    private static final Set<String> SETTINGS_TO_ADD_BACK = Set.of(IndexMetadata.SETTING_BLOCKS_WRITE, IndexMetadata.SETTING_READ_ONLY);
-
     private static final IndicesOptions IGNORE_MISSING_OPTIONS = IndicesOptions.fromOptions(true, true, false, false);
     private final ClusterService clusterService;
     private final Client client;
-    private final IndexScopedSettings indexScopedSettings;
 
     @Inject
     public ReindexDataStreamIndexTransportAction(
         TransportService transportService,
         ClusterService clusterService,
         ActionFilters actionFilters,
-        Client client,
-        IndexScopedSettings indexScopedSettings
+        Client client
     ) {
         super(
             ReindexDataStreamIndexAction.NAME,
@@ -70,7 +68,6 @@ public ReindexDataStreamIndexTransportAction(
         );
         this.clusterService = clusterService;
         this.client = client;
-        this.indexScopedSettings = indexScopedSettings;
     }
 
     @Override
@@ -81,35 +78,45 @@ protected void doExecute(
     ) {
         var sourceIndexName = request.getSourceIndex();
         var destIndexName = generateDestIndexName(sourceIndexName);
+        TaskId taskId = new TaskId(clusterService.localNode().getId(), task.getId());
         IndexMetadata sourceIndex = clusterService.state().getMetadata().index(sourceIndexName);
         Settings settingsBefore = sourceIndex.getSettings();
 
-        var hasOldVersion = ReindexDataStreamAction.getOldIndexVersionPredicate(clusterService.state().metadata());
+        var hasOldVersion = DeprecatedIndexPredicate.getReindexRequiredPredicate(clusterService.state().metadata());
         if (hasOldVersion.test(sourceIndex.getIndex()) == false) {
             logger.warn(
                 "Migrating index [{}] with version [{}] is unnecessary as its version is not before [{}]",
                 sourceIndexName,
                 sourceIndex.getCreationVersion(),
-                ReindexDataStreamAction.MINIMUM_WRITEABLE_VERSION_AFTER_UPGRADE
+                DeprecatedIndexPredicate.MINIMUM_WRITEABLE_VERSION_AFTER_UPGRADE
             );
         }
 
-        SubscribableListener.<AcknowledgedResponse>newForked(l -> setBlockWrites(sourceIndexName, l))
-            .<AcknowledgedResponse>andThen(l -> deleteDestIfExists(destIndexName, l))
-            .<CreateIndexResponse>andThen(l -> createIndex(sourceIndex, destIndexName, l))
-            .<BulkByScrollResponse>andThen(l -> reindex(sourceIndexName, destIndexName, l))
-            .<AcknowledgedResponse>andThen(l -> updateSettings(settingsBefore, destIndexName, l))
+        if (settingsBefore.getAsBoolean(IndexMetadata.SETTING_BLOCKS_READ, false)) {
+            var errorMessage = String.format(Locale.ROOT, "Cannot reindex index [%s] which has a read block.", destIndexName);
+            listener.onFailure(new ElasticsearchException(errorMessage));
+            return;
+        }
+        if (settingsBefore.getAsBoolean(IndexMetadata.SETTING_BLOCKS_METADATA, false)) {
+            var errorMessage = String.format(Locale.ROOT, "Cannot reindex index [%s] which has a metadata block.", destIndexName);
+            listener.onFailure(new ElasticsearchException(errorMessage));
+            return;
+        }
+
+        SubscribableListener.<AcknowledgedResponse>newForked(l -> setBlockWrites(sourceIndexName, l, taskId))
+            .<AcknowledgedResponse>andThen(l -> deleteDestIfExists(destIndexName, l, taskId))
+            .<AcknowledgedResponse>andThen(l -> createIndex(sourceIndex, destIndexName, l, taskId))
+            .<BulkByScrollResponse>andThen(l -> reindex(sourceIndexName, destIndexName, l, taskId))
+            .<AcknowledgedResponse>andThen(l -> copyOldSourceSettingsToDest(settingsBefore, destIndexName, l, taskId))
             .andThenApply(ignored -> new ReindexDataStreamIndexAction.Response(destIndexName))
             .addListener(listener);
     }
 
-    private void setBlockWrites(String sourceIndexName, ActionListener<AcknowledgedResponse> listener) {
+    private void setBlockWrites(String sourceIndexName, ActionListener<AcknowledgedResponse> listener, TaskId parentTaskId) {
         logger.debug("Setting write block on source index [{}]", sourceIndexName);
-        final Settings readOnlySettings = Settings.builder().put(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey(), true).build();
-        var updateSettingsRequest = new UpdateSettingsRequest(readOnlySettings, sourceIndexName);
-        client.admin().indices().updateSettings(updateSettingsRequest, new ActionListener<>() {
+        addBlockToIndex(WRITE, sourceIndexName, new ActionListener<>() {
             @Override
-            public void onResponse(AcknowledgedResponse response) {
+            public void onResponse(AddIndexBlockResponse response) {
                 if (response.isAcknowledged()) {
                     listener.onResponse(null);
                 } else {
@@ -121,91 +128,146 @@ public void onResponse(AcknowledgedResponse response) {
             @Override
             public void onFailure(Exception e) {
                 if (e instanceof ClusterBlockException || e.getCause() instanceof ClusterBlockException) {
-                    // It's fine if read-only is already set
+                    // Could fail with a cluster block exception if read-only or read-only-allow-delete is already set
+                    // In this case, we can proceed
                     listener.onResponse(null);
                 } else {
                     listener.onFailure(e);
                 }
             }
-        });
+        }, parentTaskId);
     }
 
-    private void deleteDestIfExists(String destIndexName, ActionListener<AcknowledgedResponse> listener) {
+    private void deleteDestIfExists(String destIndexName, ActionListener<AcknowledgedResponse> listener, TaskId parentTaskId) {
         logger.debug("Attempting to delete index [{}]", destIndexName);
         var deleteIndexRequest = new DeleteIndexRequest(destIndexName).indicesOptions(IGNORE_MISSING_OPTIONS)
             .masterNodeTimeout(TimeValue.MAX_VALUE);
+        deleteIndexRequest.setParentTask(parentTaskId);
         var errorMessage = String.format(Locale.ROOT, "Failed to acknowledge delete of index [%s]", destIndexName);
         client.admin().indices().delete(deleteIndexRequest, failIfNotAcknowledged(listener, errorMessage));
     }
 
-    private void createIndex(IndexMetadata sourceIndex, String destIndexName, ActionListener<CreateIndexResponse> listener) {
+    private void createIndex(
+        IndexMetadata sourceIndex,
+        String destIndexName,
+        ActionListener<AcknowledgedResponse> listener,
+        TaskId parentTaskId
+    ) {
         logger.debug("Creating destination index [{}] for source index [{}]", destIndexName, sourceIndex.getIndex().getName());
 
-        // Create destination with subset of source index settings that can be added before reindex
-        var settings = getPreSettings(sourceIndex);
-
-        var sourceMapping = sourceIndex.mapping();
-        Map<String, Object> mapping = sourceMapping != null ? sourceMapping.rawSourceAsMap() : Map.of();
-        var createIndexRequest = new CreateIndexRequest(destIndexName).settings(settings).mapping(mapping);
+        var removeReadOnlyOverride = Settings.builder()
+            // remove read-only settings if they exist
+            .putNull(IndexMetadata.SETTING_READ_ONLY)
+            .putNull(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE)
+            .putNull(IndexMetadata.SETTING_BLOCKS_WRITE)
+            // settings to optimize reindex
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1)
+            .build();
 
-        var errorMessage = String.format(Locale.ROOT, "Could not create index [%s]", destIndexName);
-        client.admin().indices().create(createIndexRequest, failIfNotAcknowledged(listener, errorMessage));
+        var request = new CreateIndexFromSourceAction.Request(
+            sourceIndex.getIndex().getName(),
+            destIndexName,
+            removeReadOnlyOverride,
+            Map.of()
+        );
+        request.setParentTask(parentTaskId);
+        var errorMessage = String.format(Locale.ROOT, "Could not create index [%s]", request.destIndex());
+        client.execute(CreateIndexFromSourceAction.INSTANCE, request, failIfNotAcknowledged(listener, errorMessage));
     }
 
-    private void reindex(String sourceIndexName, String destIndexName, ActionListener<BulkByScrollResponse> listener) {
+    private void reindex(String sourceIndexName, String destIndexName, ActionListener<BulkByScrollResponse> listener, TaskId parentTaskId) {
         logger.debug("Reindex to destination index [{}] from source index [{}]", destIndexName, sourceIndexName);
         var reindexRequest = new ReindexRequest();
         reindexRequest.setSourceIndices(sourceIndexName);
         reindexRequest.getSearchRequest().allowPartialSearchResults(false);
         reindexRequest.getSearchRequest().source().fetchSource(true);
         reindexRequest.setDestIndex(destIndexName);
+        reindexRequest.setParentTask(parentTaskId);
+        reindexRequest.setSlices(0); // equivalent to slices=auto in rest api
         client.execute(ReindexAction.INSTANCE, reindexRequest, listener);
     }
 
-    private void updateSettings(Settings settingsBefore, String destIndexName, ActionListener<AcknowledgedResponse> listener) {
-        logger.debug("Adding settings from source index that could not be added before reindex");
-
-        Settings postSettings = getPostSettings(settingsBefore);
-        if (postSettings.isEmpty()) {
+    private void addBlockIfFromSource(
+        IndexMetadata.APIBlock block,
+        Settings settingsBefore,
+        String destIndexName,
+        ActionListener<AddIndexBlockResponse> listener,
+        TaskId parentTaskId
+    ) {
+        if (settingsBefore.getAsBoolean(block.settingName(), false)) {
+            var errorMessage = String.format(Locale.ROOT, "Add [%s] block to index [%s] was not acknowledged", block.name(), destIndexName);
+            addBlockToIndex(block, destIndexName, failIfNotAcknowledged(listener, errorMessage), parentTaskId);
+        } else {
             listener.onResponse(null);
-            return;
         }
+    }
 
-        var updateSettingsRequest = new UpdateSettingsRequest(postSettings, destIndexName);
-        var errorMessage = String.format(Locale.ROOT, "Could not update settings on index [%s]", destIndexName);
+    private void updateSettings(
+        String index,
+        Settings.Builder settings,
+        ActionListener<AcknowledgedResponse> listener,
+        TaskId parentTaskId
+    ) {
+        var updateSettingsRequest = new UpdateSettingsRequest(settings.build(), index);
+        updateSettingsRequest.setParentTask(parentTaskId);
+        var errorMessage = String.format(Locale.ROOT, "Could not update settings on index [%s]", index);
         client.admin().indices().updateSettings(updateSettingsRequest, failIfNotAcknowledged(listener, errorMessage));
     }
 
-    // Filter source index settings to subset of settings that can be included during reindex.
-    // Similar to the settings filtering done when reindexing for upgrade in Kibana
-    // https://github.com/elastic/kibana/blob/8a8363f02cc990732eb9cbb60cd388643a336bed/x-pack
-    // /plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts#L155
-    private Settings getPreSettings(IndexMetadata sourceIndex) {
-        // filter settings that will be added back later
-        var filtered = sourceIndex.getSettings().filter(settingName -> SETTINGS_TO_ADD_BACK.contains(settingName) == false);
-
-        // filter private and non-copyable settings
-        var builder = MetadataCreateIndexService.copySettingsFromSource(false, filtered, indexScopedSettings, Settings.builder());
-        return builder.build();
+    private void copyOldSourceSettingsToDest(
+        Settings settingsBefore,
+        String destIndexName,
+        ActionListener<AcknowledgedResponse> listener,
+        TaskId parentTaskId
+    ) {
+        logger.debug("Updating settings on destination index after reindex completes");
+        var settings = Settings.builder();
+        copySettingOrUnset(settingsBefore, settings, IndexMetadata.SETTING_NUMBER_OF_REPLICAS);
+        copySettingOrUnset(settingsBefore, settings, IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey());
+        updateSettings(destIndexName, settings, listener, parentTaskId);
     }
 
-    private Settings getPostSettings(Settings settingsBefore) {
-        return settingsBefore.filter(SETTINGS_TO_ADD_BACK::contains);
+    private static void copySettingOrUnset(Settings settingsBefore, Settings.Builder builder, String setting) {
+        // if setting was explicitly added to the source index
+        if (settingsBefore.get(setting) != null) {
+            // copy it back to the dest index
+            builder.copy(setting, settingsBefore);
+        } else {
+            // otherwise, delete from dest index so that it loads from the settings default
+            builder.putNull(setting);
+        }
     }
 
-    public static String generateDestIndexName(String sourceIndex) {
-        return "migrated-" + sourceIndex;
+    static String generateDestIndexName(String sourceIndex) {
+        String prefix = "migrated-";
+        if (sourceIndex.startsWith(".")) {
+            return "." + prefix + sourceIndex.substring(1);
+        }
+        return prefix + sourceIndex;
     }
 
     private static <U extends AcknowledgedResponse> ActionListener<U> failIfNotAcknowledged(
         ActionListener<U> listener,
         String errorMessage
     ) {
-        return listener.delegateFailureAndWrap((delegate, response) -> {
+        return listener.delegateFailure((delegate, response) -> {
             if (response.isAcknowledged()) {
                 delegate.onResponse(null);
+            } else {
+                delegate.onFailure(new ElasticsearchException(errorMessage));
             }
-            throw new ElasticsearchException(errorMessage);
         });
     }
+
+    private void addBlockToIndex(
+        IndexMetadata.APIBlock block,
+        String index,
+        ActionListener<AddIndexBlockResponse> listener,
+        TaskId parentTaskId
+    ) {
+        AddIndexBlockRequest addIndexBlockRequest = new AddIndexBlockRequest(block, index);
+        addIndexBlockRequest.setParentTask(parentTaskId);
+        client.admin().indices().execute(TransportAddIndexBlockAction.TYPE, addIndexBlockRequest, listener);
+    }
 }
diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportAction.java
index f011c429ce79c..cc648c1984544 100644
--- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportAction.java
+++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportAction.java
@@ -26,8 +26,8 @@
 import org.elasticsearch.xpack.migrate.task.ReindexDataStreamTask;
 import org.elasticsearch.xpack.migrate.task.ReindexDataStreamTaskParams;
 
+import static org.elasticsearch.xpack.core.deprecation.DeprecatedIndexPredicate.getReindexRequiredPredicate;
 import static org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.TASK_ID_PREFIX;
-import static org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.getOldIndexVersionPredicate;
 
 /*
  * This transport action creates a new persistent task for reindexing the source data stream given in the request. On successful creation
@@ -68,7 +68,7 @@ protected void doExecute(Task task, ReindexDataStreamRequest request, ActionList
             return;
         }
         int totalIndices = dataStream.getIndices().size();
-        int totalIndicesToBeUpgraded = (int) dataStream.getIndices().stream().filter(getOldIndexVersionPredicate(metadata)).count();
+        int totalIndicesToBeUpgraded = (int) dataStream.getIndices().stream().filter(getReindexRequiredPredicate(metadata)).count();
         ReindexDataStreamTaskParams params = new ReindexDataStreamTaskParams(
             sourceDataStreamName,
             transportService.getThreadPool().absoluteTimeInMillis(),
diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/rest/RestCreateIndexFromSourceAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/rest/RestCreateIndexFromSourceAction.java
new file mode 100644
index 0000000000000..15306f65499e9
--- /dev/null
+++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/rest/RestCreateIndexFromSourceAction.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.migrate.rest;
+
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.migrate.action.CreateIndexFromSourceAction;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.rest.RestRequest.Method.PUT;
+
+public class RestCreateIndexFromSourceAction extends BaseRestHandler {
+
+    @Override
+    public String getName() {
+        return "create_index_from_source_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(PUT, "/_create_from/{source}/{dest}"), new Route(POST, "/_create_from/{source}/{dest}"));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        var createRequest = new CreateIndexFromSourceAction.Request(request.param("source"), request.param("dest"));
+        request.applyContentParser(createRequest::fromXContent);
+        return channel -> client.execute(CreateIndexFromSourceAction.INSTANCE, createRequest, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamEnrichedStatus.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamEnrichedStatus.java
new file mode 100644
index 0000000000000..9dbe1f0c8eebc
--- /dev/null
+++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamEnrichedStatus.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.migrate.task;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/*
+ * This class represents information similar to that in ReindexDataStreamStatus, but enriched from other sources besides just the task
+ * itself.
+ */
+public record ReindexDataStreamEnrichedStatus(
+    long persistentTaskStartTime,
+    int totalIndices,
+    int totalIndicesToBeUpgraded,
+    boolean complete,
+    Exception exception,
+    Map<String, Tuple<Long, Long>> inProgress,
+    int pending,
+    List<Tuple<String, Exception>> errors
+) implements ToXContentObject, Writeable {
+    public ReindexDataStreamEnrichedStatus {
+        Objects.requireNonNull(inProgress);
+        Objects.requireNonNull(errors);
+    }
+
+    public ReindexDataStreamEnrichedStatus(StreamInput in) throws IOException {
+        this(
+            in.readLong(),
+            in.readInt(),
+            in.readInt(),
+            in.readBoolean(),
+            in.readException(),
+            in.readMap(StreamInput::readString, in2 -> Tuple.tuple(in2.readLong(), in2.readLong())),
+            in.readInt(),
+            in.readCollectionAsList(in1 -> Tuple.tuple(in1.readString(), in1.readException()))
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeLong(persistentTaskStartTime);
+        out.writeInt(totalIndices);
+        out.writeInt(totalIndicesToBeUpgraded);
+        out.writeBoolean(complete);
+        out.writeException(exception);
+        out.writeMap(inProgress, StreamOutput::writeString, (out2, tuple) -> {
+            out2.writeLong(tuple.v1());
+            out2.writeLong(tuple.v2());
+        });
+        out.writeInt(pending);
+        out.writeCollection(errors, (out1, tuple) -> {
+            out1.writeString(tuple.v1());
+            out1.writeException(tuple.v2());
+        });
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.timestampFieldsFromUnixEpochMillis("start_time_millis", "start_time", persistentTaskStartTime);
+        builder.field("complete", complete);
+        builder.field("total_indices_in_data_stream", totalIndices);
+        builder.field("total_indices_requiring_upgrade", totalIndicesToBeUpgraded);
+        builder.field("successes", totalIndicesToBeUpgraded - (inProgress.size() + pending + errors.size()));
+        builder.startArray("in_progress");
+        for (Map.Entry<String, Tuple<Long, Long>> inProgressEntry : inProgress.entrySet()) {
+            builder.startObject();
+            builder.field("index", inProgressEntry.getKey());
+            builder.field("total_doc_count", inProgressEntry.getValue().v1());
+            builder.field("reindexed_doc_count", inProgressEntry.getValue().v2());
+            builder.endObject();
+        }
+        builder.endArray();
+        builder.field("pending", pending);
+        builder.startArray("errors");
+        for (Tuple<String, Exception> error : errors) {
+            builder.startObject();
+            builder.field("index", error.v1());
+            builder.field("message", error.v2() == null ? "unknown" : error.v2().getMessage());
+            builder.endObject();
+        }
+        builder.endArray();
+        if (exception != null) {
+            builder.field("exception", exception.getMessage());
+        }
+        builder.endObject();
+        return builder;
+    }
+}
diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java
index 176220a1ccae8..104f843cc7950 100644
--- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java
+++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java
@@ -7,11 +7,22 @@
 
 package org.elasticsearch.xpack.migrate.task;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.rollover.RolloverAction;
+import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.action.datastreams.GetDataStreamAction;
+import org.elasticsearch.action.datastreams.ModifyDataStreamsAction;
+import org.elasticsearch.action.support.CountDownActionListener;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamAction;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.persistent.AllocatedPersistentTask;
@@ -20,13 +31,30 @@
 import org.elasticsearch.persistent.PersistentTasksExecutor;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.migrate.action.ReindexDataStreamIndexAction;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.NoSuchElementException;
 
-import static org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.getOldIndexVersionPredicate;
+import static org.elasticsearch.xpack.core.deprecation.DeprecatedIndexPredicate.getReindexRequiredPredicate;
 
 public class ReindexDataStreamPersistentTaskExecutor extends PersistentTasksExecutor<ReindexDataStreamTaskParams> {
+    /*
+     * This setting controls how many indices we reindex concurrently for a single data stream. This is not an overall limit -- if five
+     * data streams are being reindexed, then each of them can have this many indices being reindexed at once. This setting is dynamic,
+     * but changing it does not have an impact if the task is already running (unless the task is restarted or moves to another node).
+     */
+    public static final Setting<Integer> MAX_CONCURRENT_INDICES_REINDEXED_PER_DATA_STREAM_SETTING = Setting.intSetting(
+        "migrate.max_concurrent_indices_reindexed_per_data_stream",
+        1,
+        1,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+    private static final Logger logger = LogManager.getLogger(ReindexDataStreamPersistentTaskExecutor.class);
     private static final TimeValue TASK_KEEP_ALIVE_TIME = TimeValue.timeValueDays(1);
     private final Client client;
     private final ClusterService clusterService;
@@ -50,6 +78,7 @@ protected ReindexDataStreamTask createTask(
     ) {
         ReindexDataStreamTaskParams params = taskInProgress.getParams();
         return new ReindexDataStreamTask(
+            clusterService,
             params.startTime(),
             params.totalIndices(),
             params.totalIndicesToBeUpgraded(),
@@ -63,58 +92,212 @@ protected ReindexDataStreamTask createTask(
     }
 
     @Override
-    protected void nodeOperation(AllocatedPersistentTask task, ReindexDataStreamTaskParams params, PersistentTaskState state) {
+    protected void nodeOperation(
+        AllocatedPersistentTask task,
+        ReindexDataStreamTaskParams params,
+        PersistentTaskState persistentTaskState
+    ) {
+        ReindexDataStreamPersistentTaskState state = (ReindexDataStreamPersistentTaskState) persistentTaskState;
         String sourceDataStream = params.getSourceDataStream();
+        TaskId taskId = new TaskId(clusterService.localNode().getId(), task.getId());
         GetDataStreamAction.Request request = new GetDataStreamAction.Request(TimeValue.MAX_VALUE, new String[] { sourceDataStream });
+        request.setParentTask(taskId);
         assert task instanceof ReindexDataStreamTask;
         final ReindexDataStreamTask reindexDataStreamTask = (ReindexDataStreamTask) task;
         ExecuteWithHeadersClient reindexClient = new ExecuteWithHeadersClient(client, params.headers());
         reindexClient.execute(GetDataStreamAction.INSTANCE, request, ActionListener.wrap(response -> {
             List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = response.getDataStreams();
             if (dataStreamInfos.size() == 1) {
-                List<Index> indices = dataStreamInfos.getFirst().getDataStream().getIndices();
-                List<Index> indicesToBeReindexed = indices.stream()
-                    .filter(getOldIndexVersionPredicate(clusterService.state().metadata()))
-                    .toList();
-                reindexDataStreamTask.setPendingIndicesCount(indicesToBeReindexed.size());
-                for (Index index : indicesToBeReindexed) {
-                    reindexDataStreamTask.incrementInProgressIndicesCount();
-                    // TODO This is just a placeholder. This is where the real data stream reindex logic will go
-                    reindexDataStreamTask.reindexSucceeded();
+                DataStream dataStream = dataStreamInfos.getFirst().getDataStream();
+                if (getReindexRequiredPredicate(clusterService.state().metadata()).test(dataStream.getWriteIndex())) {
+                    RolloverRequest rolloverRequest = new RolloverRequest(sourceDataStream, null);
+                    rolloverRequest.setParentTask(taskId);
+                    reindexClient.execute(
+                        RolloverAction.INSTANCE,
+                        rolloverRequest,
+                        ActionListener.wrap(
+                            rolloverResponse -> reindexIndices(
+                                dataStream,
+                                dataStream.getIndices().size() + 1,
+                                reindexDataStreamTask,
+                                params,
+                                state,
+                                reindexClient,
+                                sourceDataStream,
+                                taskId
+                            ),
+                            e -> completeFailedPersistentTask(reindexDataStreamTask, state, e)
+                        )
+                    );
+                } else {
+                    reindexIndices(
+                        dataStream,
+                        dataStream.getIndices().size(),
+                        reindexDataStreamTask,
+                        params,
+                        state,
+                        reindexClient,
+                        sourceDataStream,
+                        taskId
+                    );
                 }
-
-                completeSuccessfulPersistentTask(reindexDataStreamTask);
             } else {
-                completeFailedPersistentTask(reindexDataStreamTask, new ElasticsearchException("data stream does not exist"));
+                completeFailedPersistentTask(reindexDataStreamTask, state, new ElasticsearchException("data stream does not exist"));
             }
-        }, reindexDataStreamTask::markAsFailed));
+        }, exception -> completeFailedPersistentTask(reindexDataStreamTask, state, exception)));
     }
 
-    private void completeSuccessfulPersistentTask(ReindexDataStreamTask persistentTask) {
-        persistentTask.allReindexesCompleted(threadPool, getTimeToLive(persistentTask));
+    private void reindexIndices(
+        DataStream dataStream,
+        int totalIndicesInDataStream,
+        ReindexDataStreamTask reindexDataStreamTask,
+        ReindexDataStreamTaskParams params,
+        ReindexDataStreamPersistentTaskState state,
+        ExecuteWithHeadersClient reindexClient,
+        String sourceDataStream,
+        TaskId parentTaskId
+    ) {
+        List<Index> indices = dataStream.getIndices();
+        List<Index> indicesToBeReindexed = indices.stream().filter(getReindexRequiredPredicate(clusterService.state().metadata())).toList();
+        final ReindexDataStreamPersistentTaskState updatedState;
+        if (params.totalIndices() != totalIndicesInDataStream
+            || params.totalIndicesToBeUpgraded() != indicesToBeReindexed.size()
+            || (state != null
+                && (state.totalIndices() != null
+                    && state.totalIndicesToBeUpgraded() != null
+                    && (state.totalIndices() != totalIndicesInDataStream
+                        || state.totalIndicesToBeUpgraded() != indicesToBeReindexed.size())))) {
+            updatedState = new ReindexDataStreamPersistentTaskState(
+                totalIndicesInDataStream,
+                indicesToBeReindexed.size(),
+                state == null ? null : state.completionTime()
+            );
+            reindexDataStreamTask.updatePersistentTaskState(updatedState, ActionListener.noop());
+        } else {
+            updatedState = state;
+        }
+        reindexDataStreamTask.setPendingIndicesCount(indicesToBeReindexed.size());
+        // The CountDownActionListener is 1 more than the number of indices so that the count is not 0 if we have no indices
+        CountDownActionListener listener = new CountDownActionListener(indicesToBeReindexed.size() + 1, ActionListener.wrap(response1 -> {
+            completeSuccessfulPersistentTask(reindexDataStreamTask, updatedState);
+        }, exception -> { completeFailedPersistentTask(reindexDataStreamTask, updatedState, exception); }));
+        final int maxConcurrentIndices = clusterService.getClusterSettings().get(MAX_CONCURRENT_INDICES_REINDEXED_PER_DATA_STREAM_SETTING);
+        List<Index> indicesRemaining = Collections.synchronizedList(new ArrayList<>(indicesToBeReindexed));
+        logger.debug("Reindexing {} indices, with up to {} handled concurrently", indicesRemaining.size(), maxConcurrentIndices);
+        for (int i = 0; i < maxConcurrentIndices; i++) {
+            maybeProcessNextIndex(indicesRemaining, reindexDataStreamTask, reindexClient, sourceDataStream, listener, parentTaskId);
+        }
+        // This takes care of the additional latch count referenced above:
+        listener.onResponse(null);
     }
 
-    private void completeFailedPersistentTask(ReindexDataStreamTask persistentTask, Exception e) {
-        persistentTask.taskFailed(threadPool, getTimeToLive(persistentTask), e);
+    private void maybeProcessNextIndex(
+        List<Index> indicesRemaining,
+        ReindexDataStreamTask reindexDataStreamTask,
+        ExecuteWithHeadersClient reindexClient,
+        String sourceDataStream,
+        CountDownActionListener listener,
+        TaskId parentTaskId
+    ) {
+        if (indicesRemaining.isEmpty()) {
+            return;
+        }
+        Index index;
+        try {
+            index = indicesRemaining.removeFirst();
+        } catch (NoSuchElementException e) {
+            return;
+        }
+        reindexDataStreamTask.incrementInProgressIndicesCount(index.getName());
+        ReindexDataStreamIndexAction.Request reindexDataStreamIndexRequest = new ReindexDataStreamIndexAction.Request(index.getName());
+        reindexDataStreamIndexRequest.setParentTask(parentTaskId);
+        reindexClient.execute(ReindexDataStreamIndexAction.INSTANCE, reindexDataStreamIndexRequest, ActionListener.wrap(response1 -> {
+            updateDataStream(sourceDataStream, index.getName(), response1.getDestIndex(), ActionListener.wrap(unused -> {
+                reindexDataStreamTask.reindexSucceeded(index.getName());
+                listener.onResponse(null);
+                maybeProcessNextIndex(indicesRemaining, reindexDataStreamTask, reindexClient, sourceDataStream, listener, parentTaskId);
+            }, exception -> {
+                reindexDataStreamTask.reindexFailed(index.getName(), exception);
+                listener.onResponse(null);
+            }), reindexClient, parentTaskId);
+        }, exception -> {
+            reindexDataStreamTask.reindexFailed(index.getName(), exception);
+            listener.onResponse(null);
+        }));
     }
 
-    private TimeValue getTimeToLive(ReindexDataStreamTask reindexDataStreamTask) {
+    private void updateDataStream(
+        String dataStream,
+        String oldIndex,
+        String newIndex,
+        ActionListener<Void> listener,
+        ExecuteWithHeadersClient reindexClient,
+        TaskId parentTaskId
+    ) {
+        ModifyDataStreamsAction.Request modifyDataStreamRequest = new ModifyDataStreamsAction.Request(
+            TimeValue.MAX_VALUE,
+            TimeValue.MAX_VALUE,
+            List.of(DataStreamAction.removeBackingIndex(dataStream, oldIndex), DataStreamAction.addBackingIndex(dataStream, newIndex))
+        );
+        modifyDataStreamRequest.setParentTask(parentTaskId);
+        reindexClient.execute(ModifyDataStreamsAction.INSTANCE, modifyDataStreamRequest, new ActionListener<>() {
+            @Override
+            public void onResponse(AcknowledgedResponse response) {
+                listener.onResponse(null);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                listener.onFailure(e);
+            }
+        });
+    }
+
+    private void completeSuccessfulPersistentTask(
+        ReindexDataStreamTask persistentTask,
+        @Nullable ReindexDataStreamPersistentTaskState state
+    ) {
+        persistentTask.allReindexesCompleted(threadPool, updateCompletionTimeAndGetTimeToLive(persistentTask, state));
+    }
+
+    private void completeFailedPersistentTask(
+        ReindexDataStreamTask persistentTask,
+        @Nullable ReindexDataStreamPersistentTaskState state,
+        Exception e
+    ) {
+        persistentTask.taskFailed(threadPool, updateCompletionTimeAndGetTimeToLive(persistentTask, state), e);
+    }
+
+    private TimeValue updateCompletionTimeAndGetTimeToLive(
+        ReindexDataStreamTask reindexDataStreamTask,
+        @Nullable ReindexDataStreamPersistentTaskState state
+    ) {
         PersistentTasksCustomMetadata persistentTasksCustomMetadata = clusterService.state()
             .getMetadata()
             .custom(PersistentTasksCustomMetadata.TYPE);
         PersistentTasksCustomMetadata.PersistentTask<?> persistentTask = persistentTasksCustomMetadata.getTask(
             reindexDataStreamTask.getPersistentTaskId()
         );
-        PersistentTaskState state = persistentTask.getState();
+        if (persistentTask == null) {
+            return TimeValue.timeValueMillis(0);
+        }
         final long completionTime;
         if (state == null) {
             completionTime = threadPool.absoluteTimeInMillis();
             reindexDataStreamTask.updatePersistentTaskState(
-                new ReindexDataStreamPersistentTaskState(completionTime),
+                new ReindexDataStreamPersistentTaskState(null, null, completionTime),
                 ActionListener.noop()
             );
         } else {
-            completionTime = ((ReindexDataStreamPersistentTaskState) state).completionTime();
+            if (state.completionTime() == null) {
+                completionTime = threadPool.absoluteTimeInMillis();
+                reindexDataStreamTask.updatePersistentTaskState(
+                    new ReindexDataStreamPersistentTaskState(state.totalIndices(), state.totalIndicesToBeUpgraded(), completionTime),
+                    ActionListener.noop()
+                );
+            } else {
+                completionTime = state.completionTime();
+            }
         }
         return TimeValue.timeValueMillis(TASK_KEEP_ALIVE_TIME.millis() - (threadPool.absoluteTimeInMillis() - completionTime));
     }
diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskState.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskState.java
index 130a8f7ce372b..8ab22674082e6 100644
--- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskState.java
+++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskState.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.persistent.PersistentTaskState;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -18,22 +19,31 @@
 
 import java.io.IOException;
 
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public record ReindexDataStreamPersistentTaskState(
+    @Nullable Integer totalIndices,
+    @Nullable Integer totalIndicesToBeUpgraded,
+    @Nullable Long completionTime
+) implements Task.Status, PersistentTaskState {
 
-public record ReindexDataStreamPersistentTaskState(long completionTime) implements Task.Status, PersistentTaskState {
     public static final String NAME = ReindexDataStreamTask.TASK_NAME;
+    private static final String TOTAL_INDICES_FIELD = "total_indices_in_data_stream";
+    private static final String TOTAL_INDICES_REQUIRING_UPGRADE_FIELD = "total_indices_requiring_upgrade";
     private static final String COMPLETION_TIME_FIELD = "completion_time";
     private static final ConstructingObjectParser<ReindexDataStreamPersistentTaskState, Void> PARSER = new ConstructingObjectParser<>(
         NAME,
         true,
-        args -> new ReindexDataStreamPersistentTaskState((long) args[0])
+        args -> new ReindexDataStreamPersistentTaskState((Integer) args[0], (Integer) args[1], (Long) args[2])
     );
     static {
-        PARSER.declareLong(constructorArg(), new ParseField(COMPLETION_TIME_FIELD));
+        PARSER.declareInt(optionalConstructorArg(), new ParseField(TOTAL_INDICES_FIELD));
+        PARSER.declareInt(optionalConstructorArg(), new ParseField(TOTAL_INDICES_REQUIRING_UPGRADE_FIELD));
+        PARSER.declareLong(optionalConstructorArg(), new ParseField(COMPLETION_TIME_FIELD));
     }
 
     public ReindexDataStreamPersistentTaskState(StreamInput in) throws IOException {
-        this(in.readLong());
+        this(in.readOptionalInt(), in.readOptionalInt(), in.readOptionalLong());
     }
 
     @Override
@@ -43,13 +53,23 @@ public String getWriteableName() {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeLong(completionTime);
+        out.writeOptionalInt(totalIndices);
+        out.writeOptionalInt(totalIndicesToBeUpgraded);
+        out.writeOptionalLong(completionTime);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(COMPLETION_TIME_FIELD, completionTime);
+        if (totalIndices != null) {
+            builder.field(TOTAL_INDICES_FIELD, totalIndices);
+        }
+        if (totalIndicesToBeUpgraded != null) {
+            builder.field(TOTAL_INDICES_REQUIRING_UPGRADE_FIELD, totalIndicesToBeUpgraded);
+        }
+        if (completionTime != null) {
+            builder.field(COMPLETION_TIME_FIELD, completionTime);
+        }
         builder.endObject();
         return builder;
     }
diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamStatus.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamStatus.java
index 358062550b50a..632aea076ea5a 100644
--- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamStatus.java
+++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamStatus.java
@@ -16,6 +16,7 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
+import java.util.Set;
 
 public record ReindexDataStreamStatus(
     long persistentTaskStartTime,
@@ -23,11 +24,12 @@ public record ReindexDataStreamStatus(
     int totalIndicesToBeUpgraded,
     boolean complete,
     Exception exception,
-    int inProgress,
+    Set<String> inProgress,
     int pending,
     List<Tuple<String, Exception>> errors
 ) implements Task.Status {
     public ReindexDataStreamStatus {
+        Objects.requireNonNull(inProgress);
         Objects.requireNonNull(errors);
     }
 
@@ -40,7 +42,7 @@ public ReindexDataStreamStatus(StreamInput in) throws IOException {
             in.readInt(),
             in.readBoolean(),
             in.readException(),
-            in.readInt(),
+            in.readCollectionAsSet((Reader<String>) StreamInput::readString),
             in.readInt(),
             in.readCollectionAsList(in1 -> Tuple.tuple(in1.readString(), in1.readException()))
         );
@@ -58,7 +60,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeInt(totalIndicesToBeUpgraded);
         out.writeBoolean(complete);
         out.writeException(exception);
-        out.writeInt(inProgress);
+        out.writeStringCollection(inProgress);
         out.writeInt(pending);
         out.writeCollection(errors, (out1, tuple) -> {
             out1.writeString(tuple.v1());
@@ -69,12 +71,13 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field("start_time", persistentTaskStartTime);
+        builder.timestampFieldsFromUnixEpochMillis("start_time_millis", "start_time", persistentTaskStartTime);
         builder.field("complete", complete);
-        builder.field("total_indices", totalIndices);
+        builder.field("total_indices_in_data_stream", totalIndices);
         builder.field("total_indices_requiring_upgrade", totalIndicesToBeUpgraded);
-        builder.field("successes", totalIndicesToBeUpgraded - (inProgress + pending + errors.size()));
-        builder.field("in_progress", inProgress);
+        final int inProgressSize = inProgress.size();
+        builder.field("successes", totalIndicesToBeUpgraded - (inProgressSize + pending + errors.size()));
+        builder.field("in_progress", inProgressSize);
         builder.field("pending", pending);
         builder.startArray("errors");
         for (Tuple<String, Exception> error : errors) {
diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java
index 844f24f45ab77..04295a7521479 100644
--- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java
+++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java
@@ -7,35 +7,42 @@
 
 package org.elasticsearch.xpack.migrate.task;
 
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.util.concurrent.RunOnce;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.persistent.AllocatedPersistentTask;
+import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
 public class ReindexDataStreamTask extends AllocatedPersistentTask {
     public static final String TASK_NAME = "reindex-data-stream";
+    private final ClusterService clusterService;
     private final long persistentTaskStartTime;
-    private final int totalIndices;
-    private final int totalIndicesToBeUpgraded;
+    private final int initialTotalIndices;
+    private final int initialTotalIndicesToBeUpgraded;
     private volatile boolean complete = false;
     private volatile Exception exception;
-    private final AtomicInteger inProgress = new AtomicInteger(0);
+    private final Set<String> inProgress = Collections.synchronizedSet(new HashSet<>());
     private final AtomicInteger pending = new AtomicInteger();
-    private final List<Tuple<String, Exception>> errors = new ArrayList<>();
+    private final List<Tuple<String, Exception>> errors = Collections.synchronizedList(new ArrayList<>());
     private final RunOnce completeTask;
 
     @SuppressWarnings("this-escape")
     public ReindexDataStreamTask(
+        ClusterService clusterService,
         long persistentTaskStartTime,
-        int totalIndices,
-        int totalIndicesToBeUpgraded,
+        int initialTotalIndices,
+        int initialTotalIndicesToBeUpgraded,
         long id,
         String type,
         String action,
@@ -44,9 +51,10 @@ public ReindexDataStreamTask(
         Map<String, String> headers
     ) {
         super(id, type, action, description, parentTask, headers);
+        this.clusterService = clusterService;
         this.persistentTaskStartTime = persistentTaskStartTime;
-        this.totalIndices = totalIndices;
-        this.totalIndicesToBeUpgraded = totalIndicesToBeUpgraded;
+        this.initialTotalIndices = initialTotalIndices;
+        this.initialTotalIndicesToBeUpgraded = initialTotalIndicesToBeUpgraded;
         this.completeTask = new RunOnce(() -> {
             if (exception == null) {
                 markAsCompleted();
@@ -58,13 +66,26 @@ public ReindexDataStreamTask(
 
     @Override
     public ReindexDataStreamStatus getStatus() {
+        PersistentTasksCustomMetadata persistentTasksCustomMetadata = clusterService.state()
+            .getMetadata()
+            .custom(PersistentTasksCustomMetadata.TYPE);
+        int totalIndices = initialTotalIndices;
+        int totalIndicesToBeUpgraded = initialTotalIndicesToBeUpgraded;
+        PersistentTasksCustomMetadata.PersistentTask<?> persistentTask = persistentTasksCustomMetadata.getTask(getPersistentTaskId());
+        if (persistentTask != null) {
+            ReindexDataStreamPersistentTaskState state = (ReindexDataStreamPersistentTaskState) persistentTask.getState();
+            if (state != null && state.totalIndices() != null && state.totalIndicesToBeUpgraded() != null) {
+                totalIndices = Math.toIntExact(state.totalIndices());
+                totalIndicesToBeUpgraded = Math.toIntExact(state.totalIndicesToBeUpgraded());
+            }
+        }
         return new ReindexDataStreamStatus(
             persistentTaskStartTime,
             totalIndices,
             totalIndicesToBeUpgraded,
             complete,
             exception,
-            inProgress.get(),
+            inProgress,
             pending.get(),
             errors
         );
@@ -84,17 +105,17 @@ public void taskFailed(ThreadPool threadPool, TimeValue timeToLive, Exception e)
         allReindexesCompleted(threadPool, timeToLive);
     }
 
-    public void reindexSucceeded() {
-        inProgress.decrementAndGet();
+    public void reindexSucceeded(String index) {
+        inProgress.remove(index);
     }
 
     public void reindexFailed(String index, Exception error) {
         this.errors.add(Tuple.tuple(index, error));
-        inProgress.decrementAndGet();
+        inProgress.remove(index);
     }
 
-    public void incrementInProgressIndicesCount() {
-        inProgress.incrementAndGet();
+    public void incrementInProgressIndicesCount(String index) {
+        inProgress.add(index);
         pending.decrementAndGet();
     }
 
diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/CreateFromSourceIndexRequestTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/CreateFromSourceIndexRequestTests.java
new file mode 100644
index 0000000000000..21dab66fccd5e
--- /dev/null
+++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/CreateFromSourceIndexRequestTests.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.migrate.action;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.migrate.action.CreateIndexFromSourceAction.Request;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
+
+public class CreateFromSourceIndexRequestTests extends AbstractWireSerializingTestCase<Request> {
+
+    public void testToAndFromXContent() throws IOException {
+        var request = createTestInstance();
+
+        boolean humanReadable = randomBoolean();
+        final XContentType xContentType = randomFrom(XContentType.values());
+        BytesReference originalBytes = toShuffledXContent(request, xContentType, EMPTY_PARAMS, humanReadable);
+
+        var parsedRequest = new Request(
+            randomValueOtherThan(request.sourceIndex(), () -> randomAlphanumericOfLength(30)),
+            randomValueOtherThan(request.sourceIndex(), () -> randomAlphanumericOfLength(30))
+        );
+        try (XContentParser xParser = createParser(xContentType.xContent(), originalBytes)) {
+            parsedRequest.fromXContent(xParser);
+        }
+
+        // source and dest won't be equal
+        assertNotEquals(request, parsedRequest);
+        assertNotEquals(request.sourceIndex(), parsedRequest.sourceIndex());
+        assertNotEquals(request.destIndex(), parsedRequest.destIndex());
+
+        // but fields in xcontent will be equal
+        assertEquals(request.settingsOverride(), parsedRequest.settingsOverride());
+        assertEquals(request.mappingsOverride(), parsedRequest.mappingsOverride());
+
+        BytesReference finalBytes = toShuffledXContent(parsedRequest, xContentType, EMPTY_PARAMS, humanReadable);
+        ElasticsearchAssertions.assertToXContentEquivalent(originalBytes, finalBytes, xContentType);
+    }
+
+    @Override
+    protected Writeable.Reader<Request> instanceReader() {
+        return Request::new;
+    }
+
+    @Override
+    protected Request createTestInstance() {
+        String source = randomAlphaOfLength(30);
+        String dest = randomAlphaOfLength(30);
+        if (randomBoolean()) {
+            return new Request(source, dest);
+        } else {
+            return new Request(source, dest, randomSettings(), randomMappings());
+        }
+    }
+
+    @Override
+    protected Request mutateInstance(Request instance) throws IOException {
+
+        String sourceIndex = instance.sourceIndex();
+        String destIndex = instance.destIndex();
+        Settings settingsOverride = instance.settingsOverride();
+        Map<String, Object> mappingsOverride = instance.mappingsOverride();
+
+        switch (between(0, 3)) {
+            case 0 -> sourceIndex = randomValueOtherThan(sourceIndex, () -> randomAlphaOfLength(30));
+            case 1 -> destIndex = randomValueOtherThan(destIndex, () -> randomAlphaOfLength(30));
+            case 2 -> settingsOverride = randomValueOtherThan(settingsOverride, CreateFromSourceIndexRequestTests::randomSettings);
+            case 3 -> mappingsOverride = randomValueOtherThan(mappingsOverride, CreateFromSourceIndexRequestTests::randomMappings);
+        }
+        return new Request(sourceIndex, destIndex, settingsOverride, mappingsOverride);
+    }
+
+    public static Map<String, Object> randomMappings() {
+        var randMappings = Map.of("properties", Map.of(randomAlphaOfLength(5), Map.of("type", "keyword")));
+        return randomBoolean() ? Map.of() : Map.of("_doc", randMappings);
+    }
+
+    public static Settings randomSettings() {
+        return randomBoolean() ? Settings.EMPTY : indexSettings(randomIntBetween(1, 10), randomIntBetween(0, 5)).build();
+    }
+}
diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/GetMigrationReindexStatusActionResponseTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/GetMigrationReindexStatusActionResponseTests.java
index a18030edbf42c..1361f30840c87 100644
--- a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/GetMigrationReindexStatusActionResponseTests.java
+++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/GetMigrationReindexStatusActionResponseTests.java
@@ -7,25 +7,17 @@
 
 package org.elasticsearch.xpack.migrate.action;
 
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.network.NetworkModule;
-import org.elasticsearch.tasks.RawTaskStatus;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.tasks.TaskId;
-import org.elasticsearch.tasks.TaskInfo;
-import org.elasticsearch.tasks.TaskResult;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.migrate.action.GetMigrationReindexStatusAction.Response;
+import org.elasticsearch.xpack.migrate.task.ReindexDataStreamEnrichedStatus;
 
 import java.io.IOException;
-import java.util.Collections;
+import java.util.List;
 import java.util.Map;
-import java.util.TreeMap;
 
 public class GetMigrationReindexStatusActionResponseTests extends AbstractWireSerializingTestCase<Response> {
     @Override
@@ -35,11 +27,7 @@ protected Writeable.Reader<Response> instanceReader() {
 
     @Override
     protected Response createTestInstance() {
-        try {
-            return new Response(randomTaskResult());
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
+        return new Response(getRandomStatus());
     }
 
     @Override
@@ -47,76 +35,44 @@ protected Response mutateInstance(Response instance) throws IOException {
         return createTestInstance(); // There's only one field
     }
 
-    private static TaskResult randomTaskResult() throws IOException {
-        return switch (between(0, 2)) {
-            case 0 -> new TaskResult(randomBoolean(), randomTaskInfo());
-            case 1 -> new TaskResult(randomTaskInfo(), new RuntimeException("error"));
-            case 2 -> new TaskResult(randomTaskInfo(), randomTaskResponse());
-            default -> throw new UnsupportedOperationException("Unsupported random TaskResult constructor");
-        };
-    }
-
-    static TaskInfo randomTaskInfo() {
-        String nodeId = randomAlphaOfLength(5);
-        TaskId taskId = randomTaskId(nodeId);
-        String type = randomAlphaOfLength(5);
-        String action = randomAlphaOfLength(5);
-        Task.Status status = randomBoolean() ? randomRawTaskStatus() : null;
-        String description = randomBoolean() ? randomAlphaOfLength(5) : null;
-        long startTime = randomLong();
-        long runningTimeNanos = randomNonNegativeLong();
-        boolean cancellable = randomBoolean();
-        boolean cancelled = cancellable && randomBoolean();
-        TaskId parentTaskId = randomBoolean() ? TaskId.EMPTY_TASK_ID : randomTaskId(randomAlphaOfLength(5));
-        Map<String, String> headers = randomBoolean()
-            ? Collections.emptyMap()
-            : Collections.singletonMap(randomAlphaOfLength(5), randomAlphaOfLength(5));
-        return new TaskInfo(
-            taskId,
-            type,
-            nodeId,
-            action,
-            description,
-            status,
-            startTime,
-            runningTimeNanos,
-            cancellable,
-            cancelled,
-            parentTaskId,
-            headers
+    private ReindexDataStreamEnrichedStatus getRandomStatus() {
+        return new ReindexDataStreamEnrichedStatus(
+            randomLong(),
+            randomNegativeInt(),
+            randomNegativeInt(),
+            randomBoolean(),
+            nullableTestException(),
+            randomInProgressMap(),
+            randomNegativeInt(),
+            randomErrorList()
         );
     }
 
-    private static TaskId randomTaskId(String nodeId) {
-        return new TaskId(nodeId, randomLong());
+    private Map<String, Tuple<Long, Long>> randomInProgressMap() {
+        return randomMap(1, 50, () -> Tuple.tuple(randomAlphaOfLength(50), Tuple.tuple(randomNonNegativeLong(), randomNonNegativeLong())));
     }
 
-    private static RawTaskStatus randomRawTaskStatus() {
-        try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
-            builder.startObject();
-            int fields = between(0, 10);
-            for (int f = 0; f < fields; f++) {
-                builder.field(randomAlphaOfLength(5), randomAlphaOfLength(5));
-            }
-            builder.endObject();
-            return new RawTaskStatus(BytesReference.bytes(builder));
-        } catch (IOException e) {
-            throw new IllegalStateException(e);
+    private Exception nullableTestException() {
+        if (randomBoolean()) {
+            return testException();
         }
+        return null;
     }
 
-    private static ToXContent randomTaskResponse() {
-        Map<String, String> result = new TreeMap<>();
-        int fields = between(0, 10);
-        for (int f = 0; f < fields; f++) {
-            result.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
-        }
-        return (builder, params) -> {
-            for (Map.Entry<String, String> entry : result.entrySet()) {
-                builder.field(entry.getKey(), entry.getValue());
-            }
-            return builder;
-        };
+    private Exception testException() {
+        /*
+         * Unfortunately ElasticsearchException doesn't have an equals and just falls back to Object::equals. So we can't test for equality
+         * when we're using an exception. So always just use null.
+         */
+        return null;
+    }
+
+    private List<Tuple<String, Exception>> randomErrorList() {
+        return randomErrorList(0);
+    }
+
+    private List<Tuple<String, Exception>> randomErrorList(int minSize) {
+        return randomList(minSize, Math.max(minSize, 100), () -> Tuple.tuple(randomAlphaOfLength(30), testException()));
     }
 
     @Override
diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamEnrichedStatusTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamEnrichedStatusTests.java
new file mode 100644
index 0000000000000..993db1096aac2
--- /dev/null
+++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamEnrichedStatusTests.java
@@ -0,0 +1,165 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.migrate.task;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static java.util.Map.entry;
+import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
+import static org.hamcrest.Matchers.equalTo;
+
+public class ReindexDataStreamEnrichedStatusTests extends AbstractWireSerializingTestCase<ReindexDataStreamEnrichedStatus> {
+
+    @Override
+    protected Writeable.Reader<ReindexDataStreamEnrichedStatus> instanceReader() {
+        return ReindexDataStreamEnrichedStatus::new;
+    }
+
+    @Override
+    protected ReindexDataStreamEnrichedStatus createTestInstance() {
+        return new ReindexDataStreamEnrichedStatus(
+            randomLong(),
+            randomNegativeInt(),
+            randomNegativeInt(),
+            randomBoolean(),
+            nullableTestException(),
+            randomInProgressMap(),
+            randomNegativeInt(),
+            randomErrorList()
+        );
+    }
+
+    private Map<String, Tuple<Long, Long>> randomInProgressMap() {
+        return randomMap(1, 50, () -> Tuple.tuple(randomAlphaOfLength(50), Tuple.tuple(randomNonNegativeLong(), randomNonNegativeLong())));
+    }
+
+    private Exception nullableTestException() {
+        if (randomBoolean()) {
+            return testException();
+        }
+        return null;
+    }
+
+    private Exception testException() {
+        /*
+         * Unfortunately ElasticsearchException doesn't have an equals and just falls back to Object::equals. So we can't test for equality
+         * when we're using an exception. So always just use null.
+         */
+        return null;
+    }
+
+    private List<String> randomList() {
+        return randomList(0);
+    }
+
+    private List<String> randomList(int minSize) {
+        return randomList(minSize, Math.max(minSize, 100), () -> randomAlphaOfLength(50));
+    }
+
+    private Set<String> randomSet(int minSize) {
+        return randomSet(minSize, Math.max(minSize, 100), () -> randomAlphaOfLength(50));
+    }
+
+    private List<Tuple<String, Exception>> randomErrorList() {
+        return randomErrorList(0);
+    }
+
+    private List<Tuple<String, Exception>> randomErrorList(int minSize) {
+        return randomList(minSize, Math.max(minSize, 100), () -> Tuple.tuple(randomAlphaOfLength(30), testException()));
+    }
+
+    @Override
+    protected ReindexDataStreamEnrichedStatus mutateInstance(ReindexDataStreamEnrichedStatus instance) throws IOException {
+        long startTime = instance.persistentTaskStartTime();
+        int totalIndices = instance.totalIndices();
+        int totalIndicesToBeUpgraded = instance.totalIndicesToBeUpgraded();
+        boolean complete = instance.complete();
+        Exception exception = instance.exception();
+        Map<String, Tuple<Long, Long>> inProgress = instance.inProgress();
+        int pending = instance.pending();
+        List<Tuple<String, Exception>> errors = instance.errors();
+        switch (randomIntBetween(0, 6)) {
+            case 0 -> startTime = randomLong();
+            case 1 -> totalIndices = totalIndices + 1;
+            case 2 -> totalIndicesToBeUpgraded = totalIndicesToBeUpgraded + 1;
+            case 3 -> complete = complete == false;
+            case 4 -> inProgress = randomInProgressMap();
+            case 5 -> pending = pending + 1;
+            case 6 -> errors = randomErrorList(errors.size() + 1);
+            default -> throw new UnsupportedOperationException();
+        }
+        return new ReindexDataStreamEnrichedStatus(
+            startTime,
+            totalIndices,
+            totalIndicesToBeUpgraded,
+            complete,
+            exception,
+            inProgress,
+            pending,
+            errors
+        );
+    }
+
+    public void testToXContent() throws IOException {
+        ReindexDataStreamEnrichedStatus status = new ReindexDataStreamEnrichedStatus(
+            1234L,
+            200,
+            100,
+            true,
+            new ElasticsearchException("the whole task failed"),
+            Map.of("index-1", Tuple.tuple(10L, 8L)),
+            8,
+            List.of(
+                Tuple.tuple("index7", new ElasticsearchException("index7 failed")),
+                Tuple.tuple("index8", new ElasticsearchException("index8 " + "failed"))
+            )
+        );
+        try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent)) {
+            builder.humanReadable(true);
+            status.toXContent(builder, EMPTY_PARAMS);
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
+                Map<String, Object> parserMap = parser.map();
+                assertThat(
+                    parserMap,
+                    equalTo(
+                        Map.ofEntries(
+                            entry("start_time", "1970-01-01T00:00:01.234Z"),
+                            entry("start_time_millis", 1234),
+                            entry("total_indices_in_data_stream", 200),
+                            entry("total_indices_requiring_upgrade", 100),
+                            entry("complete", true),
+                            entry("exception", "the whole task failed"),
+                            entry("successes", 89),
+                            entry("in_progress", List.of(Map.of("index", "index-1", "total_doc_count", 10, "reindexed_doc_count", 8))),
+                            entry("pending", 8),
+                            entry(
+                                "errors",
+                                List.of(
+                                    Map.of("index", "index7", "message", "index7 failed"),
+                                    Map.of("index", "index8", "message", "index8 failed")
+                                )
+                            )
+                        )
+                    )
+                );
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskStateTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskStateTests.java
index a35cd6e5fa474..c2dee83a91dfe 100644
--- a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskStateTests.java
+++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskStateTests.java
@@ -26,11 +26,32 @@ protected Writeable.Reader<ReindexDataStreamPersistentTaskState> instanceReader(
 
     @Override
     protected ReindexDataStreamPersistentTaskState createTestInstance() {
-        return new ReindexDataStreamPersistentTaskState(randomNegativeLong());
+        return new ReindexDataStreamPersistentTaskState(
+            randomBoolean() ? null : randomNonNegativeInt(),
+            randomBoolean() ? null : randomNonNegativeInt(),
+            randomBoolean() ? null : randomNonNegativeLong()
+        );
     }
 
     @Override
     protected ReindexDataStreamPersistentTaskState mutateInstance(ReindexDataStreamPersistentTaskState instance) throws IOException {
-        return new ReindexDataStreamPersistentTaskState(instance.completionTime() + 1);
+        return switch (randomInt(2)) {
+            case 0 -> new ReindexDataStreamPersistentTaskState(
+                instance.totalIndices() == null ? randomNonNegativeInt() : instance.totalIndices() + 1,
+                instance.totalIndicesToBeUpgraded(),
+                instance.completionTime()
+            );
+            case 1 -> new ReindexDataStreamPersistentTaskState(
+                instance.totalIndices(),
+                instance.totalIndicesToBeUpgraded() == null ? randomNonNegativeInt() : instance.totalIndicesToBeUpgraded() + 1,
+                instance.completionTime()
+            );
+            case 2 -> new ReindexDataStreamPersistentTaskState(
+                instance.totalIndices(),
+                instance.totalIndicesToBeUpgraded(),
+                instance.completionTime() == null ? randomNonNegativeLong() : instance.completionTime() + 1
+            );
+            default -> throw new IllegalArgumentException("Should never get here");
+        };
     }
 }
diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamStatusTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamStatusTests.java
index d81e9d35cd490..ad47eb6a23cd7 100644
--- a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamStatusTests.java
+++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamStatusTests.java
@@ -19,6 +19,7 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static java.util.Map.entry;
 import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
@@ -39,7 +40,7 @@ protected ReindexDataStreamStatus createTestInstance() {
             randomNegativeInt(),
             randomBoolean(),
             nullableTestException(),
-            randomNegativeInt(),
+            randomSet(0),
             randomNegativeInt(),
             randomErrorList()
         );
@@ -68,6 +69,10 @@ private List<String> randomList(int minSize) {
         return randomList(minSize, Math.max(minSize, 100), () -> randomAlphaOfLength(50));
     }
 
+    private Set<String> randomSet(int minSize) {
+        return randomSet(minSize, Math.max(minSize, 100), () -> randomAlphaOfLength(50));
+    }
+
     private List<Tuple<String, Exception>> randomErrorList() {
         return randomErrorList(0);
     }
@@ -83,7 +88,7 @@ protected ReindexDataStreamStatus mutateInstance(ReindexDataStreamStatus instanc
         int totalIndicesToBeUpgraded = instance.totalIndicesToBeUpgraded();
         boolean complete = instance.complete();
         Exception exception = instance.exception();
-        int inProgress = instance.inProgress();
+        Set<String> inProgress = instance.inProgress();
         int pending = instance.pending();
         List<Tuple<String, Exception>> errors = instance.errors();
         switch (randomIntBetween(0, 6)) {
@@ -91,7 +96,7 @@ protected ReindexDataStreamStatus mutateInstance(ReindexDataStreamStatus instanc
             case 1 -> totalIndices = totalIndices + 1;
             case 2 -> totalIndicesToBeUpgraded = totalIndicesToBeUpgraded + 1;
             case 3 -> complete = complete == false;
-            case 4 -> inProgress = inProgress + 1;
+            case 4 -> inProgress = randomSet(inProgress.size() + 1);
             case 5 -> pending = pending + 1;
             case 6 -> errors = randomErrorList(errors.size() + 1);
             default -> throw new UnsupportedOperationException();
@@ -115,7 +120,7 @@ public void testToXContent() throws IOException {
             100,
             true,
             new ElasticsearchException("the whole task failed"),
-            12,
+            randomSet(12, 12, () -> randomAlphaOfLength(50)),
             8,
             List.of(
                 Tuple.tuple("index7", new ElasticsearchException("index7 failed")),
@@ -131,8 +136,9 @@ public void testToXContent() throws IOException {
                     parserMap,
                     equalTo(
                         Map.ofEntries(
-                            entry("start_time", 1234),
-                            entry("total_indices", 200),
+                            entry("start_time", "1970-01-01T00:00:01.234Z"),
+                            entry("start_time_millis", 1234),
+                            entry("total_indices_in_data_stream", 200),
                             entry("total_indices_requiring_upgrade", 100),
                             entry("complete", true),
                             entry("exception", "the whole task failed"),
diff --git a/x-pack/plugin/ml-package-loader/build.gradle b/x-pack/plugin/ml-package-loader/build.gradle
index 122ad396b507d..fa460bf5605ea 100644
--- a/x-pack/plugin/ml-package-loader/build.gradle
+++ b/x-pack/plugin/ml-package-loader/build.gradle
@@ -11,9 +11,9 @@ import org.elasticsearch.gradle.OS
 apply plugin: 'elasticsearch.internal-es-plugin'
 
 esplugin {
-  name 'ml-package-loader'
-  description 'Loader for prepackaged Machine Learning Models from Elastic'
-  classname 'org.elasticsearch.xpack.ml.packageloader.MachineLearningPackageLoader'
+  name = 'ml-package-loader'
+  description = 'Loader for prepackaged Machine Learning Models from Elastic'
+  classname ='org.elasticsearch.xpack.ml.packageloader.MachineLearningPackageLoader'
   extendedPlugins = ['x-pack-core']
 }
 
diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle
index 716c401a9fcc8..8373d9c6582e3 100644
--- a/x-pack/plugin/ml/build.gradle
+++ b/x-pack/plugin/ml/build.gradle
@@ -11,10 +11,10 @@ apply plugin: 'elasticsearch.internal-test-artifact'
 apply plugin: 'elasticsearch.dra-artifacts'
 
 esplugin {
-  name 'x-pack-ml'
-  description 'Elasticsearch Expanded Pack Plugin - Machine Learning'
-  classname 'org.elasticsearch.xpack.ml.MachineLearning'
-  hasNativeController true
+  name = 'x-pack-ml'
+  description = 'Elasticsearch Expanded Pack Plugin - Machine Learning'
+  classname ='org.elasticsearch.xpack.ml.MachineLearning'
+  hasNativeController =true
   extendedPlugins = ['x-pack-autoscaling', 'lang-painless']
 }
 
@@ -23,8 +23,8 @@ if (useDra == false) {
     exclusiveContent {
       forRepository {
         ivy {
-          name "ml-cpp"
-          url providers.systemProperty('build.ml_cpp.repo').orElse('https://prelert-artifacts.s3.amazonaws.com').get()
+          name = "ml-cpp"
+          url = providers.systemProperty('build.ml_cpp.repo').orElse('https://prelert-artifacts.s3.amazonaws.com').get()
           metadataSources {
             // no repository metadata, look directly for the artifact
             artifact()
diff --git a/x-pack/plugin/ml/qa/ml-inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelIdUniquenessIT.java b/x-pack/plugin/ml/qa/ml-inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelIdUniquenessIT.java
index 9904cfb752de5..f6f5a1299c105 100644
--- a/x-pack/plugin/ml/qa/ml-inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelIdUniquenessIT.java
+++ b/x-pack/plugin/ml/qa/ml-inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelIdUniquenessIT.java
@@ -29,9 +29,10 @@ public void testPutInferenceModelFailsWhenTrainedModelWithIdAlreadyExists() thro
         assertThat(
             e.getMessage(),
             Matchers.containsString(
-                "Model IDs must be unique. Requested model ID [" + modelId + "] matches existing model IDs but must not."
+                "Inference endpoint IDs must be unique. Requested inference endpoint ID ["
+                    + modelId
+                    + "] matches existing trained model ID(s) but must not."
             )
-
         );
     }
 
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java
index 5287d149fae3d..367c1cee8b0ee 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java
@@ -11,6 +11,8 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
+import org.elasticsearch.action.admin.cluster.node.tasks.cancel.TransportCancelTasksAction;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.ReferenceDocs;
@@ -80,6 +82,10 @@ public class DatafeedJobsIT extends MlNativeAutodetectIntegTestCase {
     public void cleanup() {
         updateClusterSettings(Settings.builder().putNull("logger.org.elasticsearch.xpack.ml.datafeed"));
         cleanUp();
+        // Race conditions between closing and killing tasks in these tests,
+        // sometimes result in lingering persistent tasks (such as "_close"),
+        // which cause subsequent tests to fail.
+        client().execute(TransportCancelTasksAction.TYPE, new CancelTasksRequest());
     }
 
     public void testLookbackOnly() throws Exception {
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java
index 2fc9a80ae3679..804033ef531b9 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java
@@ -37,7 +37,6 @@
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
 
 public class DatafeedJobsRestIT extends ESRestTestCase {
 
@@ -503,12 +502,12 @@ public void testLookbackWithIndicesOptions() throws Exception {
             }""");
         client().performRequest(createJobRequest);
         String datafeedId = jobId + "-datafeed";
-        new DatafeedBuilder(datafeedId, jobId, "*hidden-*").setIndicesOptions("""
+        new DatafeedBuilder(datafeedId, jobId, "hidden-*").setIndicesOptions("""
             {"expand_wildcards": ["all"],"allow_no_indices": true}""").build();
 
         StringBuilder bulk = new StringBuilder();
 
-        Request createGeoData = new Request("PUT", "/.hidden-index");
+        Request createGeoData = new Request("PUT", "/hidden-index");
         createGeoData.setJsonEntity("""
             {
               "mappings": {
@@ -528,23 +527,23 @@ public void testLookbackWithIndicesOptions() throws Exception {
         client().performRequest(createGeoData);
 
         bulk.append("""
-            {"index": {"_index": ".hidden-index", "_id": 1}}
+            {"index": {"_index": "hidden-index", "_id": 1}}
             {"time":"2016-06-01T00:00:00Z","value": 1000}
-            {"index": {"_index": ".hidden-index", "_id": 2}}
+            {"index": {"_index": "hidden-index", "_id": 2}}
             {"time":"2016-06-01T00:05:00Z","value":1500}
-            {"index": {"_index": ".hidden-index", "_id": 3}}
+            {"index": {"_index": "hidden-index", "_id": 3}}
             {"time":"2016-06-01T00:10:00Z","value":1600}
-            {"index": {"_index": ".hidden-index", "_id": 4}}
+            {"index": {"_index": "hidden-index", "_id": 4}}
             {"time":"2016-06-01T00:15:00Z","value":100}
-            {"index": {"_index": ".hidden-index", "_id": 5}}
+            {"index": {"_index": "hidden-index", "_id": 5}}
             {"time":"2016-06-01T00:20:00Z","value":1}
-            {"index": {"_index": ".hidden-index", "_id": 6}}
+            {"index": {"_index": "hidden-index", "_id": 6}}
             {"time":"2016-06-01T00:25:00Z","value":1500}
-            {"index": {"_index": ".hidden-index", "_id": 7}}
+            {"index": {"_index": "hidden-index", "_id": 7}}
             {"time":"2016-06-01T00:30:00Z","value":1500}
-            {"index": {"_index": ".hidden-index", "_id": 8}}
+            {"index": {"_index": "hidden-index", "_id": 8}}
             {"time":"2016-06-01T00:40:00Z","value":2100}
-            {"index": {"_index": ".hidden-index", "_id": 9}}
+            {"index": {"_index": "hidden-index", "_id": 9}}
             {"time":"2016-06-01T00:41:00Z","value":0}
             """);
         bulkIndex(bulk.toString());
@@ -1802,7 +1801,7 @@ private void bulkIndex(String bulk) throws IOException {
         bulkRequest.addParameter("refresh", "true");
         bulkRequest.addParameter("pretty", null);
         String bulkResponse = EntityUtils.toString(client().performRequest(bulkRequest).getEntity());
-        assertThat(bulkResponse, not(containsString("\"errors\": false")));
+        assertThat(bulkResponse, containsString("\"errors\" : false"));
     }
 
     private Response createJobAndDataFeed(String jobId, String datafeedId) throws IOException {
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java
index 1d67639f712a0..46510be620216 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java
@@ -214,7 +214,7 @@ private static Map<String, Settings> getIndexToSettingsMap(List<String> indexNam
     }
 
     private static Map<String, List<AliasMetadata>> getIndexToAliasesMap(List<String> indexNames) {
-        GetAliasesResponse getAliasesResponse = indicesAdmin().prepareGetAliases()
+        GetAliasesResponse getAliasesResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT)
             .setIndices(indexNames.toArray(String[]::new))
             .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
             .get();
diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
index 6572a6c286519..6f5fa72a2b2f6 100644
--- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
+++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
@@ -250,6 +250,7 @@ public void testAliasesMovedFromOldToNew() throws Exception {
         assertBusy(() -> {
             assertTrue(annotationsIndexExists(AnnotationIndex.LATEST_INDEX_NAME));
             Map<String, List<AliasMetadata>> aliases = indicesAdmin().prepareGetAliases(
+                TEST_REQUEST_TIMEOUT,
                 AnnotationIndex.READ_ALIAS_NAME,
                 AnnotationIndex.WRITE_ALIAS_NAME
             ).setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get().getAliases();
@@ -330,6 +331,7 @@ private boolean annotationsIndexExists(String expectedName) {
     private int numberOfAnnotationsAliases() {
         int count = 0;
         Map<String, List<AliasMetadata>> aliases = indicesAdmin().prepareGetAliases(
+            TEST_REQUEST_TIMEOUT,
             AnnotationIndex.READ_ALIAS_NAME,
             AnnotationIndex.WRITE_ALIAS_NAME,
             AnnotationIndex.LATEST_INDEX_NAME
diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/IndexLayoutIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/IndexLayoutIT.java
index 99052c771fb49..ec95b979fdd6d 100644
--- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/IndexLayoutIT.java
+++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/IndexLayoutIT.java
@@ -66,7 +66,7 @@ public void testCrudOnTwoJobsInSharedIndex() throws Exception {
         assertThat(
             client.admin()
                 .indices()
-                .prepareGetAliases(AnomalyDetectorsIndex.jobStateIndexPattern())
+                .prepareGetAliases(TEST_REQUEST_TIMEOUT, AnomalyDetectorsIndex.jobStateIndexPattern())
                 .get()
                 .getAliases()
                 .get(".ml-state-000001")
diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java
index 997476143dff8..7ce0fbe760644 100644
--- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java
+++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java
@@ -255,7 +255,7 @@ public void testMultipleSimultaneousJobCreations() {
 
         // Assert that the mappings contain all the additional fields: field1, field2, field3, etc.
         String sharedResultsIndex = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT;
-        GetMappingsRequest request = new GetMappingsRequest().indices(sharedResultsIndex);
+        GetMappingsRequest request = new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(sharedResultsIndex);
         GetMappingsResponse response = client().execute(GetMappingsAction.INSTANCE, request).actionGet();
         Map<String, MappingMetadata> indexMappings = response.getMappings();
         assertNotNull(indexMappings);
@@ -506,7 +506,7 @@ public void testGetDataCountsModelSizeAndTimingStatsWithSomeDocs() throws Except
     }
 
     private Map<String, Object> getIndexMappingProperties(String index) {
-        GetMappingsRequest request = new GetMappingsRequest().indices(index);
+        GetMappingsRequest request = new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(index);
         GetMappingsResponse response = client().execute(GetMappingsAction.INSTANCE, request).actionGet();
         Map<String, MappingMetadata> indexMappings = response.getMappings();
         assertNotNull(indexMappings);
@@ -530,7 +530,10 @@ private Map<String, Object> getIndexMappingProperties(String index) {
     }
 
     private Set<String> getAliases(String index) {
-        GetAliasesResponse getAliasesResponse = client().admin().indices().getAliases(new GetAliasesRequest().indices(index)).actionGet();
+        GetAliasesResponse getAliasesResponse = client().admin()
+            .indices()
+            .getAliases(new GetAliasesRequest(TEST_REQUEST_TIMEOUT).indices(index))
+            .actionGet();
         Map<String, List<AliasMetadata>> aliases = getAliasesResponse.getAliases();
         assertThat(aliases.containsKey(index), is(true));
         List<AliasMetadata> aliasMetadataList = aliases.get(index);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
index c76e43790a259..08c876dfdcc5d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
@@ -199,6 +199,7 @@
 import org.elasticsearch.xpack.core.ml.job.snapshot.upgrade.SnapshotUpgradeTaskState;
 import org.elasticsearch.xpack.core.ml.ltr.MlLTRNamedXContentProvider;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.ml.vectors.TextEmbeddingQueryVectorBuilder;
 import org.elasticsearch.xpack.core.template.TemplateUtils;
 import org.elasticsearch.xpack.ml.action.TransportAuditMlNotificationAction;
 import org.elasticsearch.xpack.ml.action.TransportCancelJobModelSnapshotUpgradeAction;
@@ -455,7 +456,6 @@
 import org.elasticsearch.xpack.ml.rest.validate.RestValidateJobConfigAction;
 import org.elasticsearch.xpack.ml.utils.NativeMemoryCalculator;
 import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService;
-import org.elasticsearch.xpack.ml.vectors.TextEmbeddingQueryVectorBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java
index 45a71a80de077..99df16bcd3dc2 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java
@@ -221,8 +221,9 @@ private void makeMlInternalIndicesHidden() {
                 logger.warn("One or more of the ML internal indices could not be made hidden.");
                 return;
             }
-            GetAliasesRequest getAliasesRequest = new GetAliasesRequest().indices(mlHiddenIndexPatterns)
-                .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN);
+            GetAliasesRequest getAliasesRequest = new GetAliasesRequest(MasterNodeRequest.INFINITE_MASTER_NODE_TIMEOUT).indices(
+                mlHiddenIndexPatterns
+            ).indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN);
             executeAsyncWithOrigin(client, ML_ORIGIN, GetAliasesAction.INSTANCE, getAliasesRequest, getAliasesResponseListener);
         }, finalListener::onFailure);
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
index 759538b4cdc63..dea5e891bd8dd 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
@@ -223,7 +223,7 @@ public void onFailure(Exception e) {
         );
 
         // Get start context
-        getStartContext(request.getId(), task, startContextListener);
+        getStartContext(request.getId(), task, startContextListener, request.masterNodeTimeout());
     }
 
     private void estimateMemoryUsageAndUpdateMemoryTracker(StartContext startContext, ActionListener<StartContext> listener) {
@@ -264,7 +264,7 @@ private void estimateMemoryUsageAndUpdateMemoryTracker(StartContext startContext
 
     }
 
-    private void getStartContext(String id, Task task, ActionListener<StartContext> finalListener) {
+    private void getStartContext(String id, Task task, ActionListener<StartContext> finalListener, TimeValue masterTimeout) {
 
         ParentTaskAssigningClient parentTaskClient = new ParentTaskAssigningClient(client, task.getParentTaskId());
 
@@ -320,6 +320,7 @@ private void getStartContext(String id, Task task, ActionListener<StartContext>
             .<StartContext>andThen(
                 (l, startContext) -> MappingsMerger.mergeMappings(
                     parentTaskClient,
+                    masterTimeout,
                     startContext.config.getHeaders(),
                     startContext.config.getSource(),
                     l.map(ignored -> startContext)
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetector.java
index e771fb3b94568..cf97ed629cfc5 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetector.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetector.java
@@ -189,7 +189,7 @@ private TestStats testTrendVs(TestStats H0, double[] values, double[] weights) {
     private TestStats testStepChangeVs(TestStats H0, double[] values, double[] weights, int[] candidateChangePoints) {
 
         double vStep = Double.MAX_VALUE;
-        int changePoint = -1;
+        int changePoint = ChangeType.NO_CHANGE_POINT;
 
         // Initialize running stats so that they are only missing the individual changepoint values
         RunningStats lowerRange = new RunningStats();
@@ -226,7 +226,7 @@ private TestStats testStepChangeVs(TestStats H0, double[] values, double[] weigh
     private TestStats testTrendChangeVs(TestStats H0, double[] values, double[] weights, int[] candidateChangePoints) {
 
         double vChange = Double.MAX_VALUE;
-        int changePoint = -1;
+        int changePoint = ChangeType.NO_CHANGE_POINT;
 
         // Initialize running stats so that they are only missing the individual changepoint values
         RunningStats lowerRange = new RunningStats();
@@ -349,7 +349,7 @@ private TestStats testDistributionChange(
     ) {
 
         double maxDiff = 0.0;
-        int changePoint = -1;
+        int changePoint = ChangeType.NO_CHANGE_POINT;
 
         // Initialize running stats so that they are only missing the individual changepoint values
         RunningStats lowerRange = new RunningStats();
@@ -378,10 +378,12 @@ private TestStats testDistributionChange(
         // before we run the tests.
         SampleData sampleData = sample(values, weights, discoveredChangePoints);
         final double[] sampleValues = sampleData.values();
-        final double[] sampleWeights = sampleData.weights();
 
         double pValue = 1;
         for (int cp : sampleData.changePoints()) {
+            if (cp == ChangeType.NO_CHANGE_POINT) {
+                continue;
+            }
             double[] x = Arrays.copyOfRange(sampleValues, 0, cp);
             double[] y = Arrays.copyOfRange(sampleValues, cp, sampleValues.length);
             double statistic = KOLMOGOROV_SMIRNOV_TEST.kolmogorovSmirnovStatistic(x, y);
@@ -451,7 +453,7 @@ private record TestStats(Type type, double pValue, double var, double nParams, i
         }
 
         TestStats(Type type, double pValue, double var, double nParams, DataStats dataStats) {
-            this(type, pValue, var, nParams, -1, dataStats);
+            this(type, pValue, var, nParams, ChangeType.NO_CHANGE_POINT, dataStats);
         }
 
         boolean accept(double pValueThreshold) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregator.java
index d643a937180a1..ce622df184617 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregator.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregator.java
@@ -47,7 +47,7 @@ public InternalAggregation doReduce(InternalAggregations aggregations, Aggregati
         ChangeType change = ChangePointDetector.getChangeType(bucketValues);
 
         ChangePointBucket changePointBucket = null;
-        if (change.changePoint() >= 0) {
+        if (change.changePoint() != ChangeType.NO_CHANGE_POINT) {
             changePointBucket = extractBucket(bucketsPaths()[0], aggregations, change.changePoint()).map(
                 b -> new ChangePointBucket(b.getKey(), b.getDocCount(), b.getAggregations())
             ).orElse(null);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeType.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeType.java
index c62355dc47451..7df542b59107b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeType.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeType.java
@@ -21,8 +21,10 @@
  */
 public interface ChangeType extends NamedWriteable, NamedXContentObject {
 
+    int NO_CHANGE_POINT = -1;
+
     default int changePoint() {
-        return -1;
+        return NO_CHANGE_POINT;
     }
 
     default double pValue() {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/SpikeAndDipDetector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/SpikeAndDipDetector.java
index 365ebe8562d6a..fa632f643ffdf 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/SpikeAndDipDetector.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/SpikeAndDipDetector.java
@@ -58,7 +58,7 @@ private SpikeOrDip findSpikeOrDip(double[] values, int extent, boolean negate) {
         int maxEnd = Math.min(maxStart + extent, values.length);
         double maxSum = sum(values, maxStart, maxEnd, negate);
         for (int start = maxStart + 1; start <= argmax; start++) {
-            if (start + extent >= values.length) {
+            if (start + extent > values.length) {
                 break;
             }
             double average = sum(values, start, start + extent, negate);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java
index ed12f54ab86b5..ba69c8733dd13 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java
@@ -38,6 +38,7 @@
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.RequiredField;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.ml.MachineLearning;
 
 import java.time.Clock;
 import java.util.Arrays;
@@ -147,6 +148,7 @@ private static void prepareCreateIndexRequest(
                 final Settings settings = settings(settingsResponse, destIndexAllowedSettings);
                 MappingsMerger.mergeMappings(
                     client,
+                    MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT,
                     config.getHeaders(),
                     config.getSource(),
                     delegate.delegateFailureAndWrap(
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java
index 8fee0160cf92e..817020d788554 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsSource;
@@ -34,6 +35,7 @@ private MappingsMerger() {}
 
     public static void mergeMappings(
         Client client,
+        TimeValue masterTimeout,
         Map<String, String> headers,
         DataFrameAnalyticsSource source,
         ActionListener<MappingMetadata> listener
@@ -43,7 +45,7 @@ public static void mergeMappings(
             listener::onFailure
         );
 
-        GetMappingsRequest getMappingsRequest = new GetMappingsRequest();
+        GetMappingsRequest getMappingsRequest = new GetMappingsRequest(masterTimeout);
         getMappingsRequest.indices(source.getIndex());
         ClientHelper.executeWithHeadersAsync(headers, ML_ORIGIN, client, GetMappingsAction.INSTANCE, getMappingsRequest, mappingsListener);
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/modelsize/ModelSizeInfo.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/modelsize/ModelSizeInfo.java
index 587e7e3815c2d..8dca78cafda47 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/modelsize/ModelSizeInfo.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/modelsize/ModelSizeInfo.java
@@ -30,7 +30,7 @@ public class ModelSizeInfo implements Accountable, ToXContentObject {
     private static final ParseField TRAINED_MODEL_SIZE = new ParseField("trained_model_size");
 
     @SuppressWarnings("unchecked")
-    public static ConstructingObjectParser<ModelSizeInfo, Void> PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<ModelSizeInfo, Void> PARSER = new ConstructingObjectParser<>(
         "model_size",
         false,
         a -> new ModelSizeInfo((EnsembleSizeInfo) a[0], (List<PreprocessorSize>) a[1])
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/Vocabulary.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/Vocabulary.java
index 0bfc64c9b0027..72fc96683b721 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/Vocabulary.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/Vocabulary.java
@@ -47,7 +47,7 @@ public static ConstructingObjectParser<Vocabulary, Void> createParser(boolean ig
         return parser;
     }
 
-    public static ConstructingObjectParser<Vocabulary, Void> PARSER = createParser(true);
+    public static final ConstructingObjectParser<Vocabulary, Void> PARSER = createParser(true);
 
     private final List<String> vocab;
     private final List<String> merges;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/results/AckResult.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/results/AckResult.java
index 06f2679c56c2d..5626270c1c6d3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/results/AckResult.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/results/AckResult.java
@@ -18,7 +18,7 @@ public record AckResult(boolean acknowledged) implements ToXContentObject {
 
     public static final ParseField ACKNOWLEDGED = new ParseField("acknowledged");
 
-    public static ConstructingObjectParser<AckResult, Void> PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<AckResult, Void> PARSER = new ConstructingObjectParser<>(
         "ack",
         a -> new AckResult((Boolean) a[0])
     );
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/results/ErrorResult.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/results/ErrorResult.java
index 68fc5cc589231..d5fd64c74c54e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/results/ErrorResult.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/results/ErrorResult.java
@@ -18,7 +18,7 @@ public record ErrorResult(String error) implements ToXContentObject {
 
     public static final ParseField ERROR = new ParseField("error");
 
-    public static ConstructingObjectParser<ErrorResult, Void> PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<ErrorResult, Void> PARSER = new ConstructingObjectParser<>(
         "error",
         a -> new ErrorResult((String) a[0])
     );
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/results/PyTorchResult.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/results/PyTorchResult.java
index 11340d0bf542d..3db738e72766b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/results/PyTorchResult.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/results/PyTorchResult.java
@@ -36,7 +36,7 @@ public record PyTorchResult(
     private static final ParseField THREAD_SETTINGS = new ParseField("thread_settings");
     private static final ParseField ACK = new ParseField("ack");
 
-    public static ConstructingObjectParser<PyTorchResult, Void> PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<PyTorchResult, Void> PARSER = new ConstructingObjectParser<>(
         "pytorch_result",
         a -> new PyTorchResult(
             (String) a[0],
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/results/ThreadSettings.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/results/ThreadSettings.java
index 9154d33c04574..af8818d540ebd 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/results/ThreadSettings.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/results/ThreadSettings.java
@@ -19,7 +19,7 @@ public record ThreadSettings(int numThreadsPerAllocation, int numAllocations) im
     private static final ParseField NUM_ALLOCATIONS = new ParseField("num_allocations");
     private static final ParseField NUM_THREADS_PER_ALLOCATION = new ParseField("num_threads_per_allocation");
 
-    public static ConstructingObjectParser<ThreadSettings, Void> PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<ThreadSettings, Void> PARSER = new ConstructingObjectParser<>(
         "thread_settings",
         a -> new ThreadSettings((int) a[0], (int) a[1])
     );
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlClassicTokenizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlClassicTokenizer.java
index 89533255528f2..c5e3754564ee3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlClassicTokenizer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlClassicTokenizer.java
@@ -15,7 +15,7 @@
  */
 public class MlClassicTokenizer extends AbstractMlTokenizer {
 
-    public static String NAME = "ml_classic";
+    public static final String NAME = "ml_classic";
 
     MlClassicTokenizer() {}
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlStandardTokenizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlStandardTokenizer.java
index ebb1b646a4c54..a3fb05efb17c4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlStandardTokenizer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlStandardTokenizer.java
@@ -18,7 +18,7 @@
  */
 public class MlStandardTokenizer extends AbstractMlTokenizer {
 
-    public static String NAME = "ml_standard";
+    public static final String NAME = "ml_standard";
 
     private int putBackChar = -1;
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java
index 0f3abe3ab8c20..a61f78e856b50 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -475,8 +476,10 @@ private void deleteAliases(@SuppressWarnings("HiddenField") String jobId, Action
         final String writeAliasName = AnomalyDetectorsIndex.resultsWriteAlias(jobId);
 
         // first find the concrete indices associated with the aliases
-        GetAliasesRequest aliasesRequest = new GetAliasesRequest().aliases(readAliasName, writeAliasName)
-            .indicesOptions(IndicesOptions.lenientExpandOpenHidden());
+        GetAliasesRequest aliasesRequest = new GetAliasesRequest(MasterNodeRequest.INFINITE_MASTER_NODE_TIMEOUT).aliases(
+            readAliasName,
+            writeAliasName
+        ).indicesOptions(IndicesOptions.lenientExpandOpenHidden());
         executeAsyncWithOrigin(
             client.threadPool().getThreadContext(),
             ML_ORIGIN,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java
index 3c0d2aca4deda..3c82841f1b99e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java
@@ -8,10 +8,12 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -102,7 +104,29 @@ public void executeRequest() {
         try (ThreadContext.StoredContext ignore = client.threadPool().getThreadContext().stashWithOrigin(ML_ORIGIN)) {
             BulkResponse addRecordsResponse = client.bulk(bulkRequest).actionGet();
             if (addRecordsResponse.hasFailures()) {
-                logger.error("[{}] Bulk index of results has errors: {}", jobId, addRecordsResponse.buildFailureMessage());
+                // Implementation note: Ignore the failures from writing to the read-only index, as it comes
+                // from changing the index format version.
+                boolean hasNonReadOnlyFailures = false;
+                for (BulkItemResponse response : addRecordsResponse.getItems()) {
+                    if (response.isFailed() == false) {
+                        continue;
+                    }
+                    if (response.getFailureMessage().contains(IndexMetadata.INDEX_READ_ONLY_BLOCK.description())) {
+                        // We expect this to happen when the old index is made read-only and being reindexed
+                        logger.debug(
+                            "[{}] Ignoring failure to write renormalized results to a read-only index [{}]: {}",
+                            jobId,
+                            response.getFailure().getIndex(),
+                            response.getFailureMessage()
+                        );
+                    } else {
+                        hasNonReadOnlyFailures = true;
+                        break;
+                    }
+                }
+                if (hasNonReadOnlyFailures) {
+                    logger.error("[{}] Bulk index of results has errors: {}", jobId, addRecordsResponse.buildFailureMessage());
+                }
             }
         }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
index 51b3e0b55d75b..23b12a23e7495 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
@@ -126,6 +126,7 @@
 import org.elasticsearch.xpack.core.ml.stats.StatsAccumulator;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.security.support.Exceptions;
+import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.job.categorization.GrokPatternCreator;
 import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder.InfluencersQuery;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams;
@@ -392,7 +393,10 @@ private void getLatestIndexMappingsAndAddTerms(String indexName, Collection<Stri
             addTermsMapping(indexMappings, indexName, termFields, listener);
         }, listener::onFailure);
 
-        GetMappingsRequest getMappingsRequest = client.admin().indices().prepareGetMappings(indexName).request();
+        GetMappingsRequest getMappingsRequest = client.admin()
+            .indices()
+            .prepareGetMappings(MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT, indexName)
+            .request();
         executeAsyncWithOrigin(
             client.threadPool().getThreadContext(),
             ML_ORIGIN,
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetectorTests.java
index 75f668a96e77e..36076bbb0ec25 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetectorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetectorTests.java
@@ -17,6 +17,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.DoubleStream;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThan;
@@ -243,4 +244,14 @@ public void testProblemDistributionChange() {
         ChangeType type = new ChangeDetector(bucketValues).detect(0.05);
         assertThat(type, instanceOf(ChangeType.DistributionChange.class));
     }
+
+    public void testUncertainNonStationary() {
+        MlAggsHelper.DoubleBucketValues bucketValues = new MlAggsHelper.DoubleBucketValues(
+            null,
+            new double[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 700, 735, 715 }
+        );
+        ChangeType type = new ChangeDetector(bucketValues).detect(0.01);
+        assertThat(type, instanceOf(ChangeType.NonStationary.class));
+        assertThat(((ChangeType.NonStationary) type).getTrend(), equalTo("increasing"));
+    }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/SpikeAndDipDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/SpikeAndDipDetectorTests.java
index b21a7c4625e83..c80cfffbd7328 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/SpikeAndDipDetectorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/SpikeAndDipDetectorTests.java
@@ -184,4 +184,14 @@ public void testMissingBuckets() {
         assertThat(change, instanceOf(ChangeType.Spike.class));
         assertThat(change.changePoint(), equalTo(10));
     }
+
+    public void testSpikeAtTail() {
+        MlAggsHelper.DoubleBucketValues bucketValues = new MlAggsHelper.DoubleBucketValues(
+            null,
+            new double[] { 2, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 9, 8 }
+        );
+        ChangeType change = new SpikeAndDipDetector(bucketValues).detect(0.01);
+        assertThat(change, instanceOf(ChangeType.Spike.class));
+        assertThat(change.changePoint(), equalTo(27));
+    }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregationTests.java
index 85219b0e62c6c..eb05d84c2140d 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregationTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregationTests.java
@@ -84,8 +84,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             public void close() throws IOException {}
         }
 
-        public static String MAP_REDUCER_NAME = "word-count-test-aggregation";
-        public static String AGG_NAME = "internal-map-reduce-aggregation-test";
+        public static final String MAP_REDUCER_NAME = "word-count-test-aggregation";
+        public static final String AGG_NAME = "internal-map-reduce-aggregation-test";
 
         WordCountMapReducer() {
             super(AGG_NAME, MAP_REDUCER_NAME);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java
index 4bb612921876e..f6cc282dde4dd 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java
@@ -348,8 +348,8 @@ public void testIndexDoesntExist() {
                     "cannot start datafeed [datafeed_id] because it failed resolving indices given [not_foo] and "
                         + "indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, "
                         + "expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, "
-                        + "forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true, include_regular_indices=true, "
-                        + "include_failure_indices=false, allow_failure_indices=true]] with exception [no such index [not_foo]]"
+                        + "forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true, "
+                        + "allow_selectors=true]] with exception [no such index [not_foo]]"
                 )
             )
         );
@@ -383,8 +383,7 @@ public void testIndexDoesntExist() {
                         + "indices given [not_foo] and indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, "
                         + "expand_wildcards_open=true, expand_wildcards_closed=false, expand_wildcards_hidden=false, "
                         + "allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false, "
-                        + "ignore_throttled=true, include_regular_indices=true, include_failure_indices=false, "
-                        + "allow_failure_indices=true]] with exception [no such index [not_foo]]]"
+                        + "ignore_throttled=true, allow_selectors=true]] with exception [no such index [not_foo]]]"
                 )
             )
         );
@@ -560,8 +559,8 @@ public void testSelectNode_GivenJobOpeningAndIndexDoesNotExist() {
                         + "[cannot start datafeed [datafeed_id] because it failed resolving indices given [not_foo] and "
                         + "indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, "
                         + "expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, "
-                        + "forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true, include_regular_indices=true, "
-                        + "include_failure_indices=false, allow_failure_indices=true]] with exception [no such index [not_foo]]]"
+                        + "forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true, "
+                        + "allow_selectors=true]] with exception [no such index [not_foo]]]"
                 )
             )
         );
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java
index 5a358ef833cb5..28d81b7464e58 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java
@@ -1556,6 +1556,7 @@ public void testAreAssignedNodesRemoved_GivenShuttingDownNodeThatIsRouted() {
                                 nodeId1,
                                 SingleNodeShutdownMetadata.builder()
                                     .setNodeId(nodeId1)
+                                    .setNodeEphemeralId(nodeId1)
                                     .setType(SingleNodeShutdownMetadata.Type.REMOVE)
                                     .setStartedAtMillis(System.currentTimeMillis())
                                     .setReason("test")
@@ -1599,6 +1600,7 @@ public void testAreAssignedNodesRemoved_GivenShuttingDownNodeThatIsNotRouted() {
                                 nodeId1,
                                 SingleNodeShutdownMetadata.builder()
                                     .setNodeId(nodeId1)
+                                    .setNodeEphemeralId(nodeId1)
                                     .setType(SingleNodeShutdownMetadata.Type.REMOVE)
                                     .setStartedAtMillis(System.currentTimeMillis())
                                     .setReason("test")
@@ -1645,6 +1647,7 @@ public void testRemoveRoutingToUnassignableNodes_RemovesRouteForRemovedNodes() {
                         nodeId3,
                         SingleNodeShutdownMetadata.builder()
                             .setNodeId(nodeId3)
+                            .setNodeEphemeralId(nodeId3)
                             .setType(SingleNodeShutdownMetadata.Type.REMOVE)
                             .setStartedAtMillis(System.currentTimeMillis())
                             .setReason("test")
@@ -1703,6 +1706,7 @@ public void testRemoveRoutingToUnassignableNodes_AddsAStoppingRouteForShuttingDo
                         nodeId3,
                         SingleNodeShutdownMetadata.builder()
                             .setNodeId(nodeId3)
+                            .setNodeEphemeralId(nodeId3)
                             .setType(SingleNodeShutdownMetadata.Type.REMOVE)
                             .setStartedAtMillis(System.currentTimeMillis())
                             .setReason("test")
@@ -1763,6 +1767,7 @@ public void testRemoveRoutingToUnassignableNodes_IgnoresARouteThatIsStoppedForSh
                         nodeId3,
                         SingleNodeShutdownMetadata.builder()
                             .setNodeId(nodeId3)
+                            .setNodeEphemeralId(nodeId3)
                             .setType(SingleNodeShutdownMetadata.Type.REMOVE)
                             .setStartedAtMillis(System.currentTimeMillis())
                             .setReason("test")
@@ -1993,6 +1998,7 @@ private NodesShutdownMetadata nodesShutdownMetadata(DiscoveryNode nodeToShutdown
                 nodeToShutdown.getId(),
                 SingleNodeShutdownMetadata.builder()
                     .setNodeId(nodeToShutdown.getId())
+                    .setNodeEphemeralId(nodeToShutdown.getEphemeralId())
                     .setStartedAtMillis(1L)
                     .setType(SingleNodeShutdownMetadata.Type.RESTART)
                     .setReason("because this cannot be null")
@@ -2049,6 +2055,7 @@ static NodesShutdownMetadata shutdownMetadata(String nodeId) {
                     .setStartedAtMillis(randomNonNegativeLong())
                     .setReason("tests")
                     .setNodeId(nodeId)
+                    .setNodeEphemeralId(nodeId)
                     .build()
             )
         );
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/XLMRobertaTestVocab.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/XLMRobertaTestVocab.java
index b0cce14c59114..3e47f0bc6cd06 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/XLMRobertaTestVocab.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/XLMRobertaTestVocab.java
@@ -41,7 +41,7 @@ public static Vocabulary loadMultiLingualTestVocab() throws IOException {
     }
 
     @SuppressWarnings("checkstyle:linelength")
-    public static String[] MULTILINUGAL_TEXTS = new String[] {
+    public static final String[] MULTILINUGAL_TEXTS = new String[] {
         "Građevne strukture Mesa Verde dokaz su akumuliranog znanja i vještina koje su se stoljećima prenosile generacijama civilizacije Anasazi. Vrhunce svojih dosega ostvarili su u 12. i 13. stoljeću, kada su sagrađene danas najpoznatije građevine na liticama. Zidali su obrađenim pješčenjakom, tvrđim kamenom oblikovanim do veličine štruce kruha. Kao žbuku između ciglā stavljali su glinu razmočenu vodom. Tim su materijalom gradili prostorije veličine do 6 četvornih metara. U potkrovljima su skladištili žitarice i druge plodine, dok su kive - ceremonijalne prostorije - gradili ispred soba, ali ukopane u zemlju, nešto poput današnjih podruma. Kiva je bila vrhunski dizajnirana prostorija okruglog oblika s prostorom za vatru zimi te s dovodom hladnog zraka za klimatizaciju ljeti. U zidane konstrukcije stavljali su i lokalno posječena stabla, što današnjim arheolozima pomaže u preciznom datiranju nastanka pojedine građevine metodom dendrokronologije. Ta stabla pridonose i teoriji o mogućem konačnom slomu ondašnjeg društva. Nakon što su, tijekom nekoliko stoljeća, šume do kraja srušene, a njihova obnova zbog sušne klime traje i po 200 godina, nije proteklo puno vremena do konačnog urušavanja civilizacije, koja se, na svojem vrhuncu osjećala nepobjedivom. 90 % sagrađenih naseobina ispod stijena ima do deset prostorija. ⅓ od ukupnog broja sagrađenih kuća ima jednu ili dvije kamene prostorije",
         "Histoarysk wie in acre in stik lân dat 40 roeden (oftewol 1 furlong of ⅛ myl of 660 foet) lang wie, en 4 roeden (of 66 foet) breed. Men is fan tinken dat dat likernôch de grûnmjitte wie dy't men mei in jok oksen yn ien dei beploegje koe.",
         "創業当初の「太平洋化学工業社」から1959年太平洋化学工業株式会社へ、1987年には太平洋化学㈱に社名を変更。 1990年以降、海外拠点を増やし本格的な国際進出を始動。 創業者がつくりあげた化粧品会社を世界企業へと成長させるべく2002年3月英文社名AMOREPACIFICに改めた。",
@@ -53,7 +53,7 @@ public static Vocabulary loadMultiLingualTestVocab() throws IOException {
         "(2) اگر زلیخا کو ملامت کرنے والی عورتیں آپ ﷺ کی جبین انور دیکھ پاتیں تو ہاتھوں کے بجائے اپنے دل کاٹنے کو ترجیح دیتیں۔صحیح بخاری میں ہے، حضرت عطاء بن یسار ؓہُنے حضرت عبداللہ بن عمرو ؓسے سیّدِ عالمﷺ کے وہ اوصاف دریافت کئے جو توریت میں مذکور ہیں تو انہوں نے فرمایا : ’’خدا کی قسم! حضور سیدُ المرسلینﷺ کے جو اوصاف قرآنِ کریم میں آئے ہیں انہیں میں سے بعض اوصاف توریت میں مذکور ہیں۔ اس کے بعد انہوں نے پڑھنا شروع کیا: اے نبی! ہم نے تمہیں شاہد و مُبَشِّر اور نذیر اور اُمِّیُّوں کا نگہبان بنا کر بھیجا، تم میرے بندے اور میرے رسول ہو، میں نے تمہارا نام متوکل رکھا،نہ بدخلق ہو نہ سخت مزاج، نہ بازاروں میں آواز بلند کرنے والے ہو نہ برائی سے برائی کو دفع کرنے والے بلکہ خطا کاروں کو معاف کرتے ہو اور ان پر احسان فرماتے ہو، اللہ تعالیٰ تمہیں نہ اٹھائے گا جب تک کہ تمہاری برکت سے غیر مستقیم ملت کو اس طرح راست نہ فرمادے کہ لوگ صدق و یقین کے ساتھ ’’ لَآاِلٰہَ اِلَّا اللہُ مُحَمَّدٌ رَّسُوْلُ اللہِ‘‘ پکارنے لگیں اور تمہاری بدولت اندھی آنکھیں بینا اور بہرے کان شنوا (سننے والے) اور پردوں میں لپٹے ہوئے دل کشادہ ہوجائیں۔ اور کعب احبارؓسے سرکارِ رسالت ﷺکی صفات میں توریت شریف کا یہ مضمون بھی منقول ہے کہ’’ اللہ تعالیٰ نے آپ ﷺکی صفت میں فرمایا کہ’’ میں اُنہیں ہر خوبی کے قابل کروں گا، اور ہر خُلقِ کریم عطا فرماؤں گا، اطمینانِ قلب اور وقار کو اُن کا لباس بناؤں گا اور طاعات وا حسان کو ان کا شعار کروں گا۔ تقویٰ کو ان کا ضمیر، حکمت کو ان کا راز، صدق و وفا کو اُن کی طبیعت ،عفوو کرم کو اُن کی عادت ، عدل کو ان کی سیرت، اظہارِ حق کو اُن کی شریعت، ہدایت کو اُن کا امام اور اسلام کو اُن کی ملت بناؤں گا۔ احمد اُن کا نام ہے، مخلوق کو اُن کے صدقے میں گمراہی کے بعد ہدایت اور جہالت کے بعد علم و معرفت اور گمنامی کے بعد رفعت و منزلت عطا کروں گا۔ اُنہیں کی برکت سے قلت کے بعد کثرت اور فقر کے بعد دولت اور تَفَرُّقے کے بعد محبت عنایت کروں گا، اُنہیں کی بدولت مختلف قبائل، غیر مجتمع خواہشوں اور اختلاف رکھنے والے دلوں میں اُلفت پیدا کروں گا اور اُن کی اُمت کو تمام اُمتوں سے بہتر کروں گا۔ ایک اور حدیث میں توریت سے حضور سید المرسلینﷺسے یہ اوصاف منقول ہیں ’’میرے بندے احمد مختار، ان کی جائے ولادت مکہ مکرمہ اور جائے ہجرت مدینہ طیبہ ہے،اُن کی اُمت ہر حال میں اللہ تعالٰی کی کثیر حمد کرنے والی ہے۔ مُنَزَّہٌ عَنْ شَرِیْکٍ فِیْ مَحَاسِنِہٖ",
         "بالآخر آنحضرتﷺ کے اس عفو وکرم نے یہ معجزہ دکھایا کہ سہیل حنین کی واپسی کے وقت آپ کے ساتھ ہوگئے اورمقام جعرانہ پہنچ کر خلعتِ اسلام سے سرفراز ہوئے آنحضرت ﷺ نے ازراہ مرحمت حنین کے مالِ غنیمت میں سے سو اونٹ عطا فرمائے، گو فتح مکہ کے بعد کے مسلمانوں کا شمار مؤلفۃ القلوب میں ہے، لیکن سہیل اس زمرہ میں اس حیثیت سے ممتاز ہیں کہ اسلام کے بعد ان سے کوئی بات اسلام کے خلاف ظہور پزیر نہیں ہوئی ،حافظ ابن حجرعسقلانی لکھتے ہیں، کان محمودالا سلام من حین اسلم۔", };
 
-    public static int[][] EXPECTED_TOKENS = new int[][] {
+    public static final int[][] EXPECTED_TOKENS = new int[][] {
         {
             0,
             910,
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/BatchedDocumentsIteratorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/BatchedDocumentsIteratorTests.java
index 4fded8ef8d05d..a624314bf22f6 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/BatchedDocumentsIteratorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/BatchedDocumentsIteratorTests.java
@@ -190,7 +190,7 @@ void assertSearchRequest(String indexName) {
             assertThat(searchRequests.size(), equalTo(1));
             SearchRequest searchRequest = searchRequests.get(0);
             assertThat(searchRequest.indices(), equalTo(new String[] { indexName }));
-            assertThat(searchRequest.scroll().keepAlive(), equalTo(TimeValue.timeValueMinutes(5)));
+            assertThat(searchRequest.scroll(), equalTo(TimeValue.timeValueMinutes(5)));
             assertThat(searchRequest.source().query(), equalTo(QueryBuilders.matchAllQuery()));
             assertThat(searchRequest.source().trackTotalHitsUpTo(), is(SearchContext.TRACK_TOTAL_HITS_ACCURATE));
         }
@@ -200,7 +200,7 @@ void assertSearchScrollRequests(int expectedCount, String scrollId) {
             assertThat(searchScrollRequests.size(), equalTo(expectedCount));
             for (SearchScrollRequest request : searchScrollRequests) {
                 assertThat(request.scrollId(), equalTo(scrollId));
-                assertThat(request.scroll().keepAlive(), equalTo(TimeValue.timeValueMinutes(5)));
+                assertThat(request.scroll(), equalTo(TimeValue.timeValueMinutes(5)));
             }
         }
     }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/vectors/TextEmbeddingQueryVectorBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/vectors/TextEmbeddingQueryVectorBuilderTests.java
index 7721c4c23953b..70f361e94c85b 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/vectors/TextEmbeddingQueryVectorBuilderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/vectors/TextEmbeddingQueryVectorBuilderTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xpack.core.ml.action.InferModelAction;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelPrefixStrings;
 import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults;
+import org.elasticsearch.xpack.core.ml.vectors.TextEmbeddingQueryVectorBuilder;
 import org.elasticsearch.xpack.ml.MachineLearningTests;
 
 import java.io.IOException;
diff --git a/x-pack/plugin/monitoring/build.gradle b/x-pack/plugin/monitoring/build.gradle
index 43f7db22cc7ef..c10f33d936530 100644
--- a/x-pack/plugin/monitoring/build.gradle
+++ b/x-pack/plugin/monitoring/build.gradle
@@ -3,9 +3,9 @@ apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-test-artifact'
 
 esplugin {
-  name 'x-pack-monitoring'
-  description 'Elasticsearch Expanded Pack Plugin - Monitoring'
-  classname 'org.elasticsearch.xpack.monitoring.Monitoring'
+  name = 'x-pack-monitoring'
+  description = 'Elasticsearch Expanded Pack Plugin - Monitoring'
+  classname ='org.elasticsearch.xpack.monitoring.Monitoring'
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporter.java
index fb24c1ad2b883..f280171c046d7 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporter.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporter.java
@@ -28,7 +28,7 @@
 
 public abstract class Exporter implements AutoCloseable {
 
-    public static Setting.AffixSettingDependency TYPE_DEPENDENCY = () -> Exporter.TYPE_SETTING;
+    public static final Setting.AffixSettingDependency TYPE_DEPENDENCY = () -> Exporter.TYPE_SETTING;
 
     private static final Setting.AffixSetting<Boolean> ENABLED_SETTING = Setting.affixKeySetting(
         "xpack.monitoring.exporters.",
diff --git a/x-pack/plugin/monitoring/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/monitoring/src/main/plugin-metadata/entitlement-policy.yaml
new file mode 100644
index 0000000000000..d826de8ca8725
--- /dev/null
+++ b/x-pack/plugin/monitoring/src/main/plugin-metadata/entitlement-policy.yaml
@@ -0,0 +1,2 @@
+ALL-UNNAMED:
+  - set_https_connection_properties # potentially required by apache.httpcomponents
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java
index 9d47fc3e973b3..8935b9f450c1f 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java
@@ -520,7 +520,7 @@ private void checkMonitoringTemplates() {
         templates.add(".monitoring-logstash");
         templates.add(".monitoring-beats");
 
-        GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates(".monitoring-*").get();
+        GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates(TEST_REQUEST_TIMEOUT, ".monitoring-*").get();
         Set<String> actualTemplates = response.getIndexTemplates().stream().map(IndexTemplateMetadata::getName).collect(Collectors.toSet());
         assertEquals(templates, actualTemplates);
     }
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java
index b6c059b7a0dcc..4811d65e6ed85 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java
@@ -389,7 +389,21 @@ private static CommonStats mockCommonStats() {
         commonStats.getStore().add(new StoreStats(++iota, no, no));
         commonStats.getRefresh().add(new RefreshStats(no, ++iota, no, ++iota, (int) no));
 
-        final IndexingStats.Stats indexingStats = new IndexingStats.Stats(++iota, ++iota, no, no, no, no, no, no, false, ++iota, no, no);
+        final IndexingStats.Stats indexingStats = new IndexingStats.Stats(
+            ++iota,
+            ++iota,
+            no,
+            no,
+            no,
+            no,
+            no,
+            no,
+            no,
+            false,
+            ++iota,
+            no,
+            no
+        );
         commonStats.getIndexing().add(new IndexingStats(indexingStats));
 
         final SearchStats.Stats searchStats = new SearchStats.Stats(++iota, ++iota, no, no, no, no, no, no, no, no, no, no, no, no);
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java
index 6822f54633bdc..ca7651ce84497 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java
@@ -183,7 +183,7 @@ private CommonStats mockCommonStats() {
         commonStats.getDocs().add(new DocsStats(1L, 0L, randomNonNegativeLong() >> 8)); // >> 8 to avoid overflow - we add these things up
         commonStats.getStore().add(new StoreStats(2L, 0L, 0L));
 
-        final IndexingStats.Stats indexingStats = new IndexingStats.Stats(3L, 4L, 0L, 0L, 0L, 0L, 0L, 0L, true, 5L, 0, 0);
+        final IndexingStats.Stats indexingStats = new IndexingStats.Stats(3L, 4L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, true, 5L, 0, 0);
         commonStats.getIndexing().add(new IndexingStats(indexingStats));
 
         final SearchStats.Stats searchStats = new SearchStats.Stats(6L, 7L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L);
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java
index 3d7f843358646..be87b92479c21 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java
@@ -344,6 +344,7 @@ private static NodeStats mockNodeStats() {
             no,
             no,
             no,
+            no,
             false,
             ++iota,
             no,
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java
index d68395ef7656f..3409fe8a69666 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java
@@ -236,7 +236,7 @@ private void checkMonitoringTemplates() {
         templates.add(".monitoring-logstash");
         templates.add(".monitoring-beats");
 
-        GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates(".monitoring-*").get();
+        GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates(TEST_REQUEST_TIMEOUT, ".monitoring-*").get();
         Set<String> actualTemplates = response.getIndexTemplates().stream().map(IndexTemplateMetadata::getName).collect(Collectors.toSet());
         assertEquals(templates, actualTemplates);
     }
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java
index d382905c1c9c2..4259fb5328435 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java
@@ -319,7 +319,11 @@ private void assertResourcesExist() throws Exception {
     private void assertTemplateNotUpdated() {
         final String name = MonitoringTemplateRegistry.getTemplateConfigForMonitoredSystem(system).getTemplateName();
 
-        for (IndexTemplateMetadata template : client().admin().indices().prepareGetTemplates(name).get().getIndexTemplates()) {
+        for (IndexTemplateMetadata template : client().admin()
+            .indices()
+            .prepareGetTemplates(TEST_REQUEST_TIMEOUT, name)
+            .get()
+            .getIndexTemplates()) {
             final String docMapping = template.getMappings().toString();
 
             assertThat(docMapping, notNullValue());
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java
index bcf942292a1f8..92e18178a667b 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java
@@ -140,7 +140,11 @@ protected List<Tuple<String, String>> monitoringWatches() {
 
     protected void assertTemplateInstalled(String name) {
         boolean found = false;
-        for (IndexTemplateMetadata template : client().admin().indices().prepareGetTemplates().get().getIndexTemplates()) {
+        for (IndexTemplateMetadata template : client().admin()
+            .indices()
+            .prepareGetTemplates(TEST_REQUEST_TIMEOUT)
+            .get()
+            .getIndexTemplates()) {
             if (Regex.simpleMatch(name, template.getName())) {
                 found = true;
             }
diff --git a/x-pack/plugin/old-lucene-versions/build.gradle b/x-pack/plugin/old-lucene-versions/build.gradle
index ea988fe5e1ff4..8d23bbea472dc 100644
--- a/x-pack/plugin/old-lucene-versions/build.gradle
+++ b/x-pack/plugin/old-lucene-versions/build.gradle
@@ -3,9 +3,9 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-test-artifact'
 
 esplugin {
-  name 'old-lucene-versions'
-  description 'A plugin for accessing older Lucene indices'
-  classname 'org.elasticsearch.xpack.lucene.bwc.OldLuceneVersions'
+  name = 'old-lucene-versions'
+  description = 'A plugin for accessing older Lucene indices'
+  classname = 'org.elasticsearch.xpack.lucene.bwc.OldLuceneVersions'
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/otel-data/build.gradle b/x-pack/plugin/otel-data/build.gradle
index 126875bf183f8..0ae8022bff617 100644
--- a/x-pack/plugin/otel-data/build.gradle
+++ b/x-pack/plugin/otel-data/build.gradle
@@ -9,9 +9,9 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  name 'x-pack-otel-data'
-  description 'The OTEL plugin defines OTEL data streams and ingest pipelines.'
-  classname 'org.elasticsearch.xpack.oteldata.OTelPlugin'
+  name = 'x-pack-otel-data'
+  description = 'The OTEL plugin defines OTEL data streams and ingest pipelines.'
+  classname ='org.elasticsearch.xpack.oteldata.OTelPlugin'
   extendedPlugins = ['x-pack-core']
 }
 
diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml
index 5f4dcbd416720..9c47cfad993e2 100644
--- a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml
+++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml
@@ -9,6 +9,7 @@ template:
       mode: logsdb
       sort:
         field: [ "resource.attributes.host.name", "@timestamp" ]
+        order: [ "asc", "desc" ]
   mappings:
     properties:
       attributes:
@@ -39,6 +40,8 @@ template:
       log.level:
         type: alias
         path: severity_text
+      event_name:
+        type: keyword
       body:
         type: object
         properties:
diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml
index 95a42b137df52..635ba386f739c 100644
--- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml
+++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml
@@ -105,6 +105,7 @@ Event body:
                 service.name: my-service
             attributes:
               event.name: foo
+            event_name: foo
             body:
               structured:
                 foo:
@@ -119,6 +120,7 @@ Event body:
         index: $datastream-backing-index
   - is_true: $datastream-backing-index
   - match: { .$datastream-backing-index.mappings.properties.body.properties.structured.properties.foo\.bar.type: "keyword" }
+  - match: { .$datastream-backing-index.mappings.properties.event_name.type: "keyword" }
 ---
 Structured log body:
   - do:
diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml
index 1aafe3765813b..c8330a44cd7ce 100644
--- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml
+++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml
@@ -148,9 +148,6 @@ setup:
   - match: { .$idx0name.mappings.properties.attributes.properties.foo.type: "keyword" }
 ---
 IP dimensions:
-  - requires:
-      cluster_features: ["routing.multi_value_routing_path"]
-      reason: support for multi-value dimensions
   - do:
       bulk:
         index: metrics-generic.otel-default
diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml
index d5b87c9b45116..82d0672438ce5 100644
--- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml
+++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml
@@ -120,3 +120,23 @@ Conflicting attribute types:
   - is_true: $datastream-backing-index
   - match: { .$datastream-backing-index.settings.index.sort.field.0: "resource.attributes.host.name" }
   - match: { .$datastream-backing-index.settings.index.sort.field.1: "@timestamp" }
+
+---
+traces@lifecycle:
+  - do:
+      bulk:
+        index: traces-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-07-18T14:49:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "span_id":"1"}'
+  - is_false: errors
+  - do:
+      indices.get_data_stream:
+        name: traces-generic.otel-default
+  - set: { data_streams.0.indices.0.index_name: datastream-backing-index }
+  - do:
+      indices.get_settings:
+        index: $datastream-backing-index
+  - is_true: $datastream-backing-index
+  - match: { .$datastream-backing-index.settings.index.lifecycle.name: "traces@lifecycle" }
diff --git a/x-pack/plugin/profiling/build.gradle b/x-pack/plugin/profiling/build.gradle
index d291a1a25dfbc..954cdcb670a8f 100644
--- a/x-pack/plugin/profiling/build.gradle
+++ b/x-pack/plugin/profiling/build.gradle
@@ -8,9 +8,9 @@ apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-es-plugin'
 
 esplugin {
-  name 'x-pack-profiling'
-  description 'The profiling plugin adds support for retrieving data from Universal Profiling.'
-  classname 'org.elasticsearch.xpack.profiling.ProfilingPlugin'
+  name = 'x-pack-profiling'
+  description = 'The profiling plugin adds support for retrieving data from Universal Profiling.'
+  classname = 'org.elasticsearch.xpack.profiling.ProfilingPlugin'
   extendedPlugins = ['x-pack-core']
 }
 
diff --git a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetFlameGraphActionIT.java b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetFlameGraphActionIT.java
index db343b62c5a1d..49a5cfa7ca067 100644
--- a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetFlameGraphActionIT.java
+++ b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetFlameGraphActionIT.java
@@ -22,7 +22,6 @@ public void testGetStackTracesUnfiltered() throws Exception {
             null,
             null,
             null,
-            null,
             null
         );
         GetFlamegraphResponse response = client().execute(GetFlamegraphAction.INSTANCE, request).get();
diff --git a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetStackTracesActionIT.java b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetStackTracesActionIT.java
index 4b3a4fb0108f7..4a9f1146fe95c 100644
--- a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetStackTracesActionIT.java
+++ b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetStackTracesActionIT.java
@@ -28,7 +28,6 @@ public void testGetStackTracesUnfiltered() throws Exception {
             null,
             null,
             null,
-            null,
             null
         );
         request.setAdjustSampleCount(true);
@@ -68,8 +67,7 @@ public void testGetStackTracesGroupedByServiceName() throws Exception {
             null,
             null,
             null,
-            "service.name",
-            null,
+            new String[] { "service.name" },
             null,
             null,
             null,
@@ -117,8 +115,7 @@ public void testGetStackTracesFromAPMWithMatchNoDownsampling() throws Exception
             // also match an index that does not contain stacktrace ids to ensure it is ignored
             new String[] { "apm-test-*", "apm-legacy-test-*" },
             "transaction.profiler_stack_trace_ids",
-            "transaction.name",
-            null,
+            new String[] { "transaction.name" },
             null,
             null,
             null,
@@ -168,7 +165,6 @@ public void testGetStackTracesFromAPMWithMatchAndDownsampling() throws Exception
             null,
             null,
             null,
-            null,
             null
         );
         // ensures consistent results in the random sampler aggregation that is used internally
@@ -219,7 +215,6 @@ public void testGetStackTracesFromAPMNoMatch() throws Exception {
             null,
             null,
             null,
-            null,
             null
         );
         GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
@@ -242,7 +237,6 @@ public void testGetStackTracesFromAPMIndexNotAvailable() throws Exception {
             null,
             null,
             null,
-            null,
             null
         );
         GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
@@ -265,7 +259,6 @@ public void testGetStackTracesFromAPMStackTraceFieldNotAvailable() throws Except
             null,
             null,
             null,
-            null,
             null
         );
         GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
diff --git a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetTopNFunctionsActionIT.java b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetTopNFunctionsActionIT.java
index c6250dae4d649..4a3e200af6739 100644
--- a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetTopNFunctionsActionIT.java
+++ b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetTopNFunctionsActionIT.java
@@ -25,7 +25,6 @@ public void testGetTopNFunctionsUnfiltered() throws Exception {
             null,
             null,
             null,
-            null,
             null
         );
         request.setAdjustSampleCount(true);
@@ -42,8 +41,7 @@ public void testGetTopNFunctionsGroupedByServiceName() throws Exception {
             null,
             null,
             null,
-            "service.name",
-            null,
+            new String[] { "service.name" },
             null,
             null,
             null,
@@ -70,8 +68,7 @@ public void testGetTopNFunctionsFromAPM() throws Exception {
             // also match an index that does not contain stacktrace ids to ensure it is ignored
             new String[] { "apm-test-*", "apm-legacy-test-*" },
             "transaction.profiler_stack_trace_ids",
-            "transaction.name",
-            null,
+            new String[] { "transaction.name" },
             null,
             null,
             null,
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetFlamegraphResponse.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetFlamegraphResponse.java
index 5d32c39e350a5..2419e2e1dc0c2 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetFlamegraphResponse.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetFlamegraphResponse.java
@@ -11,13 +11,12 @@
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -177,56 +176,89 @@ public long getTotalSamples() {
     @UpdateForV9(owner = UpdateForV9.Owner.PROFILING) // change casing from Camel Case to Snake Case (requires updates in Kibana as well)
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).object(ob -> {
-            ob.array("Edges", edges.iterator(), (eb, e) -> eb.array(intValues(e.values())));
-            ob.array("FileID", fileIds.toArray(String[]::new));
-            ob.array("FrameType", intValues(frameTypes));
-            ob.array("Inline", inlineFrames.iterator(), e -> (b, p) -> b.value(e));
-            ob.array("ExeFilename", fileNames.toArray(String[]::new));
-            ob.array("AddressOrLine", intValues(addressOrLines));
-            ob.array("FunctionName", functionNames.toArray(String[]::new));
-            ob.array("FunctionOffset", intValues(functionOffsets));
-            ob.array("SourceFilename", sourceFileNames.toArray(String[]::new));
-            ob.array("SourceLine", intValues(sourceLines));
-            ob.array("CountInclusive", longValues(countInclusive));
-            ob.array("CountExclusive", longValues(countExclusive));
-            ob.array("AnnualCO2TonsInclusive", doubleValues(annualCO2TonsInclusive));
-            ob.array("AnnualCO2TonsExclusive", doubleValues(annualCO2TonsExclusive));
-            ob.array("AnnualCostsUSDInclusive", doubleValues(annualCostsUSDInclusive));
-            ob.array("AnnualCostsUSDExclusive", doubleValues(annualCostsUSDExclusive));
-            ob.field("Size", size);
-            ob.field("SamplingRate", samplingRate);
-            ob.field("SelfCPU", selfCPU);
-            ob.field("TotalCPU", totalCPU);
-            ob.field("TotalSamples", totalSamples);
-        });
-    }
-
-    private static Iterator<ToXContent> intValues(Collection<Integer> values) {
-        return Iterators.single((b, p) -> {
-            for (Integer i : values) {
-                b.value(i);
-            }
-            return b;
-        });
-    }
-
-    private static Iterator<ToXContent> longValues(Collection<Long> values) {
-        return Iterators.single((b, p) -> {
-            for (Long l : values) {
-                b.value(l);
-            }
-            return b;
-        });
-    }
-
-    private static Iterator<ToXContent> doubleValues(Collection<Double> values) {
-        return Iterators.single((b, p) -> {
-            for (Double d : values) {
-                // write as raw value - we need direct control over the output representation (here: limit to 4 decimal places)
-                b.rawValue(NumberUtils.doubleToString(d));
-            }
-            return b;
-        });
+        return Iterators.concat(
+            ChunkedToXContentHelper.startObject(),
+            ChunkedToXContentHelper.array(
+                "Edges",
+                Iterators.flatMap(
+                    edges.iterator(),
+                    perNodeEdges -> Iterators.concat(
+                        ChunkedToXContentHelper.startArray(),
+                        Iterators.map(perNodeEdges.entrySet().iterator(), edge -> (b, p) -> b.value(edge.getValue())),
+                        ChunkedToXContentHelper.endArray()
+                    )
+                )
+            ),
+            ChunkedToXContentHelper.array("FileID", Iterators.map(fileIds.iterator(), e -> (b, p) -> b.value(e))),
+            ChunkedToXContentHelper.array("FrameType", Iterators.map(frameTypes.iterator(), e -> (b, p) -> b.value(e))),
+            ChunkedToXContentHelper.array("Inline", Iterators.map(inlineFrames.iterator(), e -> (b, p) -> b.value(e))),
+            ChunkedToXContentHelper.array("ExeFilename", Iterators.map(fileNames.iterator(), e -> (b, p) -> b.value(e))),
+            ChunkedToXContentHelper.array("AddressOrLine", Iterators.map(addressOrLines.iterator(), e -> (b, p) -> b.value(e))),
+            ChunkedToXContentHelper.array("FunctionName", Iterators.map(functionNames.iterator(), e -> (b, p) -> b.value(e))),
+            ChunkedToXContentHelper.chunk((b, p) -> {
+                b.startArray("FunctionOffset");
+                for (int functionOffset : functionOffsets) {
+                    b.value(functionOffset);
+                }
+                return b.endArray();
+            }),
+            ChunkedToXContentHelper.array("SourceFilename", Iterators.map(sourceFileNames.iterator(), e -> (b, p) -> b.value(e))),
+            ChunkedToXContentHelper.chunk((b, p) -> {
+                b.startArray("SourceLine");
+                for (int sourceLine : sourceLines) {
+                    b.value(sourceLine);
+                }
+                return b.endArray();
+            }),
+            ChunkedToXContentHelper.chunk((b, p) -> {
+                b.startArray("CountInclusive");
+                for (long countInclusive : countInclusive) {
+                    b.value(countInclusive);
+                }
+                return b.endArray();
+            }),
+            ChunkedToXContentHelper.chunk((b, p) -> {
+                b.startArray("CountExclusive");
+                for (long c : countExclusive) {
+                    b.value(c);
+                }
+                return b.endArray();
+            }),
+            ChunkedToXContentHelper.chunk((b, p) -> {
+                b.startArray("AnnualCO2TonsInclusive");
+                for (double co2Tons : annualCO2TonsInclusive) {
+                    // write as raw value - we need direct control over the output representation (here: limit to 4 decimal places)
+                    b.rawValue(NumberUtils.doubleToString(co2Tons));
+                }
+                return b.endArray();
+            }),
+            ChunkedToXContentHelper.chunk((b, p) -> {
+                b.startArray("AnnualCO2TonsExclusive");
+                for (double co2Tons : annualCO2TonsExclusive) {
+                    b.rawValue(NumberUtils.doubleToString(co2Tons));
+                }
+                return b.endArray();
+            }),
+            ChunkedToXContentHelper.chunk((b, p) -> {
+                b.startArray("AnnualCostsUSDInclusive");
+                for (double costs : annualCostsUSDInclusive) {
+                    b.rawValue(NumberUtils.doubleToString(costs));
+                }
+                return b.endArray();
+            }),
+            ChunkedToXContentHelper.chunk((b, p) -> {
+                b.startArray("AnnualCostsUSDExclusive");
+                for (double costs : annualCostsUSDExclusive) {
+                    b.rawValue(NumberUtils.doubleToString(costs));
+                }
+                return b.endArray();
+            }),
+            Iterators.single((b, p) -> b.field("Size", size)),
+            Iterators.single((b, p) -> b.field("SamplingRate", samplingRate)),
+            Iterators.single((b, p) -> b.field("SelfCPU", selfCPU)),
+            Iterators.single((b, p) -> b.field("TotalCPU", totalCPU)),
+            Iterators.single((b, p) -> b.field("TotalSamples", totalSamples)),
+            ChunkedToXContentHelper.endObject()
+        );
     }
 }
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesRequest.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesRequest.java
index fb780a3f6f98a..82acaa0020ad8 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesRequest.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesRequest.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
@@ -43,8 +42,6 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque
     public static final ParseField LIMIT_FIELD = new ParseField("limit");
     public static final ParseField INDICES_FIELD = new ParseField("indices");
     public static final ParseField STACKTRACE_IDS_FIELD = new ParseField("stacktrace_ids_field");
-    @UpdateForV9(owner = UpdateForV9.Owner.PROFILING) // Remove this BWC layer and allow only AGGREGATION_FIELDS
-    public static final ParseField AGGREGATION_FIELD = new ParseField("aggregation_field");
     public static final ParseField AGGREGATION_FIELDS = new ParseField("aggregation_fields");
     public static final ParseField REQUESTED_DURATION_FIELD = new ParseField("requested_duration");
     public static final ParseField AWS_COST_FACTOR_FIELD = new ParseField("aws_cost_factor");
@@ -62,8 +59,6 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque
     private String[] indices;
     private boolean userProvidedIndices;
     private String stackTraceIdsField;
-    @UpdateForV9(owner = UpdateForV9.Owner.PROFILING) // Remove this BWC layer and allow only aggregationFields
-    private String aggregationField;
     private String[] aggregationFields;
     private Double requestedDuration;
     private Double awsCostFactor;
@@ -83,7 +78,7 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque
     private Integer shardSeed;
 
     public GetStackTracesRequest() {
-        this(null, null, null, null, null, null, null, null, null, null, null, null, null, null);
+        this(null, null, null, null, null, null, null, null, null, null, null, null, null);
     }
 
     public GetStackTracesRequest(
@@ -94,7 +89,6 @@ public GetStackTracesRequest(
         QueryBuilder query,
         String[] indices,
         String stackTraceIdsField,
-        String aggregationField,
         String[] aggregationFields,
         Double customCO2PerKWH,
         Double customDatacenterPUE,
@@ -110,7 +104,6 @@ public GetStackTracesRequest(
         this.indices = indices;
         this.userProvidedIndices = indices != null && indices.length > 0;
         this.stackTraceIdsField = stackTraceIdsField;
-        this.aggregationField = aggregationField;
         this.aggregationFields = aggregationFields;
         this.customCO2PerKWH = customCO2PerKWH;
         this.customDatacenterPUE = customDatacenterPUE;
@@ -184,12 +177,8 @@ public String getStackTraceIdsField() {
         return stackTraceIdsField;
     }
 
-    public String getAggregationField() {
-        return aggregationField;
-    }
-
     public String[] getAggregationFields() {
-        return aggregationField != null ? new String[] { aggregationField } : aggregationFields;
+        return aggregationFields;
     }
 
     public boolean hasAggregationFields() {
@@ -197,10 +186,6 @@ public boolean hasAggregationFields() {
         return f != null && f.length > 0;
     }
 
-    public boolean isLegacyAggregationField() {
-        return aggregationField != null;
-    }
-
     public boolean isAdjustSampleCount() {
         return Boolean.TRUE.equals(adjustSampleCount);
     }
@@ -237,8 +222,6 @@ public void parseXContent(XContentParser parser) throws IOException {
                     this.limit = parser.intValue();
                 } else if (STACKTRACE_IDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     this.stackTraceIdsField = parser.text();
-                } else if (AGGREGATION_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                    this.aggregationField = parser.text();
                 } else if (REQUESTED_DURATION_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     this.requestedDuration = parser.doubleValue();
                 } else if (AWS_COST_FACTOR_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
@@ -322,17 +305,6 @@ public ActionRequestValidationException validate() {
                 );
             }
         }
-        if (aggregationField != null && aggregationFields != null) {
-            validationException = addValidationError(
-                "["
-                    + AGGREGATION_FIELD.getPreferredName()
-                    + "] must not be set when ["
-                    + AGGREGATION_FIELDS.getPreferredName()
-                    + "] is also set",
-                validationException
-            );
-
-        }
         if (aggregationFields != null) {
             // limit so we avoid an explosion of buckets
             if (aggregationFields.length < 1 || aggregationFields.length > 2) {
@@ -348,13 +320,6 @@ public ActionRequestValidationException validate() {
 
         }
 
-        if (aggregationField != null && aggregationField.isBlank()) {
-            validationException = addValidationError(
-                "[" + AGGREGATION_FIELD.getPreferredName() + "] must be non-empty",
-                validationException
-            );
-        }
-
         validationException = requirePositive(SAMPLE_SIZE_FIELD, sampleSize, validationException);
         validationException = requirePositive(LIMIT_FIELD, limit, validationException);
         validationException = requirePositive(REQUESTED_DURATION_FIELD, requestedDuration, validationException);
@@ -386,7 +351,6 @@ public String getDescription() {
                 StringBuilder sb = new StringBuilder();
                 appendField(sb, "indices", indices);
                 appendField(sb, "stacktrace_ids_field", stackTraceIdsField);
-                appendField(sb, "aggregation_field", aggregationField);
                 appendField(sb, "aggregation_fields", aggregationFields);
                 appendField(sb, "sample_size", sampleSize);
                 appendField(sb, "limit", limit);
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java
index 19747ec6e49f3..6ec1f9a7aa7e1 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java
@@ -8,20 +8,19 @@
 
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.TriConsumer;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
-import org.elasticsearch.common.xcontent.ChunkedToXContentBuilder;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.xcontent.ToXContent;
 
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Objects;
-import java.util.function.Consumer;
+import java.util.function.BiFunction;
 
 public class GetStackTracesResponse extends ActionResponse implements ChunkedToXContentObject {
     @Nullable
@@ -91,33 +90,34 @@ public long getTotalSamples() {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).object(ob -> {
-            ob.execute(optional("stack_traces", stackTraces, ChunkedToXContentBuilder::xContentObjectFields));
-            ob.execute(optional("stack_frames", stackFrames, ChunkedToXContentBuilder::xContentObjectFields));
-            ob.execute(optional("executables", executables, ChunkedToXContentBuilder::object));
+        return Iterators.concat(
+            ChunkedToXContentHelper.startObject(),
+            optional("stack_traces", stackTraces, ChunkedToXContentHelper::xContentObjectFields),
+            optional("stack_frames", stackFrames, ChunkedToXContentHelper::xContentObjectFields),
+            optional("executables", executables, ChunkedToXContentHelper::object),
             // render only count for backwards-compatibility
-            ob.execute(
-                optional(
-                    "stack_trace_events",
-                    stackTraceEvents,
-                    (steb, n, v) -> steb.object(n, v.entrySet().iterator(), e -> (b, p) -> b.field(e.getKey(), e.getValue().count))
-                )
-            );
-            ob.field("total_frames", totalFrames);
-            ob.field("sampling_rate", samplingRate);
+            optional(
+                "stack_trace_events",
+                stackTraceEvents,
+                (n, v) -> ChunkedToXContentHelper.object(n, v, entry -> (b, p) -> b.field(entry.getKey(), entry.getValue().count))
+            ),
+            Iterators.single((b, p) -> b.field("total_frames", totalFrames)),
+            Iterators.single((b, p) -> b.field("sampling_rate", samplingRate)),
             // the following fields are intentionally not written to the XContent representation (only needed on the transport layer):
+            //
             // * start
             // * end
             // * totalSamples
-        });
+            ChunkedToXContentHelper.endObject()
+        );
     }
 
-    private static <T> Consumer<ChunkedToXContentBuilder> optional(
+    private static <T> Iterator<? extends ToXContent> optional(
         String name,
         Map<String, T> values,
-        TriConsumer<ChunkedToXContentBuilder, String, Map<String, T>> function
+        BiFunction<String, Map<String, T>, Iterator<? extends ToXContent>> supplier
     ) {
-        return values != null ? b -> function.apply(b, name, values) : b -> {};
+        return (values != null) ? supplier.apply(name, values) : Collections.emptyIterator();
     }
 
     @Override
@@ -141,9 +141,4 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(stackTraces, stackFrames, executables, stackTraceEvents, totalFrames, samplingRate);
     }
-
-    @Override
-    public String toString() {
-        return Strings.toString(this, true, true);
-    }
 }
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/SubGroup.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/SubGroup.java
index 513ec8fe1bc4c..9c953eea04967 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/SubGroup.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/SubGroup.java
@@ -20,23 +20,21 @@ public class SubGroup implements ToXContentFragment {
     private final String name;
     private Long count;
     @UpdateForV9(owner = UpdateForV9.Owner.PROFILING) // remove legacy XContent rendering
-    private final boolean renderLegacyXContent;
     private final Map<String, SubGroup> subgroups;
 
-    public static SubGroup root(String name, boolean renderLegacyXContent) {
-        return new SubGroup(name, null, renderLegacyXContent, new HashMap<>());
+    public static SubGroup root(String name) {
+        return new SubGroup(name, null, new HashMap<>());
     }
 
-    public SubGroup(String name, Long count, boolean renderLegacyXContent, Map<String, SubGroup> subgroups) {
+    public SubGroup(String name, Long count, Map<String, SubGroup> subgroups) {
         this.name = name;
         this.count = count;
-        this.renderLegacyXContent = renderLegacyXContent;
         this.subgroups = subgroups;
     }
 
     public SubGroup addCount(String name, long count) {
         if (this.subgroups.containsKey(name) == false) {
-            this.subgroups.put(name, new SubGroup(name, count, renderLegacyXContent, new HashMap<>()));
+            this.subgroups.put(name, new SubGroup(name, count, new HashMap<>()));
         } else {
             SubGroup s = this.subgroups.get(name);
             s.count += count;
@@ -46,7 +44,7 @@ public SubGroup addCount(String name, long count) {
 
     public SubGroup getOrAddChild(String name) {
         if (subgroups.containsKey(name) == false) {
-            this.subgroups.put(name, new SubGroup(name, null, renderLegacyXContent, new HashMap<>()));
+            this.subgroups.put(name, new SubGroup(name, null, new HashMap<>()));
         }
         return this.subgroups.get(name);
     }
@@ -65,32 +63,22 @@ public SubGroup copy() {
         for (Map.Entry<String, SubGroup> subGroup : subgroups.entrySet()) {
             copy.put(subGroup.getKey(), subGroup.getValue().copy());
         }
-        return new SubGroup(name, count, renderLegacyXContent, copy);
+        return new SubGroup(name, count, copy);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        if (renderLegacyXContent) {
-            // This assumes that we only have one level of sub groups
-            if (subgroups != null && subgroups.isEmpty() == false) {
-                for (SubGroup subgroup : subgroups.values()) {
-                    builder.field(subgroup.name, subgroup.count);
-                }
-            }
-            return builder;
-        } else {
-            builder.startObject(name);
-            // only the root node has no count
-            if (count != null) {
-                builder.field("count", count);
-            }
-            if (subgroups != null && subgroups.isEmpty() == false) {
-                for (SubGroup subgroup : subgroups.values()) {
-                    subgroup.toXContent(builder, params);
-                }
+        builder.startObject(name);
+        // only the root node has no count
+        if (count != null) {
+            builder.field("count", count);
+        }
+        if (subgroups != null && subgroups.isEmpty() == false) {
+            for (SubGroup subgroup : subgroups.values()) {
+                subgroup.toXContent(builder, params);
             }
-            return builder.endObject();
         }
+        return builder.endObject();
     }
 
     @Override
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/SubGroupCollector.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/SubGroupCollector.java
index 63491a63243dc..6eaa346be3e43 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/SubGroupCollector.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/SubGroupCollector.java
@@ -26,21 +26,15 @@ public final class SubGroupCollector {
     private static final Logger log = LogManager.getLogger(SubGroupCollector.class);
 
     private final String[] aggregationFields;
-    private final boolean legacyAggregationField;
-
-    public static SubGroupCollector attach(
-        AbstractAggregationBuilder<?> parentAggregation,
-        String[] aggregationFields,
-        boolean legacyAggregationField
-    ) {
-        SubGroupCollector c = new SubGroupCollector(aggregationFields, legacyAggregationField);
+
+    public static SubGroupCollector attach(AbstractAggregationBuilder<?> parentAggregation, String[] aggregationFields) {
+        SubGroupCollector c = new SubGroupCollector(aggregationFields);
         c.addAggregations(parentAggregation);
         return c;
     }
 
-    private SubGroupCollector(String[] aggregationFields, boolean legacyAggregationField) {
+    private SubGroupCollector(String[] aggregationFields) {
         this.aggregationFields = aggregationFields;
-        this.legacyAggregationField = legacyAggregationField;
     }
 
     private boolean hasAggregationFields() {
@@ -68,7 +62,7 @@ void collectResults(MultiBucketsAggregation.Bucket bucket, TraceEvent event) {
     void collectResults(Bucket bucket, TraceEvent event) {
         if (hasAggregationFields()) {
             if (event.subGroups == null) {
-                event.subGroups = SubGroup.root(aggregationFields[0], legacyAggregationField);
+                event.subGroups = SubGroup.root(aggregationFields[0]);
             }
             collectInternal(bucket.getAggregations(), event.subGroups, 0);
         }
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java
index f447f67b4cdd2..982410e8a5345 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java
@@ -255,11 +255,7 @@ private void searchGenericEventGroupedByStackTrace(
         CountedTermsAggregationBuilder groupByStackTraceId = new CountedTermsAggregationBuilder("group_by").size(
             MAX_TRACE_EVENTS_RESULT_SIZE
         ).field(request.getStackTraceIdsField());
-        SubGroupCollector subGroups = SubGroupCollector.attach(
-            groupByStackTraceId,
-            request.getAggregationFields(),
-            request.isLegacyAggregationField()
-        );
+        SubGroupCollector subGroups = SubGroupCollector.attach(groupByStackTraceId, request.getAggregationFields());
         RandomSamplerAggregationBuilder randomSampler = new RandomSamplerAggregationBuilder("sample").setSeed(request.hashCode())
             .setProbability(responseBuilder.getSamplingRate())
             .subAggregation(groupByStackTraceId);
@@ -326,11 +322,7 @@ private void searchEventGroupedByStackTrace(
             // Especially with high cardinality fields, this makes aggregations really slow.
             .executionHint("map")
             .subAggregation(new SumAggregationBuilder("count").field("Stacktrace.count"));
-        SubGroupCollector subGroups = SubGroupCollector.attach(
-            groupByStackTraceId,
-            request.getAggregationFields(),
-            request.isLegacyAggregationField()
-        );
+        SubGroupCollector subGroups = SubGroupCollector.attach(groupByStackTraceId, request.getAggregationFields());
         client.prepareSearch(eventsIndex.getName())
             .setTrackTotalHits(false)
             .setSize(0)
diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/GetStackTracesRequestTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/GetStackTracesRequestTests.java
index 82544f7cb7acf..bcbf0f2e0a922 100644
--- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/GetStackTracesRequestTests.java
+++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/GetStackTracesRequestTests.java
@@ -51,7 +51,6 @@ public void testParseValidXContent() throws IOException {
             // Expect the default values
             assertNull(request.getIndices());
             assertNull(request.getStackTraceIdsField());
-            assertFalse(request.isLegacyAggregationField());
             assertNull(request.getAggregationFields());
             assertNull(request.getAwsCostFactor());
             assertNull(request.getAzureCostFactor());
@@ -92,7 +91,6 @@ public void testParseValidXContentWithCustomIndex() throws IOException {
 
             // Expect the default values
             assertNull(request.getRequestedDuration());
-            assertFalse(request.isLegacyAggregationField());
             assertNull(request.getAggregationFields());
             assertNull(request.getAwsCostFactor());
             assertNull(request.getAzureCostFactor());
@@ -111,7 +109,7 @@ public void testParseValidXContentWithOneAggregationField() throws IOException {
                 .field("sample_size", 2000)
                 .field("indices", new String[] {"my-traces"})
                 .field("stacktrace_ids_field", "stacktraces")
-                .field("aggregation_field", "service")
+                .field("aggregation_fields", new String[] {"service"})
                 .startObject("query")
                     .startObject("range")
                         .startObject("@timestamp")
@@ -130,7 +128,6 @@ public void testParseValidXContentWithOneAggregationField() throws IOException {
             assertArrayEquals(new String[] { "my-traces" }, request.getIndices());
             assertEquals("stacktraces", request.getStackTraceIdsField());
             assertArrayEquals(new String[] { "service" }, request.getAggregationFields());
-            assertTrue(request.isLegacyAggregationField());
             // a basic check suffices here
             assertEquals("@timestamp", ((RangeQueryBuilder) request.getQuery()).fieldName());
 
@@ -177,7 +174,6 @@ public void testParseValidXContentWithMultipleAggregationFields() throws IOExcep
 
             // Expect the default values
             assertNull(request.getRequestedDuration());
-            assertFalse(request.isLegacyAggregationField());
             assertNull(request.getAwsCostFactor());
             assertNull(request.getAzureCostFactor());
             assertNull(request.getCustomCO2PerKWH());
@@ -231,7 +227,6 @@ public void testParseValidXContentWithCustomCostAndCO2Data() throws IOException
             // Expect the default values
             assertNull(request.getIndices());
             assertNull(request.getStackTraceIdsField());
-            assertFalse(request.isLegacyAggregationField());
             assertNull(request.getAggregationFields());
         }
     }
@@ -345,7 +340,6 @@ public void testValidateWrongSampleSize() {
             null,
             null,
             null,
-            null,
             null
         );
         List<String> validationErrors = request.validate().validationErrors();
@@ -367,7 +361,6 @@ public void testValidateSampleSizeIsValidWithCustomIndices() {
             null,
             null,
             null,
-            null,
             null
         );
         assertNull("Expecting no validation errors", request.validate());
@@ -387,7 +380,6 @@ public void testValidateStacktraceWithoutIndices() {
             null,
             null,
             null,
-            null,
             null
         );
         List<String> validationErrors = request.validate().validationErrors();
@@ -409,7 +401,6 @@ public void testValidateIndicesWithoutStacktraces() {
             null,
             null,
             null,
-            null,
             null
         );
         List<String> validationErrors = request.validate().validationErrors();
@@ -417,50 +408,6 @@ public void testValidateIndicesWithoutStacktraces() {
         assertEquals("[stacktrace_ids_field] is mandatory", validationErrors.get(0));
     }
 
-    public void testValidateEmptyAggregationField() {
-        GetStackTracesRequest request = new GetStackTracesRequest(
-            null,
-            1.0d,
-            1.0d,
-            1.0d,
-            null,
-            new String[] { randomAlphaOfLength(5) },
-            randomAlphaOfLength(5),
-            "",
-            null,
-            null,
-            null,
-            null,
-            null,
-            null
-        );
-        List<String> validationErrors = request.validate().validationErrors();
-        assertEquals(1, validationErrors.size());
-        assertEquals("[aggregation_field] must be non-empty", validationErrors.get(0));
-    }
-
-    public void testValidateAggregationFieldAndAggregationFields() {
-        GetStackTracesRequest request = new GetStackTracesRequest(
-            null,
-            1.0d,
-            1.0d,
-            1.0d,
-            null,
-            new String[] { randomAlphaOfLength(5) },
-            randomAlphaOfLength(5),
-            "transaction.name",
-            new String[] { "transaction.name", "service.name" },
-            null,
-            null,
-            null,
-            null,
-            null
-        );
-        List<String> validationErrors = request.validate().validationErrors();
-        assertEquals(1, validationErrors.size());
-        assertEquals("[aggregation_field] must not be set when [aggregation_fields] is also set", validationErrors.get(0));
-    }
-
     public void testValidateAggregationFieldsContainsTooFewElements() {
         GetStackTracesRequest request = new GetStackTracesRequest(
             null,
@@ -470,7 +417,6 @@ public void testValidateAggregationFieldsContainsTooFewElements() {
             null,
             new String[] { randomAlphaOfLength(5) },
             randomAlphaOfLength(5),
-            null,
             new String[] {},
             null,
             null,
@@ -492,7 +438,6 @@ public void testValidateAggregationFieldsContainsTooManyElements() {
             null,
             new String[] { randomAlphaOfLength(5) },
             randomAlphaOfLength(5),
-            null,
             new String[] { "application", "service", "transaction" },
             null,
             null,
@@ -514,7 +459,6 @@ public void testValidateAggregationFieldsContainsEnoughElements() {
             null,
             new String[] { randomAlphaOfLength(5) },
             randomAlphaOfLength(5),
-            null,
             new String[] { "service", "service" },
             null,
             null,
@@ -540,7 +484,6 @@ public void testConsidersCustomIndicesInRelatedIndices() {
             null,
             null,
             null,
-            null,
             null
         );
         String[] indices = request.indices();
@@ -562,7 +505,6 @@ public void testConsidersDefaultIndicesInRelatedIndices() {
             null,
             null,
             null,
-            null,
             null
         );
         String[] indices = request.indices();
diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/GetTopNFunctionsResponseTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/GetTopNFunctionsResponseTests.java
index 24f8ea85212d8..cf8742b5ddf62 100644
--- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/GetTopNFunctionsResponseTests.java
+++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/GetTopNFunctionsResponseTests.java
@@ -90,7 +90,7 @@ public void testToXContent() throws IOException {
             22.0d,
             12.0d,
             120.0d,
-            SubGroup.root("transaction.name", false).addCount("basket", 7L)
+            SubGroup.root("transaction.name").addCount("basket", 7L)
         );
         GetTopNFunctionsResponse response = new GetTopNFunctionsResponse(1, 10, 2.2d, 12.0d, List.of(topNFunction));
         response.toXContent(actualResponse, ToXContent.EMPTY_PARAMS);
diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/ResamplerTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/ResamplerTests.java
index fec9704dc8c02..c2537edab6bbd 100644
--- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/ResamplerTests.java
+++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/ResamplerTests.java
@@ -43,7 +43,6 @@ public void testNoResamplingNoSampleRateAdjustment() {
             null,
             null,
             null,
-            null,
             null
         );
         request.setAdjustSampleCount(false);
@@ -73,7 +72,6 @@ public void testNoResamplingButAdjustSampleRate() {
             null,
             null,
             null,
-            null,
             null
         );
         request.setAdjustSampleCount(true);
@@ -103,7 +101,6 @@ public void testResamplingNoSampleRateAdjustment() {
             null,
             null,
             null,
-            null,
             null
         );
         request.setAdjustSampleCount(false);
@@ -136,7 +133,6 @@ public void testResamplingNoSampleRateAdjustmentWithQuery() {
             null,
             null,
             null,
-            null,
             null
         );
 
@@ -166,7 +162,6 @@ public void testResamplingAndSampleRateAdjustment() {
             null,
             null,
             null,
-            null,
             null
         );
         request.setAdjustSampleCount(true);
diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupCollectorTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupCollectorTests.java
index 5d6022f322762..73164ed9069d9 100644
--- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupCollectorTests.java
+++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupCollectorTests.java
@@ -21,7 +21,7 @@ public void testNoAggs() {
         TermsAggregationBuilder stackTraces = new TermsAggregationBuilder("stacktraces").field("stacktrace.id");
         TraceEvent traceEvent = new TraceEvent("1");
 
-        SubGroupCollector collector = SubGroupCollector.attach(stackTraces, new String[0], false);
+        SubGroupCollector collector = SubGroupCollector.attach(stackTraces, new String[0]);
         assertTrue("Sub aggregations attached", stackTraces.getSubAggregations().isEmpty());
 
         SubGroupCollector.Bucket currentStackTrace = bucket("1", 5);
@@ -34,7 +34,7 @@ public void testMultipleAggsInSingleStackTrace() {
         TermsAggregationBuilder stackTraces = new TermsAggregationBuilder("stacktraces").field("stacktrace.id");
         TraceEvent traceEvent = new TraceEvent("1");
 
-        SubGroupCollector collector = SubGroupCollector.attach(stackTraces, new String[] { "service.name", "transaction.name" }, false);
+        SubGroupCollector collector = SubGroupCollector.attach(stackTraces, new String[] { "service.name", "transaction.name" });
         assertFalse("No sub aggregations attached", stackTraces.getSubAggregations().isEmpty());
 
         StaticAgg services = new StaticAgg();
@@ -73,7 +73,7 @@ public void testSingleAggInMultipleStackTraces() {
         TermsAggregationBuilder stackTraces = new TermsAggregationBuilder("stacktraces").field("stacktrace.id");
         TraceEvent traceEvent = new TraceEvent("1");
 
-        SubGroupCollector collector = SubGroupCollector.attach(stackTraces, new String[] { "service.name" }, false);
+        SubGroupCollector collector = SubGroupCollector.attach(stackTraces, new String[] { "service.name" });
         assertFalse("No sub aggregations attached", stackTraces.getSubAggregations().isEmpty());
 
         StaticAgg services1 = new StaticAgg();
diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupTests.java
index c571d7c03c252..0d85b62be1c82 100644
--- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupTests.java
+++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupTests.java
@@ -34,25 +34,7 @@ public void testToXContent() throws IOException {
 
         XContentBuilder actualRequest = XContentFactory.contentBuilder(contentType);
         actualRequest.startObject();
-        SubGroup g = SubGroup.root("transaction.name", false).addCount("basket", 7L);
-        g.toXContent(actualRequest, ToXContent.EMPTY_PARAMS);
-        actualRequest.endObject();
-
-        assertToXContentEquivalent(BytesReference.bytes(expectedRequest), BytesReference.bytes(actualRequest), contentType);
-    }
-
-    public void testRenderLegacyXContent() throws IOException {
-        XContentType contentType = randomFrom(XContentType.values());
-        // tag::noformat
-        XContentBuilder expectedRequest = XContentFactory.contentBuilder(contentType)
-            .startObject()
-                .field("basket", 7L)
-            .endObject();
-        // end::noformat
-
-        XContentBuilder actualRequest = XContentFactory.contentBuilder(contentType);
-        actualRequest.startObject();
-        SubGroup g = SubGroup.root("transaction.name", true).addCount("basket", 7L);
+        SubGroup g = SubGroup.root("transaction.name").addCount("basket", 7L);
         g.toXContent(actualRequest, ToXContent.EMPTY_PARAMS);
         actualRequest.endObject();
 
@@ -60,8 +42,8 @@ public void testRenderLegacyXContent() throws IOException {
     }
 
     public void testMergeNoCommonRoot() {
-        SubGroup root1 = SubGroup.root("transaction.name", false);
-        SubGroup root2 = SubGroup.root("service.name", false);
+        SubGroup root1 = SubGroup.root("transaction.name");
+        SubGroup root2 = SubGroup.root("service.name");
 
         SubGroup toMerge = root1.copy();
 
@@ -71,7 +53,7 @@ public void testMergeNoCommonRoot() {
     }
 
     public void testMergeIdenticalTree() {
-        SubGroup g = SubGroup.root("transaction.name", false);
+        SubGroup g = SubGroup.root("transaction.name");
         g.addCount("basket", 5L);
         g.addCount("checkout", 7L);
 
@@ -84,11 +66,11 @@ public void testMergeIdenticalTree() {
     }
 
     public void testMergeMixedTree() {
-        SubGroup g1 = SubGroup.root("transaction.name", false);
+        SubGroup g1 = SubGroup.root("transaction.name");
         g1.addCount("basket", 5L);
         g1.addCount("checkout", 7L);
 
-        SubGroup g2 = SubGroup.root("transaction.name", false);
+        SubGroup g2 = SubGroup.root("transaction.name");
         g2.addCount("catalog", 8L);
         g2.addCount("basket", 5L);
         g2.addCount("checkout", 2L);
diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/TopNFunctionTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/TopNFunctionTests.java
index 76379adcd3b8a..b63cca21f9b46 100644
--- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/TopNFunctionTests.java
+++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/TopNFunctionTests.java
@@ -81,7 +81,7 @@ public void testToXContent() throws IOException {
             22.0d,
             12.0d,
             120.0d,
-            SubGroup.root("transaction.name", false).addCount("basket", 7L)
+            SubGroup.root("transaction.name").addCount("basket", 7L)
         );
         topNFunction.toXContent(actualRequest, ToXContent.EMPTY_PARAMS);
 
@@ -116,7 +116,7 @@ public void testEquality() {
             4.0d,
             23.2d,
             12.0d,
-            SubGroup.root("transaction.name", false).addCount("checkout", 4L).addCount("basket", 12L)
+            SubGroup.root("transaction.name").addCount("checkout", 4L).addCount("basket", 12L)
         );
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(topNFunction, (TopNFunction::copy));
     }
diff --git a/x-pack/plugin/ql/build.gradle b/x-pack/plugin/ql/build.gradle
index c9d86cdd63e43..947b4d8a0d0ce 100644
--- a/x-pack/plugin/ql/build.gradle
+++ b/x-pack/plugin/ql/build.gradle
@@ -2,9 +2,9 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-test-artifact'
 
 esplugin {
-  name 'x-pack-ql'
-  description 'Elasticsearch infrastructure plugin for EQL and SQL for Elasticsearch'
-  classname 'org.elasticsearch.xpack.ql.plugin.QlPlugin'
+  name = 'x-pack-ql'
+  description = 'Elasticsearch infrastructure plugin for EQL and SQL for Elasticsearch'
+  classname ='org.elasticsearch.xpack.ql.plugin.QlPlugin'
   extendedPlugins = ['x-pack-core']
 }
 
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/processor/ConstantProcessor.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/processor/ConstantProcessor.java
index cf87d01b87a50..3d5f63db23c5d 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/processor/ConstantProcessor.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/processor/ConstantProcessor.java
@@ -19,7 +19,7 @@
 
 public class ConstantProcessor implements Processor {
 
-    public static String NAME = "c";
+    public static final String NAME = "c";
 
     private Object constant;
     private final Type type;
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java
index 318f3888ac9b3..b50ca4d928040 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
 import org.elasticsearch.common.Strings;
@@ -694,7 +695,7 @@ public void resolveAsSeparateMappings(
     }
 
     private static GetAliasesRequest createGetAliasesRequest(FieldCapabilitiesResponse response, boolean includeFrozen) {
-        return new GetAliasesRequest().aliases("*")
+        return new GetAliasesRequest(MasterNodeRequest.INFINITE_MASTER_NODE_TIMEOUT).aliases("*")
             .indices(response.getIndices())
             .indicesOptions(includeFrozen ? FIELD_CAPS_FROZEN_INDICES_OPTIONS : FIELD_CAPS_INDICES_OPTIONS);
     }
@@ -1037,7 +1038,7 @@ public interface ExistingFieldInvalidCallback extends BiConsumer<EsField, Invali
     /**
      * Preserve the properties (sub fields) of an existing field even when marking it as invalid.
      */
-    public static ExistingFieldInvalidCallback PRESERVE_PROPERTIES = (oldField, newField) -> {
+    public static final ExistingFieldInvalidCallback PRESERVE_PROPERTIES = (oldField, newField) -> {
         var oldProps = oldField.getProperties();
         if (oldProps.size() > 0) {
             newField.getProperties().putAll(oldProps);
diff --git a/x-pack/plugin/rank-rrf/build.gradle b/x-pack/plugin/rank-rrf/build.gradle
index 2c3f217243aa4..216e85f48f56f 100644
--- a/x-pack/plugin/rank-rrf/build.gradle
+++ b/x-pack/plugin/rank-rrf/build.gradle
@@ -10,9 +10,9 @@ apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
 esplugin {
-  name 'rank-rrf'
-  description 'Reciprocal rank fusion in search.'
-  classname 'org.elasticsearch.xpack.rank.rrf.RRFRankPlugin'
+  name = 'rank-rrf'
+  description = 'Reciprocal rank fusion in search.'
+  classname ='org.elasticsearch.xpack.rank.rrf.RRFRankPlugin'
   extendedPlugins = ['x-pack-core']
 }
 
@@ -22,6 +22,7 @@ dependencies {
   testImplementation(testArtifact(project(xpackModule('core'))))
   testImplementation(testArtifact(project(':server')))
 
+  clusterModules project(':modules:mapper-extras')
   clusterModules project(xpackModule('rank-rrf'))
   clusterModules project(xpackModule('inference'))
   clusterModules project(':modules:lang-painless')
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFFeatures.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFFeatures.java
index bb61fa951948d..494eaa508c14a 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFFeatures.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFFeatures.java
@@ -13,18 +13,12 @@
 import java.util.Set;
 
 import static org.elasticsearch.search.retriever.CompoundRetrieverBuilder.INNER_RETRIEVERS_FILTER_SUPPORT;
-import static org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilder.RRF_RETRIEVER_COMPOSITION_SUPPORTED;
 
 /**
  * A set of features specifically for the rrf plugin.
  */
 public class RRFFeatures implements FeatureSpecification {
 
-    @Override
-    public Set<NodeFeature> getFeatures() {
-        return Set.of(RRFRetrieverBuilder.RRF_RETRIEVER_SUPPORTED, RRF_RETRIEVER_COMPOSITION_SUPPORTED);
-    }
-
     @Override
     public Set<NodeFeature> getTestFeatures() {
         return Set.of(INNER_RETRIEVERS_FILTER_SUPPORT);
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
index b5bca57478684..91bc19a3e0903 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
@@ -194,9 +194,6 @@ public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorCo
 
     @Override
     public RetrieverBuilder toRetriever(SearchSourceBuilder source, Predicate<NodeFeature> clusterSupportsFeature) {
-        if (false == clusterSupportsFeature.test(RRFRetrieverBuilder.RRF_RETRIEVER_COMPOSITION_SUPPORTED)) {
-            return null;
-        }
         int totalQueries = source.subSearches().size() + source.knnSearch().size();
         if (totalQueries < 2) {
             throw new IllegalArgumentException("[rrf] requires at least 2 sub-queries to be defined");
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java
index f1171b74f7468..a749a7c402c30 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java
@@ -8,9 +8,7 @@
 package org.elasticsearch.xpack.rank.rrf;
 
 import org.apache.lucene.search.ScoreDoc;
-import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.search.rank.RankBuilder;
@@ -44,11 +42,8 @@
 public final class RRFRetrieverBuilder extends CompoundRetrieverBuilder<RRFRetrieverBuilder> {
 
     public static final String NAME = "rrf";
-    public static final NodeFeature RRF_RETRIEVER_SUPPORTED = new NodeFeature("rrf_retriever_supported");
-    public static final NodeFeature RRF_RETRIEVER_COMPOSITION_SUPPORTED = new NodeFeature("rrf_retriever_composition_supported");
 
     public static final ParseField RETRIEVERS_FIELD = new ParseField("retrievers");
-    public static final ParseField RANK_WINDOW_SIZE_FIELD = new ParseField("rank_window_size");
     public static final ParseField RANK_CONSTANT_FIELD = new ParseField("rank_constant");
 
     public static final int DEFAULT_RANK_CONSTANT = 60;
@@ -80,12 +75,6 @@ public final class RRFRetrieverBuilder extends CompoundRetrieverBuilder<RRFRetri
     }
 
     public static RRFRetrieverBuilder fromXContent(XContentParser parser, RetrieverParserContext context) throws IOException {
-        if (context.clusterSupportsFeature(RRF_RETRIEVER_SUPPORTED) == false) {
-            throw new ParsingException(parser.getTokenLocation(), "unknown retriever [" + NAME + "]");
-        }
-        if (context.clusterSupportsFeature(RRF_RETRIEVER_COMPOSITION_SUPPORTED) == false) {
-            throw new IllegalArgumentException("[rrf] retriever composition feature is not supported by all nodes in the cluster");
-        }
         if (RRFRankPlugin.RANK_RRF_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) {
             throw LicenseUtils.newComplianceException("Reciprocal Rank Fusion (RRF)");
         }
diff --git a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderTests.java b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderTests.java
index bdd6d73ec0fbf..3a77b733d6129 100644
--- a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderTests.java
+++ b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderTests.java
@@ -7,10 +7,8 @@
 
 package org.elasticsearch.xpack.rank.rrf;
 
-import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.builder.PointInTimeBuilder;
@@ -29,18 +27,6 @@
 /** Tests for the rrf retriever. */
 public class RRFRetrieverBuilderTests extends ESTestCase {
 
-    /** Tests the rrf retriever validates on its own {@link NodeFeature} */
-    public void testRetrieverVersions() throws IOException {
-        try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"retriever\":{\"rrf\":{}}}")) {
-            SearchSourceBuilder ssb = new SearchSourceBuilder();
-            ParsingException iae = expectThrows(
-                ParsingException.class,
-                () -> ssb.parseXContent(parser, true, nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED)
-            );
-            assertEquals("unknown retriever [rrf]", iae.getMessage());
-        }
-    }
-
     /** Tests extraction errors related to compound retrievers. These tests require a compound retriever which is why they are here. */
     public void testRetrieverExtractionErrors() throws IOException {
         try (
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankClientYamlTestSuiteIT.java b/x-pack/plugin/rank-rrf/src/yamlRestTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankClientYamlTestSuiteIT.java
index 32b5aedd5d99a..1a22f8738a26a 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankClientYamlTestSuiteIT.java
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankClientYamlTestSuiteIT.java
@@ -21,6 +21,7 @@ public class RRFRankClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     @ClassRule
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .nodes(2)
+        .module("mapper-extras")
         .module("rank-rrf")
         .module("lang-painless")
         .module("x-pack-inference")
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml
index 258ab70cd09bd..ac328967d9fcf 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml
@@ -1,8 +1,4 @@
 setup:
-  - requires:
-      cluster_features: 'rrf_retriever_supported'
-      reason: 'test requires rrf retriever implementation'
-
   - do:
       indices.create:
         index: test
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/350_rrf_retriever_pagination.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/350_rrf_retriever_pagination.yml
index d5d7a5de1dc71..e7a4bf8a5b34e 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/350_rrf_retriever_pagination.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/350_rrf_retriever_pagination.yml
@@ -4,10 +4,6 @@ setup:
         - close_to
         - contains
 
-  - requires:
-      cluster_features: 'rrf_retriever_composition_supported'
-      reason: 'test requires rrf retriever composition support'
-
   - do:
       indices.create:
         index: test
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/400_rrf_retriever_script.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/400_rrf_retriever_script.yml
index bbc1087b05cc3..4dea4c3568739 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/400_rrf_retriever_script.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/400_rrf_retriever_script.yml
@@ -2,10 +2,6 @@ setup:
   - skip:
       features: close_to
 
-  - requires:
-      cluster_features: 'rrf_retriever_supported'
-      reason: 'test requires rrf retriever implementation'
-
   - do:
       indices.create:
         index: test
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml
index 24259e3aa2a85..b1c5ec1664c8f 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml
@@ -1,7 +1,5 @@
 setup:
   - requires:
-      cluster_features: 'rrf_retriever_composition_supported'
-      reason: 'test requires rrf retriever composition support'
       test_runner_features: close_to
 
   - do:
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml
index cb30542d80003..01d645fbfb4f5 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml
@@ -1,9 +1,6 @@
 setup:
   - skip:
       features: close_to
-  - requires:
-      cluster_features: 'rrf_retriever_composition_supported'
-      reason: 'test requires rrf retriever composition support'
 
   - do:
       indices.create:
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml
index d9db1fe387625..98c4ae9e642d1 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml
@@ -5,8 +5,6 @@ setup:
         - contains
 
   - requires:
-      cluster_features: ['rrf_retriever_composition_supported', 'text_similarity_reranker_retriever_supported']
-      reason: need to have support for rrf and semantic reranking composition
       test_runner_features: "close_to"
 
   - do:
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/900_rrf_with_rescorer.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/900_rrf_with_rescorer.yml
new file mode 100644
index 0000000000000..c825ede6fccf0
--- /dev/null
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/900_rrf_with_rescorer.yml
@@ -0,0 +1,409 @@
+setup:
+  - requires:
+      cluster_features: [ "search.retriever.rescorer.enabled" ]
+      reason: "Support for rescorer retriever"
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 3
+          mappings:
+            properties:
+              available:
+                type: boolean
+              features:
+                type: rank_features
+
+  - do:
+      bulk:
+        refresh: true
+        index: test
+        body:
+          - '{"index": {"_id": 1 }}'
+          - '{"features": { "first_query": 1, "second_query": 3, "final_score": 7}, "available": true}'
+          - '{"index": {"_id": 2 }}'
+          - '{"features": { "first_query": 5, "second_query": 7, "final_score": 4}, "available": false}'
+          - '{"index": {"_id": 3 }}'
+          - '{"features": { "first_query": 6, "second_query": 5, "final_score": 3}, "available": false}'
+          - '{"index": {"_id": 4 }}'
+          - '{"features": { "first_query": 3, "second_query": 2, "final_score": 2}, "available": true}'
+          - '{"index": {"_id": 5 }}'
+          - '{"features": { "first_query": 2, "second_query": 1, "final_score": 1}, "available": true}'
+          - '{"index": {"_id": 6 }}'
+          - '{"features": { "first_query": 4, "second_query": 4, "final_score": 8}, "available": false}'
+          - '{"index": {"_id": 7 }}'
+          - '{"features": { "first_query": 7, "second_query": 10, "final_score": 9}, "available": true}'
+          - '{"index": {"_id": 8 }}'
+          - '{"features": { "first_query": 8, "second_query": 8, "final_score": 10}, "available": true}'
+          - '{"index": {"_id": 9 }}'
+          - '{"features": { "first_query": 9, "second_query": 9, "final_score": 5}, "available": true}'
+          - '{"index": {"_id": 10 }}'
+          - '{"features": { "first_query": 10, "second_query": 6, "final_score": 6}, "available": false}'
+
+---
+"RRF with rescorer retriever basic":
+  - do:
+      search:
+        index: test
+        body:
+          retriever:
+            rescorer:
+              rescore:
+                window_size: 10
+                query:
+                  rescore_query:
+                    rank_feature:
+                      field: "features.final_score"
+                      linear: { }
+                  query_weight: 0
+              retriever:
+                rrf:
+                  rank_window_size: 10
+                  retrievers: [
+                    {
+                      standard: {
+                        query: {
+                          rank_feature: {
+                            field: "features.first_query",
+                            linear: { }
+                          }
+                        }
+                      }
+                    },
+                    {
+                      standard: {
+                        query: {
+                          rank_feature: {
+                            field: "features.second_query",
+                            linear: { }
+                          }
+                        }
+                      }
+                    }
+                  ]
+          size: 3
+
+  - match:  { hits.total.value: 10 }
+  - length: { hits.hits: 3}
+  - match:  { hits.hits.0._id: "8" }
+  - match:  { hits.hits.0._score: 10.0 }
+  - match:  { hits.hits.1._id: "7" }
+  - match:  { hits.hits.1._score: 9.0 }
+  - match:  { hits.hits.2._id: "6" }
+  - match:  { hits.hits.2._score: 8.0 }
+
+  - do:
+      search:
+        index: test
+        body:
+          retriever:
+            rescorer:
+              rescore:
+                window_size: 5
+                query:
+                  rescore_query:
+                    rank_feature:
+                      field: "features.final_score"
+                      linear: { }
+                  query_weight: 0
+              retriever:
+                rrf:
+                  rank_window_size: 5
+                  retrievers: [
+                    {
+                      standard: {
+                        query: {
+                          rank_feature: {
+                            field: "features.first_query",
+                            linear: { }
+                          }
+                        }
+                      }
+                    },
+                    {
+                      standard: {
+                        query: {
+                          rank_feature: {
+                            field: "features.second_query",
+                            linear: { }
+                          }
+                        }
+                      }
+                    }
+                  ]
+          size: 3
+
+  - match:  { hits.total.value: 10 }
+  - length: { hits.hits: 3}
+  - match:  { hits.hits.0._id: "8" }
+  - match:  { hits.hits.0._score: 10.0 }
+  - match:  { hits.hits.1._id: "7" }
+  - match:  { hits.hits.1._score: 9.0 }
+  - match:  { hits.hits.2._id: "10" }
+  - match:  { hits.hits.2._score: 6.0 }
+
+---
+"RRF with rescorer retriever and prefilters":
+  - do:
+      search:
+        index: test
+        body:
+          retriever:
+            rescorer:
+              filter:
+                match:
+                  available: true
+              rescore:
+                window_size: 5
+                query:
+                  rescore_query:
+                    rank_feature:
+                      field: "features.final_score"
+                      linear: { }
+                  query_weight: 0
+              retriever:
+                rrf:
+                  rank_window_size: 5
+                  retrievers: [
+                    {
+                      standard: {
+                        query: {
+                          rank_feature: {
+                            field: "features.first_query",
+                            linear: { }
+                          }
+                        }
+                      }
+                    },
+                    {
+                      standard: {
+                        query: {
+                          rank_feature: {
+                            field: "features.second_query",
+                            linear: { }
+                          }
+                        }
+                      }
+                    }
+                  ]
+          size: 3
+
+  - match:  { hits.total.value: 6 }
+  - length: { hits.hits: 3}
+  - match:  { hits.hits.0._id: "8" }
+  - match:  { hits.hits.0._score: 10.0 }
+  - match:  { hits.hits.1._id: "7" }
+  - match:  { hits.hits.1._score: 9.0 }
+  - match:  { hits.hits.2._id: "1" }
+  - match:  { hits.hits.2._score: 7.0 }
+
+  - do:
+      search:
+        index: test
+        body:
+          retriever:
+            rescorer:
+              filter:
+                match:
+                  available: true
+              rescore:
+                window_size: 5
+                query:
+                  rescore_query:
+                    rank_feature:
+                      field: "features.final_score"
+                      linear: { }
+                  query_weight: 0
+              retriever:
+                rrf:
+                  rank_window_size: 5
+                  retrievers: [
+                    {
+                      standard: {
+                        query: {
+                          rank_feature: {
+                            field: "features.first_query",
+                            linear: { }
+                          }
+                        }
+                      }
+                    },
+                    {
+                      standard: {
+                        filter: {
+                          match: {
+                            available: true
+                          }
+                        },
+                        query: {
+                          rank_feature: {
+                            field: "features.second_query",
+                            linear: { }
+                          }
+                        }
+                      }
+                    }
+                  ]
+          size: 3
+
+  - match:  { hits.total.value: 6 }
+  - length: { hits.hits: 3}
+  - match:  { hits.hits.0._id: "8" }
+  - match:  { hits.hits.0._score: 10.0 }
+  - match:  { hits.hits.1._id: "7" }
+  - match:  { hits.hits.1._score: 9.0 }
+  - match:  { hits.hits.2._id: "1" }
+  - match:  { hits.hits.2._score: 7.0 }
+
+---
+"RRF with rescorer retriever and aggs":
+  - do:
+      search:
+        index: test
+        body:
+          aggs:
+            1:
+              terms:
+                field: available
+          retriever:
+            rescorer:
+              rescore:
+                window_size: 5
+                query:
+                  rescore_query:
+                    rank_feature:
+                      field: "features.final_score"
+                      linear: { }
+                  query_weight: 0
+              retriever:
+                rrf:
+                  rank_window_size: 5
+                  retrievers: [
+                    {
+                      standard: {
+                        query: {
+                          rank_feature: {
+                            field: "features.first_query",
+                            linear: { }
+                          }
+                        }
+                      }
+                    },
+                    {
+                      standard: {
+                        filter: {
+                          match: {
+                            available: true
+                          }
+                        },
+                        query: {
+                          rank_feature: {
+                            field: "features.second_query",
+                            linear: { }
+                          }
+                        }
+                      }
+                    }
+                  ]
+          size: 3
+
+  - match:  { hits.total.value: 10 }
+  - length: { hits.hits: 3}
+  - match:  { hits.hits.0._id: "8" }
+  - match:  { hits.hits.0._score: 10.0 }
+  - match:  { hits.hits.1._id: "7" }
+  - match:  { hits.hits.1._score: 9.0 }
+  - match:  { hits.hits.2._id: "1" }
+  - match:  { hits.hits.2._score: 7.0 }
+  - length: { aggregations.1.buckets: 2}
+  - match:  { aggregations.1.buckets.0.key: 1}
+  - match:  { aggregations.1.buckets.0.doc_count: 6}
+  - match:  { aggregations.1.buckets.1.key: 0 }
+  - match:  { aggregations.1.buckets.1.doc_count: 4 }
+
+---
+"RRF with rescorer retriever and invalid window size":
+  - do:
+      catch: "/\\[rescorer\\] requires \\[window_size: 5\\] be greater than or equal to \\[size: 10\\]/"
+      search:
+        index: test
+        body:
+          retriever:
+            rescorer:
+              rescore:
+                window_size: 5
+                query:
+                  rescore_query:
+                    rank_feature:
+                      field: "features.final_score"
+                      linear: { }
+                  query_weight: 0
+              retriever:
+                rrf:
+                  rank_window_size: 5
+                  retrievers: [
+                    {
+                      standard: {
+                        query: {
+                          rank_feature: {
+                            field: "features.first_query",
+                            linear: { }
+                          }
+                        }
+                      }
+                    },
+                    {
+                      standard: {
+                        query: {
+                          rank_feature: {
+                            field: "features.second_query",
+                            linear: { }
+                          }
+                        }
+                      }
+                    }
+                  ]
+          size: 10
+
+  - do:
+      catch: "/\\[rescorer\\] requires \\[window_size: 10\\] to be smaller than or equal to its sub retriever's rrf \\[rank_window_size: 5\\]/"
+      search:
+        index: test
+        body:
+          retriever:
+            rescorer:
+              rescore:
+                window_size: 10
+                query:
+                  rescore_query:
+                    rank_feature:
+                      field: "features.final_score"
+                      linear: { }
+                  query_weight: 0
+              retriever:
+                rrf:
+                  rank_window_size: 5
+                  retrievers: [
+                    {
+                      standard: {
+                        query: {
+                          rank_feature: {
+                            field: "features.first_query",
+                            linear: { }
+                          }
+                        }
+                      }
+                    },
+                    {
+                      standard: {
+                        query: {
+                          rank_feature: {
+                            field: "features.second_query",
+                            linear: { }
+                          }
+                        }
+                      }
+                    }
+                  ]
+          size: 5
diff --git a/x-pack/plugin/rank-vectors/build.gradle b/x-pack/plugin/rank-vectors/build.gradle
new file mode 100644
index 0000000000000..53aabb8fdbf74
--- /dev/null
+++ b/x-pack/plugin/rank-vectors/build.gradle
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+apply plugin: 'elasticsearch.internal-es-plugin'
+apply plugin: 'elasticsearch.internal-cluster-test'
+
+esplugin {
+  name = 'rank-vectors'
+  description = 'Rank vectors in search.'
+  classname = 'org.elasticsearch.xpack.rank.vectors.RankVectorsPlugin'
+  extendedPlugins = ['x-pack-core', 'lang-painless']
+}
+
+dependencies {
+  compileOnly project(path: xpackModule('core'))
+  compileOnly(project(':modules:lang-painless:spi'))
+
+  testImplementation(testArtifact(project(xpackModule('core'))))
+  testImplementation(testArtifact(project(':server')))
+}
diff --git a/x-pack/plugin/rank-vectors/src/main/java/module-info.java b/x-pack/plugin/rank-vectors/src/main/java/module-info.java
new file mode 100644
index 0000000000000..4af3c994edd38
--- /dev/null
+++ b/x-pack/plugin/rank-vectors/src/main/java/module-info.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module org.elasticsearch.rank.vectors {
+    requires org.elasticsearch.xcore;
+    requires org.elasticsearch.painless.spi;
+    requires org.elasticsearch.server;
+    requires org.apache.lucene.core;
+    requires org.elasticsearch.xcontent;
+
+    exports org.elasticsearch.xpack.rank.vectors;
+    exports org.elasticsearch.xpack.rank.vectors.mapper;
+    exports org.elasticsearch.xpack.rank.vectors.script;
+
+    // whitelist resource access
+    opens org.elasticsearch.xpack.rank.vectors.script to org.elasticsearch.painless.spi;
+
+    provides org.elasticsearch.painless.spi.PainlessExtension with org.elasticsearch.xpack.rank.vectors.script.RankVectorsPainlessExtension;
+    provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.rank.vectors.RankVectorsFeatures;
+
+}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKitFeatures.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsFeatures.java
similarity index 51%
rename from x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKitFeatures.java
rename to x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsFeatures.java
index cc513a948519b..44b1b7a068860 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKitFeatures.java
+++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsFeatures.java
@@ -5,18 +5,19 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.xpack.rank.vectors;
 
 import org.elasticsearch.features.FeatureSpecification;
 import org.elasticsearch.features.NodeFeature;
 
 import java.util.Set;
 
-import static org.elasticsearch.repositories.blobstore.testkit.integrity.RestRepositoryVerifyIntegrityAction.REPOSITORY_VERIFY_INTEGRITY_FEATURE;
+public class RankVectorsFeatures implements FeatureSpecification {
+    public static final NodeFeature RANK_VECTORS_FEATURE = new NodeFeature("rank_vectors");
 
-public class SnapshotRepositoryTestKitFeatures implements FeatureSpecification {
     @Override
-    public Set<NodeFeature> getFeatures() {
-        return Set.of(REPOSITORY_VERIFY_INTEGRITY_FEATURE);
+    public Set<NodeFeature> getTestFeatures() {
+        return Set.of(RANK_VECTORS_FEATURE);
     }
+
 }
diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java
new file mode 100644
index 0000000000000..35c87f1fc1847
--- /dev/null
+++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.rank.vectors;
+
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.license.License;
+import org.elasticsearch.license.LicenseUtils;
+import org.elasticsearch.license.LicensedFeature;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsFieldMapper;
+
+import java.util.Map;
+
+import static org.elasticsearch.index.mapper.FieldMapper.notInMultiFields;
+import static org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsFieldMapper.CONTENT_TYPE;
+
+public class RankVectorsPlugin extends Plugin implements MapperPlugin {
+    public static final LicensedFeature.Momentary RANK_VECTORS_FEATURE = LicensedFeature.momentary(
+        null,
+        "rank-vectors",
+        License.OperationMode.ENTERPRISE
+    );
+
+    @Override
+    public Map<String, Mapper.TypeParser> getMappers() {
+        return Map.of(CONTENT_TYPE, new FieldMapper.TypeParser((n, c) -> {
+            if (RANK_VECTORS_FEATURE.check(getLicenseState()) == false) {
+                throw LicenseUtils.newComplianceException("Rank Vectors");
+            }
+            return new RankVectorsFieldMapper.Builder(n, c.indexVersionCreated(), getLicenseState());
+        }, notInMultiFields(CONTENT_TYPE)));
+    }
+
+    protected XPackLicenseState getLicenseState() {
+        return XPackPlugin.getSharedLicenseState();
+    }
+}
diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java
new file mode 100644
index 0000000000000..b858b935c1483
--- /dev/null
+++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java
@@ -0,0 +1,158 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.rank.vectors.mapper;
+
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.fielddata.FormattedDocValues;
+import org.elasticsearch.index.fielddata.LeafFieldData;
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
+import org.elasticsearch.script.field.DocValuesScriptFieldFactory;
+import org.elasticsearch.script.field.vectors.BitRankVectorsDocValuesField;
+import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField;
+import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField;
+import org.elasticsearch.script.field.vectors.VectorIterator;
+import org.elasticsearch.search.DocValueFormat;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+final class RankVectorsDVLeafFieldData implements LeafFieldData {
+    private final LeafReader reader;
+    private final String field;
+    private final DenseVectorFieldMapper.ElementType elementType;
+    private final int dims;
+
+    RankVectorsDVLeafFieldData(LeafReader reader, String field, DenseVectorFieldMapper.ElementType elementType, int dims) {
+        this.reader = reader;
+        this.field = field;
+        this.elementType = elementType;
+        this.dims = dims;
+    }
+
+    @Override
+    public FormattedDocValues getFormattedValues(DocValueFormat format) {
+        int dims = elementType == DenseVectorFieldMapper.ElementType.BIT ? this.dims / Byte.SIZE : this.dims;
+        return switch (elementType) {
+            case BYTE, BIT -> new FormattedDocValues() {
+                private final byte[] vector = new byte[dims];
+                private BytesRef ref = null;
+                private int numVecs = -1;
+                private final BinaryDocValues binary;
+                {
+                    try {
+                        binary = DocValues.getBinary(reader, field);
+                    } catch (IOException e) {
+                        throw new IllegalStateException("Cannot load doc values", e);
+                    }
+                }
+
+                @Override
+                public boolean advanceExact(int docId) throws IOException {
+                    if (binary == null || binary.advanceExact(docId) == false) {
+                        return false;
+                    }
+                    ref = binary.binaryValue();
+                    assert ref.length % dims == 0;
+                    numVecs = ref.length / dims;
+                    return true;
+                }
+
+                @Override
+                public int docValueCount() {
+                    return 1;
+                }
+
+                public Object nextValue() {
+                    // Boxed to keep from `byte[]` being transformed into a string
+                    List<Byte[]> vectors = new ArrayList<>(numVecs);
+                    VectorIterator<byte[]> iterator = new ByteRankVectorsDocValuesField.ByteVectorIterator(ref, vector, numVecs);
+                    while (iterator.hasNext()) {
+                        byte[] v = iterator.next();
+                        Byte[] vec = new Byte[dims];
+                        for (int i = 0; i < dims; i++) {
+                            vec[i] = v[i];
+                        }
+                        vectors.add(vec);
+                    }
+                    return vectors;
+                }
+            };
+            case FLOAT -> new FormattedDocValues() {
+                private final float[] vector = new float[dims];
+                private BytesRef ref = null;
+                private int numVecs = -1;
+                private final BinaryDocValues binary;
+                {
+                    try {
+                        binary = DocValues.getBinary(reader, field);
+                    } catch (IOException e) {
+                        throw new IllegalStateException("Cannot load doc values", e);
+                    }
+                }
+
+                @Override
+                public boolean advanceExact(int docId) throws IOException {
+                    if (binary == null || binary.advanceExact(docId) == false) {
+                        return false;
+                    }
+                    ref = binary.binaryValue();
+                    assert ref.length % (Float.BYTES * dims) == 0;
+                    numVecs = ref.length / (Float.BYTES * dims);
+                    return true;
+                }
+
+                @Override
+                public int docValueCount() {
+                    return 1;
+                }
+
+                @Override
+                public Object nextValue() {
+                    List<float[]> vectors = new ArrayList<>(numVecs);
+                    VectorIterator<float[]> iterator = new FloatRankVectorsDocValuesField.FloatVectorIterator(ref, vector, numVecs);
+                    while (iterator.hasNext()) {
+                        float[] v = iterator.next();
+                        vectors.add(Arrays.copyOf(v, v.length));
+                    }
+                    return vectors;
+                }
+            };
+        };
+    }
+
+    @Override
+    public DocValuesScriptFieldFactory getScriptFieldFactory(String name) {
+        try {
+            BinaryDocValues values = DocValues.getBinary(reader, field);
+            BinaryDocValues magnitudeValues = DocValues.getBinary(reader, field + RankVectorsFieldMapper.VECTOR_MAGNITUDES_SUFFIX);
+            return switch (elementType) {
+                case BYTE -> new ByteRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims);
+                case FLOAT -> new FloatRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims);
+                case BIT -> new BitRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims);
+            };
+        } catch (IOException e) {
+            throw new IllegalStateException("Cannot load doc values for multi-vector field!", e);
+        }
+    }
+
+    @Override
+    public SortedBinaryDocValues getBytesValues() {
+        throw new UnsupportedOperationException("String representation of doc values for multi-vector fields is not supported");
+    }
+
+    @Override
+    public long ramBytesUsed() {
+        return 0;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapper.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java
similarity index 88%
rename from server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapper.java
rename to x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java
index d57dbf79b450c..873d67e76b04a 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapper.java
+++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.index.mapper.vectors;
+package org.elasticsearch.xpack.rank.vectors.mapper;
 
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.index.BinaryDocValues;
@@ -15,7 +13,6 @@
 import org.apache.lucene.search.FieldExistsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.fielddata.FieldDataContext;
@@ -31,7 +28,10 @@
 import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.TextSearchInfo;
 import org.elasticsearch.index.mapper.ValueFetcher;
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.license.LicenseUtils;
+import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.vectors.VectorData;
@@ -50,11 +50,11 @@
 import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT;
 import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT_BIT;
 import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.namesToElementType;
+import static org.elasticsearch.xpack.rank.vectors.RankVectorsPlugin.RANK_VECTORS_FEATURE;
 
 public class RankVectorsFieldMapper extends FieldMapper {
 
     public static final String VECTOR_MAGNITUDES_SUFFIX = "._magnitude";
-    public static final FeatureFlag FEATURE_FLAG = new FeatureFlag("rank_vectors");
     public static final String CONTENT_TYPE = "rank_vectors";
 
     private static RankVectorsFieldMapper toType(FieldMapper in) {
@@ -111,10 +111,12 @@ public static class Builder extends FieldMapper.Builder {
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
 
         private final IndexVersion indexCreatedVersion;
+        private final XPackLicenseState licenseState;
 
-        public Builder(String name, IndexVersion indexCreatedVersion) {
+        public Builder(String name, IndexVersion indexCreatedVersion, XPackLicenseState licenseState) {
             super(name);
             this.indexCreatedVersion = indexCreatedVersion;
+            this.licenseState = licenseState;
         }
 
         @Override
@@ -122,18 +124,12 @@ protected Parameter<?>[] getParameters() {
             return new Parameter<?>[] { elementType, dims, meta };
         }
 
-        public RankVectorsFieldMapper.Builder dimensions(int dimensions) {
-            this.dims.setValue(dimensions);
-            return this;
-        }
-
-        public RankVectorsFieldMapper.Builder elementType(DenseVectorFieldMapper.ElementType elementType) {
-            this.elementType.setValue(elementType);
-            return this;
-        }
-
         @Override
         public RankVectorsFieldMapper build(MapperBuilderContext context) {
+            // Validate on Mapping creation
+            if (RANK_VECTORS_FEATURE.check(licenseState) == false) {
+                throw LicenseUtils.newComplianceException("Rank Vectors");
+            }
             // Validate again here because the dimensions or element type could have been set programmatically,
             // which affects index option validity
             validate();
@@ -143,36 +139,32 @@ public RankVectorsFieldMapper build(MapperBuilderContext context) {
                     context.buildFullName(leafName()),
                     elementType.getValue(),
                     dims.getValue(),
-                    indexCreatedVersion,
+                    licenseState,
                     meta.getValue()
                 ),
                 builderParams(this, context),
-                indexCreatedVersion
+                indexCreatedVersion,
+                licenseState
             );
         }
     }
 
-    public static final TypeParser PARSER = new TypeParser(
-        (n, c) -> new RankVectorsFieldMapper.Builder(n, c.indexVersionCreated()),
-        notInMultiFields(CONTENT_TYPE)
-    );
-
     public static final class RankVectorsFieldType extends SimpleMappedFieldType {
         private final DenseVectorFieldMapper.ElementType elementType;
         private final Integer dims;
-        private final IndexVersion indexCreatedVersion;
+        private final XPackLicenseState licenseState;
 
         public RankVectorsFieldType(
             String name,
             DenseVectorFieldMapper.ElementType elementType,
             Integer dims,
-            IndexVersion indexCreatedVersion,
+            XPackLicenseState licenseState,
             Map<String, String> meta
         ) {
             super(name, false, false, true, TextSearchInfo.NONE, meta);
             this.elementType = elementType;
             this.dims = dims;
-            this.indexCreatedVersion = indexCreatedVersion;
+            this.licenseState = licenseState;
         }
 
         @Override
@@ -195,9 +187,7 @@ protected Object parseSourceValue(Object value) {
 
         @Override
         public DocValueFormat docValueFormat(String format, ZoneId timeZone) {
-            throw new IllegalArgumentException(
-                "Field [" + name() + "] of type [" + typeName() + "] doesn't support docvalue_fields or aggregations"
-            );
+            return DocValueFormat.DENSE_VECTOR;
         }
 
         @Override
@@ -207,7 +197,10 @@ public boolean isAggregatable() {
 
         @Override
         public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
-            return new RankVectorsIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, indexCreatedVersion, dims, elementType);
+            if (RANK_VECTORS_FEATURE.check(licenseState) == false) {
+                throw LicenseUtils.newComplianceException("Rank Vectors");
+            }
+            return new RankVectorsIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, dims, elementType);
         }
 
         @Override
@@ -230,10 +223,18 @@ DenseVectorFieldMapper.ElementType getElementType() {
     }
 
     private final IndexVersion indexCreatedVersion;
-
-    private RankVectorsFieldMapper(String simpleName, MappedFieldType fieldType, BuilderParams params, IndexVersion indexCreatedVersion) {
+    private final XPackLicenseState licenseState;
+
+    private RankVectorsFieldMapper(
+        String simpleName,
+        MappedFieldType fieldType,
+        BuilderParams params,
+        IndexVersion indexCreatedVersion,
+        XPackLicenseState licenseState
+    ) {
         super(simpleName, fieldType, params);
         this.indexCreatedVersion = indexCreatedVersion;
+        this.licenseState = licenseState;
     }
 
     @Override
@@ -248,6 +249,9 @@ public boolean parsesArrayValue() {
 
     @Override
     public void parse(DocumentParserContext context) throws IOException {
+        if (RANK_VECTORS_FEATURE.check(licenseState) == false) {
+            throw LicenseUtils.newComplianceException("Rank Vectors");
+        }
         if (context.doc().getByKey(fieldType().name()) != null) {
             throw new IllegalArgumentException(
                 "Field ["
@@ -281,10 +285,10 @@ public void parse(DocumentParserContext context) throws IOException {
                 fieldType().name(),
                 fieldType().elementType,
                 currentDims,
-                indexCreatedVersion,
+                licenseState,
                 fieldType().meta()
             );
-            Mapper update = new RankVectorsFieldMapper(leafName(), updatedFieldType, builderParams, indexCreatedVersion);
+            Mapper update = new RankVectorsFieldMapper(leafName(), updatedFieldType, builderParams, indexCreatedVersion, licenseState);
             context.addDynamicMapper(update);
             return;
         }
@@ -366,12 +370,12 @@ protected String contentType() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new RankVectorsFieldMapper.Builder(leafName(), indexCreatedVersion).init(this);
+        return new Builder(leafName(), indexCreatedVersion, licenseState).init(this);
     }
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport() {
-        return new SyntheticSourceSupport.Native(new RankVectorsFieldMapper.DocValuesSyntheticFieldLoader());
+        return new SyntheticSourceSupport.Native(new DocValuesSyntheticFieldLoader());
     }
 
     private class DocValuesSyntheticFieldLoader extends SourceLoader.DocValuesBasedSyntheticFieldLoader {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsIndexFieldData.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsIndexFieldData.java
similarity index 76%
rename from server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsIndexFieldData.java
rename to x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsIndexFieldData.java
index 7f54d2b9a8ad8..7ec426b904ce0 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsIndexFieldData.java
+++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsIndexFieldData.java
@@ -1,20 +1,18 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.index.mapper.vectors;
+package org.elasticsearch.xpack.rank.vectors.mapper;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.MultiValueMode;
@@ -26,19 +24,16 @@ public class RankVectorsIndexFieldData implements IndexFieldData<RankVectorsDVLe
     protected final String fieldName;
     protected final ValuesSourceType valuesSourceType;
     private final int dims;
-    private final IndexVersion indexVersion;
     private final DenseVectorFieldMapper.ElementType elementType;
 
     public RankVectorsIndexFieldData(
         String fieldName,
         int dims,
         ValuesSourceType valuesSourceType,
-        IndexVersion indexVersion,
         DenseVectorFieldMapper.ElementType elementType
     ) {
         this.fieldName = fieldName;
         this.valuesSourceType = valuesSourceType;
-        this.indexVersion = indexVersion;
         this.elementType = elementType;
         this.dims = dims;
     }
@@ -88,27 +83,19 @@ public static class Builder implements IndexFieldData.Builder {
 
         private final String name;
         private final ValuesSourceType valuesSourceType;
-        private final IndexVersion indexVersion;
         private final int dims;
         private final DenseVectorFieldMapper.ElementType elementType;
 
-        public Builder(
-            String name,
-            ValuesSourceType valuesSourceType,
-            IndexVersion indexVersion,
-            int dims,
-            DenseVectorFieldMapper.ElementType elementType
-        ) {
+        public Builder(String name, ValuesSourceType valuesSourceType, int dims, DenseVectorFieldMapper.ElementType elementType) {
             this.name = name;
             this.valuesSourceType = valuesSourceType;
-            this.indexVersion = indexVersion;
             this.dims = dims;
             this.elementType = elementType;
         }
 
         @Override
         public IndexFieldData<?> build(IndexFieldDataCache cache, CircuitBreakerService breakerService) {
-            return new RankVectorsIndexFieldData(name, dims, valuesSourceType, indexVersion, elementType);
+            return new RankVectorsIndexFieldData(name, dims, valuesSourceType, elementType);
         }
     }
 }
diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsPainlessExtension.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsPainlessExtension.java
new file mode 100644
index 0000000000000..5893452f5fdf8
--- /dev/null
+++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsPainlessExtension.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.rank.vectors.script;
+
+import org.elasticsearch.painless.spi.PainlessExtension;
+import org.elasticsearch.painless.spi.Whitelist;
+import org.elasticsearch.painless.spi.WhitelistLoader;
+import org.elasticsearch.script.ScoreScript;
+import org.elasticsearch.script.ScriptContext;
+
+import java.util.List;
+import java.util.Map;
+
+public class RankVectorsPainlessExtension implements PainlessExtension {
+    private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles(
+        RankVectorsPainlessExtension.class,
+        "rank_vector_whitelist.txt"
+    );
+
+    @Override
+    public Map<ScriptContext<?>, List<Whitelist>> getContextWhitelists() {
+        return Map.of(ScoreScript.CONTEXT, List.of(WHITELIST));
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/script/RankVectorsScoreScriptUtils.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java
similarity index 97%
rename from server/src/main/java/org/elasticsearch/script/RankVectorsScoreScriptUtils.java
rename to x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java
index 2d11641cb5aa7..3d692db08ff9e 100644
--- a/server/src/main/java/org/elasticsearch/script/RankVectorsScoreScriptUtils.java
+++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java
@@ -1,16 +1,15 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.script;
+package org.elasticsearch.xpack.rank.vectors.script;
 
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
+import org.elasticsearch.script.ScoreScript;
 import org.elasticsearch.script.field.vectors.DenseVector;
 import org.elasticsearch.script.field.vectors.RankVectorsDocValuesField;
 
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
similarity index 74%
rename from x-pack/plugin/snapshot-repo-test-kit/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
rename to x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
index ae11c3bb39d0b..388dbe7dead96 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
+++ b/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
@@ -5,4 +5,4 @@
 # 2.0.
 #
 
-org.elasticsearch.repositories.blobstore.testkit.SnapshotRepositoryTestKitFeatures
+org.elasticsearch.xpack.rank.vectors.RankVectorsFeatures
diff --git a/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension
new file mode 100644
index 0000000000000..06c77434c77a6
--- /dev/null
+++ b/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension
@@ -0,0 +1 @@
+org.elasticsearch.xpack.rank.vectors.script.RankVectorsPainlessExtension
diff --git a/x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/rank_vector_whitelist.txt b/x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/rank_vector_whitelist.txt
new file mode 100644
index 0000000000000..ce3d04f73dea8
--- /dev/null
+++ b/x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/rank_vector_whitelist.txt
@@ -0,0 +1,13 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0; you may not use this file except in compliance with the Elastic License
+# 2.0.
+#
+
+# This file contains a rank vectors whitelist for functions to be used in Score context
+
+static_import {
+    double maxSimDotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.xpack.rank.vectors.script.RankVectorsScoreScriptUtils$MaxSimDotProduct
+    double maxSimInvHamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.xpack.rank.vectors.script.RankVectorsScoreScriptUtils$MaxSimInvHamming
+}
diff --git a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/LocalStateRankVectors.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/LocalStateRankVectors.java
new file mode 100644
index 0000000000000..ba085b8f0d705
--- /dev/null
+++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/LocalStateRankVectors.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.rank.vectors;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.license.License;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.license.internal.XPackLicenseStatus;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+
+import java.util.Map;
+
+public class LocalStateRankVectors extends LocalStateCompositeXPackPlugin {
+
+    private final RankVectorsPlugin rankVectorsPlugin;
+    private final XPackLicenseState licenseState = new XPackLicenseState(
+        System::currentTimeMillis,
+        new XPackLicenseStatus(License.OperationMode.TRIAL, true, null)
+    );
+
+    public LocalStateRankVectors(Settings settings) {
+        super(settings, null);
+        LocalStateRankVectors thisVar = this;
+        rankVectorsPlugin = new RankVectorsPlugin() {
+            @Override
+            protected XPackLicenseState getLicenseState() {
+                return licenseState;
+            }
+        };
+    }
+
+    @Override
+    public Map<String, Mapper.TypeParser> getMappers() {
+        return rankVectorsPlugin.getMappers();
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapperTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java
similarity index 97%
rename from server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapperTests.java
rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java
index e81c28cbcc444..25264976c58a5 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapperTests.java
+++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java
@@ -1,13 +1,11 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.index.mapper.vectors;
+package org.elasticsearch.xpack.rank.vectors.mapper;
 
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.index.IndexableField;
@@ -29,18 +27,21 @@
 import org.elasticsearch.index.mapper.ValueFetcher;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.rank.vectors.LocalStateRankVectors;
 import org.junit.AssumptionViolatedException;
-import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.nio.FloatBuffer;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Stream;
@@ -53,11 +54,6 @@
 
 public class RankVectorsFieldMapperTests extends MapperTestCase {
 
-    @BeforeClass
-    public static void setup() {
-        assumeTrue("Requires rank vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled());
-    }
-
     private final ElementType elementType;
     private final int dims;
 
@@ -66,6 +62,11 @@ public RankVectorsFieldMapperTests() {
         this.dims = ElementType.BIT == elementType ? 4 * Byte.SIZE : 4;
     }
 
+    @Override
+    protected Collection<? extends Plugin> getPlugins() {
+        return Collections.singletonList(new LocalStateRankVectors(SETTINGS));
+    }
+
     @Override
     protected void minimalMapping(XContentBuilder b) throws IOException {
         indexMapping(b, IndexVersion.current());
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldTypeTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java
similarity index 67%
rename from server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldTypeTests.java
rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java
index b4cbbc4730d7c..59c5b0414910d 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldTypeTests.java
+++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java
@@ -1,20 +1,21 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.index.mapper.vectors;
+package org.elasticsearch.xpack.rank.vectors.mapper;
 
-import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
 import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper.RankVectorsFieldType;
-import org.junit.BeforeClass;
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
+import org.elasticsearch.license.License;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.license.internal.XPackLicenseStatus;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsFieldMapper.RankVectorsFieldType;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -25,23 +26,17 @@
 
 public class RankVectorsFieldTypeTests extends FieldTypeTestCase {
 
-    @BeforeClass
-    public static void setup() {
-        assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled());
-    }
+    private final XPackLicenseState licenseState = new XPackLicenseState(
+        System::currentTimeMillis,
+        new XPackLicenseStatus(License.OperationMode.TRIAL, true, null)
+    );
 
     private RankVectorsFieldType createFloatFieldType() {
-        return new RankVectorsFieldType(
-            "f",
-            DenseVectorFieldMapper.ElementType.FLOAT,
-            BBQ_MIN_DIMS,
-            IndexVersion.current(),
-            Collections.emptyMap()
-        );
+        return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.FLOAT, BBQ_MIN_DIMS, licenseState, Collections.emptyMap());
     }
 
-    private RankVectorsFieldType createByteFieldType() {
-        return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.BYTE, 5, IndexVersion.current(), Collections.emptyMap());
+    private RankVectorsFieldMapper.RankVectorsFieldType createByteFieldType() {
+        return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.BYTE, 5, licenseState, Collections.emptyMap());
     }
 
     public void testHasDocValues() {
@@ -84,9 +79,9 @@ public void testFielddataBuilder() {
 
     public void testDocValueFormat() {
         RankVectorsFieldType fft = createFloatFieldType();
-        expectThrows(IllegalArgumentException.class, () -> fft.docValueFormat(null, null));
+        assertEquals(DocValueFormat.DENSE_VECTOR, fft.docValueFormat(null, null));
         RankVectorsFieldType bft = createByteFieldType();
-        expectThrows(IllegalArgumentException.class, () -> bft.docValueFormat(null, null));
+        assertEquals(DocValueFormat.DENSE_VECTOR, bft.docValueFormat(null, null));
     }
 
     public void testFetchSourceValue() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValuesTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsScriptDocValuesTests.java
similarity index 95%
rename from server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValuesTests.java
rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsScriptDocValuesTests.java
index c38ed0f60f0ae..f0b00849557bd 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValuesTests.java
+++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsScriptDocValuesTests.java
@@ -1,24 +1,22 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.index.mapper.vectors;
+package org.elasticsearch.xpack.rank.vectors.mapper;
 
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType;
+import org.elasticsearch.index.mapper.vectors.RankVectorsScriptDocValues;
 import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField;
 import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField;
 import org.elasticsearch.script.field.vectors.RankVectors;
 import org.elasticsearch.script.field.vectors.RankVectorsDocValuesField;
 import org.elasticsearch.test.ESTestCase;
-import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -29,11 +27,6 @@
 
 public class RankVectorsScriptDocValuesTests extends ESTestCase {
 
-    @BeforeClass
-    public static void setup() {
-        assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled());
-    }
-
     public void testFloatGetVectorValueAndGetMagnitude() throws IOException {
         int dims = 3;
         float[][][] vectors = { { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } }, { { 1, 0, 2 } } };
diff --git a/server/src/test/java/org/elasticsearch/script/RankVectorsScoreScriptUtilsTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtilsTests.java
similarity index 93%
rename from server/src/test/java/org/elasticsearch/script/RankVectorsScoreScriptUtilsTests.java
rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtilsTests.java
index 917cc2069a293..da0340a22c074 100644
--- a/server/src/test/java/org/elasticsearch/script/RankVectorsScoreScriptUtilsTests.java
+++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtilsTests.java
@@ -1,26 +1,23 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.script;
+package org.elasticsearch.xpack.rank.vectors.script;
 
 import org.apache.lucene.util.VectorUtil;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType;
-import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper;
-import org.elasticsearch.index.mapper.vectors.RankVectorsScriptDocValuesTests;
-import org.elasticsearch.script.RankVectorsScoreScriptUtils.MaxSimDotProduct;
-import org.elasticsearch.script.RankVectorsScoreScriptUtils.MaxSimInvHamming;
+import org.elasticsearch.script.ScoreScript;
 import org.elasticsearch.script.field.vectors.BitRankVectorsDocValuesField;
 import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField;
 import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField;
 import org.elasticsearch.script.field.vectors.RankVectorsDocValuesField;
 import org.elasticsearch.test.ESTestCase;
-import org.junit.BeforeClass;
+import org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsScriptDocValuesTests;
+import org.elasticsearch.xpack.rank.vectors.script.RankVectorsScoreScriptUtils.MaxSimDotProduct;
+import org.elasticsearch.xpack.rank.vectors.script.RankVectorsScoreScriptUtils.MaxSimInvHamming;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -33,11 +30,6 @@
 
 public class RankVectorsScoreScriptUtilsTests extends ESTestCase {
 
-    @BeforeClass
-    public static void setup() {
-        assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled());
-    }
-
     public void testFloatMultiVectorClassBindings() throws IOException {
         String fieldName = "vector";
         int dims = 5;
@@ -88,7 +80,7 @@ public void testFloatMultiVectorClassBindings() throws IOException {
             // Check each function rejects query vectors with the wrong dimension
             IllegalArgumentException e = expectThrows(
                 IllegalArgumentException.class,
-                () -> new RankVectorsScoreScriptUtils.MaxSimDotProduct(scoreScript, invalidQueryVector, fieldName)
+                () -> new MaxSimDotProduct(scoreScript, invalidQueryVector, fieldName)
             );
             assertThat(
                 e.getMessage(),
@@ -336,7 +328,4 @@ public void testByteBoundaries() throws IOException {
         }
     }
 
-    public void testDimMismatch() throws IOException {
-
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/script/field/vectors/RankVectorsTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsTests.java
similarity index 80%
rename from server/src/test/java/org/elasticsearch/script/field/vectors/RankVectorsTests.java
rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsTests.java
index ca7608b10aed9..570e2a59926aa 100644
--- a/server/src/test/java/org/elasticsearch/script/field/vectors/RankVectorsTests.java
+++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsTests.java
@@ -1,19 +1,19 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
  */
 
-package org.elasticsearch.script.field.vectors;
+package org.elasticsearch.xpack.rank.vectors.script;
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.VectorUtil;
-import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper;
+import org.elasticsearch.script.field.vectors.ByteRankVectors;
+import org.elasticsearch.script.field.vectors.FloatRankVectors;
+import org.elasticsearch.script.field.vectors.RankVectors;
+import org.elasticsearch.script.field.vectors.VectorIterator;
 import org.elasticsearch.test.ESTestCase;
-import org.junit.BeforeClass;
 
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -21,11 +21,6 @@
 
 public class RankVectorsTests extends ESTestCase {
 
-    @BeforeClass
-    public static void setup() {
-        assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled());
-    }
-
     public void testByteUnsupported() {
         int count = randomIntBetween(1, 16);
         int dims = randomIntBetween(1, 16);
diff --git a/x-pack/plugin/redact/build.gradle b/x-pack/plugin/redact/build.gradle
index f179ce3a88cfa..e03567a4fe5aa 100644
--- a/x-pack/plugin/redact/build.gradle
+++ b/x-pack/plugin/redact/build.gradle
@@ -1,9 +1,9 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 esplugin {
-  name 'x-pack-redact'
-  description 'Elasticsearch Expanded Pack Plugin - Redact'
-  classname 'org.elasticsearch.xpack.redact.RedactPlugin'
+  name = 'x-pack-redact'
+  description = 'Elasticsearch Expanded Pack Plugin - Redact'
+  classname ='org.elasticsearch.xpack.redact.RedactPlugin'
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/repositories-metering-api/build.gradle b/x-pack/plugin/repositories-metering-api/build.gradle
index c9c3cbf842565..a9c706741e55c 100644
--- a/x-pack/plugin/repositories-metering-api/build.gradle
+++ b/x-pack/plugin/repositories-metering-api/build.gradle
@@ -2,9 +2,9 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-test-artifact'
 
 esplugin {
-  name 'repositories-metering-api'
-  description 'Repositories metering API'
-  classname 'org.elasticsearch.xpack.repositories.metering.RepositoriesMeteringPlugin'
+  name = 'repositories-metering-api'
+  description = 'Repositories metering API'
+  classname = 'org.elasticsearch.xpack.repositories.metering.RepositoriesMeteringPlugin'
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/rollup/build.gradle b/x-pack/plugin/rollup/build.gradle
index 4fca8e15d2472..83fd08cb18f96 100644
--- a/x-pack/plugin/rollup/build.gradle
+++ b/x-pack/plugin/rollup/build.gradle
@@ -1,8 +1,8 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 esplugin {
-  name 'x-pack-rollup'
-  description 'Elasticsearch Expanded Pack Plugin - Rollup'
-  classname 'org.elasticsearch.xpack.rollup.Rollup'
+  name = 'x-pack-rollup'
+  description = 'Elasticsearch Expanded Pack Plugin - Rollup'
+  classname ='org.elasticsearch.xpack.rollup.Rollup'
   extendedPlugins = ['x-pack-core']
 }
 
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java
index 3035bb98a3a93..3d04c9ed0739d 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java
@@ -199,7 +199,7 @@ static void createIndex(
             ActionListener.wrap(createIndexResponse -> startPersistentTask(job, listener, persistentTasksService), e -> {
                 if (e instanceof ResourceAlreadyExistsException) {
                     logger.debug("Rolled index already exists for rollup job [" + job.getConfig().getId() + "], updating metadata.");
-                    updateMapping(job, listener, persistentTasksService, client, logger);
+                    updateMapping(job, listener, persistentTasksService, client, logger, request.masterNodeTimeout());
                 } else {
                     String msg = "Could not create index for rollup job [" + job.getConfig().getId() + "]";
                     logger.error(msg);
@@ -249,7 +249,8 @@ static void updateMapping(
         ActionListener<AcknowledgedResponse> listener,
         PersistentTasksService persistentTasksService,
         Client client,
-        Logger logger
+        Logger logger,
+        TimeValue masterTimeout
     ) {
 
         final String indexName = job.getConfig().getRollupIndex();
@@ -304,7 +305,7 @@ static void updateMapping(
             );
         };
 
-        GetMappingsRequest request = new GetMappingsRequest();
+        GetMappingsRequest request = new GetMappingsRequest(masterTimeout);
         client.execute(GetMappingsAction.INSTANCE, request, ActionListener.wrap(getMappingResponseHandler, e -> {
             String msg = "Could not update mappings for rollup job [" + job.getConfig().getId() + "]";
             logger.error(msg);
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java
index ee8b4c79d1893..fed2439e513c8 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java
@@ -166,7 +166,14 @@ public void testGetMappingFails() {
             return null;
         }).when(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), requestCaptor.capture());
 
-        TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger);
+        TransportPutRollupJobAction.updateMapping(
+            job,
+            testListener,
+            mock(PersistentTasksService.class),
+            client,
+            logger,
+            TEST_REQUEST_TIMEOUT
+        );
         verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any());
     }
 
@@ -201,7 +208,14 @@ public void testNoMetadataInMapping() {
             return null;
         }).when(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), requestCaptor.capture());
 
-        TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger);
+        TransportPutRollupJobAction.updateMapping(
+            job,
+            testListener,
+            mock(PersistentTasksService.class),
+            client,
+            logger,
+            TEST_REQUEST_TIMEOUT
+        );
         verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any());
     }
 
@@ -238,7 +252,14 @@ public void testMetadataButNotRollup() {
             return null;
         }).when(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), requestCaptor.capture());
 
-        TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger);
+        TransportPutRollupJobAction.updateMapping(
+            job,
+            testListener,
+            mock(PersistentTasksService.class),
+            client,
+            logger,
+            TEST_REQUEST_TIMEOUT
+        );
         verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any());
     }
 
@@ -272,7 +293,14 @@ public void testJobAlreadyInMapping() {
             return null;
         }).when(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), requestCaptor.capture());
 
-        TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger);
+        TransportPutRollupJobAction.updateMapping(
+            job,
+            testListener,
+            mock(PersistentTasksService.class),
+            client,
+            logger,
+            TEST_REQUEST_TIMEOUT
+        );
         verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any());
     }
 
@@ -318,7 +346,14 @@ public void testAddJobToMapping() {
             return null;
         }).when(client).execute(eq(TransportPutMappingAction.TYPE), any(PutMappingRequest.class), requestCaptor2.capture());
 
-        TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger);
+        TransportPutRollupJobAction.updateMapping(
+            job,
+            testListener,
+            mock(PersistentTasksService.class),
+            client,
+            logger,
+            TEST_REQUEST_TIMEOUT
+        );
         verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any());
         verify(client).execute(eq(TransportPutMappingAction.TYPE), any(PutMappingRequest.class), any());
     }
diff --git a/x-pack/plugin/search-business-rules/build.gradle b/x-pack/plugin/search-business-rules/build.gradle
index 11e1207f7f3c7..cce6e15aedc06 100644
--- a/x-pack/plugin/search-business-rules/build.gradle
+++ b/x-pack/plugin/search-business-rules/build.gradle
@@ -2,9 +2,9 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  name 'search-business-rules'
-  description 'A plugin for applying business rules to search result rankings'
-  classname 'org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRules'
+  name = 'search-business-rules'
+  description = 'A plugin for applying business rules to search result rankings'
+  classname ='org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRules'
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/searchable-snapshots/build.gradle b/x-pack/plugin/searchable-snapshots/build.gradle
index 747e94b0e8d8d..571ee3c1101cb 100644
--- a/x-pack/plugin/searchable-snapshots/build.gradle
+++ b/x-pack/plugin/searchable-snapshots/build.gradle
@@ -3,9 +3,9 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-test-artifact'
 
 esplugin {
-  name 'searchable-snapshots'
-  description 'A plugin for the searchable snapshots functionality'
-  classname 'org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots'
+  name = 'searchable-snapshots'
+  description = 'A plugin for the searchable snapshots functionality'
+  classname = 'org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots'
   extendedPlugins = ['x-pack-core', 'blob-cache']
 }
 base {
diff --git a/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle b/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle
index b41e0f8dcc1cf..4577935e4e08d 100644
--- a/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle
+++ b/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle
@@ -12,7 +12,8 @@ apply plugin: 'elasticsearch.internal-available-ports'
 dependencies {
   clusterPlugins project(':plugins:repository-hdfs')
   javaRestTestImplementation(testArtifact(project(xpackModule('searchable-snapshots'))))
-  javaRestTestImplementation project(path: ':test:fixtures:hdfs-fixture', configuration:"shadowedHdfs2")
+  javaRestTestCompileOnly project(path: ':test:fixtures:hdfs-fixture')
+  javaRestTestRuntimeOnly project(path: ':test:fixtures:hdfs-fixture', configuration:"shadowedHdfs2")
   javaRestTestImplementation project(':test:fixtures:krb5kdc-fixture')
   javaRestTestRuntimeOnly "com.google.guava:guava:16.0.1"
   javaRestTestRuntimeOnly "commons-cli:commons-cli:1.2"
diff --git a/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle
index 1659c592e5e64..b95e07090baf7 100644
--- a/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle
+++ b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle
@@ -13,6 +13,7 @@ apply plugin: 'elasticsearch.rest-resources'
 dependencies {
   javaRestTestImplementation(testArtifact(project(xpackModule('searchable-snapshots'))))
   javaRestTestImplementation project(":test:framework")
+  javaRestTestImplementation project(':test:fixtures:aws-fixture-utils')
   javaRestTestImplementation project(':test:fixtures:s3-fixture')
 }
 
diff --git a/x-pack/plugin/searchable-snapshots/qa/s3/src/javaRestTest/java/org/elasticsearch/xpack/searchablesnapshots/s3/S3SearchableSnapshotsCredentialsReloadIT.java b/x-pack/plugin/searchable-snapshots/qa/s3/src/javaRestTest/java/org/elasticsearch/xpack/searchablesnapshots/s3/S3SearchableSnapshotsCredentialsReloadIT.java
index 989e5468c4fb3..bb5df5d37dba3 100644
--- a/x-pack/plugin/searchable-snapshots/qa/s3/src/javaRestTest/java/org/elasticsearch/xpack/searchablesnapshots/s3/S3SearchableSnapshotsCredentialsReloadIT.java
+++ b/x-pack/plugin/searchable-snapshots/qa/s3/src/javaRestTest/java/org/elasticsearch/xpack/searchablesnapshots/s3/S3SearchableSnapshotsCredentialsReloadIT.java
@@ -36,6 +36,7 @@
 import java.io.IOException;
 import java.util.function.UnaryOperator;
 
+import static fixture.aws.AwsCredentialsUtils.mutableAccessKey;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.allOf;
 
@@ -46,12 +47,7 @@ public class S3SearchableSnapshotsCredentialsReloadIT extends ESRestTestCase {
 
     private static volatile String repositoryAccessKey;
 
-    public static final S3HttpFixture s3Fixture = new S3HttpFixture(
-        true,
-        BUCKET,
-        BASE_PATH,
-        S3HttpFixture.mutableAccessKey(() -> repositoryAccessKey)
-    );
+    public static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, mutableAccessKey(() -> repositoryAccessKey));
 
     private static final MutableSettingsProvider keystoreSettings = new MutableSettingsProvider();
 
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java
index 2797202e5f24e..da60f9a84c3ba 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java
@@ -318,17 +318,17 @@ public void testCreateAndRestorePartialSearchableSnapshot() throws Exception {
         );
 
         if (deletedBeforeMount) {
-            assertThat(indicesAdmin().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(0));
+            assertThat(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, aliasName).get().getAliases().size(), equalTo(0));
             assertAcked(indicesAdmin().prepareAliases().addAlias(restoredIndexName, aliasName));
         } else if (indexName.equals(restoredIndexName) == false) {
-            assertThat(indicesAdmin().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(1));
+            assertThat(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, aliasName).get().getAliases().size(), equalTo(1));
             assertAcked(
                 indicesAdmin().prepareAliases()
                     .addAliasAction(IndicesAliasesRequest.AliasActions.remove().index(indexName).alias(aliasName).mustExist(true))
                     .addAlias(restoredIndexName, aliasName)
             );
         }
-        assertThat(indicesAdmin().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(1));
+        assertThat(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, aliasName).get().getAliases().size(), equalTo(1));
         assertTotalHits(aliasName, originalAllHits, originalBarHits);
 
         final var request = new ClusterAllocationExplainRequest(TEST_REQUEST_TIMEOUT).setIndex(restoredIndexName)
@@ -422,7 +422,7 @@ public void testCreateAndRestorePartialSearchableSnapshot() throws Exception {
         assertFalse(clonedIndexSettings.hasValue(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey()));
 
         assertAcked(indicesAdmin().prepareDelete(restoredIndexName));
-        assertThat(indicesAdmin().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(0));
+        assertThat(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, aliasName).get().getAliases().size(), equalTo(0));
         assertAcked(indicesAdmin().prepareAliases().addAlias(clonedIndexName, aliasName));
         assertTotalHits(aliasName, originalAllHits, originalBarHits);
     }
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java
index 84a632a419ead..25617028fe544 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java
@@ -264,17 +264,17 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception {
         assertThat(indexMetadata.getEventIngestedRange(), sameInstance(IndexLongFieldRange.UNKNOWN));
 
         if (deletedBeforeMount) {
-            assertThat(indicesAdmin().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(0));
+            assertThat(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, aliasName).get().getAliases().size(), equalTo(0));
             assertAcked(indicesAdmin().prepareAliases().addAlias(restoredIndexName, aliasName));
         } else if (indexName.equals(restoredIndexName) == false) {
-            assertThat(indicesAdmin().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(1));
+            assertThat(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, aliasName).get().getAliases().size(), equalTo(1));
             assertAcked(
                 indicesAdmin().prepareAliases()
                     .addAliasAction(IndicesAliasesRequest.AliasActions.remove().index(indexName).alias(aliasName).mustExist(true))
                     .addAlias(restoredIndexName, aliasName)
             );
         }
-        assertThat(indicesAdmin().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(1));
+        assertThat(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, aliasName).get().getAliases().size(), equalTo(1));
         assertTotalHits(aliasName, originalAllHits, originalBarHits);
 
         internalCluster().fullRestart();
@@ -347,7 +347,7 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception {
         assertFalse(clonedIndexSettings.hasValue(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey()));
 
         assertAcked(indicesAdmin().prepareDelete(restoredIndexName));
-        assertThat(indicesAdmin().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(0));
+        assertThat(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, aliasName).get().getAliases().size(), equalTo(0));
         assertAcked(indicesAdmin().prepareAliases().addAlias(clonedIndexName, aliasName));
         assertTotalHits(aliasName, originalAllHits, originalBarHits);
     }
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/CachedBlob.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/CachedBlob.java
index a434ac39f59e9..bacd0c750f525 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/CachedBlob.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/CachedBlob.java
@@ -7,10 +7,8 @@
 
 package org.elasticsearch.xpack.searchablesnapshots.cache.blob;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -60,14 +58,10 @@ private CachedBlob(Instant creationTime, String repository, String name, String
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_FOUNDATIONS)
-        // we can remove the version field when we no longer need to keep compatibility with <8.12
-        final int version = Version.CURRENT.id;
         builder.startObject();
         {
             builder.field("type", TYPE);
             builder.field(CREATION_TIME_FIELD, creationTime.toEpochMilli());
-            builder.field("version", version);
             builder.field("repository", repository);
             builder.startObject("blob");
             {
diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle
index ebd435bf653a2..d3425c1e42e5b 100644
--- a/x-pack/plugin/security/build.gradle
+++ b/x-pack/plugin/security/build.gradle
@@ -4,10 +4,10 @@ apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-test-artifact'
 
 esplugin {
-  name 'x-pack-security'
-  description 'Elasticsearch Expanded Pack Plugin - Security'
-  classname 'org.elasticsearch.xpack.security.Security'
-  requiresKeystore true
+  name = 'x-pack-security'
+  description = 'Elasticsearch Expanded Pack Plugin - Security'
+  classname = 'org.elasticsearch.xpack.security.Security'
+  requiresKeystore =true
   extendedPlugins = ['x-pack-core']
 }
 
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1EnrichUnavailableRemotesIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1EnrichUnavailableRemotesIT.java
new file mode 100644
index 0000000000000..19d742e32aa08
--- /dev/null
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1EnrichUnavailableRemotesIT.java
@@ -0,0 +1,358 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.remotecluster;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.json.JsonXContent;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.oneOf;
+
+public class CrossClusterEsqlRCS1EnrichUnavailableRemotesIT extends AbstractRemoteClusterSecurityTestCase {
+    private static final AtomicBoolean SSL_ENABLED_REF = new AtomicBoolean();
+
+    static {
+        fulfillingCluster = ElasticsearchCluster.local()
+            .name("fulfilling-cluster")
+            .nodes(1)
+            .module("x-pack-autoscaling")
+            .module("x-pack-esql")
+            .module("x-pack-enrich")
+            .module("x-pack-ml")
+            .module("ingest-common")
+            .apply(commonClusterConfig)
+            .setting("remote_cluster.port", "0")
+            .setting("xpack.ml.enabled", "false")
+            .setting("xpack.security.remote_cluster_server.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get()))
+            .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key")
+            .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt")
+            .setting("xpack.security.authc.token.enabled", "true")
+            .keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password")
+            .node(0, spec -> spec.setting("remote_cluster_server.enabled", "true"))
+            .build();
+
+        queryCluster = ElasticsearchCluster.local()
+            .name("query-cluster")
+            .module("x-pack-autoscaling")
+            .module("x-pack-esql")
+            .module("x-pack-enrich")
+            .module("x-pack-ml")
+            .module("ingest-common")
+            .apply(commonClusterConfig)
+            .setting("xpack.ml.enabled", "false")
+            .setting("xpack.security.remote_cluster_client.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get()))
+            .build();
+    }
+
+    @ClassRule
+    public static TestRule clusterRule = RuleChain.outerRule(fulfillingCluster).around(queryCluster);
+
+    private final String[] modes = { "_coordinator", "_remote" };
+
+    @Before
+    public void setupPreRequisites() throws IOException {
+        setupRolesAndPrivileges();
+        setSourceData();
+
+        var policy = createPolicy("employees-policy", "employees", "email", new String[] { "id", "designation" });
+        // Create the enrich policy on both clusters.
+        assertOK(client().performRequest(policy));
+        assertOK(performRequestAgainstFulfillingCluster(policy));
+
+        // Execute the enrich policy on both clusters.
+        var exec = executePolicy("employees-policy");
+        assertOK(client().performRequest(exec));
+        assertOK(performRequestAgainstFulfillingCluster(exec));
+    }
+
+    public void testEsqlEnrichWithSkipUnavailable() throws Exception {
+        esqlEnrichWithRandomSkipUnavailable();
+        esqlEnrichWithSkipUnavailableTrue();
+        esqlEnrichWithSkipUnavailableFalse();
+    }
+
+    private void esqlEnrichWithRandomSkipUnavailable() throws Exception {
+        configureRemoteCluster("my_remote_cluster", fulfillingCluster, true, randomBoolean(), randomBoolean());
+
+        String query = "FROM to-be-enr*,my_remote_cluster:to-be-enr* | ENRICH " + randomFrom(modes) + ":employees-policy | LIMIT 10";
+        Response response = client().performRequest(esqlRequest(query));
+
+        Map<String, Object> map = responseAsMap(response);
+        ArrayList<?> values = (ArrayList<?>) map.get("values");
+        Map<?, ?> clusters = (Map<?, ?>) map.get("_clusters");
+        Map<?, ?> clusterDetails = (Map<?, ?>) clusters.get("details");
+        Map<?, ?> localClusterDetails = (Map<?, ?>) clusterDetails.get("(local)");
+        Map<?, ?> remoteClusterDetails = (Map<?, ?>) clusterDetails.get("my_remote_cluster");
+
+        assertOK(response);
+        assertThat((int) map.get("took"), greaterThan(0));
+        assertThat(values.size(), is(6));
+        for (int i = 0; i < 6; i++) {
+            ArrayList<?> value = (ArrayList<?>) values.get(i);
+            // Size is 3: ID, Email, Designation.
+            assertThat(value.size(), is(3));
+            // Email
+            assertThat((String) value.get(0), endsWith("@corp.co"));
+            // ID
+            assertThat(value.get(1), is(i + 1));
+        }
+
+        assertThat((int) clusters.get("total"), is(2));
+        assertThat((int) clusters.get("successful"), is(2));
+        assertThat((int) clusters.get("running"), is(0));
+        assertThat((int) clusters.get("skipped"), is(0));
+        assertThat((int) clusters.get("partial"), is(0));
+        assertThat((int) clusters.get("failed"), is(0));
+
+        assertThat(clusterDetails.size(), is(2));
+        assertThat((int) localClusterDetails.get("took"), greaterThan(0));
+        assertThat(localClusterDetails.get("status"), is("successful"));
+
+        assertThat((int) remoteClusterDetails.get("took"), greaterThan(0));
+        assertThat(remoteClusterDetails.get("status"), is("successful"));
+    }
+
+    @SuppressWarnings("unchecked")
+    private void esqlEnrichWithSkipUnavailableTrue() throws Exception {
+        configureRemoteCluster("my_remote_cluster", fulfillingCluster, true, randomBoolean(), true);
+
+        try {
+            fulfillingCluster.stop(true);
+
+            String query = "FROM to-be-enriched,my_remote_cluster:to-be-enriched | ENRICH employees-policy | LIMIT 10";
+            Response response = client().performRequest(esqlRequest(query));
+
+            Map<String, Object> map = responseAsMap(response);
+            ArrayList<?> values = (ArrayList<?>) map.get("values");
+            Map<?, ?> clusters = (Map<?, ?>) map.get("_clusters");
+            Map<?, ?> clusterDetails = (Map<?, ?>) clusters.get("details");
+            Map<?, ?> localClusterDetails = (Map<?, ?>) clusterDetails.get("(local)");
+            Map<?, ?> remoteClusterDetails = (Map<?, ?>) clusterDetails.get("my_remote_cluster");
+
+            assertOK(response);
+            assertThat((int) map.get("took"), greaterThan(0));
+            assertThat(values.size(), is(3));
+
+            // We only have 3 values since the remote cluster is turned off.
+            for (int i = 0; i < 3; i++) {
+                ArrayList<?> value = (ArrayList<?>) values.get(i);
+                // Size is 3: ID, Email, Designation.
+                assertThat(value.size(), is(3));
+                // Email
+                assertThat((String) value.get(0), endsWith("@corp.co"));
+                // ID
+                assertThat(value.get(1), is(i + 1));
+            }
+
+            assertThat((int) clusters.get("total"), is(2));
+            assertThat((int) clusters.get("successful"), is(1));
+            assertThat((int) clusters.get("running"), is(0));
+            assertThat((int) clusters.get("skipped"), is(1));
+            assertThat((int) clusters.get("partial"), is(0));
+            assertThat((int) clusters.get("failed"), is(0));
+
+            assertThat(clusterDetails.size(), is(2));
+            assertThat((int) localClusterDetails.get("took"), greaterThan(0));
+            assertThat(localClusterDetails.get("status"), is("successful"));
+
+            assertThat((int) remoteClusterDetails.get("took"), greaterThan(0));
+            assertThat(remoteClusterDetails.get("status"), is("skipped"));
+
+            ArrayList<?> remoteClusterFailures = (ArrayList<?>) remoteClusterDetails.get("failures");
+            assertThat(remoteClusterFailures.size(), equalTo(1));
+            Map<String, ?> failuresMap = (Map<String, ?>) remoteClusterFailures.get(0);
+
+            Map<String, ?> reason = (Map<String, ?>) failuresMap.get("reason");
+            assertThat(reason.get("type").toString(), oneOf("node_disconnected_exception", "connect_transport_exception"));
+        } finally {
+            fulfillingCluster.start();
+            closeFulfillingClusterClient();
+            initFulfillingClusterClient();
+        }
+    }
+
+    private void esqlEnrichWithSkipUnavailableFalse() throws Exception {
+        configureRemoteCluster("my_remote_cluster", fulfillingCluster, true, randomBoolean(), false);
+
+        try {
+            fulfillingCluster.stop(true);
+
+            String query = "FROM to-be-enr*,my_remote_cluster:to-be-enr* | ENRICH " + randomFrom(modes) + ":employees-policy | LIMIT 10";
+            ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(query)));
+
+            assertThat(
+                ex.getMessage(),
+                anyOf(containsString("connect_transport_exception"), containsString("node_disconnected_exception"))
+            );
+        } finally {
+            fulfillingCluster.start();
+            closeFulfillingClusterClient();
+            initFulfillingClusterClient();
+        }
+    }
+
+    private void setupRolesAndPrivileges() throws IOException {
+        var putUserRequest = new Request("PUT", "/_security/user/" + REMOTE_SEARCH_USER);
+        putUserRequest.setJsonEntity("""
+            {
+              "password": "x-pack-test-password",
+              "roles" : ["remote_search"]
+            }""");
+        assertOK(adminClient().performRequest(putUserRequest));
+
+        var putRoleOnRemoteClusterRequest = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE);
+        putRoleOnRemoteClusterRequest.setJsonEntity("""
+            {
+              "indices": [
+                {
+                  "names": ["*"],
+                  "privileges": ["read"]
+                }
+              ],
+              "cluster": [ "monitor_enrich", "manage_own_api_key" ],
+              "remote_indices": [
+                {
+                  "names": ["*"],
+                  "privileges": ["read"],
+                  "clusters": ["my_remote_cluster"]
+                }
+              ],
+              "remote_cluster": [
+                {
+                  "privileges": ["monitor_enrich"],
+                  "clusters": ["my_remote_cluster"]
+                }
+              ]
+            }""");
+        assertOK(adminClient().performRequest(putRoleOnRemoteClusterRequest));
+    }
+
+    private void setSourceData() throws IOException {
+        Request createIndex = new Request("PUT", "employees");
+        createIndex.setJsonEntity("""
+            {
+                "mappings": {
+                    "properties": {
+                      "id": { "type": "integer" },
+                      "email": { "type": "text" },
+                      "designation": { "type": "text" }
+                    }
+                }
+            }
+            """);
+        assertOK(client().performRequest(createIndex));
+        assertOK(performRequestAgainstFulfillingCluster(createIndex));
+
+        Request bulkRequest = new Request("POST", "/_bulk?refresh=true");
+        bulkRequest.setJsonEntity("""
+            { "index": { "_index": "employees" } }
+            { "id": 1, "email": "a@corp.co", "designation": "SDE intern"}
+            { "index": { "_index": "employees" } }
+            { "id": 2, "email": "b@corp.co", "designation": "SDE 1"}
+            { "index": { "_index": "employees" } }
+            { "id": 3, "email": "c@corp.co", "designation": "SDE 2"}
+            { "index": { "_index": "employees" } }
+            { "id": 4, "email": "d@corp.co", "designation": "SSE"}
+            { "index": { "_index": "employees" } }
+            { "id": 5, "email": "e@corp.co", "designation": "PSE 1"}
+            { "index": { "_index": "employees" } }
+            { "id": 6, "email": "f@corp.co", "designation": "PSE 2"}
+            """);
+        assertOK(client().performRequest(bulkRequest));
+        assertOK(performRequestAgainstFulfillingCluster(bulkRequest));
+
+        createIndex = new Request("PUT", "to-be-enriched");
+        createIndex.setJsonEntity("""
+            {
+                "mappings": {
+                    "properties": {
+                      "email": { "type": "text" }
+                    }
+                }
+            }
+            """);
+        assertOK(client().performRequest(createIndex));
+        assertOK(performRequestAgainstFulfillingCluster(createIndex));
+
+        bulkRequest = new Request("POST", "/_bulk?refresh=true");
+        bulkRequest.setJsonEntity("""
+            { "index": { "_index": "to-be-enriched" } }
+            { "email": "a@corp.co"}
+            { "index": { "_index": "to-be-enriched" } }
+            { "email": "b@corp.co"}
+            { "index": { "_index": "to-be-enriched" } }
+            { "email": "c@corp.co"}
+            """);
+        assertOK(client().performRequest(bulkRequest));
+
+        bulkRequest = new Request("POST", "/_bulk?refresh=true");
+        bulkRequest.setJsonEntity("""
+            { "index": { "_index": "to-be-enriched" } }
+            { "email": "d@corp.co"}
+            { "index": { "_index": "to-be-enriched" } }
+            { "email": "e@corp.co"}
+            { "index": { "_index": "to-be-enriched" } }
+            { "email": "f@corp.co"}
+            """);
+        assertOK(performRequestAgainstFulfillingCluster(bulkRequest));
+    }
+
+    private Request createPolicy(String policyName, String matchIndex, String matchField, String[] enrichFields) throws IOException {
+        XContentBuilder body = JsonXContent.contentBuilder();
+        body.startObject();
+        body.startObject("match");
+        body.field("indices", matchIndex);
+        body.field("match_field", matchField);
+        body.field("enrich_fields", enrichFields);
+
+        body.endObject();
+        body.endObject();
+
+        return makeRequest("PUT", "_enrich/policy/" + policyName, body);
+    }
+
+    private Request executePolicy(String policyName) {
+        return new Request("PUT", "_enrich/policy/employees-policy/_execute");
+    }
+
+    private Request esqlRequest(String query) throws IOException {
+        XContentBuilder body = JsonXContent.contentBuilder();
+
+        body.startObject();
+        body.field("query", query);
+        body.field("include_ccs_metadata", true);
+        body.endObject();
+
+        return makeRequest("POST", "_query", body);
+    }
+
+    private Request makeRequest(String method, String endpoint, XContentBuilder requestBody) {
+        Request request = new Request(method, endpoint);
+        request.setJsonEntity(Strings.toString(requestBody));
+        return request;
+    }
+}
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS2EnrichUnavailableRemotesIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS2EnrichUnavailableRemotesIT.java
new file mode 100644
index 0000000000000..3f068cf3a04cc
--- /dev/null
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS2EnrichUnavailableRemotesIT.java
@@ -0,0 +1,384 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.remotecluster;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.json.JsonXContent;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.oneOf;
+
+public class CrossClusterEsqlRCS2EnrichUnavailableRemotesIT extends AbstractRemoteClusterSecurityTestCase {
+    private static final AtomicReference<Map<String, Object>> API_KEY_MAP_REF = new AtomicReference<>();
+
+    static {
+        fulfillingCluster = ElasticsearchCluster.local()
+            .name("fulfilling-cluster")
+            .nodes(1)
+            .module("x-pack-autoscaling")
+            .module("x-pack-esql")
+            .module("x-pack-enrich")
+            .module("x-pack-ml")
+            .module("ingest-common")
+            .apply(commonClusterConfig)
+            .setting("remote_cluster.port", "0")
+            .setting("xpack.ml.enabled", "false")
+            .setting("xpack.security.remote_cluster_server.ssl.enabled", "true")
+            .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key")
+            .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt")
+            .setting("xpack.security.authc.token.enabled", "true")
+            .keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password")
+            .node(0, spec -> spec.setting("remote_cluster_server.enabled", "true"))
+            .build();
+
+        queryCluster = ElasticsearchCluster.local()
+            .name("query-cluster")
+            .module("x-pack-autoscaling")
+            .module("x-pack-esql")
+            .module("x-pack-enrich")
+            .module("x-pack-ml")
+            .module("ingest-common")
+            .apply(commonClusterConfig)
+            .setting("xpack.ml.enabled", "false")
+            .setting("xpack.security.remote_cluster_client.ssl.enabled", "true")
+            .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt")
+            .setting("xpack.security.authc.token.enabled", "true")
+            .keystore("cluster.remote.my_remote_cluster.credentials", () -> {
+                if (API_KEY_MAP_REF.get() == null) {
+                    final Map<String, Object> apiKeyMap = createCrossClusterAccessApiKey("""
+                        {
+                          "search": [
+                            {
+                                "names": ["*"]
+                            }
+                          ]
+                        }""");
+                    API_KEY_MAP_REF.set(apiKeyMap);
+                }
+                return (String) API_KEY_MAP_REF.get().get("encoded");
+            })
+            .rolesFile(Resource.fromClasspath("roles.yml"))
+            .user(REMOTE_METRIC_USER, PASS.toString(), "read_remote_shared_metrics", false)
+            .build();
+    }
+
+    @ClassRule
+    public static TestRule clusterRule = RuleChain.outerRule(fulfillingCluster).around(queryCluster);
+
+    private String[] modes = { "_coordinator", "_remote" };
+
+    @Before
+    public void setupPreRequisites() throws IOException {
+        setupRolesAndPrivileges();
+        setSourceData();
+
+        var policy = createPolicy("employees-policy", "employees", "email", new String[] { "id", "designation" });
+        // Create the enrich policy on both clusters.
+        assertOK(client().performRequest(policy));
+        assertOK(performRequestAgainstFulfillingCluster(policy));
+
+        // Execute the enrich policy on both clusters.
+        var exec = executePolicy("employees-policy");
+        assertOK(client().performRequest(exec));
+        assertOK(performRequestAgainstFulfillingCluster(exec));
+    }
+
+    public void testEsqlEnrichWithSkipUnavailable() throws Exception {
+        esqlEnrichWithRandomSkipUnavailable();
+        esqlEnrichWithSkipUnavailableTrue();
+        esqlEnrichWithSkipUnavailableFalse();
+    }
+
+    private void esqlEnrichWithRandomSkipUnavailable() throws Exception {
+        configureRemoteCluster("my_remote_cluster", fulfillingCluster, false, randomBoolean(), randomBoolean());
+
+        String query = "FROM to-be-enr*,my_remote_cluster:to-be-enr* | ENRICH " + randomFrom(modes) + ":employees-policy | LIMIT 10";
+        Response response = performRequestWithRemoteSearchUser(esqlRequest(query));
+
+        Map<String, Object> map = responseAsMap(response);
+        ArrayList<?> values = (ArrayList<?>) map.get("values");
+        Map<?, ?> clusters = (Map<?, ?>) map.get("_clusters");
+        Map<?, ?> clusterDetails = (Map<?, ?>) clusters.get("details");
+        Map<?, ?> localClusterDetails = (Map<?, ?>) clusterDetails.get("(local)");
+        Map<?, ?> remoteClusterDetails = (Map<?, ?>) clusterDetails.get("my_remote_cluster");
+
+        assertOK(response);
+        assertThat((int) map.get("took"), greaterThan(0));
+        assertThat(values.size(), is(6));
+        for (int i = 0; i < 6; i++) {
+            ArrayList<?> value = (ArrayList<?>) values.get(i);
+            // Size is 3: ID, Email, Designation.
+            assertThat(value.size(), is(3));
+            // Email
+            assertThat((String) value.get(0), endsWith("@corp.co"));
+            // ID
+            assertThat(value.get(1), is(i + 1));
+        }
+
+        assertThat((int) clusters.get("total"), is(2));
+        assertThat((int) clusters.get("successful"), is(2));
+        assertThat((int) clusters.get("running"), is(0));
+        assertThat((int) clusters.get("skipped"), is(0));
+        assertThat((int) clusters.get("partial"), is(0));
+        assertThat((int) clusters.get("failed"), is(0));
+
+        assertThat(clusterDetails.size(), is(2));
+        assertThat((int) localClusterDetails.get("took"), greaterThan(0));
+        assertThat(localClusterDetails.get("status"), is("successful"));
+
+        assertThat((int) remoteClusterDetails.get("took"), greaterThan(0));
+        assertThat(remoteClusterDetails.get("status"), is("successful"));
+    }
+
+    @SuppressWarnings("unchecked")
+    private void esqlEnrichWithSkipUnavailableTrue() throws Exception {
+        configureRemoteCluster("my_remote_cluster", fulfillingCluster, false, randomBoolean(), true);
+
+        try {
+            fulfillingCluster.stop(true);
+
+            String query = "FROM to-be-enriched,my_remote_cluster:to-be-enriched | ENRICH employees-policy | LIMIT 10";
+            Response response = performRequestWithRemoteSearchUser(esqlRequest(query));
+
+            Map<String, Object> map = responseAsMap(response);
+            ArrayList<?> values = (ArrayList<?>) map.get("values");
+            Map<?, ?> clusters = (Map<?, ?>) map.get("_clusters");
+            Map<?, ?> clusterDetails = (Map<?, ?>) clusters.get("details");
+            Map<?, ?> localClusterDetails = (Map<?, ?>) clusterDetails.get("(local)");
+            Map<?, ?> remoteClusterDetails = (Map<?, ?>) clusterDetails.get("my_remote_cluster");
+
+            assertOK(response);
+            assertThat((int) map.get("took"), greaterThan(0));
+            assertThat(values.size(), is(3));
+
+            // We only have 3 values since the remote cluster is turned off.
+            for (int i = 0; i < 3; i++) {
+                ArrayList<?> value = (ArrayList<?>) values.get(i);
+                // Size is 3: ID, Email, Designation.
+                assertThat(value.size(), is(3));
+                // Email
+                assertThat((String) value.get(0), endsWith("@corp.co"));
+                // ID
+                assertThat(value.get(1), is(i + 1));
+            }
+
+            assertThat((int) clusters.get("total"), is(2));
+            assertThat((int) clusters.get("successful"), is(1));
+            assertThat((int) clusters.get("running"), is(0));
+            assertThat((int) clusters.get("skipped"), is(1));
+            assertThat((int) clusters.get("partial"), is(0));
+            assertThat((int) clusters.get("failed"), is(0));
+
+            assertThat(clusterDetails.size(), is(2));
+            assertThat((int) localClusterDetails.get("took"), greaterThan(0));
+            assertThat(localClusterDetails.get("status"), is("successful"));
+
+            assertThat((int) remoteClusterDetails.get("took"), greaterThan(0));
+            assertThat(remoteClusterDetails.get("status"), is("skipped"));
+
+            ArrayList<?> remoteClusterFailures = (ArrayList<?>) remoteClusterDetails.get("failures");
+            assertThat(remoteClusterFailures.size(), equalTo(1));
+            Map<String, ?> failuresMap = (Map<String, ?>) remoteClusterFailures.get(0);
+
+            Map<String, ?> reason = (Map<String, ?>) failuresMap.get("reason");
+            assertThat(reason.get("type").toString(), oneOf("node_disconnected_exception", "connect_transport_exception"));
+        } finally {
+            fulfillingCluster.start();
+            closeFulfillingClusterClient();
+            initFulfillingClusterClient();
+        }
+    }
+
+    private void esqlEnrichWithSkipUnavailableFalse() throws Exception {
+        configureRemoteCluster("my_remote_cluster", fulfillingCluster, false, randomBoolean(), false);
+
+        try {
+            fulfillingCluster.stop(true);
+
+            String query = "FROM to-be-enr*,my_remote_cluster:to-be-enr* | ENRICH " + randomFrom(modes) + ":employees-policy | LIMIT 10";
+            ResponseException ex = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(esqlRequest(query)));
+            assertThat(
+                ex.getMessage(),
+                anyOf(containsString("connect_transport_exception"), containsString("node_disconnected_exception"))
+            );
+        } finally {
+            fulfillingCluster.start();
+            closeFulfillingClusterClient();
+            initFulfillingClusterClient();
+        }
+    }
+
+    private void setupRolesAndPrivileges() throws IOException {
+        var putUserRequest = new Request("PUT", "/_security/user/" + REMOTE_SEARCH_USER);
+        putUserRequest.setJsonEntity("""
+            {
+              "password": "x-pack-test-password",
+              "roles" : ["remote_search"]
+            }""");
+        assertOK(adminClient().performRequest(putUserRequest));
+
+        var putRoleOnRemoteClusterRequest = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE);
+        putRoleOnRemoteClusterRequest.setJsonEntity("""
+            {
+              "indices": [
+                {
+                  "names": ["*"],
+                  "privileges": ["read"]
+                }
+              ],
+              "cluster": [ "monitor_enrich", "manage_own_api_key" ],
+              "remote_indices": [
+                {
+                  "names": ["*"],
+                  "privileges": ["read"],
+                  "clusters": ["my_remote_cluster"]
+                }
+              ],
+              "remote_cluster": [
+                {
+                  "privileges": ["monitor_enrich"],
+                  "clusters": ["my_remote_cluster"]
+                }
+              ]
+            }""");
+        assertOK(adminClient().performRequest(putRoleOnRemoteClusterRequest));
+    }
+
+    private void setSourceData() throws IOException {
+        Request createIndex = new Request("PUT", "employees");
+        createIndex.setJsonEntity("""
+            {
+                "mappings": {
+                    "properties": {
+                      "id": { "type": "integer" },
+                      "email": { "type": "text" },
+                      "designation": { "type": "text" }
+                    }
+                }
+            }
+            """);
+        assertOK(client().performRequest(createIndex));
+        assertOK(performRequestAgainstFulfillingCluster(createIndex));
+
+        Request bulkRequest = new Request("POST", "/_bulk?refresh=true");
+        bulkRequest.setJsonEntity("""
+            { "index": { "_index": "employees" } }
+            { "id": 1, "email": "a@corp.co", "designation": "SDE intern"}
+            { "index": { "_index": "employees" } }
+            { "id": 2, "email": "b@corp.co", "designation": "SDE 1"}
+            { "index": { "_index": "employees" } }
+            { "id": 3, "email": "c@corp.co", "designation": "SDE 2"}
+            { "index": { "_index": "employees" } }
+            { "id": 4, "email": "d@corp.co", "designation": "SSE"}
+            { "index": { "_index": "employees" } }
+            { "id": 5, "email": "e@corp.co", "designation": "PSE 1"}
+            { "index": { "_index": "employees" } }
+            { "id": 6, "email": "f@corp.co", "designation": "PSE 2"}
+            """);
+        assertOK(client().performRequest(bulkRequest));
+        assertOK(performRequestAgainstFulfillingCluster(bulkRequest));
+
+        createIndex = new Request("PUT", "to-be-enriched");
+        createIndex.setJsonEntity("""
+            {
+                "mappings": {
+                    "properties": {
+                      "email": { "type": "text" }
+                    }
+                }
+            }
+            """);
+        assertOK(client().performRequest(createIndex));
+        assertOK(performRequestAgainstFulfillingCluster(createIndex));
+
+        bulkRequest = new Request("POST", "/_bulk?refresh=true");
+        bulkRequest.setJsonEntity("""
+            { "index": { "_index": "to-be-enriched" } }
+            { "email": "a@corp.co"}
+            { "index": { "_index": "to-be-enriched" } }
+            { "email": "b@corp.co"}
+            { "index": { "_index": "to-be-enriched" } }
+            { "email": "c@corp.co"}
+            """);
+        assertOK(client().performRequest(bulkRequest));
+
+        bulkRequest = new Request("POST", "/_bulk?refresh=true");
+        bulkRequest.setJsonEntity("""
+            { "index": { "_index": "to-be-enriched" } }
+            { "email": "d@corp.co"}
+            { "index": { "_index": "to-be-enriched" } }
+            { "email": "e@corp.co"}
+            { "index": { "_index": "to-be-enriched" } }
+            { "email": "f@corp.co"}
+            """);
+        assertOK(performRequestAgainstFulfillingCluster(bulkRequest));
+    }
+
+    private Request createPolicy(String policyName, String matchIndex, String matchField, String[] enrichFields) throws IOException {
+        XContentBuilder body = JsonXContent.contentBuilder();
+        body.startObject();
+        body.startObject("match");
+        body.field("indices", matchIndex);
+        body.field("match_field", matchField);
+        body.field("enrich_fields", enrichFields);
+
+        body.endObject();
+        body.endObject();
+
+        return makeRequest("PUT", "_enrich/policy/" + policyName, body);
+    }
+
+    private Request executePolicy(String policyName) {
+        return new Request("PUT", "_enrich/policy/employees-policy/_execute");
+    }
+
+    private Request esqlRequest(String query) throws IOException {
+        XContentBuilder body = JsonXContent.contentBuilder();
+
+        body.startObject();
+        body.field("query", query);
+        body.field("include_ccs_metadata", true);
+        body.endObject();
+
+        return makeRequest("POST", "_query", body);
+    }
+
+    private Request makeRequest(String method, String endpoint, XContentBuilder requestBody) {
+        Request request = new Request(method, endpoint);
+        request.setJsonEntity(Strings.toString(requestBody));
+        return request;
+    }
+
+    private Response performRequestWithRemoteSearchUser(final Request request) throws IOException {
+        request.setOptions(
+            RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", headerFromRandomAuthMethod(REMOTE_SEARCH_USER, PASS))
+        );
+        return client().performRequest(request);
+    }
+}
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRCS1ResolveClusterIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRCS1ResolveClusterIT.java
index 813739a8e0d06..f22f642bb9151 100644
--- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRCS1ResolveClusterIT.java
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRCS1ResolveClusterIT.java
@@ -93,7 +93,9 @@ public void testResolveClusterUnderRCS1() throws Exception {
             assertLocalMatching(responseMap);
 
             Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
-            assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
+            // with security exceptions, the remote should be marked as connected=false, since you can't tell whether a security
+            // exception comes from the local cluster (intercepted) or the remote
+            assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false));
             assertThat((String) remoteClusterResponse.get("error"), containsString("unauthorized for user [remote_search_user]"));
 
             // TEST CASE 2: Query cluster -> add user role and user on remote cluster and try resolve again
@@ -171,7 +173,7 @@ public void testResolveClusterUnderRCS1() throws Exception {
             Map<String, Object> responseMap = responseAsMap(response);
             assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue());
             Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
-            assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
+            assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false));
             assertThat((String) remoteClusterResponse.get("error"), containsString("unauthorized for user [remote_search_user]"));
             assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [secretindex]"));
         }
@@ -183,7 +185,7 @@ public void testResolveClusterUnderRCS1() throws Exception {
             Map<String, Object> responseMap = responseAsMap(response);
             assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue());
             Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
-            assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
+            assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false));
             assertThat((String) remoteClusterResponse.get("error"), containsString("unauthorized for user [remote_search_user]"));
             assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [doesnotexist]"));
         }
@@ -195,6 +197,7 @@ public void testResolveClusterUnderRCS1() throws Exception {
             Map<String, Object> responseMap = responseAsMap(response);
             assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue());
             Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
+            // with IndexNotFoundExceptions, we know that error came from the remote cluster, so we can mark the remote as connected=true
             assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
             assertThat((String) remoteClusterResponse.get("error"), containsString("no such index [index99]"));
         }
@@ -210,7 +213,7 @@ public void testResolveClusterUnderRCS1() throws Exception {
             Map<String, Object> responseMap = responseAsMap(response);
             assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue());
             Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
-            assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
+            assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false));
             assertThat((String) remoteClusterResponse.get("error"), containsString("unauthorized for user [remote_search_user]"));
             assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [secretindex]"));
         }
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRCS2ResolveClusterIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRCS2ResolveClusterIT.java
index a3bc56dafce98..a1f897b5b0503 100644
--- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRCS2ResolveClusterIT.java
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRCS2ResolveClusterIT.java
@@ -180,7 +180,9 @@ public void testResolveCluster() throws Exception {
             assertLocalMatching(responseMap);
 
             Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
-            assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
+            // with security exceptions, the remote should be marked as connected=false, since you can't tell whether a security
+            // exception comes from the local cluster (intercepted) or the remote
+            assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false));
             assertThat((String) remoteClusterResponse.get("error"), containsString("is unauthorized for user"));
             assertThat(
                 (String) remoteClusterResponse.get("error"),
@@ -261,7 +263,7 @@ public void testResolveCluster() throws Exception {
             Map<String, Object> responseMap = responseAsMap(response);
             assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue());
             Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
-            assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
+            assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false));
             assertThat((String) remoteClusterResponse.get("error"), containsString("is unauthorized for user"));
             assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [secretindex]"));
         }
@@ -273,7 +275,7 @@ public void testResolveCluster() throws Exception {
             Map<String, Object> responseMap = responseAsMap(response);
             assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue());
             Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
-            assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
+            assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false));
             assertThat((String) remoteClusterResponse.get("error"), containsString("is unauthorized for user"));
             assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [doesnotexist]"));
         }
@@ -285,6 +287,7 @@ public void testResolveCluster() throws Exception {
             Map<String, Object> responseMap = responseAsMap(response);
             assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue());
             Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
+            // with IndexNotFoundExceptions, we know that error came from the remote cluster, so we can mark the remote as connected=true
             assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
             assertThat((Boolean) remoteClusterResponse.get("skip_unavailable"), equalTo(false));
             assertThat((String) remoteClusterResponse.get("error"), containsString("no such index [index99]"));
@@ -301,7 +304,7 @@ public void testResolveCluster() throws Exception {
             Map<String, Object> responseMap = responseAsMap(response);
             assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue());
             Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
-            assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
+            assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false));
             assertThat((String) remoteClusterResponse.get("error"), containsString("is unauthorized for user"));
             assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [secretindex]"));
         }
diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle b/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle
index e9abb413402de..e3bad85a4256d 100644
--- a/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle
+++ b/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle
@@ -3,9 +3,9 @@ apply plugin: 'elasticsearch.base-internal-es-plugin'
 apply plugin: 'elasticsearch.internal-java-rest-test'
 
 esplugin {
-  name 'operator-privileges-test'
-  description 'An test plugin for testing hard to get internals'
-  classname 'org.elasticsearch.xpack.security.operator.OperatorPrivilegesTestPlugin'
+  name = 'operator-privileges-test'
+  description = 'An test plugin for testing hard to get internals'
+  classname = 'org.elasticsearch.xpack.security.operator.OperatorPrivilegesTestPlugin'
 }
 
 dependencies {
diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
index c07e4b2c541a2..82bad85fa34dc 100644
--- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
+++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
@@ -115,6 +115,7 @@ public class Constants {
         "cluster:admin/transform/schedule_now",
         "cluster:admin/transform/update",
         "cluster:admin/transform/upgrade",
+        "cluster:admin/transform/upgrade_mode",
         "cluster:admin/transform/validate",
         // "cluster:admin/voting_config/add_exclusions",
         // "cluster:admin/voting_config/clear_exclusions",
@@ -133,40 +134,40 @@ public class Constants {
         "cluster:admin/xpack/ccr/auto_follow_pattern/put",
         "cluster:admin/xpack/ccr/pause_follow",
         "cluster:admin/xpack/ccr/resume_follow",
-        "indices:data/write/xpack/connector/delete",
-        "indices:data/read/xpack/connector/get",
-        "indices:data/read/xpack/connector/list",
-        "indices:data/write/xpack/connector/post",
-        "indices:data/write/xpack/connector/put",
-        "indices:data/write/xpack/connector/update_api_key_id",
-        "indices:data/write/xpack/connector/update_configuration",
-        "indices:data/write/xpack/connector/update_error",
-        "indices:data/write/xpack/connector/update_features",
-        "indices:data/write/xpack/connector/update_filtering",
-        "indices:data/write/xpack/connector/update_filtering/activate",
-        "indices:data/write/xpack/connector/update_filtering/draft_validation",
-        "indices:data/write/xpack/connector/update_index_name",
-        "indices:data/write/xpack/connector/update_last_seen",
-        "indices:data/write/xpack/connector/update_last_sync_stats",
-        "indices:data/write/xpack/connector/update_name",
-        "indices:data/write/xpack/connector/update_native",
-        "indices:data/write/xpack/connector/update_pipeline",
-        "indices:data/write/xpack/connector/update_scheduling",
-        "indices:data/write/xpack/connector/update_service_type",
-        "indices:data/write/xpack/connector/update_status",
+        "cluster:admin/xpack/connector/delete",
+        "cluster:admin/xpack/connector/get",
+        "cluster:admin/xpack/connector/list",
+        "cluster:admin/xpack/connector/post",
+        "cluster:admin/xpack/connector/put",
+        "cluster:admin/xpack/connector/update_api_key_id",
+        "cluster:admin/xpack/connector/update_configuration",
+        "cluster:admin/xpack/connector/update_error",
+        "cluster:admin/xpack/connector/update_features",
+        "cluster:admin/xpack/connector/update_filtering",
+        "cluster:admin/xpack/connector/update_filtering/activate",
+        "cluster:admin/xpack/connector/update_filtering/draft_validation",
+        "cluster:admin/xpack/connector/update_index_name",
+        "cluster:admin/xpack/connector/update_last_seen",
+        "cluster:admin/xpack/connector/update_last_sync_stats",
+        "cluster:admin/xpack/connector/update_name",
+        "cluster:admin/xpack/connector/update_native",
+        "cluster:admin/xpack/connector/update_pipeline",
+        "cluster:admin/xpack/connector/update_scheduling",
+        "cluster:admin/xpack/connector/update_service_type",
+        "cluster:admin/xpack/connector/update_status",
         "cluster:admin/xpack/connector/secret/delete",
         "cluster:admin/xpack/connector/secret/get",
         "cluster:admin/xpack/connector/secret/post",
         "cluster:admin/xpack/connector/secret/put",
-        "indices:data/write/xpack/connector/sync_job/cancel",
-        "indices:data/write/xpack/connector/sync_job/check_in",
-        "indices:data/write/xpack/connector/sync_job/claim",
-        "indices:data/write/xpack/connector/sync_job/delete",
-        "indices:data/read/xpack/connector/sync_job/get",
-        "indices:data/read/xpack/connector/sync_job/list",
-        "indices:data/write/xpack/connector/sync_job/post",
-        "indices:data/write/xpack/connector/sync_job/update_error",
-        "indices:data/write/xpack/connector/sync_job/update_stats",
+        "cluster:admin/xpack/connector/sync_job/cancel",
+        "cluster:admin/xpack/connector/sync_job/check_in",
+        "cluster:admin/xpack/connector/sync_job/claim",
+        "cluster:admin/xpack/connector/sync_job/delete",
+        "cluster:admin/xpack/connector/sync_job/get",
+        "cluster:admin/xpack/connector/sync_job/list",
+        "cluster:admin/xpack/connector/sync_job/post",
+        "cluster:admin/xpack/connector/sync_job/update_error",
+        "cluster:admin/xpack/connector/sync_job/update_stats",
         "cluster:admin/xpack/deprecation/info",
         "cluster:admin/xpack/deprecation/nodes/info",
         "cluster:admin/xpack/enrich/delete",
@@ -641,6 +642,7 @@ public class Constants {
         new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/data_stream/index/reindex" : null,
         new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/data_stream/reindex" : null,
         new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/data_stream/reindex_cancel" : null,
+        new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/index/create_from_source" : null,
         "internal:admin/repository/verify",
         "internal:admin/repository/verify/coordinate"
     ).filter(Objects::nonNull).collect(Collectors.toUnmodifiableSet());
diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/OperatorPrivilegesIT.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/OperatorPrivilegesIT.java
index 9e680688edeaa..16485479e8a45 100644
--- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/OperatorPrivilegesIT.java
+++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/OperatorPrivilegesIT.java
@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.security.operator;
 
+import org.elasticsearch.Build;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
@@ -165,6 +166,7 @@ public void testServiceAccountUpdateOperatorSettings() throws IOException {
 
     @SuppressWarnings("unchecked")
     public void testEveryActionIsEitherOperatorOnlyOrNonOperator() throws IOException {
+        assumeTrue("Exclude release builds due to maintenance burden around feature flags", Build.current().isSnapshot());
         final String message = "An action should be declared to be either operator-only in ["
             + DefaultOperatorOnlyRegistry.class.getName()
             + "] or non-operator in ["
diff --git a/x-pack/plugin/security/qa/secondary-auth-actions/build.gradle b/x-pack/plugin/security/qa/secondary-auth-actions/build.gradle
index f805dc74c4ca0..b99bc6594a5e2 100644
--- a/x-pack/plugin/security/qa/secondary-auth-actions/build.gradle
+++ b/x-pack/plugin/security/qa/secondary-auth-actions/build.gradle
@@ -4,9 +4,9 @@ apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.base-internal-es-plugin'
 
 esplugin {
-  name 'secondary-auth-actions-extension'
-  description 'Spi extension plugin for security to enforce custom secondary auth actions'
-  classname 'org.elasticsearch.secondary.auth.actions.SecondaryAuthActionsPlugin'
+  name = 'secondary-auth-actions-extension'
+  description = 'Spi extension plugin for security to enforce custom secondary auth actions'
+  classname ='org.elasticsearch.secondary.auth.actions.SecondaryAuthActionsPlugin'
   extendedPlugins = ['x-pack-security']
 }
 
diff --git a/x-pack/plugin/security/qa/security-basic/build.gradle b/x-pack/plugin/security/qa/security-basic/build.gradle
index 8740354646346..766598edc6f01 100644
--- a/x-pack/plugin/security/qa/security-basic/build.gradle
+++ b/x-pack/plugin/security/qa/security-basic/build.gradle
@@ -4,20 +4,31 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
+apply plugin: 'elasticsearch.base-internal-es-plugin'
 apply plugin: 'elasticsearch.internal-java-rest-test'
 
+
+esplugin {
+  name = 'queryable-reserved-roles-test'
+  description = 'A test plugin for testing that changes to reserved roles are made queryable'
+  classname = 'org.elasticsearch.xpack.security.role.QueryableBuiltInRolesTestPlugin'
+  extendedPlugins = ['x-pack-core', 'x-pack-security']
+}
 dependencies {
   javaRestTestImplementation(testArtifact(project(xpackModule('security'))))
   javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
+  compileOnly project(':x-pack:plugin:core')
+  compileOnly project(':x-pack:plugin:security')
+  clusterPlugins project(':x-pack:plugin:security:qa:security-basic')
 }
 
 tasks.named('javaRestTest') {
   usesDefaultDistribution()
 }
 
+tasks.named("javadoc").configure { enabled = false }
 
-if (buildParams.inFipsJvm){
+if (buildParams.inFipsJvm) {
   // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC
-  tasks.named("javaRestTest").configure{enabled = false }
+  tasks.named("javaRestTest").configure { enabled = false }
 }
diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryRoleIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryRoleIT.java
index 1588749b9a331..311510352d805 100644
--- a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryRoleIT.java
+++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryRoleIT.java
@@ -496,7 +496,7 @@ private RoleDescriptor createRole(
         );
     }
 
-    private void assertQuery(String body, int total, Consumer<List<Map<String, Object>>> roleVerifier) throws IOException {
+    static void assertQuery(String body, int total, Consumer<List<Map<String, Object>>> roleVerifier) throws IOException {
         assertQuery(client(), body, total, roleVerifier);
     }
 
diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryableReservedRolesIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryableReservedRolesIT.java
new file mode 100644
index 0000000000000..7adff21d8df4f
--- /dev/null
+++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryableReservedRolesIT.java
@@ -0,0 +1,354 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.test.AnnotationTestOrdering;
+import org.elasticsearch.test.AnnotationTestOrdering.Order;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.MutableSettingsProvider;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.local.model.User;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.ObjectPath;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
+import org.elasticsearch.xpack.core.security.test.TestRestrictedIndices;
+import org.elasticsearch.xpack.security.support.QueryableBuiltInRolesSynchronizer;
+import org.elasticsearch.xpack.security.support.SecurityMigrations;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7;
+import static org.elasticsearch.xpack.security.QueryRoleIT.assertQuery;
+import static org.elasticsearch.xpack.security.QueryRoleIT.waitForMigrationCompletion;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.iterableWithSize;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.oneOf;
+
+@TestCaseOrdering(AnnotationTestOrdering.class)
+public class QueryableReservedRolesIT extends ESRestTestCase {
+
+    protected static final String REST_USER = "security_test_user";
+    private static final SecureString REST_PASSWORD = new SecureString("security-test-password".toCharArray());
+    private static final String ADMIN_USER = "admin_user";
+    private static final SecureString ADMIN_PASSWORD = new SecureString("admin-password".toCharArray());
+    protected static final String READ_SECURITY_USER = "read_security_user";
+    private static final SecureString READ_SECURITY_PASSWORD = new SecureString("read-security-password".toCharArray());
+
+    @BeforeClass
+    public static void setup() {
+        new ReservedRolesStore();
+    }
+
+    @Override
+    protected boolean preserveClusterUponCompletion() {
+        return true;
+    }
+
+    private static MutableSettingsProvider clusterSettings = new MutableSettingsProvider() {
+        {
+            put("xpack.license.self_generated.type", "basic");
+            put("xpack.security.enabled", "true");
+            put("xpack.security.http.ssl.enabled", "false");
+            put("xpack.security.transport.ssl.enabled", "false");
+        }
+    };
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .nodes(2)
+        .settings(clusterSettings)
+        .rolesFile(Resource.fromClasspath("roles.yml"))
+        .user(ADMIN_USER, ADMIN_PASSWORD.toString(), User.ROOT_USER_ROLE, true)
+        .user(REST_USER, REST_PASSWORD.toString(), "security_test_role", false)
+        .user(READ_SECURITY_USER, READ_SECURITY_PASSWORD.toString(), "read_security_user_role", false)
+        .systemProperty("es.queryable_built_in_roles_enabled", "true")
+        .plugin("queryable-reserved-roles-test")
+        .build();
+
+    private static Set<String> PREVIOUS_RESERVED_ROLES;
+    private static Set<String> CONFIGURED_RESERVED_ROLES;
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue(ADMIN_USER, ADMIN_PASSWORD);
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue(REST_USER, REST_PASSWORD);
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+    @Order(10)
+    public void testQueryDeleteOrUpdateReservedRoles() throws Exception {
+        waitForMigrationCompletion(adminClient(), SecurityMigrations.ROLE_METADATA_FLATTENED_MIGRATION_VERSION);
+
+        final String[] allReservedRoles = ReservedRolesStore.names().toArray(new String[0]);
+        assertQuery(client(), """
+            { "query": { "bool": { "must": { "term": { "metadata._reserved": true } } } }, "size": 100 }
+            """, allReservedRoles.length, roles -> {
+            assertThat(roles, iterableWithSize(allReservedRoles.length));
+            for (var role : roles) {
+                assertThat((String) role.get("name"), is(oneOf(allReservedRoles)));
+            }
+        });
+
+        final String roleName = randomFrom(allReservedRoles);
+        assertQuery(client(), String.format("""
+            { "query": { "bool": { "must": { "term": { "name": "%s" } } } } }
+            """, roleName), 1, roles -> {
+            assertThat(roles, iterableWithSize(1));
+            assertThat((String) roles.get(0).get("name"), equalTo(roleName));
+        });
+
+        assertCannotDeleteReservedRoles();
+        assertCannotCreateOrUpdateReservedRole(roleName);
+    }
+
+    @Order(11)
+    public void testGetReservedRoles() throws Exception {
+        final String[] allReservedRoles = ReservedRolesStore.names().toArray(new String[0]);
+        final String roleName = randomFrom(allReservedRoles);
+        Request request = new Request("GET", "/_security/role/" + roleName);
+        Response response = adminClient().performRequest(request);
+        assertOK(response);
+        var responseMap = responseAsMap(response);
+        assertThat(responseMap.size(), equalTo(1));
+        assertThat(responseMap.containsKey(roleName), is(true));
+    }
+
+    @Order(20)
+    public void testRestartForConfiguringReservedRoles() throws Exception {
+        configureReservedRoles(List.of("editor", "viewer", "kibana_system", "apm_system", "beats_system", "logstash_system"));
+        cluster.restart(false);
+        closeClients();
+    }
+
+    @Order(30)
+    public void testConfiguredReservedRoles() throws Exception {
+        assert CONFIGURED_RESERVED_ROLES != null;
+
+        // Test query roles API
+        assertBusy(() -> {
+            assertQuery(client(), """
+                { "query": { "bool": { "must": { "term": { "metadata._reserved": true } } } }, "size": 100 }
+                """, CONFIGURED_RESERVED_ROLES.size(), roles -> {
+                assertThat(roles, iterableWithSize(CONFIGURED_RESERVED_ROLES.size()));
+                for (var role : roles) {
+                    assertThat((String) role.get("name"), is(oneOf(CONFIGURED_RESERVED_ROLES.toArray(new String[0]))));
+                }
+            });
+        }, 30, TimeUnit.SECONDS);
+
+        // Test get roles API
+        assertBusy(() -> {
+            final Response response = adminClient().performRequest(new Request("GET", "/_security/role"));
+            assertOK(response);
+            final Map<String, Object> responseMap = responseAsMap(response);
+            assertThat(responseMap.keySet(), equalTo(CONFIGURED_RESERVED_ROLES));
+        });
+    }
+
+    @Order(40)
+    public void testRestartForConfiguringReservedRolesAndClosingIndex() throws Exception {
+        configureReservedRoles(List.of("editor", "viewer"));
+        closeSecurityIndex();
+        cluster.restart(false);
+        closeClients();
+    }
+
+    @Order(50)
+    public void testConfiguredReservedRolesAfterClosingAndOpeningIndex() throws Exception {
+        assert CONFIGURED_RESERVED_ROLES != null;
+        assert PREVIOUS_RESERVED_ROLES != null;
+        assertThat(PREVIOUS_RESERVED_ROLES, is(not(equalTo(CONFIGURED_RESERVED_ROLES))));
+
+        // Test configured roles did not get updated because the security index is closed
+        assertMetadataContainsBuiltInRoles(PREVIOUS_RESERVED_ROLES);
+
+        // Open the security index
+        openSecurityIndex();
+
+        // Test that the roles are now updated after index got opened
+        assertBusy(() -> {
+            assertQuery(client(), """
+                { "query": { "bool": { "must": { "term": { "metadata._reserved": true } } } }, "size": 100 }
+                """, CONFIGURED_RESERVED_ROLES.size(), roles -> {
+                assertThat(roles, iterableWithSize(CONFIGURED_RESERVED_ROLES.size()));
+                for (var role : roles) {
+                    assertThat((String) role.get("name"), is(oneOf(CONFIGURED_RESERVED_ROLES.toArray(new String[0]))));
+                }
+            });
+        }, 30, TimeUnit.SECONDS);
+
+    }
+
+    @Order(60)
+    public void testDeletingAndCreatingSecurityIndexTriggersSynchronization() throws Exception {
+        deleteSecurityIndex();
+
+        assertBusy(this::assertSecurityIndexDeleted, 30, TimeUnit.SECONDS);
+
+        // Creating a user will trigger .security index creation
+        createUser("superman", "superman", "superuser");
+
+        // Test that the roles are now updated after index got created
+        assertBusy(() -> {
+            assertQuery(client(), """
+                { "query": { "bool": { "must": { "term": { "metadata._reserved": true } } } }, "size": 100 }
+                """, CONFIGURED_RESERVED_ROLES.size(), roles -> {
+                assertThat(roles, iterableWithSize(CONFIGURED_RESERVED_ROLES.size()));
+                for (var role : roles) {
+                    assertThat((String) role.get("name"), is(oneOf(CONFIGURED_RESERVED_ROLES.toArray(new String[0]))));
+                }
+            });
+        }, 30, TimeUnit.SECONDS);
+    }
+
+    private void createUser(String name, String password, String role) throws IOException {
+        Request request = new Request("PUT", "/_security/user/" + name);
+        request.setJsonEntity("{ \"password\": \"" + password + "\", \"roles\": [ \"" + role + "\"] }");
+        assertOK(adminClient().performRequest(request));
+    }
+
+    private void deleteSecurityIndex() throws IOException {
+        final Request deleteRequest = new Request("DELETE", INTERNAL_SECURITY_MAIN_INDEX_7);
+        deleteRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(ESRestTestCase::ignoreSystemIndexAccessWarnings));
+        final Response response = adminClient().performRequest(deleteRequest);
+        try (InputStream is = response.getEntity().getContent()) {
+            assertTrue((boolean) XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true).get("acknowledged"));
+        }
+    }
+
+    private void assertMetadataContainsBuiltInRoles(Set<String> builtInRoles) throws IOException {
+        final Request request = new Request("GET", "_cluster/state/metadata/" + INTERNAL_SECURITY_MAIN_INDEX_7);
+        final Response response = adminClient().performRequest(request);
+        assertOK(response);
+        final Map<String, String> builtInRolesDigests = ObjectPath.createFromResponse(response)
+            .evaluate("metadata.indices.\\.security-7." + QueryableBuiltInRolesSynchronizer.METADATA_QUERYABLE_BUILT_IN_ROLES_DIGEST_KEY);
+        assertThat(builtInRolesDigests.keySet(), equalTo(builtInRoles));
+    }
+
+    private void assertSecurityIndexDeleted() throws IOException {
+        final Request request = new Request("GET", "_cluster/state/metadata/" + INTERNAL_SECURITY_MAIN_INDEX_7);
+        final Response response = adminClient().performRequest(request);
+        assertOK(response);
+        final Map<String, String> securityIndexMetadata = ObjectPath.createFromResponse(response)
+            .evaluate("metadata.indices.\\.security-7");
+        assertThat(securityIndexMetadata, is(nullValue()));
+    }
+
+    private void configureReservedRoles(List<String> reservedRoles) throws Exception {
+        PREVIOUS_RESERVED_ROLES = CONFIGURED_RESERVED_ROLES;
+        CONFIGURED_RESERVED_ROLES = new HashSet<>();
+        CONFIGURED_RESERVED_ROLES.add("superuser"); // superuser must always be included
+        CONFIGURED_RESERVED_ROLES.addAll(reservedRoles);
+        clusterSettings.put("xpack.security.reserved_roles.include", Strings.collectionToCommaDelimitedString(CONFIGURED_RESERVED_ROLES));
+    }
+
+    private void closeSecurityIndex() throws Exception {
+        Request request = new Request("POST", "/" + TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7 + "/_close");
+        request.setOptions(
+            expectWarnings(
+                "this request accesses system indices: [.security-7], but in a future major version, "
+                    + "direct access to system indices will be prevented by default"
+            )
+        );
+        Response response = adminClient().performRequest(request);
+        assertOK(response);
+    }
+
+    private void openSecurityIndex() throws Exception {
+        Request request = new Request("POST", "/" + TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7 + "/_open");
+        request.setOptions(
+            expectWarnings(
+                "this request accesses system indices: [.security-7], but in a future major version, "
+                    + "direct access to system indices will be prevented by default"
+            )
+        );
+        Response response = adminClient().performRequest(request);
+        assertOK(response);
+    }
+
+    private void assertCannotDeleteReservedRoles() throws Exception {
+        {
+            String roleName = randomFrom(ReservedRolesStore.names());
+            Request request = new Request("DELETE", "/_security/role/" + roleName);
+            var e = expectThrows(ResponseException.class, () -> adminClient().performRequest(request));
+            assertThat(e.getMessage(), containsString("role [" + roleName + "] is reserved and cannot be deleted"));
+        }
+        {
+            Request request = new Request("DELETE", "/_security/role/");
+            request.setJsonEntity(
+                """
+                    {
+                      "names": [%s]
+                    }
+                    """.formatted(
+                    ReservedRolesStore.names().stream().map(name -> "\"" + name + "\"").reduce((a, b) -> a + ", " + b).orElse("")
+                )
+            );
+            Response response = adminClient().performRequest(request);
+            assertOK(response);
+            String responseAsString = responseAsMap(response).toString();
+            for (String roleName : ReservedRolesStore.names()) {
+                assertThat(responseAsString, containsString("role [" + roleName + "] is reserved and cannot be deleted"));
+            }
+        }
+    }
+
+    private void assertCannotCreateOrUpdateReservedRole(String roleName) throws Exception {
+        Request request = new Request(randomBoolean() ? "PUT" : "POST", "/_security/role/" + roleName);
+        request.setJsonEntity("""
+            {
+              "cluster": ["all"],
+              "indices": [
+                {
+                  "names": ["*"],
+                  "privileges": ["all"]
+                }
+              ]
+            }
+            """);
+        var e = expectThrows(ResponseException.class, () -> adminClient().performRequest(request));
+        assertThat(e.getMessage(), containsString("Role [" + roleName + "] is reserved and may not be used."));
+    }
+
+}
diff --git a/x-pack/plugin/security/qa/security-basic/src/main/java/module-info.java b/x-pack/plugin/security/qa/security-basic/src/main/java/module-info.java
new file mode 100644
index 0000000000000..00c8e480cfbaf
--- /dev/null
+++ b/x-pack/plugin/security/qa/security-basic/src/main/java/module-info.java
@@ -0,0 +1,6 @@
+module org.elasticsearch.internal.security {
+    requires org.elasticsearch.base;
+    requires org.elasticsearch.server;
+    requires org.elasticsearch.xcore;
+    requires org.elasticsearch.security;
+}
diff --git a/x-pack/plugin/security/qa/security-basic/src/main/java/org/elasticsearch/xpack/security/role/QueryableBuiltInRolesTestPlugin.java b/x-pack/plugin/security/qa/security-basic/src/main/java/org/elasticsearch/xpack/security/role/QueryableBuiltInRolesTestPlugin.java
new file mode 100644
index 0000000000000..ba5538d992cfb
--- /dev/null
+++ b/x-pack/plugin/security/qa/security-basic/src/main/java/org/elasticsearch/xpack/security/role/QueryableBuiltInRolesTestPlugin.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.role;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
+
+import java.util.List;
+
+public class QueryableBuiltInRolesTestPlugin extends Plugin {
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return List.of(ReservedRolesStore.INCLUDED_RESERVED_ROLES_SETTING);
+    }
+}
diff --git a/x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java b/x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java
index b7fb8c37f4c45..77bde274f9a1c 100644
--- a/x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java
+++ b/x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java
@@ -316,6 +316,21 @@ public class ServiceAccountIT extends ESRestTestCase {
                     "view_index_metadata"
                   ],
                   "allow_restricted_indices": false
+                },
+                {
+                  "names": [
+                    "agentless-*"
+                  ],
+                  "privileges": [
+                    "read",
+                    "write",
+                    "monitor",
+                    "create_index",
+                    "auto_configure",
+                    "maintenance",
+                    "view_index_metadata"
+                  ],
+                  "allow_restricted_indices": false
                 }
               ],
               "applications": [        {
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java
index 57d18abaf1a92..f7361480f0a51 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java
@@ -287,28 +287,28 @@ public void testGetMappingsIsFiltered() {
         {
             GetMappingsResponse getMappingsResponse = client().filterWithHeader(
                 Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD))
-            ).admin().indices().prepareGetMappings("test").get();
+            ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get();
             assertExpectedMetadataFields(getMappingsResponse.getMappings(), "field1");
         }
 
         {
             GetMappingsResponse getMappingsResponse = client().filterWithHeader(
                 Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD))
-            ).admin().indices().prepareGetMappings("test").get();
+            ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get();
             assertExpectedMetadataFields(getMappingsResponse.getMappings(), "field2");
         }
 
         {
             GetMappingsResponse getMappingsResponse = client().filterWithHeader(
                 Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD))
-            ).admin().indices().prepareGetMappings("test").get();
+            ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get();
             assertExpectedMetadataFields(getMappingsResponse.getMappings(), "field1");
         }
 
         {
             GetMappingsResponse getMappingsResponse = client().filterWithHeader(
                 Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD))
-            ).admin().indices().prepareGetMappings("test").get();
+            ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get();
             assertExpectedMetadataFields(getMappingsResponse.getMappings(), "field1", "field2");
         }
     }
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java
index bc730b5695c19..b501ba69d673b 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java
@@ -166,7 +166,7 @@ public void testGetMappings() throws Exception {
 
         GetMappingsResponse response = client().filterWithHeader(
             singletonMap("Authorization", UsernamePasswordToken.basicAuthHeaderValue("kibana_user", USERS_PASSWD))
-        ).admin().indices().prepareGetMappings("logstash-*").get();
+        ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "logstash-*").get();
         Map<String, MappingMetadata> mappingsMap = response.getMappings();
         assertNotNull(mappingsMap);
         assertNotNull(mappingsMap.get(index));
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/PermissionPrecedenceTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/PermissionPrecedenceTests.java
index f80c232e40f73..60923ac023474 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/PermissionPrecedenceTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/PermissionPrecedenceTests.java
@@ -89,7 +89,7 @@ public void testDifferentCombinationsOfIndices() throws Exception {
         ).admin().indices().preparePutTemplate("template1").setPatterns(Collections.singletonList("test_*")).get();
         assertAcked(putResponse);
 
-        GetIndexTemplatesResponse getResponse = client.admin().indices().prepareGetTemplates("template1").get();
+        GetIndexTemplatesResponse getResponse = client.admin().indices().prepareGetTemplates(TEST_REQUEST_TIMEOUT, "template1").get();
         List<IndexTemplateMetadata> templates = getResponse.getIndexTemplates();
         assertThat(templates, hasSize(1));
 
@@ -114,7 +114,7 @@ public void testDifferentCombinationsOfIndices() throws Exception {
             basicAuthHeaderValue("user", SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)
         );
         assertThrowsAuthorizationException(
-            client.filterWithHeader(headers).admin().indices().prepareGetTemplates("template1")::get,
+            client.filterWithHeader(headers).admin().indices().prepareGetTemplates(TEST_REQUEST_TIMEOUT, "template1")::get,
             GetIndexTemplatesAction.NAME,
             "user"
         );
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/TemplateUpgraderTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/TemplateUpgraderTests.java
index d73719bb0cac4..416c4212acc49 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/TemplateUpgraderTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/TemplateUpgraderTests.java
@@ -79,7 +79,7 @@ public void testTemplatesWorkAsExpected() throws Exception {
     }
 
     private void assertTemplates(String existingTemplate, String deletedTemplate) {
-        GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
+        GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates(TEST_REQUEST_TIMEOUT).get();
         List<String> templateNames = response.getIndexTemplates().stream().map(IndexTemplateMetadata::name).collect(Collectors.toList());
         assertThat(templateNames, hasItem(existingTemplate));
         assertThat(templateNames, not(hasItem(deletedTemplate)));
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java
index b56ea7ae3e456..fef1a98ca67e9 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java
@@ -327,8 +327,8 @@ public void testInvalidateNotValidAccessTokens() throws Exception {
             ResponseException.class,
             () -> invalidateAccessToken(
                 tokenService.prependVersionAndEncodeAccessToken(
-                    TransportVersions.MINIMUM_COMPATIBLE,
-                    tokenService.getRandomTokenBytes(TransportVersions.MINIMUM_COMPATIBLE, randomBoolean()).v1()
+                    TransportVersions.V_7_3_2,
+                    tokenService.getRandomTokenBytes(TransportVersions.V_7_3_2, randomBoolean()).v1()
                 )
             )
         );
@@ -347,7 +347,7 @@ public void testInvalidateNotValidAccessTokens() throws Exception {
         byte[] longerAccessToken = new byte[randomIntBetween(17, 24)];
         random().nextBytes(longerAccessToken);
         invalidateResponse = invalidateAccessToken(
-            tokenService.prependVersionAndEncodeAccessToken(TransportVersions.MINIMUM_COMPATIBLE, longerAccessToken)
+            tokenService.prependVersionAndEncodeAccessToken(TransportVersions.V_7_3_2, longerAccessToken)
         );
         assertThat(invalidateResponse.invalidated(), equalTo(0));
         assertThat(invalidateResponse.previouslyInvalidated(), equalTo(0));
@@ -365,7 +365,7 @@ public void testInvalidateNotValidAccessTokens() throws Exception {
         byte[] shorterAccessToken = new byte[randomIntBetween(12, 15)];
         random().nextBytes(shorterAccessToken);
         invalidateResponse = invalidateAccessToken(
-            tokenService.prependVersionAndEncodeAccessToken(TransportVersions.MINIMUM_COMPATIBLE, shorterAccessToken)
+            tokenService.prependVersionAndEncodeAccessToken(TransportVersions.V_7_3_2, shorterAccessToken)
         );
         assertThat(invalidateResponse.invalidated(), equalTo(0));
         assertThat(invalidateResponse.previouslyInvalidated(), equalTo(0));
@@ -394,8 +394,8 @@ public void testInvalidateNotValidAccessTokens() throws Exception {
 
         invalidateResponse = invalidateAccessToken(
             tokenService.prependVersionAndEncodeAccessToken(
-                TransportVersions.MINIMUM_COMPATIBLE,
-                tokenService.getRandomTokenBytes(TransportVersions.MINIMUM_COMPATIBLE, randomBoolean()).v1()
+                TransportVersions.V_7_3_2,
+                tokenService.getRandomTokenBytes(TransportVersions.V_7_3_2, randomBoolean()).v1()
             )
         );
         assertThat(invalidateResponse.invalidated(), equalTo(0));
@@ -420,8 +420,8 @@ public void testInvalidateNotValidRefreshTokens() throws Exception {
             ResponseException.class,
             () -> invalidateRefreshToken(
                 TokenService.prependVersionAndEncodeRefreshToken(
-                    TransportVersions.MINIMUM_COMPATIBLE,
-                    tokenService.getRandomTokenBytes(TransportVersions.MINIMUM_COMPATIBLE, true).v2()
+                    TransportVersions.V_7_3_2,
+                    tokenService.getRandomTokenBytes(TransportVersions.V_7_3_2, true).v2()
                 )
             )
         );
@@ -441,7 +441,7 @@ public void testInvalidateNotValidRefreshTokens() throws Exception {
         byte[] longerRefreshToken = new byte[randomIntBetween(17, 24)];
         random().nextBytes(longerRefreshToken);
         invalidateResponse = invalidateRefreshToken(
-            TokenService.prependVersionAndEncodeRefreshToken(TransportVersions.MINIMUM_COMPATIBLE, longerRefreshToken)
+            TokenService.prependVersionAndEncodeRefreshToken(TransportVersions.V_7_3_2, longerRefreshToken)
         );
         assertThat(invalidateResponse.invalidated(), equalTo(0));
         assertThat(invalidateResponse.previouslyInvalidated(), equalTo(0));
@@ -459,7 +459,7 @@ public void testInvalidateNotValidRefreshTokens() throws Exception {
         byte[] shorterRefreshToken = new byte[randomIntBetween(12, 15)];
         random().nextBytes(shorterRefreshToken);
         invalidateResponse = invalidateRefreshToken(
-            TokenService.prependVersionAndEncodeRefreshToken(TransportVersions.MINIMUM_COMPATIBLE, shorterRefreshToken)
+            TokenService.prependVersionAndEncodeRefreshToken(TransportVersions.V_7_3_2, shorterRefreshToken)
         );
         assertThat(invalidateResponse.invalidated(), equalTo(0));
         assertThat(invalidateResponse.previouslyInvalidated(), equalTo(0));
@@ -488,8 +488,8 @@ public void testInvalidateNotValidRefreshTokens() throws Exception {
 
         invalidateResponse = invalidateRefreshToken(
             TokenService.prependVersionAndEncodeRefreshToken(
-                TransportVersions.MINIMUM_COMPATIBLE,
-                tokenService.getRandomTokenBytes(TransportVersions.MINIMUM_COMPATIBLE, true).v2()
+                TransportVersions.V_7_3_2,
+                tokenService.getRandomTokenBytes(TransportVersions.V_7_3_2, true).v2()
             )
         );
         assertThat(invalidateResponse.invalidated(), equalTo(0));
@@ -758,11 +758,18 @@ public void testAuthenticateWithWrongToken() throws Exception {
         assertAuthenticateWithToken(response.accessToken(), TEST_USER_NAME);
         // Now attempt to authenticate with an invalid access token string
         assertUnauthorizedToken(randomAlphaOfLengthBetween(0, 128));
-        // Now attempt to authenticate with an invalid access token with valid structure (after 8.0 pre 8.10)
+        // Now attempt to authenticate with an invalid access token with valid structure (pre 7.2)
         assertUnauthorizedToken(
             tokenService.prependVersionAndEncodeAccessToken(
-                TransportVersions.V_8_0_0,
-                tokenService.getRandomTokenBytes(TransportVersions.V_8_0_0, randomBoolean()).v1()
+                TransportVersions.V_7_1_0,
+                tokenService.getRandomTokenBytes(TransportVersions.V_7_1_0, randomBoolean()).v1()
+            )
+        );
+        // Now attempt to authenticate with an invalid access token with valid structure (after 7.2 pre 8.10)
+        assertUnauthorizedToken(
+            tokenService.prependVersionAndEncodeAccessToken(
+                TransportVersions.V_7_4_0,
+                tokenService.getRandomTokenBytes(TransportVersions.V_7_4_0, randomBoolean()).v1()
             )
         );
         // Now attempt to authenticate with an invalid access token with valid structure (current version)
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java
index 0a9995761cc42..c1dbb004196ac 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java
@@ -195,19 +195,31 @@ public void testGetAliasesCreateOnlyPermissionStrict() {
         final Client client = client().filterWithHeader(headers);
 
         assertThrowsAuthorizationException(
-            client.admin().indices().prepareGetAliases("test_1").setIndices("test_1").setIndicesOptions(IndicesOptions.strictExpand())::get,
+            client.admin()
+                .indices()
+                .prepareGetAliases(TEST_REQUEST_TIMEOUT, "test_1")
+                .setIndices("test_1")
+                .setIndicesOptions(IndicesOptions.strictExpand())::get,
             GetAliasesAction.NAME,
             "create_only"
         );
 
         assertThrowsAuthorizationException(
-            client.admin().indices().prepareGetAliases("_all").setIndices("test_1").setIndicesOptions(IndicesOptions.strictExpand())::get,
+            client.admin()
+                .indices()
+                .prepareGetAliases(TEST_REQUEST_TIMEOUT, "_all")
+                .setIndices("test_1")
+                .setIndicesOptions(IndicesOptions.strictExpand())::get,
             GetAliasesAction.NAME,
             "create_only"
         );
 
         assertThrowsAuthorizationException(
-            client.admin().indices().prepareGetAliases().setIndices("test_1").setIndicesOptions(IndicesOptions.strictExpand())::get,
+            client.admin()
+                .indices()
+                .prepareGetAliases(TEST_REQUEST_TIMEOUT)
+                .setIndices("test_1")
+                .setIndicesOptions(IndicesOptions.strictExpand())::get,
             GetAliasesAction.NAME,
             "create_only"
         );
@@ -215,7 +227,7 @@ public void testGetAliasesCreateOnlyPermissionStrict() {
         assertThrowsAuthorizationException(
             client.admin()
                 .indices()
-                .prepareGetAliases("test_alias")
+                .prepareGetAliases(TEST_REQUEST_TIMEOUT, "test_alias")
                 .setIndices("test_*")
                 .setIndicesOptions(IndicesOptions.strictExpand())::get,
             GetAliasesAction.NAME,
@@ -224,7 +236,11 @@ public void testGetAliasesCreateOnlyPermissionStrict() {
 
         // this throws exception no matter what the indices options are because the aliases part cannot be resolved to any alias
         // and there is no way to "allow_no_aliases" like we can do with indices.
-        assertThrowsAuthorizationException(client.admin().indices().prepareGetAliases()::get, GetAliasesAction.NAME, "create_only");
+        assertThrowsAuthorizationException(
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT)::get,
+            GetAliasesAction.NAME,
+            "create_only"
+        );
     }
 
     public void testGetAliasesCreateOnlyPermissionIgnoreUnavailable() {
@@ -238,7 +254,7 @@ public void testGetAliasesCreateOnlyPermissionIgnoreUnavailable() {
         assertThrowsAuthorizationException(
             client.admin()
                 .indices()
-                .prepareGetAliases("test_1")
+                .prepareGetAliases(TEST_REQUEST_TIMEOUT, "test_1")
                 .setIndices("test_1")
                 .setIndicesOptions(IndicesOptions.lenientExpandOpen())::get,
             GetAliasesAction.NAME,
@@ -248,17 +264,25 @@ public void testGetAliasesCreateOnlyPermissionIgnoreUnavailable() {
         assertThrowsAuthorizationException(
             client.admin()
                 .indices()
-                .prepareGetAliases("_all")
+                .prepareGetAliases(TEST_REQUEST_TIMEOUT, "_all")
                 .setIndices("test_1")
                 .setIndicesOptions(IndicesOptions.lenientExpandOpen())::get,
             GetAliasesAction.NAME,
             "create_only"
         );
 
-        assertThrowsAuthorizationException(client.admin().indices().prepareGetAliases("alias*")::get, GetAliasesAction.NAME, "create_only");
+        assertThrowsAuthorizationException(
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias*")::get,
+            GetAliasesAction.NAME,
+            "create_only"
+        );
 
         assertThrowsAuthorizationException(
-            client.admin().indices().prepareGetAliases().setIndices("test_1").setIndicesOptions(IndicesOptions.lenientExpandOpen())::get,
+            client.admin()
+                .indices()
+                .prepareGetAliases(TEST_REQUEST_TIMEOUT)
+                .setIndices("test_1")
+                .setIndicesOptions(IndicesOptions.lenientExpandOpen())::get,
             GetAliasesAction.NAME,
             "create_only"
         );
@@ -266,7 +290,7 @@ public void testGetAliasesCreateOnlyPermissionIgnoreUnavailable() {
         assertThrowsAuthorizationException(
             client.admin()
                 .indices()
-                .prepareGetAliases("test_alias")
+                .prepareGetAliases(TEST_REQUEST_TIMEOUT, "test_alias")
                 .setIndices("test_*")
                 .setIndicesOptions(IndicesOptions.lenientExpandOpen())::get,
             GetAliasesAction.NAME,
@@ -274,7 +298,11 @@ public void testGetAliasesCreateOnlyPermissionIgnoreUnavailable() {
         );
 
         assertThrowsAuthorizationException(
-            client.admin().indices().prepareGetAliases().setIndices("test_*").setIndicesOptions(IndicesOptions.lenientExpandOpen())::get,
+            client.admin()
+                .indices()
+                .prepareGetAliases(TEST_REQUEST_TIMEOUT)
+                .setIndices("test_*")
+                .setIndicesOptions(IndicesOptions.lenientExpandOpen())::get,
             GetAliasesAction.NAME,
             "create_only"
         );
@@ -282,7 +310,7 @@ public void testGetAliasesCreateOnlyPermissionIgnoreUnavailable() {
         // this throws exception no matter what the indices options are because the aliases part cannot be resolved to any alias
         // and there is no way to "allow_no_aliases" like we can do with indices.
         assertThrowsAuthorizationException(
-            client.admin().indices().prepareGetAliases().setIndicesOptions(IndicesOptions.lenientExpandOpen())::get,
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setIndicesOptions(IndicesOptions.lenientExpandOpen())::get,
             GetAliasesAction.NAME,
             "create_only"
         );
@@ -415,43 +443,63 @@ public void testGetAliasesCreateAndAliasesPermission() {
         assertAcked(client.admin().indices().prepareCreate("test_1").addAlias(new Alias("test_alias")).get());
 
         // ok: user has manage_aliases on test_*
-        assertAliases(client.admin().indices().prepareGetAliases().setAliases("test_alias").setIndices("test_1"), "test_1", "test_alias");
+        assertAliases(
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("test_alias").setIndices("test_1"),
+            "test_1",
+            "test_alias"
+        );
 
         // ok: user has manage_aliases on test_*, test_* gets resolved to test_1
-        assertAliases(client.admin().indices().prepareGetAliases().setAliases("test_alias").setIndices("test_*"), "test_1", "test_alias");
+        assertAliases(
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("test_alias").setIndices("test_*"),
+            "test_1",
+            "test_alias"
+        );
 
         // ok: user has manage_aliases on test_*, empty indices gets resolved to _all indices (thus test_1)
-        assertAliases(client.admin().indices().prepareGetAliases().setAliases("test_alias"), "test_1", "test_alias");
+        assertAliases(client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("test_alias"), "test_1", "test_alias");
 
         // ok: user has manage_aliases on test_*, _all aliases gets resolved to test_alias and empty indices gets resolved to _all
         // indices (thus test_1)
-        assertAliases(client.admin().indices().prepareGetAliases().setAliases("_all").setIndices("test_1"), "test_1", "test_alias");
+        assertAliases(
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("_all").setIndices("test_1"),
+            "test_1",
+            "test_alias"
+        );
 
         // ok: user has manage_aliases on test_*, empty aliases gets resolved to test_alias and empty indices gets resolved to _all
         // indices (thus test_1)
-        assertAliases(client.admin().indices().prepareGetAliases().setIndices("test_1"), "test_1", "test_alias");
+        assertAliases(client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setIndices("test_1"), "test_1", "test_alias");
 
         // ok: user has manage_aliases on test_*, test_* aliases gets resolved to test_alias and empty indices gets resolved to _all
         // indices (thus test_1)
-        assertAliases(client.admin().indices().prepareGetAliases().setAliases("test_*").setIndices("test_1"), "test_1", "test_alias");
+        assertAliases(
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("test_*").setIndices("test_1"),
+            "test_1",
+            "test_alias"
+        );
 
         // ok: user has manage_aliases on test_*, _all aliases gets resolved to test_alias and _all indices becomes test_1
-        assertAliases(client.admin().indices().prepareGetAliases().setAliases("_all").setIndices("_all"), "test_1", "test_alias");
+        assertAliases(
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("_all").setIndices("_all"),
+            "test_1",
+            "test_alias"
+        );
 
         // ok: user has manage_aliases on test_*, empty aliases gets resolved to test_alias and empty indices becomes test_1
-        assertAliases(client.admin().indices().prepareGetAliases(), "test_1", "test_alias");
+        assertAliases(client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT), "test_1", "test_alias");
 
         // fails: user has manage_aliases on test_*, although _all aliases and empty indices can be resolved, the explicit non
         // authorized alias (alias_1) causes the request to fail
         assertThrowsAuthorizationException(
-            client.admin().indices().prepareGetAliases().setAliases("_all", "alias_1")::get,
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("_all", "alias_1")::get,
             GetAliasesAction.NAME,
             "create_test_aliases_test"
         );
 
         // fails: user doesn't have manage_aliases on alias_1
         assertThrowsAuthorizationException(
-            client.admin().indices().prepareGetAliases().setAliases("alias_1")::get,
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("alias_1")::get,
             GetAliasesAction.NAME,
             "create_test_aliases_test"
         );
@@ -542,7 +590,7 @@ public void testGetAliasesCreateAndAliasesPermission2() {
 
         // fails: user doesn't have manage_aliases aliases on test_1, nor test_alias
         assertThrowsAuthorizationException(
-            client.admin().indices().prepareGetAliases().setAliases("test_alias").setIndices("test_1")::get,
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("test_alias").setIndices("test_1")::get,
             GetAliasesAction.NAME,
             "create_test_aliases_alias"
         );
@@ -550,20 +598,20 @@ public void testGetAliasesCreateAndAliasesPermission2() {
         // user doesn't have manage_aliases aliases on test_*, no matching indices to replace wildcards
         GetAliasesResponse getAliasesResponse = client.admin()
             .indices()
-            .prepareGetAliases()
+            .prepareGetAliases(TEST_REQUEST_TIMEOUT)
             .setIndices("test_*")
             .setAliases("test_alias")
             .get();
         assertEquals(0, getAliasesResponse.getAliases().size());
 
         // no existing indices to replace empty indices (thus _all)
-        getAliasesResponse = client.admin().indices().prepareGetAliases().setAliases("test_alias").get();
+        getAliasesResponse = client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("test_alias").get();
         assertEquals(0, getAliasesResponse.getAliases().size());
 
         {
             // fails: no existing aliases to replace wildcards
             assertThrowsAuthorizationException(
-                client.admin().indices().prepareGetAliases().setIndices("test_1").setAliases("test_*")::get,
+                client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setIndices("test_1").setAliases("test_*")::get,
                 GetAliasesAction.NAME,
                 "create_test_aliases_alias"
             );
@@ -571,7 +619,7 @@ public void testGetAliasesCreateAndAliasesPermission2() {
         {
             // fails: no existing aliases to replace _all
             assertThrowsAuthorizationException(
-                client.admin().indices().prepareGetAliases().setIndices("test_1").setAliases("_all")::get,
+                client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setIndices("test_1").setAliases("_all")::get,
                 GetAliasesAction.NAME,
                 "create_test_aliases_alias"
             );
@@ -579,14 +627,14 @@ public void testGetAliasesCreateAndAliasesPermission2() {
         {
             // fails: no existing aliases to replace empty aliases
             assertThrowsAuthorizationException(
-                client.admin().indices().prepareGetAliases().setIndices("test_1")::get,
+                client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setIndices("test_1")::get,
                 GetAliasesAction.NAME,
                 "create_test_aliases_alias"
             );
         }
         {
             // fails: no existing aliases to replace empty aliases
-            GetAliasesResponse response = client.admin().indices().prepareGetAliases().get();
+            GetAliasesResponse response = client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).get();
             assertThat(response.getAliases().size(), equalTo(0));
         }
     }
@@ -668,28 +716,58 @@ public void testGetAliasesCreateAndAliasesPermission3() {
         // user has create permission on test_* and manage_aliases permission on test_*,alias_*. All good.
         assertAcked(client.admin().indices().prepareCreate("test_1").addAlias(new Alias("test_alias")).addAlias(new Alias("alias_1")));
 
-        assertAliases(client.admin().indices().prepareGetAliases().setAliases("test_alias").setIndices("test_1"), "test_1", "test_alias");
+        assertAliases(
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("test_alias").setIndices("test_1"),
+            "test_1",
+            "test_alias"
+        );
 
-        assertAliases(client.admin().indices().prepareGetAliases().setAliases("alias_1").setIndices("test_1"), "test_1", "alias_1");
+        assertAliases(
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("alias_1").setIndices("test_1"),
+            "test_1",
+            "alias_1"
+        );
 
-        assertAliases(client.admin().indices().prepareGetAliases().setAliases("alias_1").setIndices("test_*"), "test_1", "alias_1");
+        assertAliases(
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("alias_1").setIndices("test_*"),
+            "test_1",
+            "alias_1"
+        );
 
-        assertAliases(client.admin().indices().prepareGetAliases().setAliases("test_*").setIndices("test_1"), "test_1", "test_alias");
+        assertAliases(
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("test_*").setIndices("test_1"),
+            "test_1",
+            "test_alias"
+        );
 
         assertAliases(
-            client.admin().indices().prepareGetAliases().setAliases("_all").setIndices("test_1"),
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("_all").setIndices("test_1"),
             "test_1",
             "alias_1",
             "test_alias"
         );
 
-        assertAliases(client.admin().indices().prepareGetAliases().setAliases("_all"), "test_1", "alias_1", "test_alias");
+        assertAliases(
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("_all"),
+            "test_1",
+            "alias_1",
+            "test_alias"
+        );
 
-        assertAliases(client.admin().indices().prepareGetAliases().setIndices("test_1"), "test_1", "alias_1", "test_alias");
+        assertAliases(
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setIndices("test_1"),
+            "test_1",
+            "alias_1",
+            "test_alias"
+        );
 
-        assertAliases(client.admin().indices().prepareGetAliases(), "test_1", "alias_1", "test_alias");
+        assertAliases(client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT), "test_1", "alias_1", "test_alias");
 
-        assertAliases(client.admin().indices().prepareGetAliases().setAliases("alias_*").setIndices("test_*"), "test_1", "alias_1");
+        assertAliases(
+            client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("alias_*").setIndices("test_*"),
+            "test_1",
+            "alias_1"
+        );
     }
 
     public void testCreateIndexAliasesOnlyPermission() {
@@ -718,7 +796,7 @@ public void testGetAliasesAliasesOnlyPermissionStrict() {
             IndexNotFoundException.class,
             client.admin()
                 .indices()
-                .prepareGetAliases("alias_1")
+                .prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias_1")
                 .setIndices("test_1")
                 .setIndicesOptions(IndicesOptions.strictExpandOpen())::get
         );
@@ -728,7 +806,7 @@ public void testGetAliasesAliasesOnlyPermissionStrict() {
         assertThrowsAuthorizationException(
             client.admin()
                 .indices()
-                .prepareGetAliases("non_authorized")
+                .prepareGetAliases(TEST_REQUEST_TIMEOUT, "non_authorized")
                 .setIndices("test_1")
                 .setIndicesOptions(IndicesOptions.strictExpandOpen())::get,
             GetAliasesAction.NAME,
@@ -739,7 +817,7 @@ public void testGetAliasesAliasesOnlyPermissionStrict() {
         assertThrowsAuthorizationException(
             client.admin()
                 .indices()
-                .prepareGetAliases("alias_1")
+                .prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias_1")
                 .setIndices("non_authorized")
                 .setIndicesOptions(IndicesOptions.strictExpandOpen())::get,
             GetAliasesAction.NAME,
@@ -758,7 +836,7 @@ public void testGetAliasesAliasesOnlyPermissionIgnoreUnavailable() {
         // ok: manage_aliases on both test_* and alias_*
         GetAliasesResponse getAliasesResponse = client.admin()
             .indices()
-            .prepareGetAliases("alias_1")
+            .prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias_1")
             .setIndices("test_1")
             .setIndicesOptions(IndicesOptions.lenientExpandOpen())
             .get();
@@ -768,7 +846,7 @@ public void testGetAliasesAliasesOnlyPermissionIgnoreUnavailable() {
         assertThrowsAuthorizationException(
             client.admin()
                 .indices()
-                .prepareGetAliases("non_authorized")
+                .prepareGetAliases(TEST_REQUEST_TIMEOUT, "non_authorized")
                 .setIndices("test_1")
                 .setIndicesOptions(IndicesOptions.lenientExpandOpen())::get,
             GetAliasesAction.NAME,
@@ -778,7 +856,7 @@ public void testGetAliasesAliasesOnlyPermissionIgnoreUnavailable() {
         // no manage_aliases privilege on non_authorized index
         getAliasesResponse = client.admin()
             .indices()
-            .prepareGetAliases("alias_1")
+            .prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias_1")
             .setIndices("non_authorized")
             .setIndicesOptions(IndicesOptions.lenientExpandOpen())
             .get();
@@ -796,13 +874,18 @@ public void testRemoveIndex() {
         assertAcked(client.admin().indices().prepareCreate("test_1").addAlias(new Alias("test_alias_1")));
 
         assertAcked(client.admin().indices().prepareAliases().removeIndex("test_delete_*").get());
-        assertAliases(client.admin().indices().prepareGetAliases().setAliases("*"), "test_1", "test_alias_1");
+        assertAliases(client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("*"), "test_1", "test_alias_1");
 
         // test that the remove index wildcard expacnds only to authorized indices
         assertAcked(client.admin().indices().prepareAliases().removeIndex("*").get());
-        GetAliasesResponse getAliasesResponse = client.admin().indices().prepareGetAliases().setAliases("*").get();
+        GetAliasesResponse getAliasesResponse = client.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("*").get();
         assertThat(getAliasesResponse.getAliases().size(), equalTo(0));
-        assertAliases(indicesAdmin().prepareGetAliases().setAliases("*"), "bogus_index_1", "bogus_alias_1", "bogus_alias_2");
+        assertAliases(
+            indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT).setAliases("*"),
+            "bogus_index_1",
+            "bogus_alias_1",
+            "bogus_alias_2"
+        );
     }
 
     public void testAliasesForHiddenIndices() {
@@ -834,13 +917,13 @@ public void testAliasesForHiddenIndices() {
         );
 
         // The index should be returned here when queried by name or by wildcard because the alias is visible
-        final GetAliasesRequestBuilder req = aliasesClient.admin().indices().prepareGetAliases(visibleAlias);
+        final GetAliasesRequestBuilder req = aliasesClient.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT, visibleAlias);
         GetAliasesResponse response = req.get();
         assertThat(response.getAliases().get(hiddenIndex), hasSize(1));
         assertThat(response.getAliases().get(hiddenIndex).get(0).alias(), Matchers.equalTo(visibleAlias));
         assertThat(response.getAliases().get(hiddenIndex).get(0).isHidden(), nullValue());
 
-        response = indicesAdmin().prepareGetAliases("alias*").get();
+        response = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias*").get();
         assertThat(response.getAliases().get(hiddenIndex), hasSize(1));
         assertThat(response.getAliases().get(hiddenIndex).get(0).alias(), Matchers.equalTo(visibleAlias));
         assertThat(response.getAliases().get(hiddenIndex).get(0).isHidden(), nullValue());
@@ -855,19 +938,23 @@ public void testAliasesForHiddenIndices() {
         );
 
         // Querying by name directly should get the right result
-        response = aliasesClient.admin().indices().prepareGetAliases(hiddenAlias).get();
+        response = aliasesClient.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT, hiddenAlias).get();
         assertThat(response.getAliases().get(hiddenIndex), hasSize(1));
         assertThat(response.getAliases().get(hiddenIndex).get(0).alias(), Matchers.equalTo(hiddenAlias));
         assertThat(response.getAliases().get(hiddenIndex).get(0).isHidden(), Matchers.equalTo(true));
 
         // querying by wildcard should get the right result because the indices options include hidden by default
-        response = aliasesClient.admin().indices().prepareGetAliases("alias*").get();
+        response = aliasesClient.admin().indices().prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias*").get();
         assertThat(response.getAliases().get(hiddenIndex), hasSize(1));
         assertThat(response.getAliases().get(hiddenIndex).get(0).alias(), Matchers.equalTo(hiddenAlias));
         assertThat(response.getAliases().get(hiddenIndex).get(0).isHidden(), Matchers.equalTo(true));
 
         // But we should get no results if we specify indices options that don't include hidden
-        response = aliasesClient.admin().indices().prepareGetAliases("alias*").setIndicesOptions(IndicesOptions.strictExpandOpen()).get();
+        response = aliasesClient.admin()
+            .indices()
+            .prepareGetAliases(TEST_REQUEST_TIMEOUT, "alias*")
+            .setIndicesOptions(IndicesOptions.strictExpandOpen())
+            .get();
         assertThat(response.getAliases().get(hiddenIndex), nullValue());
     }
 
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
index 695ea611e599e..81d117df75c24 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.license.MockLicenseState;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.SearchContextMissingException;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
 import org.elasticsearch.search.internal.LegacyReaderContext;
@@ -64,7 +63,7 @@ public void setupShard() {
 
     public void testOnNewContextSetsAuthentication() throws Exception {
         final ShardSearchRequest shardSearchRequest = mock(ShardSearchRequest.class);
-        when(shardSearchRequest.scroll()).thenReturn(new Scroll(TimeValue.timeValueMinutes(between(1, 10))));
+        when(shardSearchRequest.scroll()).thenReturn(TimeValue.timeValueMinutes(between(1, 10)));
         try (
             LegacyReaderContext readerContext = new LegacyReaderContext(
                 new ShardSearchContextId(UUIDs.randomBase64UUID(), 0L),
@@ -100,7 +99,7 @@ public void testOnNewContextSetsAuthentication() throws Exception {
 
     public void testValidateSearchContext() throws Exception {
         final ShardSearchRequest shardSearchRequest = mock(ShardSearchRequest.class);
-        when(shardSearchRequest.scroll()).thenReturn(new Scroll(TimeValue.timeValueMinutes(between(1, 10))));
+        when(shardSearchRequest.scroll()).thenReturn(TimeValue.timeValueMinutes(between(1, 10)));
         try (
             LegacyReaderContext readerContext = new LegacyReaderContext(
                 new ShardSearchContextId(UUIDs.randomBase64UUID(), 0L),
@@ -243,7 +242,7 @@ public void testValidateSearchContext() throws Exception {
 
     public void testValidateResourceAccessCheck() throws Exception {
         final ShardSearchRequest shardSearchRequest = mock(ShardSearchRequest.class);
-        when(shardSearchRequest.scroll()).thenReturn(new Scroll(TimeValue.timeValueMinutes(between(1, 10))));
+        when(shardSearchRequest.scroll()).thenReturn(TimeValue.timeValueMinutes(between(1, 10)));
         final ShardSearchContextId shardSearchContextId = new ShardSearchContextId(UUIDs.randomBase64UUID(), randomLong());
         try (
             LegacyReaderContext readerContext = new LegacyReaderContext(
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java
index cb4dde4e1cf0b..9d72b9eb5c526 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java
@@ -93,7 +93,6 @@ protected boolean transportSSLEnabled() {
     }
 
     public void testThatSSLConfigurationReloadsOnModification() throws Exception {
-        assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm());
         Path keyPath = createTempDir().resolve("testnode_updated.pem");
         Path certPath = createTempDir().resolve("testnode_updated.crt");
         Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.pem"), keyPath);
@@ -127,6 +126,9 @@ public void testThatSSLConfigurationReloadsOnModification() throws Exception {
             fail("handshake should not have been successful!");
         } catch (SSLException | SocketException expected) {
             logger.trace("expected exception", expected);
+        } catch (IOException e) {
+            // BouncyCastle throws a different exception
+            assertThat(e.getClass().getName(), is("org.bouncycastle.tls.TlsFatalAlertReceived"));
         }
         // Copy testnode_updated.crt to the placeholder updateable.crt so that the nodes will start trusting it now
         try {
diff --git a/x-pack/plugin/security/src/main/java/module-info.java b/x-pack/plugin/security/src/main/java/module-info.java
index a072b34da7e96..947211559b0c2 100644
--- a/x-pack/plugin/security/src/main/java/module-info.java
+++ b/x-pack/plugin/security/src/main/java/module-info.java
@@ -70,6 +70,8 @@
     exports org.elasticsearch.xpack.security.slowlog to org.elasticsearch.server;
     exports org.elasticsearch.xpack.security.authc.support to org.elasticsearch.internal.security;
     exports org.elasticsearch.xpack.security.rest.action.apikey to org.elasticsearch.internal.security;
+    exports org.elasticsearch.xpack.security.support to org.elasticsearch.internal.security;
+    exports org.elasticsearch.xpack.security.authz.store to org.elasticsearch.internal.security;
 
     provides org.elasticsearch.index.SlowLogFieldProvider with org.elasticsearch.xpack.security.slowlog.SecuritySlowLogFieldProvider;
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index ef66392a87260..fd530a338b26c 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -411,6 +411,8 @@
 import org.elasticsearch.xpack.security.rest.action.user.RestSetEnabledAction;
 import org.elasticsearch.xpack.security.support.CacheInvalidatorRegistry;
 import org.elasticsearch.xpack.security.support.ExtensionComponents;
+import org.elasticsearch.xpack.security.support.QueryableBuiltInRolesProviderFactory;
+import org.elasticsearch.xpack.security.support.QueryableBuiltInRolesSynchronizer;
 import org.elasticsearch.xpack.security.support.ReloadableSecurityComponent;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.elasticsearch.xpack.security.support.SecurityMigrationExecutor;
@@ -461,6 +463,7 @@
 import static org.elasticsearch.xpack.core.security.SecurityField.FIELD_LEVEL_SECURITY_FEATURE;
 import static org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore.INCLUDED_RESERVED_ROLES_SETTING;
 import static org.elasticsearch.xpack.security.operator.OperatorPrivileges.OPERATOR_PRIVILEGES_ENABLED;
+import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesSynchronizer.QUERYABLE_BUILT_IN_ROLES_ENABLED;
 import static org.elasticsearch.xpack.security.transport.SSLEngineUtils.extractClientCertificates;
 
 public class Security extends Plugin
@@ -631,7 +634,7 @@ public class Security extends Plugin
     private final SetOnce<ReservedRoleNameChecker.Factory> reservedRoleNameCheckerFactory = new SetOnce<>();
     private final SetOnce<FileRoleValidator> fileRoleValidator = new SetOnce<>();
     private final SetOnce<SecondaryAuthActions> secondaryAuthActions = new SetOnce<>();
-
+    private final SetOnce<QueryableBuiltInRolesProviderFactory> queryableRolesProviderFactory = new SetOnce<>();
     private final SetOnce<SecurityMigrationExecutor> securityMigrationExecutor = new SetOnce<>();
 
     // Node local retry count for migration jobs that's checked only on the master node to make sure
@@ -1202,6 +1205,23 @@ Collection<Object> createComponents(
 
         reservedRoleMappingAction.set(new ReservedRoleMappingAction());
 
+        if (QUERYABLE_BUILT_IN_ROLES_ENABLED) {
+            if (queryableRolesProviderFactory.get() == null) {
+                queryableRolesProviderFactory.set(new QueryableBuiltInRolesProviderFactory.Default());
+            }
+            components.add(
+                new QueryableBuiltInRolesSynchronizer(
+                    clusterService,
+                    featureService,
+                    queryableRolesProviderFactory.get(),
+                    nativeRolesStore,
+                    reservedRolesStore,
+                    fileRolesStore.get(),
+                    threadPool
+                )
+            );
+        }
+
         cacheInvalidatorRegistry.validate();
 
         final List<ReloadableSecurityComponent> reloadableComponents = new ArrayList<>();
@@ -2317,6 +2337,7 @@ public void loadExtensions(ExtensionLoader loader) {
         loadSingletonExtensionAndSetOnce(loader, grantApiKeyRequestTranslator, RestGrantApiKeyAction.RequestTranslator.class);
         loadSingletonExtensionAndSetOnce(loader, fileRoleValidator, FileRoleValidator.class);
         loadSingletonExtensionAndSetOnce(loader, secondaryAuthActions, SecondaryAuthActions.class);
+        loadSingletonExtensionAndSetOnce(loader, queryableRolesProviderFactory, QueryableBuiltInRolesProviderFactory.class);
     }
 
     private <T> void loadSingletonExtensionAndSetOnce(ExtensionLoader loader, SetOnce<T> setOnce, Class<T> clazz) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java
index 53ecafa280715..409ab62ae3e70 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java
@@ -12,14 +12,12 @@
 
 import java.util.Set;
 
-import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MIGRATION_FRAMEWORK;
-import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_ROLES_METADATA_FLATTENED;
-import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_ROLE_MAPPING_CLEANUP;
+import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesSynchronizer.QUERYABLE_BUILT_IN_ROLES_FEATURE;
 
 public class SecurityFeatures implements FeatureSpecification {
 
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(SECURITY_ROLE_MAPPING_CLEANUP, SECURITY_ROLES_METADATA_FLATTENED, SECURITY_MIGRATION_FRAMEWORK);
+        return Set.of(QUERYABLE_BUILT_IN_ROLES_FEATURE);
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesAction.java
index e019f168cf8c0..cdeac51e1f492 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesAction.java
@@ -20,11 +20,9 @@
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
+import java.util.LinkedHashSet;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -51,8 +49,8 @@ protected void doExecute(Task task, final GetRolesRequest request, final ActionL
             return;
         }
 
-        final Set<String> rolesToSearchFor = new HashSet<>();
-        final List<RoleDescriptor> reservedRoles = new ArrayList<>();
+        final Set<String> rolesToSearchFor = new LinkedHashSet<>();
+        final Set<RoleDescriptor> reservedRoles = new LinkedHashSet<>();
         if (specificRolesRequested) {
             for (String role : requestedRoles) {
                 if (ReservedRolesStore.isReserved(role)) {
@@ -80,10 +78,10 @@ protected void doExecute(Task task, final GetRolesRequest request, final ActionL
     }
 
     private void getNativeRoles(Set<String> rolesToSearchFor, ActionListener<GetRolesResponse> listener) {
-        getNativeRoles(rolesToSearchFor, new ArrayList<>(), listener);
+        getNativeRoles(rolesToSearchFor, new LinkedHashSet<>(), listener);
     }
 
-    private void getNativeRoles(Set<String> rolesToSearchFor, List<RoleDescriptor> foundRoles, ActionListener<GetRolesResponse> listener) {
+    private void getNativeRoles(Set<String> rolesToSearchFor, Set<RoleDescriptor> foundRoles, ActionListener<GetRolesResponse> listener) {
         nativeRolesStore.getRoleDescriptors(rolesToSearchFor, ActionListener.wrap((retrievalResult) -> {
             if (retrievalResult.isSuccess()) {
                 foundRoles.addAll(retrievalResult.getDescriptors());
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
index 900436a1fd874..4f7ba7808b823 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
@@ -48,7 +48,9 @@
 import org.elasticsearch.common.cache.CacheBuilder;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
+import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -57,6 +59,7 @@
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Streams;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
@@ -90,8 +93,10 @@
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
 import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.Closeable;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.io.UncheckedIOException;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
@@ -127,6 +132,7 @@
 
 import javax.crypto.Cipher;
 import javax.crypto.CipherInputStream;
+import javax.crypto.CipherOutputStream;
 import javax.crypto.NoSuchPaddingException;
 import javax.crypto.SecretKey;
 import javax.crypto.SecretKeyFactory;
@@ -195,8 +201,14 @@ public class TokenService {
     // UUIDs are 16 bytes encoded base64 without padding, therefore the length is (16 / 3) * 4 + ((16 % 3) * 8 + 5) / 6 chars
     private static final int TOKEN_LENGTH = 22;
     private static final String TOKEN_DOC_ID_PREFIX = TOKEN_DOC_TYPE + "_";
+    static final int LEGACY_MINIMUM_BYTES = VERSION_BYTES + SALT_BYTES + IV_BYTES + 1;
     static final int MINIMUM_BYTES = VERSION_BYTES + TOKEN_LENGTH + 1;
+    static final int LEGACY_MINIMUM_BASE64_BYTES = Double.valueOf(Math.ceil((4 * LEGACY_MINIMUM_BYTES) / 3)).intValue();
     public static final int MINIMUM_BASE64_BYTES = Double.valueOf(Math.ceil((4 * MINIMUM_BYTES) / 3)).intValue();
+    static final TransportVersion VERSION_HASHED_TOKENS = TransportVersions.V_7_2_0;
+    static final TransportVersion VERSION_TOKENS_INDEX_INTRODUCED = TransportVersions.V_7_2_0;
+    static final TransportVersion VERSION_ACCESS_TOKENS_AS_UUIDS = TransportVersions.V_7_2_0;
+    static final TransportVersion VERSION_MULTIPLE_CONCURRENT_REFRESHES = TransportVersions.V_7_2_0;
     static final TransportVersion VERSION_CLIENT_AUTH_FOR_REFRESH = TransportVersions.V_8_2_0;
     static final TransportVersion VERSION_GET_TOKEN_DOC_FOR_REFRESH = TransportVersions.V_8_10_X;
 
@@ -261,7 +273,8 @@ public TokenService(
 
     /**
      * Creates an access token and optionally a refresh token as well, based on the provided authentication and metadata with
-     * auto-generated values. The created tokens are stored a specific security tokens index.
+     * auto-generated values. The created tokens are stored in the security index for versions up to
+     * {@link #VERSION_TOKENS_INDEX_INTRODUCED} and to a specific security tokens index for later versions.
      */
     public void createOAuth2Tokens(
         Authentication authentication,
@@ -278,7 +291,8 @@ public void createOAuth2Tokens(
 
     /**
      * Creates an access token and optionally a refresh token as well from predefined values, based on the provided authentication and
-     * metadata. The created tokens are stored in a specific security tokens index.
+     * metadata. The created tokens are stored in the security index for versions up to {@link #VERSION_TOKENS_INDEX_INTRODUCED} and to a
+     * specific security tokens index for later versions.
      */
     // public for testing
     public void createOAuth2Tokens(
@@ -300,15 +314,21 @@ public void createOAuth2Tokens(
      *
      * @param accessTokenBytes      The predefined seed value for the access token. This will then be
      *                              <ul>
-     *                                <li> Hashed before stored </li>
-     *                                <li> Stored in a specific security tokens index </li>
+     *                                <li> Encrypted before stored for versions before {@link #VERSION_TOKENS_INDEX_INTRODUCED} </li>
+     *                                <li> Hashed before stored for versions after {@link #VERSION_TOKENS_INDEX_INTRODUCED} </li>
+     *                                <li> Stored in the security index for versions up to {@link #VERSION_TOKENS_INDEX_INTRODUCED}</li>
+     *                                <li> Stored in a specific security tokens index for versions after
+     *                                {@link #VERSION_TOKENS_INDEX_INTRODUCED}</li>
      *                                <li> Prepended with a version ID and Base64 encoded before returned to the caller of the APIs</li>
      *                              </ul>
      * @param refreshTokenBytes     The predefined seed value for the access token. This will then be
      *                              <ul>
-     *                                <li> Hashed before stored </li>
-     *                                <li> Stored in a specific security tokens index </li>
-     *                                <li> Prepended with a version ID and Base64 encoded before returned to the caller of the APIs </li>
+     *                                <li> Hashed before stored for versions after {@link #VERSION_TOKENS_INDEX_INTRODUCED} </li>
+     *                                <li> Stored in the security index for versions up to {@link #VERSION_TOKENS_INDEX_INTRODUCED}</li>
+     *                                <li> Stored in a specific security tokens index for versions after
+     *                                {@link #VERSION_TOKENS_INDEX_INTRODUCED}</li>
+     *                                <li> Prepended with a version ID and encoded with Base64 before returned to the caller of the APIs
+     *                                for versions after {@link #VERSION_TOKENS_INDEX_INTRODUCED}</li>
      *                              </ul>
      * @param tokenVersion          The version of the nodes with which these tokens will be compatible.
      * @param authentication        The authentication object representing the user for which the tokens are created
@@ -364,7 +384,7 @@ private void createOAuth2Tokens(
                     } else {
                         refreshTokenToStore = refreshTokenToReturn = null;
                     }
-                } else {
+                } else if (tokenVersion.onOrAfter(VERSION_HASHED_TOKENS)) {
                     assert accessTokenBytes.length == RAW_TOKEN_BYTES_LENGTH;
                     userTokenId = hashTokenString(Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(accessTokenBytes));
                     accessTokenToStore = null;
@@ -375,6 +395,18 @@ private void createOAuth2Tokens(
                     } else {
                         refreshTokenToStore = refreshTokenToReturn = null;
                     }
+                } else {
+                    assert accessTokenBytes.length == RAW_TOKEN_BYTES_LENGTH;
+                    userTokenId = Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(accessTokenBytes);
+                    accessTokenToStore = null;
+                    if (refreshTokenBytes != null) {
+                        assert refreshTokenBytes.length == RAW_TOKEN_BYTES_LENGTH;
+                        refreshTokenToStore = refreshTokenToReturn = Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(
+                            refreshTokenBytes
+                        );
+                    } else {
+                        refreshTokenToStore = refreshTokenToReturn = null;
+                    }
                 }
                 UserToken userToken = new UserToken(userTokenId, tokenVersion, tokenAuth, getExpirationTime(), metadata);
                 tokenDocument = createTokenDocument(userToken, accessTokenToStore, refreshTokenToStore, originatingClientAuth);
@@ -387,22 +419,23 @@ private void createOAuth2Tokens(
             final RefreshPolicy tokenCreationRefreshPolicy = tokenVersion.onOrAfter(VERSION_GET_TOKEN_DOC_FOR_REFRESH)
                 ? RefreshPolicy.NONE
                 : RefreshPolicy.WAIT_UNTIL;
+            final SecurityIndexManager tokensIndex = getTokensIndexForVersion(tokenVersion);
             logger.debug(
                 () -> format(
                     "Using refresh policy [%s] when creating token doc [%s] in the security index [%s]",
                     tokenCreationRefreshPolicy,
                     documentId,
-                    securityTokensIndex.aliasName()
+                    tokensIndex.aliasName()
                 )
             );
-            final IndexRequest indexTokenRequest = client.prepareIndex(securityTokensIndex.aliasName())
+            final IndexRequest indexTokenRequest = client.prepareIndex(tokensIndex.aliasName())
                 .setId(documentId)
                 .setOpType(OpType.CREATE)
                 .setSource(tokenDocument, XContentType.JSON)
                 .setRefreshPolicy(tokenCreationRefreshPolicy)
                 .request();
-            securityTokensIndex.prepareIndexIfNeededThenExecute(
-                ex -> listener.onFailure(traceLog("prepare tokens index [" + securityTokensIndex.aliasName() + "]", documentId, ex)),
+            tokensIndex.prepareIndexIfNeededThenExecute(
+                ex -> listener.onFailure(traceLog("prepare tokens index [" + tokensIndex.aliasName() + "]", documentId, ex)),
                 () -> executeAsyncWithOrigin(
                     client,
                     SECURITY_ORIGIN,
@@ -521,16 +554,17 @@ private void getTokenDocById(
         @Nullable String storedRefreshToken,
         ActionListener<Doc> listener
     ) {
-        final SecurityIndexManager frozenTokensIndex = securityTokensIndex.defensiveCopy();
+        final SecurityIndexManager tokensIndex = getTokensIndexForVersion(tokenVersion);
+        final SecurityIndexManager frozenTokensIndex = tokensIndex.defensiveCopy();
         if (frozenTokensIndex.isAvailable(PRIMARY_SHARDS) == false) {
-            logger.warn("failed to get access token [{}] because index [{}] is not available", tokenId, securityTokensIndex.aliasName());
+            logger.warn("failed to get access token [{}] because index [{}] is not available", tokenId, tokensIndex.aliasName());
             listener.onFailure(frozenTokensIndex.getUnavailableReason(PRIMARY_SHARDS));
             return;
         }
-        final GetRequest getRequest = client.prepareGet(securityTokensIndex.aliasName(), getTokenDocumentId(tokenId)).request();
+        final GetRequest getRequest = client.prepareGet(tokensIndex.aliasName(), getTokenDocumentId(tokenId)).request();
         final Consumer<Exception> onFailure = ex -> listener.onFailure(traceLog("get token from id", tokenId, ex));
-        securityTokensIndex.checkIndexVersionThenExecute(
-            ex -> listener.onFailure(traceLog("prepare tokens index [" + securityTokensIndex.aliasName() + "]", tokenId, ex)),
+        tokensIndex.checkIndexVersionThenExecute(
+            ex -> listener.onFailure(traceLog("prepare tokens index [" + tokensIndex.aliasName() + "]", tokenId, ex)),
             () -> executeAsyncWithOrigin(
                 client.threadPool().getThreadContext(),
                 SECURITY_ORIGIN,
@@ -576,11 +610,7 @@ private void getTokenDocById(
                     // if the index or the shard is not there / available we assume that
                     // the token is not valid
                     if (isShardNotAvailableException(e)) {
-                        logger.warn(
-                            "failed to get token doc [{}] because index [{}] is not available",
-                            tokenId,
-                            securityTokensIndex.aliasName()
-                        );
+                        logger.warn("failed to get token doc [{}] because index [{}] is not available", tokenId, tokensIndex.aliasName());
                     } else {
                         logger.error(() -> "failed to get token doc [" + tokenId + "]", e);
                     }
@@ -620,7 +650,7 @@ void decodeToken(String token, boolean validateUserToken, ActionListener<UserTok
                 final String userTokenId = Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(userTokenIdDigest.digest());
                 final String storedAccessToken = Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(sha256().digest(accessTokenBytes));
                 getAndValidateUserToken(userTokenId, version, storedAccessToken, validateUserToken, listener);
-            } else {
+            } else if (version.onOrAfter(VERSION_ACCESS_TOKENS_AS_UUIDS)) {
                 // The token was created in a > VERSION_ACCESS_TOKENS_UUIDS cluster
                 if (in.available() < MINIMUM_BYTES) {
                     logger.debug("invalid token, smaller than [{}] bytes", MINIMUM_BYTES);
@@ -630,6 +660,41 @@ void decodeToken(String token, boolean validateUserToken, ActionListener<UserTok
                 final String accessToken = in.readString();
                 final String userTokenId = hashTokenString(accessToken);
                 getAndValidateUserToken(userTokenId, version, null, validateUserToken, listener);
+            } else {
+                // The token was created in a < VERSION_ACCESS_TOKENS_UUIDS cluster so we need to decrypt it to get the tokenId
+                if (in.available() < LEGACY_MINIMUM_BYTES) {
+                    logger.debug("invalid token, smaller than [{}] bytes", LEGACY_MINIMUM_BYTES);
+                    listener.onResponse(null);
+                    return;
+                }
+                final BytesKey decodedSalt = new BytesKey(in.readByteArray());
+                final BytesKey passphraseHash = new BytesKey(in.readByteArray());
+                final byte[] iv = in.readByteArray();
+                final BytesStreamOutput out = new BytesStreamOutput();
+                Streams.copy(in, out);
+                final byte[] encryptedTokenId = BytesReference.toBytes(out.bytes());
+                final KeyAndCache keyAndCache = keyCache.get(passphraseHash);
+                if (keyAndCache != null) {
+                    getKeyAsync(decodedSalt, keyAndCache, ActionListener.wrap(decodeKey -> {
+                        if (decodeKey != null) {
+                            try {
+                                final Cipher cipher = getDecryptionCipher(iv, decodeKey, version, decodedSalt);
+                                final String tokenId = decryptTokenId(encryptedTokenId, cipher, version);
+                                getAndValidateUserToken(tokenId, version, null, validateUserToken, listener);
+                            } catch (IOException | GeneralSecurityException e) {
+                                // could happen with a token that is not ours
+                                logger.warn("invalid token", e);
+                                listener.onResponse(null);
+                            }
+                        } else {
+                            // could happen with a token that is not ours
+                            listener.onResponse(null);
+                        }
+                    }, listener::onFailure));
+                } else {
+                    logger.debug(() -> format("invalid key %s key: %s", passphraseHash, keyCache.cache.keySet()));
+                    listener.onResponse(null);
+                }
             }
         } catch (Exception e) {
             // could happen with a token that is not ours
@@ -787,7 +852,11 @@ private void indexInvalidation(
         final Set<String> idsOfOlderTokens = new HashSet<>();
         boolean anyOlderTokensBeforeRefreshViaGet = false;
         for (UserToken userToken : userTokens) {
-            idsOfRecentTokens.add(userToken.getId());
+            if (userToken.getTransportVersion().onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) {
+                idsOfRecentTokens.add(userToken.getId());
+            } else {
+                idsOfOlderTokens.add(userToken.getId());
+            }
             anyOlderTokensBeforeRefreshViaGet |= userToken.getTransportVersion().before(VERSION_GET_TOKEN_DOC_FOR_REFRESH);
         }
         final RefreshPolicy tokensInvalidationRefreshPolicy = anyOlderTokensBeforeRefreshViaGet
@@ -1055,7 +1124,7 @@ private void findTokenFromRefreshToken(String refreshToken, Iterator<TimeValue>
                         );
                         getTokenDocById(userTokenId, version, null, storedRefreshToken, listener);
                     }
-                } else {
+                } else if (version.onOrAfter(VERSION_HASHED_TOKENS)) {
                     final String unencodedRefreshToken = in.readString();
                     if (unencodedRefreshToken.length() != TOKEN_LENGTH) {
                         logger.debug("Decoded refresh token [{}] with version [{}] is invalid.", unencodedRefreshToken, version);
@@ -1064,6 +1133,9 @@ private void findTokenFromRefreshToken(String refreshToken, Iterator<TimeValue>
                         final String hashedRefreshToken = hashTokenString(unencodedRefreshToken);
                         findTokenFromRefreshToken(hashedRefreshToken, securityTokensIndex, backoff, listener);
                     }
+                } else {
+                    logger.debug("Unrecognized refresh token version [{}].", version);
+                    listener.onResponse(null);
                 }
             } catch (IOException e) {
                 logger.debug(() -> "Could not decode refresh token [" + refreshToken + "].", e);
@@ -1178,6 +1250,7 @@ private void innerRefresh(
             return;
         }
         final RefreshTokenStatus refreshTokenStatus = checkRefreshResult.v1();
+        final SecurityIndexManager refreshedTokenIndex = getTokensIndexForVersion(refreshTokenStatus.getTransportVersion());
         if (refreshTokenStatus.isRefreshed()) {
             logger.debug(
                 "Token document [{}] was recently refreshed, when a new token document was generated. Reusing that result.",
@@ -1185,29 +1258,31 @@ private void innerRefresh(
             );
             final Tuple<UserToken, String> parsedTokens = parseTokensFromDocument(tokenDoc.sourceAsMap(), null);
             Authentication authentication = parsedTokens.v1().getAuthentication();
-            decryptAndReturnSupersedingTokens(refreshToken, refreshTokenStatus, securityTokensIndex, authentication, listener);
+            decryptAndReturnSupersedingTokens(refreshToken, refreshTokenStatus, refreshedTokenIndex, authentication, listener);
         } else {
             final TransportVersion newTokenVersion = getTokenVersionCompatibility();
             final Tuple<byte[], byte[]> newTokenBytes = getRandomTokenBytes(newTokenVersion, true);
             final Map<String, Object> updateMap = new HashMap<>();
             updateMap.put("refreshed", true);
-            updateMap.put("refresh_time", clock.instant().toEpochMilli());
-            try {
-                final byte[] iv = getRandomBytes(IV_BYTES);
-                final byte[] salt = getRandomBytes(SALT_BYTES);
-                String encryptedAccessAndRefreshToken = encryptSupersedingTokens(
-                    newTokenBytes.v1(),
-                    newTokenBytes.v2(),
-                    refreshToken,
-                    iv,
-                    salt
-                );
-                updateMap.put("superseding.encrypted_tokens", encryptedAccessAndRefreshToken);
-                updateMap.put("superseding.encryption_iv", Base64.getEncoder().encodeToString(iv));
-                updateMap.put("superseding.encryption_salt", Base64.getEncoder().encodeToString(salt));
-            } catch (GeneralSecurityException e) {
-                logger.warn("could not encrypt access token and refresh token string", e);
-                onFailure.accept(invalidGrantException("could not refresh the requested token"));
+            if (newTokenVersion.onOrAfter(VERSION_MULTIPLE_CONCURRENT_REFRESHES)) {
+                updateMap.put("refresh_time", clock.instant().toEpochMilli());
+                try {
+                    final byte[] iv = getRandomBytes(IV_BYTES);
+                    final byte[] salt = getRandomBytes(SALT_BYTES);
+                    String encryptedAccessAndRefreshToken = encryptSupersedingTokens(
+                        newTokenBytes.v1(),
+                        newTokenBytes.v2(),
+                        refreshToken,
+                        iv,
+                        salt
+                    );
+                    updateMap.put("superseding.encrypted_tokens", encryptedAccessAndRefreshToken);
+                    updateMap.put("superseding.encryption_iv", Base64.getEncoder().encodeToString(iv));
+                    updateMap.put("superseding.encryption_salt", Base64.getEncoder().encodeToString(salt));
+                } catch (GeneralSecurityException e) {
+                    logger.warn("could not encrypt access token and refresh token string", e);
+                    onFailure.accept(invalidGrantException("could not refresh the requested token"));
+                }
             }
             assert tokenDoc.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO : "expected an assigned sequence number";
             assert tokenDoc.primaryTerm() != SequenceNumbers.UNASSIGNED_PRIMARY_TERM : "expected an assigned primary term";
@@ -1218,17 +1293,17 @@ private void innerRefresh(
                     "Using refresh policy [%s] when updating token doc [%s] for refresh in the security index [%s]",
                     tokenRefreshUpdateRefreshPolicy,
                     tokenDoc.id(),
-                    securityTokensIndex.aliasName()
+                    refreshedTokenIndex.aliasName()
                 )
             );
-            final UpdateRequestBuilder updateRequest = client.prepareUpdate(securityTokensIndex.aliasName(), tokenDoc.id())
+            final UpdateRequestBuilder updateRequest = client.prepareUpdate(refreshedTokenIndex.aliasName(), tokenDoc.id())
                 .setDoc("refresh_token", updateMap)
                 .setFetchSource(logger.isDebugEnabled())
                 .setRefreshPolicy(tokenRefreshUpdateRefreshPolicy)
                 .setIfSeqNo(tokenDoc.seqNo())
                 .setIfPrimaryTerm(tokenDoc.primaryTerm());
-            securityTokensIndex.prepareIndexIfNeededThenExecute(
-                ex -> listener.onFailure(traceLog("prepare index [" + securityTokensIndex.aliasName() + "]", ex)),
+            refreshedTokenIndex.prepareIndexIfNeededThenExecute(
+                ex -> listener.onFailure(traceLog("prepare index [" + refreshedTokenIndex.aliasName() + "]", ex)),
                 () -> executeAsyncWithOrigin(
                     client.threadPool().getThreadContext(),
                     SECURITY_ORIGIN,
@@ -1274,7 +1349,7 @@ private void innerRefresh(
                         if (cause instanceof VersionConflictEngineException) {
                             // The document has been updated by another thread, get it again.
                             logger.debug("version conflict while updating document [{}], attempting to get it again", tokenDoc.id());
-                            getTokenDocAsync(tokenDoc.id(), securityTokensIndex, true, new ActionListener<>() {
+                            getTokenDocAsync(tokenDoc.id(), refreshedTokenIndex, true, new ActionListener<>() {
                                 @Override
                                 public void onResponse(GetResponse response) {
                                     if (response.isExists()) {
@@ -1293,7 +1368,7 @@ public void onFailure(Exception e) {
                                             logger.info("could not get token document [{}] for refresh, retrying", tokenDoc.id());
                                             client.threadPool()
                                                 .schedule(
-                                                    () -> getTokenDocAsync(tokenDoc.id(), securityTokensIndex, true, this),
+                                                    () -> getTokenDocAsync(tokenDoc.id(), refreshedTokenIndex, true, this),
                                                     backoff.next(),
                                                     client.threadPool().generic()
                                                 );
@@ -1614,13 +1689,17 @@ private static Optional<ElasticsearchSecurityException> checkMultipleRefreshes(
         RefreshTokenStatus refreshTokenStatus
     ) {
         if (refreshTokenStatus.isRefreshed()) {
-            if (refreshRequested.isAfter(refreshTokenStatus.getRefreshInstant().plus(30L, ChronoUnit.SECONDS))) {
-                return Optional.of(invalidGrantException("token has already been refreshed more than 30 seconds in the past"));
-            }
-            if (refreshRequested.isBefore(refreshTokenStatus.getRefreshInstant().minus(30L, ChronoUnit.SECONDS))) {
-                return Optional.of(
-                    invalidGrantException("token has been refreshed more than 30 seconds in the future, clock skew too great")
-                );
+            if (refreshTokenStatus.getTransportVersion().onOrAfter(VERSION_MULTIPLE_CONCURRENT_REFRESHES)) {
+                if (refreshRequested.isAfter(refreshTokenStatus.getRefreshInstant().plus(30L, ChronoUnit.SECONDS))) {
+                    return Optional.of(invalidGrantException("token has already been refreshed more than 30 seconds in the past"));
+                }
+                if (refreshRequested.isBefore(refreshTokenStatus.getRefreshInstant().minus(30L, ChronoUnit.SECONDS))) {
+                    return Optional.of(
+                        invalidGrantException("token has been refreshed more than 30 seconds in the future, clock skew too great")
+                    );
+                }
+            } else {
+                return Optional.of(invalidGrantException("token has already been refreshed"));
             }
         }
         return Optional.empty();
@@ -1900,6 +1979,21 @@ private void ensureEnabled() {
         }
     }
 
+    /**
+     * In version {@code #VERSION_TOKENS_INDEX_INTRODUCED} security tokens were moved into a separate index, away from the other entities in
+     * the main security index, due to their ephemeral nature. They moved "seamlessly" - without manual user intervention. In this way, new
+     * tokens are created in the new index, while the existing ones were left in place - to be accessed from the old index - and due to be
+     * removed automatically by the {@code ExpiredTokenRemover} periodic job. Therefore, in general, when searching for a token we need to
+     * consider both the new and the old indices.
+     */
+    private SecurityIndexManager getTokensIndexForVersion(TransportVersion version) {
+        if (version.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) {
+            return securityTokensIndex;
+        } else {
+            return securityMainIndex;
+        }
+    }
+
     public TimeValue getExpirationDelay() {
         return expirationDelay;
     }
@@ -1928,13 +2022,41 @@ public String prependVersionAndEncodeAccessToken(TransportVersion version, byte[
                 out.writeByteArray(accessTokenBytes);
                 return Base64.getEncoder().encodeToString(out.bytes().toBytesRef().bytes);
             }
-        } else {
+        } else if (version.onOrAfter(VERSION_ACCESS_TOKENS_AS_UUIDS)) {
             try (BytesStreamOutput out = new BytesStreamOutput(MINIMUM_BASE64_BYTES)) {
                 out.setTransportVersion(version);
                 TransportVersion.writeVersion(version, out);
                 out.writeString(Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(accessTokenBytes));
                 return Base64.getEncoder().encodeToString(out.bytes().toBytesRef().bytes);
             }
+        } else {
+            // we know that the minimum length is larger than the default of the ByteArrayOutputStream so set the size to this explicitly
+            try (
+                ByteArrayOutputStream os = new ByteArrayOutputStream(LEGACY_MINIMUM_BASE64_BYTES);
+                OutputStream base64 = Base64.getEncoder().wrap(os);
+                StreamOutput out = new OutputStreamStreamOutput(base64)
+            ) {
+                out.setTransportVersion(version);
+                KeyAndCache keyAndCache = keyCache.activeKeyCache;
+                TransportVersion.writeVersion(version, out);
+                out.writeByteArray(keyAndCache.getSalt().bytes);
+                out.writeByteArray(keyAndCache.getKeyHash().bytes);
+                final byte[] initializationVector = getRandomBytes(IV_BYTES);
+                out.writeByteArray(initializationVector);
+                try (
+                    CipherOutputStream encryptedOutput = new CipherOutputStream(
+                        out,
+                        getEncryptionCipher(initializationVector, keyAndCache, version)
+                    );
+                    StreamOutput encryptedStreamOutput = new OutputStreamStreamOutput(encryptedOutput)
+                ) {
+                    encryptedStreamOutput.setTransportVersion(version);
+                    encryptedStreamOutput.writeString(Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(accessTokenBytes));
+                    // StreamOutput needs to be closed explicitly because it wraps CipherOutputStream
+                    encryptedStreamOutput.close();
+                    return new String(os.toByteArray(), StandardCharsets.UTF_8);
+                }
+            }
         }
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java
index 12f51e0ec7912..57911737cd232 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java
@@ -698,7 +698,6 @@ private boolean usingGlobalCatalog(LDAPConnection ldapConnection) {
      */
     static class UpnADAuthenticator extends ADAuthenticator {
         static final String UPN_USER_FILTER = "(&(objectClass=user)(userPrincipalName={1}))";
-        private final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(logger.getName());
 
         UpnADAuthenticator(
             RealmConfig config,
@@ -725,13 +724,17 @@ static class UpnADAuthenticator extends ADAuthenticator {
                 bindRequestSupplier
             );
             if (userSearchFilter.contains("{0}")) {
-                deprecationLogger.warn(
-                    DeprecationCategory.SECURITY,
-                    "ldap_settings",
-                    "The use of the account name variable {0} in the setting ["
-                        + RealmSettings.getFullSettingKey(config, ActiveDirectorySessionFactorySettings.AD_UPN_USER_SEARCH_FILTER_SETTING)
-                        + "] has been deprecated and will be removed in a future version!"
-                );
+                DeprecationLogger.getLogger(logger.getName())
+                    .warn(
+                        DeprecationCategory.SECURITY,
+                        "ldap_settings",
+                        "The use of the account name variable {0} in the setting ["
+                            + RealmSettings.getFullSettingKey(
+                                config,
+                                ActiveDirectorySessionFactorySettings.AD_UPN_USER_SEARCH_FILTER_SETTING
+                            )
+                            + "] has been deprecated and will be removed in a future version!"
+                    );
             }
         }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/PoolingSessionFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/PoolingSessionFactory.java
index 4f87ac27be141..f31d5647b4cc5 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/PoolingSessionFactory.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/PoolingSessionFactory.java
@@ -16,7 +16,6 @@
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -74,7 +73,7 @@ abstract class PoolingSessionFactory extends SessionFactory implements Releasabl
         super(config, sslService, threadPool);
         this.groupResolver = groupResolver;
         this.bindDn = bindDn;
-        this.bindRequest = new AtomicReference<>(buildBindRequest(config.settings(), false));
+        this.bindRequest = new AtomicReference<>(buildBindRequest(config.settings()));
         this.useConnectionPool = config.getSetting(poolingEnabled);
         if (useConnectionPool) {
             this.connectionPool = createConnectionPool(config, serverSet, timeout, logger, bindRequest.get(), healthCheckDNSupplier);
@@ -93,11 +92,9 @@ abstract class PoolingSessionFactory extends SessionFactory implements Releasabl
      * will perform a setting consistency validation and throw {@link SettingsException} in case of violation.
      * Due to legacy reasons and BWC, when {@code reloadRequest} is se to {@code false}, this method will only log a warning message.
      *
-     * @param reloadRequest {@code true} if this method is called during reloading of secure settings,
-     *                      {@code false} if it is called during bootstrapping.
      * @return A new {@link SimpleBindRequest} that contains configured bind DN and password.
      */
-    private SimpleBindRequest buildBindRequest(Settings settings, boolean reloadRequest) {
+    private SimpleBindRequest buildBindRequest(Settings settings) {
         final byte[] bindPassword;
         final Setting<SecureString> legacyPasswordSetting = config.getConcreteSetting(LEGACY_BIND_PASSWORD);
         final Setting<SecureString> securePasswordSetting = config.getConcreteSetting(SECURE_BIND_PASSWORD);
@@ -119,27 +116,13 @@ private SimpleBindRequest buildBindRequest(Settings settings, boolean reloadRequ
             return new SimpleBindRequest();
         } else {
             if (bindPassword == null) {
-                if (reloadRequest) {
-                    throw new SettingsException(
-                        "[{}] is set but no bind password is specified. Without a corresponding bind password, "
-                            + "all {} realm authentication will fail. Specify a bind password via [{}].",
-                        RealmSettings.getFullSettingKey(config, PoolingSessionFactorySettings.BIND_DN),
-                        config.type(),
-                        RealmSettings.getFullSettingKey(config, SECURE_BIND_PASSWORD)
-                    );
-                } else {
-                    deprecationLogger.critical(
-                        DeprecationCategory.SECURITY,
-                        "bind_dn_set_without_password",
-                        "[{}] is set but no bind password is specified. Without a corresponding bind password, "
-                            + "all {} realm authentication will fail. Specify a bind password via [{}] or [{}]. "
-                            + "In the next major release, nodes with incomplete bind credentials will fail to start.",
-                        RealmSettings.getFullSettingKey(config, PoolingSessionFactorySettings.BIND_DN),
-                        config.type(),
-                        RealmSettings.getFullSettingKey(config, SECURE_BIND_PASSWORD),
-                        RealmSettings.getFullSettingKey(config, LEGACY_BIND_PASSWORD)
-                    );
-                }
+                throw new SettingsException(
+                    "[{}] is set but no bind password is specified. Without a corresponding bind password, "
+                        + "all {} realm authentication will fail. Specify a bind password via [{}].",
+                    RealmSettings.getFullSettingKey(config, PoolingSessionFactorySettings.BIND_DN),
+                    config.type(),
+                    RealmSettings.getFullSettingKey(config, SECURE_BIND_PASSWORD)
+                );
             }
             return new SimpleBindRequest(this.bindDn, bindPassword);
         }
@@ -148,7 +131,7 @@ private SimpleBindRequest buildBindRequest(Settings settings, boolean reloadRequ
     @Override
     public void reload(Settings settings) {
         final SimpleBindRequest oldRequest = bindRequest.get();
-        final SimpleBindRequest newRequest = buildBindRequest(settings, true);
+        final SimpleBindRequest newRequest = buildBindRequest(settings);
         if (bindRequestEquals(newRequest, oldRequest) == false) {
             if (bindRequest.compareAndSet(oldRequest, newRequest)) {
                 if (connectionPool != null) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/LdapMetadataResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/LdapMetadataResolver.java
index 229b517be5292..43615f3eb605d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/LdapMetadataResolver.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/LdapMetadataResolver.java
@@ -109,7 +109,7 @@ private static Attribute findAttribute(Collection<Attribute> attributes, String
 
     public static class LdapMetadataResult {
 
-        public static LdapMetadataResult EMPTY = new LdapMetadataResult(null, null, Map.of());
+        public static final LdapMetadataResult EMPTY = new LdapMetadataResult(null, null, Map.of());
 
         private final String fullName;
         private final String email;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactory.java
index 2a8625b2d93fb..b5e1343382b05 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactory.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactory.java
@@ -56,7 +56,6 @@ public abstract class SessionFactory implements Closeable, ReloadableSecurityCom
     private static final Pattern STARTS_WITH_LDAP = Pattern.compile("^ldap:.*", Pattern.CASE_INSENSITIVE);
 
     protected final Logger logger;
-    protected final DeprecationLogger deprecationLogger;
     protected final RealmConfig config;
     protected final TimeValue timeout;
     protected final SSLService sslService;
@@ -71,7 +70,6 @@ public abstract class SessionFactory implements Closeable, ReloadableSecurityCom
     protected SessionFactory(RealmConfig config, SSLService sslService, ThreadPool threadPool) {
         this.config = config;
         this.logger = LogManager.getLogger(getClass());
-        this.deprecationLogger = DeprecationLogger.getLogger(logger.getName());
         TimeValue searchTimeout = config.getSetting(
             SessionFactorySettings.TIMEOUT_LDAP_SETTING,
             () -> SessionFactorySettings.TIMEOUT_DEFAULT
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java
index 67211ec6135be..dd671ebef824e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java
@@ -166,6 +166,12 @@ final class ElasticServiceAccounts {
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices("content-*", ".search-acl-filter-*")
                     .privileges("read", "write", "monitor", "create_index", "auto_configure", "maintenance", "view_index_metadata")
+                    .build(),
+                // Custom permissions required for stateful agentless integrations
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices("agentless-*")
+                    .privileges("read", "write", "monitor", "create_index", "auto_configure", "maintenance", "view_index_metadata")
+                    .allowRestrictedIndices(false)
                     .build(), },
             new RoleDescriptor.ApplicationResourcePrivileges[] {
                 RoleDescriptor.ApplicationResourcePrivileges.builder()
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
index d5cbbe8b349a7..e2a52b1c83131 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.search.SearchContextId;
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
@@ -25,6 +26,7 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.transport.NoSuchRemoteClusterException;
@@ -90,6 +92,15 @@ class IndicesAndAliasesResolver {
      * be added to the <em>remote</em> index list.
      * <br>
      * Otherwise, <em>N</em> will be added to the <em>local</em> index list.
+     * <br><br>
+     * <p>
+     * If the provided <code>request</code> is of a type that {@link IndicesOptions#allowSelectors() allows selectors}, then the given index
+     * names may be modified to include any {@link IndexComponentSelector selectors} that should be
+     * returned. If a wildcard selector is present on the index name then the wildcard will be resolved to
+     * {@link IndexComponentSelector#values() all applicable concrete selector names} for a given index abstraction. Selectors that are
+     * not applicable for an index abstraction (such as the <code>::failures</code> selector on non-data streams) are not returned when
+     * resolving wildcards.
+     * </p>
      */
 
     ResolvedIndices resolve(
@@ -151,7 +162,12 @@ ResolvedIndices resolveIndicesAndAliasesWithoutWildcards(String action, IndicesR
         if (indices == null || indices.length == 0) {
             throw new IllegalArgumentException("the action " + action + " requires explicit index names, but none were provided");
         }
-        if (IndexNameExpressionResolver.isAllIndices(Arrays.asList(indices))) {
+        // We need to parse off any potential selector suffixes to see if the index part of an expression is a match all
+        // Do so lazily to avoid extra work in case of a large number of indices
+        if (IndexNameExpressionResolver.isAllIndices(
+            Arrays.asList(indices),
+            (expr) -> IndexNameExpressionResolver.splitSelectorExpression(expr).v1()
+        )) {
             throw new IllegalArgumentException(
                 "the action "
                     + action
@@ -186,11 +202,25 @@ ResolvedIndices resolveIndicesAndAliasesWithoutWildcards(String action, IndicesR
         // shard level requests.
         final List<String> localIndices = new ArrayList<>(split.getLocal().size());
         for (String localName : split.getLocal()) {
+            // TODO: Shard level requests should already have their selectors resolved to concrete indices by their parent requests
+            // see https://github.com/elastic/elasticsearch/issues/119629
+            // Resolve them here as long as they are non-wildcard, or missing from the expression
+            Tuple<String, String> expressionTuple = IndexNameExpressionResolver.splitSelectorExpression(localName);
+            String selector = expressionTuple.v2();
+            if (selector != null && Regex.isSimpleMatchPattern(selector)) {
+                throwOnUnexpectedWildcardSelectors(action, split.getLocal());
+            }
+            String localExpression = expressionTuple.v1();
             // TODO: Shard level requests have wildcard expanded already and do not need go through this check
-            if (Regex.isSimpleMatchPattern(localName)) {
+            // see https://github.com/elastic/elasticsearch/issues/119629
+            if (Regex.isSimpleMatchPattern(localExpression)) {
                 throwOnUnexpectedWildcards(action, split.getLocal());
             }
-            localIndices.add(IndexNameExpressionResolver.resolveDateMathExpression(localName));
+            String finalExpression = IndexNameExpressionResolver.resolveDateMathExpression(localExpression);
+            if (selector != null) {
+                finalExpression = IndexNameExpressionResolver.combineSelectorExpression(finalExpression, selector);
+            }
+            localIndices.add(finalExpression);
         }
 
         return new ResolvedIndices(localIndices, split.getRemote());
@@ -227,6 +257,19 @@ private static void throwOnUnexpectedWildcards(String action, List<String> indic
         );
     }
 
+    private static void throwOnUnexpectedWildcardSelectors(String action, List<String> indices) {
+        final List<String> selectorExpressions = indices.stream().filter(IndexNameExpressionResolver::hasSelectorSuffix).toList();
+        assert selectorExpressions.isEmpty() == false : "we already know that there's at least one selector in the indices";
+        throw new IllegalArgumentException(
+            "the action "
+                + action
+                + " does not support wildcard selectors;"
+                + " the provided index expression(s) ["
+                + Strings.collectionToCommaDelimitedString(selectorExpressions)
+                + "] are not allowed"
+        );
+    }
+
     ResolvedIndices resolveIndicesAndAliases(
         String action,
         IndicesRequest indicesRequest,
@@ -250,18 +293,48 @@ ResolvedIndices resolveIndicesAndAliases(
             final IndicesOptions indicesOptions = indicesRequest.indicesOptions();
 
             // check for all and return list of authorized indices
-            if (IndexNameExpressionResolver.isAllIndices(indicesList(indicesRequest.indices()))) {
+            boolean isAllIndices;
+            String allIndicesPatternSelector = null;
+            if (indicesRequest.indices() != null && indicesRequest.indices().length > 0) {
+                // Always parse selectors, but do so lazily so that we don't spend a lot of time splitting strings each resolution
+                isAllIndices = IndexNameExpressionResolver.isAllIndices(
+                    indicesList(indicesRequest.indices()),
+                    (expr) -> IndexNameExpressionResolver.splitSelectorExpression(expr).v1()
+                );
+                if (isAllIndices) {
+                    // This parses the single all-indices expression for a second time in this conditional branch, but this is better than
+                    // parsing a potentially big list of indices on every request.
+                    allIndicesPatternSelector = IndexNameExpressionResolver.splitSelectorExpression(
+                        indicesList(indicesRequest.indices()).getFirst()
+                    ).v2();
+                }
+            } else {
+                isAllIndices = IndexNameExpressionResolver.isAllIndices(indicesList(indicesRequest.indices()));
+            }
+            if (isAllIndices) {
+                // First, if a selector is present, check to make sure that selectors are even allowed here
+                if (indicesOptions.allowSelectors() == false && allIndicesPatternSelector != null) {
+                    String originalIndexExpression = indicesRequest.indices()[0];
+                    throw new IllegalArgumentException(
+                        "Index component selectors are not supported in this context but found selector in expression ["
+                            + originalIndexExpression
+                            + "]"
+                    );
+                }
                 if (indicesOptions.expandWildcardExpressions()) {
                     for (String authorizedIndex : authorizedIndices.all().get()) {
                         if (IndexAbstractionResolver.isIndexVisible(
                             "*",
+                            allIndicesPatternSelector,
                             authorizedIndex,
                             indicesOptions,
                             metadata,
                             nameExpressionResolver,
                             indicesRequest.includeDataStreams()
                         )) {
-                            resolvedIndicesBuilder.addLocal(authorizedIndex);
+                            resolvedIndicesBuilder.addLocal(
+                                IndexNameExpressionResolver.combineSelectorExpression(authorizedIndex, allIndicesPatternSelector)
+                            );
                         }
                     }
                 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
index 2353d710059ff..c5f4af93ab4b8 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.action.search.TransportMultiSearchAction;
 import org.elasticsearch.action.search.TransportSearchScrollAction;
 import org.elasticsearch.action.termvectors.MultiTermVectorsAction;
+import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -528,7 +529,12 @@ private static boolean isChildActionAuthorizedByParentOnLocalNode(RequestInfo re
                 + Arrays.stream(indices).filter(Regex::isSimpleMatchPattern).toList();
 
         // Check if the parent context has already successfully authorized access to the child's indices
-        return Arrays.stream(indices).allMatch(indicesAccessControl::hasIndexPermissions);
+        for (String index : indices) {
+            if (indicesAccessControl.hasIndexPermissions(index) == false) {
+                return false;
+            }
+        }
+        return true;
     }
 
     @Override
@@ -875,6 +881,10 @@ static AuthorizedIndices resolveAuthorizedIndicesFromRole(
                             for (Index index : indexAbstraction.getIndices()) {
                                 indicesAndAliases.add(index.getName());
                             }
+                            // TODO: We need to limit if a data stream's failure indices should return here.
+                            for (Index index : ((DataStream) indexAbstraction).getFailureIndices().getIndices()) {
+                                indicesAndAliases.add(index.getName());
+                            }
                         }
                     }
                 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java
index 7618135c8662f..87378ac0b9f25 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java
@@ -44,6 +44,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -173,6 +174,14 @@ public Path getFile() {
         return file;
     }
 
+    /**
+     * @return a map of all file role definitions. The returned map is unmodifiable.
+     */
+    public Map<String, RoleDescriptor> getAllRoleDescriptors() {
+        final Map<String, RoleDescriptor> localPermissions = permissions;
+        return Collections.unmodifiableMap(localPermissions);
+    }
+
     // package private for testing
     Set<String> getAllRoleNames() {
         return permissions.keySet();
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
index 23a1fc188e4a0..dc20b1e28ba78 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
@@ -63,13 +63,13 @@
 import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges;
 import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult;
 import org.elasticsearch.xpack.core.security.authz.support.DLSRoleQueryValidator;
-import org.elasticsearch.xpack.core.security.support.NativeRealmValidationUtil;
 import org.elasticsearch.xpack.security.authz.ReservedRoleNameChecker;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -96,7 +96,6 @@
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.SEARCH_SHARDS;
 import static org.elasticsearch.xpack.security.support.SecurityMigrations.ROLE_METADATA_FLATTENED_MIGRATION_VERSION;
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS;
-import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_ROLES_METADATA_FLATTENED;
 
 /**
  * NativeRolesStore is a {@code RolesStore} that, instead of reading from a
@@ -169,6 +168,10 @@ public NativeRolesStore(
         this.enabled = settings.getAsBoolean(NATIVE_ROLES_ENABLED, true);
     }
 
+    public boolean isEnabled() {
+        return enabled;
+    }
+
     @Override
     public void accept(Set<String> names, ActionListener<RoleRetrievalResult> listener) {
         getRoleDescriptors(names, listener);
@@ -263,6 +266,10 @@ public boolean isMetadataSearchable() {
     }
 
     public void queryRoleDescriptors(SearchSourceBuilder searchSourceBuilder, ActionListener<QueryRoleResult> listener) {
+        if (enabled == false) {
+            listener.onResponse(QueryRoleResult.EMPTY);
+            return;
+        }
         SearchRequest searchRequest = new SearchRequest(new String[] { SECURITY_MAIN_ALIAS }, searchSourceBuilder);
         SecurityIndexManager frozenSecurityIndex = securityIndex.defensiveCopy();
         if (frozenSecurityIndex.indexExists() == false) {
@@ -345,6 +352,15 @@ public void deleteRoles(
         final List<String> roleNames,
         WriteRequest.RefreshPolicy refreshPolicy,
         final ActionListener<BulkRolesResponse> listener
+    ) {
+        deleteRoles(roleNames, refreshPolicy, true, listener);
+    }
+
+    public void deleteRoles(
+        final Collection<String> roleNames,
+        WriteRequest.RefreshPolicy refreshPolicy,
+        boolean validateRoleNames,
+        final ActionListener<BulkRolesResponse> listener
     ) {
         if (enabled == false) {
             listener.onFailure(new IllegalStateException("Native role management is disabled"));
@@ -355,7 +371,7 @@ public void deleteRoles(
         Map<String, Exception> validationErrorByRoleName = new HashMap<>();
 
         for (String roleName : roleNames) {
-            if (reservedRoleNameChecker.isReserved(roleName)) {
+            if (validateRoleNames && reservedRoleNameChecker.isReserved(roleName)) {
                 validationErrorByRoleName.put(
                     roleName,
                     new IllegalArgumentException("role [" + roleName + "] is reserved and cannot be deleted")
@@ -402,7 +418,7 @@ public void onFailure(Exception e) {
     }
 
     private void bulkResponseAndRefreshRolesCache(
-        List<String> roleNames,
+        Collection<String> roleNames,
         BulkResponse bulkResponse,
         Map<String, Exception> validationErrorByRoleName,
         ActionListener<BulkRolesResponse> listener
@@ -430,7 +446,7 @@ private void bulkResponseAndRefreshRolesCache(
     }
 
     private void bulkResponseWithOnlyValidationErrors(
-        List<String> roleNames,
+        Collection<String> roleNames,
         Map<String, Exception> validationErrorByRoleName,
         ActionListener<BulkRolesResponse> listener
     ) {
@@ -542,7 +558,16 @@ public void onFailure(Exception e) {
 
     public void putRoles(
         final WriteRequest.RefreshPolicy refreshPolicy,
-        final List<RoleDescriptor> roles,
+        final Collection<RoleDescriptor> roles,
+        final ActionListener<BulkRolesResponse> listener
+    ) {
+        putRoles(refreshPolicy, roles, true, listener);
+    }
+
+    public void putRoles(
+        final WriteRequest.RefreshPolicy refreshPolicy,
+        final Collection<RoleDescriptor> roles,
+        boolean validateRoleDescriptors,
         final ActionListener<BulkRolesResponse> listener
     ) {
         if (enabled == false) {
@@ -555,7 +580,7 @@ public void putRoles(
         for (RoleDescriptor role : roles) {
             Exception validationException;
             try {
-                validationException = validateRoleDescriptor(role);
+                validationException = validateRoleDescriptors ? validateRoleDescriptor(role) : null;
             } catch (Exception e) {
                 validationException = e;
             }
@@ -621,16 +646,12 @@ private DeleteRequest createRoleDeleteRequest(final String roleName) {
 
     // Package private for testing
     XContentBuilder createRoleXContentBuilder(RoleDescriptor role) throws IOException {
-        assert NativeRealmValidationUtil.validateRoleName(role.getName(), false) == null
-            : "Role name was invalid or reserved: " + role.getName();
         assert false == role.hasRestriction() : "restriction is not supported for native roles";
 
         XContentBuilder builder = jsonBuilder().startObject();
         role.innerToXContent(builder, ToXContent.EMPTY_PARAMS, true);
 
-        if (featureService.clusterHasFeature(clusterService.state(), SECURITY_ROLES_METADATA_FLATTENED)) {
-            builder.field(RoleDescriptor.Fields.METADATA_FLATTENED.getPreferredName(), role.getMetadata());
-        }
+        builder.field(RoleDescriptor.Fields.METADATA_FLATTENED.getPreferredName(), role.getMetadata());
 
         // When role descriptor XContent is generated for the security index all empty fields need to have default values to make sure
         // existing values are overwritten if not present since the request to update could be an UpdateRequest
@@ -671,7 +692,11 @@ public void usageStats(ActionListener<Map<String, Object>> listener) {
                     client.prepareMultiSearch()
                         .add(
                             client.prepareSearch(SECURITY_MAIN_ALIAS)
-                                .setQuery(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE))
+                                .setQuery(
+                                    QueryBuilders.boolQuery()
+                                        .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE))
+                                        .mustNot(QueryBuilders.termQuery("metadata_flattened._reserved", true))
+                                )
                                 .setTrackTotalHits(true)
                                 .setSize(0)
                         )
@@ -680,6 +705,7 @@ public void usageStats(ActionListener<Map<String, Object>> listener) {
                                 .setQuery(
                                     QueryBuilders.boolQuery()
                                         .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE))
+                                        .mustNot(QueryBuilders.termQuery("metadata_flattened._reserved", true))
                                         .must(
                                             QueryBuilders.boolQuery()
                                                 .should(existsQuery("indices.field_security.grant"))
@@ -697,6 +723,7 @@ public void usageStats(ActionListener<Map<String, Object>> listener) {
                                 .setQuery(
                                     QueryBuilders.boolQuery()
                                         .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE))
+                                        .mustNot(QueryBuilders.termQuery("metadata_flattened._reserved", true))
                                         .filter(existsQuery("indices.query"))
                                 )
                                 .setTrackTotalHits(true)
@@ -708,6 +735,7 @@ public void usageStats(ActionListener<Map<String, Object>> listener) {
                                 .setQuery(
                                     QueryBuilders.boolQuery()
                                         .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE))
+                                        .mustNot(QueryBuilders.termQuery("metadata_flattened._reserved", true))
                                         .filter(existsQuery("remote_indices"))
                                 )
                                 .setTrackTotalHits(true)
@@ -718,6 +746,7 @@ public void usageStats(ActionListener<Map<String, Object>> listener) {
                                 .setQuery(
                                     QueryBuilders.boolQuery()
                                         .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE))
+                                        .mustNot(QueryBuilders.termQuery("metadata_flattened._reserved", true))
                                         .filter(existsQuery("remote_cluster"))
                                 )
                                 .setTrackTotalHits(true)
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestQueryRoleAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestQueryRoleAction.java
index c2dc7166bd3b6..862ff2552b4e3 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestQueryRoleAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestQueryRoleAction.java
@@ -14,6 +14,8 @@
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.Scope;
+import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.search.searchafter.SearchAfterBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
@@ -32,6 +34,7 @@
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
+@ServerlessScope(Scope.PUBLIC)
 public final class RestQueryRoleAction extends NativeRoleBaseRestHandler {
 
     @SuppressWarnings("unchecked")
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/slowlog/SecuritySlowLogFieldProvider.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/slowlog/SecuritySlowLogFieldProvider.java
index 1610aedd1d363..b5327b6779656 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/slowlog/SecuritySlowLogFieldProvider.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/slowlog/SecuritySlowLogFieldProvider.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.SlowLogFieldProvider;
+import org.elasticsearch.index.SlowLogFields;
 import org.elasticsearch.xpack.security.Security;
 
 import java.util.Map;
@@ -18,8 +19,36 @@
 
 public class SecuritySlowLogFieldProvider implements SlowLogFieldProvider {
     private final Security plugin;
-    private boolean includeUserInIndexing = false;
-    private boolean includeUserInSearch = false;
+
+    private class SecuritySlowLogFields implements SlowLogFields {
+        private boolean includeUserInIndexing = false;
+        private boolean includeUserInSearch = false;
+
+        SecuritySlowLogFields(IndexSettings indexSettings) {
+            indexSettings.getScopedSettings()
+                .addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_INCLUDE_USER_SETTING, newValue -> this.includeUserInSearch = newValue);
+            this.includeUserInSearch = indexSettings.getValue(INDEX_SEARCH_SLOWLOG_INCLUDE_USER_SETTING);
+            indexSettings.getScopedSettings()
+                .addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_INCLUDE_USER_SETTING, newValue -> this.includeUserInIndexing = newValue);
+            this.includeUserInIndexing = indexSettings.getValue(INDEX_INDEXING_SLOWLOG_INCLUDE_USER_SETTING);
+        }
+
+        @Override
+        public Map<String, String> indexFields() {
+            if (includeUserInIndexing) {
+                return plugin.getAuthContextForSlowLog();
+            }
+            return Map.of();
+        }
+
+        @Override
+        public Map<String, String> searchFields() {
+            if (includeUserInSearch) {
+                return plugin.getAuthContextForSlowLog();
+            }
+            return Map.of();
+        }
+    }
 
     public SecuritySlowLogFieldProvider() {
         throw new IllegalStateException("Provider must be constructed using PluginsService");
@@ -30,28 +59,7 @@ public SecuritySlowLogFieldProvider(Security plugin) {
     }
 
     @Override
-    public void init(IndexSettings indexSettings) {
-        indexSettings.getScopedSettings()
-            .addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_INCLUDE_USER_SETTING, newValue -> this.includeUserInSearch = newValue);
-        this.includeUserInSearch = indexSettings.getValue(INDEX_SEARCH_SLOWLOG_INCLUDE_USER_SETTING);
-        indexSettings.getScopedSettings()
-            .addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_INCLUDE_USER_SETTING, newValue -> this.includeUserInIndexing = newValue);
-        this.includeUserInIndexing = indexSettings.getValue(INDEX_INDEXING_SLOWLOG_INCLUDE_USER_SETTING);
-    }
-
-    @Override
-    public Map<String, String> indexSlowLogFields() {
-        if (includeUserInIndexing) {
-            return plugin.getAuthContextForSlowLog();
-        }
-        return Map.of();
-    }
-
-    @Override
-    public Map<String, String> searchSlowLogFields() {
-        if (includeUserInSearch) {
-            return plugin.getAuthContextForSlowLog();
-        }
-        return Map.of();
+    public SlowLogFields create(IndexSettings indexSettings) {
+        return new SecuritySlowLogFields(indexSettings);
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/FeatureNotEnabledException.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/FeatureNotEnabledException.java
index 87c23284c5819..8ba3ebad8a851 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/FeatureNotEnabledException.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/FeatureNotEnabledException.java
@@ -29,6 +29,7 @@ public enum Feature {
         }
     }
 
+    @SuppressWarnings("this-escape")
     public FeatureNotEnabledException(Feature feature, String message, Object... args) {
         super(message, args);
         addMetadata(DISABLED_FEATURE_METADATA, feature.featureName);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRoles.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRoles.java
new file mode 100644
index 0000000000000..ec38e4951f45c
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRoles.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.support;
+
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * A class that holds the built-in roles and their hash digests.
+ */
+public record QueryableBuiltInRoles(Map<String, String> rolesDigest, Collection<RoleDescriptor> roleDescriptors) {
+
+    /**
+     * A listener that is notified when the built-in roles change.
+     */
+    public interface Listener {
+
+        /**
+         * Called when the built-in roles change.
+         *
+         * @param roles the new built-in roles.
+         */
+        void onRolesChanged(QueryableBuiltInRoles roles);
+
+    }
+
+    /**
+     * A provider that provides the built-in roles and can notify subscribed listeners when the built-in roles change.
+     */
+    public interface Provider {
+
+        /**
+         * @return the built-in roles.
+         */
+        QueryableBuiltInRoles getRoles();
+
+        /**
+         * Adds a listener to be notified when the built-in roles change.
+         *
+         * @param listener the listener to add.
+         */
+        void addListener(Listener listener);
+
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesProviderFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesProviderFactory.java
new file mode 100644
index 0000000000000..c29b64836d1a5
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesProviderFactory.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.support;
+
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
+import org.elasticsearch.xpack.security.authz.store.FileRolesStore;
+
+public interface QueryableBuiltInRolesProviderFactory {
+
+    QueryableBuiltInRoles.Provider createProvider(ReservedRolesStore reservedRolesStore, FileRolesStore fileRolesStore);
+
+    class Default implements QueryableBuiltInRolesProviderFactory {
+        @Override
+        public QueryableBuiltInRoles.Provider createProvider(ReservedRolesStore reservedRolesStore, FileRolesStore fileRolesStore) {
+            return new QueryableReservedRolesProvider(reservedRolesStore);
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesSynchronizer.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesSynchronizer.java
new file mode 100644
index 0000000000000..2173e74fb8b93
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesSynchronizer.java
@@ -0,0 +1,549 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.support;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.TransportActions;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.ClusterStateTaskListener;
+import org.elasticsearch.cluster.NotMasterException;
+import org.elasticsearch.cluster.SimpleBatchedExecutor;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.component.LifecycleListener;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.features.FeatureService;
+import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.engine.DocumentMissingException;
+import org.elasticsearch.index.engine.VersionConflictEngineException;
+import org.elasticsearch.indices.IndexClosedException;
+import org.elasticsearch.indices.IndexPrimaryShardNotAllocatedException;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.security.action.role.BulkRolesResponse;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
+import org.elasticsearch.xpack.security.authz.store.FileRolesStore;
+import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static java.util.stream.Collectors.toMap;
+import static java.util.stream.Collectors.toSet;
+import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesUtils.determineRolesToDelete;
+import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesUtils.determineRolesToUpsert;
+import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS;
+
+/**
+ * Synchronizes built-in roles to the .security index.
+ * The .security index is created if it does not exist.
+ * <p>
+ * The synchronization is executed only on the elected master node
+ * after the cluster has recovered and roles need to be synced.
+ * The goal is to reduce the potential for conflicting operations.
+ * While in most cases, there should be only a single node that’s
+ * attempting to create/update/delete roles, it’s still possible
+ * that the master node changes in the middle of the syncing process.
+ */
+public final class QueryableBuiltInRolesSynchronizer implements ClusterStateListener {
+
+    private static final Logger logger = LogManager.getLogger(QueryableBuiltInRolesSynchronizer.class);
+
+    /**
+     * This is a temporary feature flag to allow enabling the synchronization of built-in roles to the .security index.
+     * Initially, it is disabled by default due to the number of tests that need to be adjusted now that .security index
+     * is created earlier in the cluster lifecycle.
+     * <p>
+     * Once all tests are adjusted, this flag will be set to enabled by default and later removed altogether.
+     */
+    public static final boolean QUERYABLE_BUILT_IN_ROLES_ENABLED;
+    static {
+        final var propertyValue = System.getProperty("es.queryable_built_in_roles_enabled");
+        if (propertyValue == null || propertyValue.isEmpty() || "false".equals(propertyValue)) {
+            QUERYABLE_BUILT_IN_ROLES_ENABLED = false;
+        } else if ("true".equals(propertyValue)) {
+            QUERYABLE_BUILT_IN_ROLES_ENABLED = true;
+        } else {
+            throw new IllegalStateException(
+                "system property [es.queryable_built_in_roles_enabled] may only be set to [true] or [false], but was ["
+                    + propertyValue
+                    + "]"
+            );
+        }
+    }
+
+    public static final NodeFeature QUERYABLE_BUILT_IN_ROLES_FEATURE = new NodeFeature("security.queryable_built_in_roles");
+
+    /**
+     * Index metadata key of the digest of built-in roles indexed in the .security index.
+     * <p>
+     * The value is a map of built-in role names to their digests (calculated by sha256 of the role definition).
+     */
+    public static final String METADATA_QUERYABLE_BUILT_IN_ROLES_DIGEST_KEY = "queryable_built_in_roles_digest";
+
+    private static final SimpleBatchedExecutor<MarkRolesAsSyncedTask, Map<String, String>> MARK_ROLES_AS_SYNCED_TASK_EXECUTOR =
+        new SimpleBatchedExecutor<>() {
+            @Override
+            public Tuple<ClusterState, Map<String, String>> executeTask(MarkRolesAsSyncedTask task, ClusterState clusterState) {
+                return task.execute(clusterState);
+            }
+
+            @Override
+            public void taskSucceeded(MarkRolesAsSyncedTask task, Map<String, String> value) {
+                task.success(value);
+            }
+        };
+
+    private final MasterServiceTaskQueue<MarkRolesAsSyncedTask> markRolesAsSyncedTaskQueue;
+
+    private final ClusterService clusterService;
+    private final FeatureService featureService;
+    private final QueryableBuiltInRoles.Provider rolesProvider;
+    private final NativeRolesStore nativeRolesStore;
+    private final Executor executor;
+    private final AtomicBoolean synchronizationInProgress = new AtomicBoolean(false);
+
+    private volatile boolean securityIndexDeleted = false;
+
+    /**
+     * Constructs a new built-in roles synchronizer.
+     *
+     * @param clusterService the cluster service to register as a listener
+     * @param featureService the feature service to check if the cluster has the queryable built-in roles feature
+     * @param rolesProviderFactory the factory to create the built-in roles provider
+     * @param nativeRolesStore the native roles store to sync the built-in roles to
+     * @param reservedRolesStore the reserved roles store to fetch the built-in roles from
+     * @param fileRolesStore the file roles store to fetch the built-in roles from
+     * @param threadPool the thread pool
+     */
+    public QueryableBuiltInRolesSynchronizer(
+        ClusterService clusterService,
+        FeatureService featureService,
+        QueryableBuiltInRolesProviderFactory rolesProviderFactory,
+        NativeRolesStore nativeRolesStore,
+        ReservedRolesStore reservedRolesStore,
+        FileRolesStore fileRolesStore,
+        ThreadPool threadPool
+    ) {
+        this.clusterService = clusterService;
+        this.featureService = featureService;
+        this.rolesProvider = rolesProviderFactory.createProvider(reservedRolesStore, fileRolesStore);
+        this.nativeRolesStore = nativeRolesStore;
+        this.executor = threadPool.generic();
+        this.markRolesAsSyncedTaskQueue = clusterService.createTaskQueue(
+            "mark-built-in-roles-as-synced-task-queue",
+            Priority.LOW,
+            MARK_ROLES_AS_SYNCED_TASK_EXECUTOR
+        );
+        this.rolesProvider.addListener(this::builtInRolesChanged);
+        this.clusterService.addLifecycleListener(new LifecycleListener() {
+            @Override
+            public void beforeStop() {
+                clusterService.removeListener(QueryableBuiltInRolesSynchronizer.this);
+            }
+
+            @Override
+            public void beforeStart() {
+                clusterService.addListener(QueryableBuiltInRolesSynchronizer.this);
+            }
+        });
+    }
+
+    private void builtInRolesChanged(QueryableBuiltInRoles roles) {
+        logger.debug("Built-in roles changed, attempting to sync to .security index");
+        final ClusterState state = clusterService.state();
+        if (shouldSyncBuiltInRoles(state)) {
+            syncBuiltInRoles(roles);
+        }
+    }
+
+    @Override
+    public void clusterChanged(ClusterChangedEvent event) {
+        final ClusterState state = event.state();
+        if (isSecurityIndexDeleted(event)) {
+            this.securityIndexDeleted = true;
+            logger.trace("Received security index deletion event, skipping built-in roles synchronization");
+            return;
+        } else if (isSecurityIndexCreatedOrRecovered(event)) {
+            this.securityIndexDeleted = false;
+            logger.trace("Security index has been created/recovered, attempting to sync built-in roles");
+        }
+        if (shouldSyncBuiltInRoles(state)) {
+            final QueryableBuiltInRoles roles = rolesProvider.getRoles();
+            syncBuiltInRoles(roles);
+        }
+    }
+
+    /**
+     * @return {@code true} if the synchronization of built-in roles is in progress, {@code false} otherwise
+     */
+    public boolean isSynchronizationInProgress() {
+        return synchronizationInProgress.get();
+    }
+
+    private void syncBuiltInRoles(final QueryableBuiltInRoles roles) {
+        if (synchronizationInProgress.compareAndSet(false, true)) {
+            try {
+                final Map<String, String> indexedRolesDigests = readIndexedBuiltInRolesDigests(clusterService.state());
+                if (roles.rolesDigest().equals(indexedRolesDigests)) {
+                    logger.debug("Security index already contains the latest built-in roles indexed, skipping roles synchronization");
+                    synchronizationInProgress.set(false);
+                } else {
+                    executor.execute(() -> doSyncBuiltinRoles(indexedRolesDigests, roles, ActionListener.wrap(v -> {
+                        logger.info("Successfully synced [" + roles.roleDescriptors().size() + "] built-in roles to .security index");
+                        synchronizationInProgress.set(false);
+                    }, e -> {
+                        handleException(e);
+                        synchronizationInProgress.set(false);
+                    })));
+                }
+            } catch (Exception e) {
+                logger.error("Failed to sync built-in roles", e);
+                synchronizationInProgress.set(false);
+            }
+        }
+    }
+
+    private static void handleException(Exception e) {
+        if (e instanceof BulkRolesResponseException bulkException) {
+            final boolean isBulkDeleteFailure = bulkException instanceof BulkDeleteRolesResponseException;
+            for (final Map.Entry<String, Exception> bulkFailure : bulkException.getFailures().entrySet()) {
+                final String logMessage = Strings.format(
+                    "Failed to [%s] built-in role [%s]",
+                    isBulkDeleteFailure ? "delete" : "create/update",
+                    bulkFailure.getKey()
+                );
+                if (isExpectedFailure(bulkFailure.getValue())) {
+                    logger.info(logMessage, bulkFailure.getValue());
+                } else {
+                    logger.warn(logMessage, bulkFailure.getValue());
+                }
+            }
+        } else if (isExpectedFailure(e)) {
+            logger.info("Failed to sync built-in roles to .security index", e);
+        } else {
+            logger.warn("Failed to sync built-in roles to .security index due to unexpected exception", e);
+        }
+    }
+
+    /**
+     * Some failures are expected and should not be logged as errors.
+     * These exceptions are either:
+     * - transient (e.g. connection errors),
+     * - recoverable (e.g. no longer master, index reallocating or caused by concurrent operations)
+     * - not recoverable but expected (e.g. index closed).
+     *
+     * @param e to check
+     * @return {@code true} if the exception is expected and should not be logged as an error
+     */
+    private static boolean isExpectedFailure(final Exception e) {
+        final Throwable cause = ExceptionsHelper.unwrapCause(e);
+        return ExceptionsHelper.isNodeOrShardUnavailableTypeException(cause)
+            || TransportActions.isShardNotAvailableException(cause)
+            || cause instanceof IndexClosedException
+            || cause instanceof IndexPrimaryShardNotAllocatedException
+            || cause instanceof NotMasterException
+            || cause instanceof ResourceAlreadyExistsException
+            || cause instanceof VersionConflictEngineException
+            || cause instanceof DocumentMissingException
+            || cause instanceof FailedToMarkBuiltInRolesAsSyncedException;
+    }
+
+    private boolean shouldSyncBuiltInRoles(final ClusterState state) {
+        if (false == state.nodes().isLocalNodeElectedMaster()) {
+            logger.trace("Local node is not the master, skipping built-in roles synchronization");
+            return false;
+        }
+        if (false == state.clusterRecovered()) {
+            logger.trace("Cluster state has not recovered yet, skipping built-in roles synchronization");
+            return false;
+        }
+        if (nativeRolesStore.isEnabled() == false) {
+            logger.trace("Native roles store is not enabled, skipping built-in roles synchronization");
+            return false;
+        }
+        if (state.nodes().getDataNodes().isEmpty()) {
+            logger.trace("No data nodes in the cluster, skipping built-in roles synchronization");
+            return false;
+        }
+        if (state.nodes().isMixedVersionCluster()) {
+            // To keep things simple and avoid potential overwrites with an older version of built-in roles,
+            // we only sync built-in roles if all nodes are on the same version.
+            logger.trace("Not all nodes are on the same version, skipping built-in roles synchronization");
+            return false;
+        }
+        if (false == featureService.clusterHasFeature(state, QUERYABLE_BUILT_IN_ROLES_FEATURE)) {
+            logger.trace("Not all nodes support queryable built-in roles feature, skipping built-in roles synchronization");
+            return false;
+        }
+        if (securityIndexDeleted) {
+            logger.trace("Security index is deleted, skipping built-in roles synchronization");
+            return false;
+        }
+        if (isSecurityIndexClosed(state)) {
+            logger.trace("Security index is closed, skipping built-in roles synchronization");
+            return false;
+        }
+        return true;
+    }
+
+    private void doSyncBuiltinRoles(
+        final Map<String, String> indexedRolesDigests,
+        final QueryableBuiltInRoles roles,
+        final ActionListener<Void> listener
+    ) {
+        final Set<RoleDescriptor> rolesToUpsert = determineRolesToUpsert(roles, indexedRolesDigests);
+        final Set<String> rolesToDelete = determineRolesToDelete(roles, indexedRolesDigests);
+
+        assert Sets.intersection(rolesToUpsert.stream().map(RoleDescriptor::getName).collect(toSet()), rolesToDelete).isEmpty()
+            : "The roles to upsert and delete should not have any common roles";
+
+        if (rolesToUpsert.isEmpty() && rolesToDelete.isEmpty()) {
+            logger.debug("No changes to built-in roles to sync to .security index");
+            listener.onResponse(null);
+            return;
+        }
+
+        indexRoles(rolesToUpsert, listener.delegateFailureAndWrap((l1, indexResponse) -> {
+            deleteRoles(rolesToDelete, l1.delegateFailureAndWrap((l2, deleteResponse) -> {
+                markRolesAsSynced(indexedRolesDigests, roles.rolesDigest(), l2);
+            }));
+        }));
+    }
+
+    private void deleteRoles(final Set<String> rolesToDelete, final ActionListener<Void> listener) {
+        if (rolesToDelete.isEmpty()) {
+            listener.onResponse(null);
+            return;
+        }
+        nativeRolesStore.deleteRoles(rolesToDelete, WriteRequest.RefreshPolicy.IMMEDIATE, false, ActionListener.wrap(deleteResponse -> {
+            final Map<String, Exception> deleteFailure = deleteResponse.getItems()
+                .stream()
+                .filter(BulkRolesResponse.Item::isFailed)
+                .collect(toMap(BulkRolesResponse.Item::getRoleName, BulkRolesResponse.Item::getCause));
+            if (deleteFailure.isEmpty()) {
+                listener.onResponse(null);
+            } else {
+                listener.onFailure(new BulkDeleteRolesResponseException(deleteFailure));
+            }
+        }, listener::onFailure));
+    }
+
+    private void indexRoles(final Collection<RoleDescriptor> rolesToUpsert, final ActionListener<Void> listener) {
+        if (rolesToUpsert.isEmpty()) {
+            listener.onResponse(null);
+            return;
+        }
+        nativeRolesStore.putRoles(WriteRequest.RefreshPolicy.IMMEDIATE, rolesToUpsert, false, ActionListener.wrap(response -> {
+            final Map<String, Exception> indexFailures = response.getItems()
+                .stream()
+                .filter(BulkRolesResponse.Item::isFailed)
+                .collect(toMap(BulkRolesResponse.Item::getRoleName, BulkRolesResponse.Item::getCause));
+            if (indexFailures.isEmpty()) {
+                listener.onResponse(null);
+            } else {
+                listener.onFailure(new BulkIndexRolesResponseException(indexFailures));
+            }
+        }, listener::onFailure));
+    }
+
+    private boolean isSecurityIndexDeleted(final ClusterChangedEvent event) {
+        final IndexMetadata previousSecurityIndexMetadata = resolveSecurityIndexMetadata(event.previousState().metadata());
+        final IndexMetadata currentSecurityIndexMetadata = resolveSecurityIndexMetadata(event.state().metadata());
+        return previousSecurityIndexMetadata != null && currentSecurityIndexMetadata == null;
+    }
+
+    private boolean isSecurityIndexCreatedOrRecovered(final ClusterChangedEvent event) {
+        final IndexMetadata previousSecurityIndexMetadata = resolveSecurityIndexMetadata(event.previousState().metadata());
+        final IndexMetadata currentSecurityIndexMetadata = resolveSecurityIndexMetadata(event.state().metadata());
+        return previousSecurityIndexMetadata == null && currentSecurityIndexMetadata != null;
+    }
+
+    private boolean isSecurityIndexClosed(final ClusterState state) {
+        final IndexMetadata indexMetadata = resolveSecurityIndexMetadata(state.metadata());
+        return indexMetadata != null && indexMetadata.getState() == IndexMetadata.State.CLOSE;
+    }
+
+    /**
+     * This method marks the built-in roles as synced in the .security index
+     * by setting the new roles digests in the metadata of the .security index.
+     * <p>
+     * The marking is done as a compare and swap operation to ensure that the roles
+     * are marked as synced only when new roles are indexed. The operation is idempotent
+     * and will succeed if the expected roles digests are equal to the digests in the
+     * .security index or if they are equal to the new roles digests.
+     */
+    private void markRolesAsSynced(
+        final Map<String, String> expectedRolesDigests,
+        final Map<String, String> newRolesDigests,
+        final ActionListener<Void> listener
+    ) {
+        final IndexMetadata securityIndexMetadata = resolveSecurityIndexMetadata(clusterService.state().metadata());
+        if (securityIndexMetadata == null) {
+            listener.onFailure(new IndexNotFoundException(SECURITY_MAIN_ALIAS));
+            return;
+        }
+        final Index concreteSecurityIndex = securityIndexMetadata.getIndex();
+        markRolesAsSyncedTaskQueue.submitTask(
+            "mark built-in roles as synced task",
+            new MarkRolesAsSyncedTask(listener.delegateFailureAndWrap((l, response) -> {
+                if (newRolesDigests.equals(response) == false) {
+                    logger.debug(
+                        () -> Strings.format(
+                            "Another master node most probably indexed a newer versions of built-in roles in the meantime. "
+                                + "Expected: [%s], Actual: [%s]",
+                            newRolesDigests,
+                            response
+                        )
+                    );
+                    l.onFailure(
+                        new FailedToMarkBuiltInRolesAsSyncedException(
+                            "Failed to mark built-in roles as synced. The expected role digests have changed."
+                        )
+                    );
+                } else {
+                    l.onResponse(null);
+                }
+            }), concreteSecurityIndex.getName(), expectedRolesDigests, newRolesDigests),
+            null
+        );
+    }
+
+    private Map<String, String> readIndexedBuiltInRolesDigests(final ClusterState state) {
+        final IndexMetadata indexMetadata = resolveSecurityIndexMetadata(state.metadata());
+        if (indexMetadata == null) {
+            return null;
+        }
+        return indexMetadata.getCustomData(METADATA_QUERYABLE_BUILT_IN_ROLES_DIGEST_KEY);
+    }
+
+    private static IndexMetadata resolveSecurityIndexMetadata(final Metadata metadata) {
+        return SecurityIndexManager.resolveConcreteIndex(SECURITY_MAIN_ALIAS, metadata);
+    }
+
+    static class MarkRolesAsSyncedTask implements ClusterStateTaskListener {
+
+        private final ActionListener<Map<String, String>> listener;
+        private final String concreteSecurityIndexName;
+        private final Map<String, String> expectedRoleDigests;
+        private final Map<String, String> newRoleDigests;
+
+        MarkRolesAsSyncedTask(
+            ActionListener<Map<String, String>> listener,
+            String concreteSecurityIndexName,
+            @Nullable Map<String, String> expectedRoleDigests,
+            @Nullable Map<String, String> newRoleDigests
+        ) {
+            this.listener = listener;
+            this.concreteSecurityIndexName = concreteSecurityIndexName;
+            this.expectedRoleDigests = expectedRoleDigests;
+            this.newRoleDigests = newRoleDigests;
+        }
+
+        public Map<String, String> getNewRoleDigests() {
+            return newRoleDigests;
+        }
+
+        Tuple<ClusterState, Map<String, String>> execute(ClusterState state) {
+            IndexMetadata indexMetadata = state.metadata().index(concreteSecurityIndexName);
+            if (indexMetadata == null) {
+                throw new IndexNotFoundException(concreteSecurityIndexName);
+            }
+            Map<String, String> existingRoleDigests = indexMetadata.getCustomData(METADATA_QUERYABLE_BUILT_IN_ROLES_DIGEST_KEY);
+            if (Objects.equals(expectedRoleDigests, existingRoleDigests)) {
+                IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexMetadata);
+                if (newRoleDigests != null) {
+                    indexMetadataBuilder.putCustom(METADATA_QUERYABLE_BUILT_IN_ROLES_DIGEST_KEY, newRoleDigests);
+                } else {
+                    indexMetadataBuilder.removeCustom(METADATA_QUERYABLE_BUILT_IN_ROLES_DIGEST_KEY);
+                }
+                indexMetadataBuilder.version(indexMetadataBuilder.version() + 1);
+                ImmutableOpenMap.Builder<String, IndexMetadata> builder = ImmutableOpenMap.builder(state.metadata().indices());
+                builder.put(concreteSecurityIndexName, indexMetadataBuilder.build());
+                return new Tuple<>(
+                    ClusterState.builder(state).metadata(Metadata.builder(state.metadata()).indices(builder.build()).build()).build(),
+                    newRoleDigests
+                );
+            } else {
+                // returns existing value when expectation is not met
+                return new Tuple<>(state, existingRoleDigests);
+            }
+        }
+
+        void success(Map<String, String> value) {
+            listener.onResponse(value);
+        }
+
+        @Override
+        public void onFailure(Exception e) {
+            listener.onFailure(e);
+        }
+    }
+
+    private static class BulkDeleteRolesResponseException extends BulkRolesResponseException {
+
+        BulkDeleteRolesResponseException(Map<String, Exception> failures) {
+            super("Failed to bulk delete built-in roles", failures);
+        }
+
+    }
+
+    private static class BulkIndexRolesResponseException extends BulkRolesResponseException {
+
+        BulkIndexRolesResponseException(Map<String, Exception> failures) {
+            super("Failed to bulk create/update built-in roles", failures);
+        }
+
+    }
+
+    private abstract static class BulkRolesResponseException extends RuntimeException {
+
+        private final Map<String, Exception> failures;
+
+        BulkRolesResponseException(String message, Map<String, Exception> failures) {
+            super(message);
+            assert failures != null && failures.isEmpty() == false;
+            this.failures = failures;
+            failures.values().forEach(this::addSuppressed);
+        }
+
+        Map<String, Exception> getFailures() {
+            return failures;
+        }
+
+    }
+
+    private static class FailedToMarkBuiltInRolesAsSyncedException extends RuntimeException {
+
+        FailedToMarkBuiltInRolesAsSyncedException(String message) {
+            super(message);
+        }
+
+    }
+
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesUtils.java
new file mode 100644
index 0000000000000..2d2eb345594ed
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesUtils.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.support;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
+
+/**
+ * Utility class which provides helper method for calculating the hash of a role descriptor,
+ * determining the roles to upsert and the roles to delete.
+ */
+public final class QueryableBuiltInRolesUtils {
+
+    /**
+     * Calculates the hash of the given role descriptor by serializing it by calling {@link RoleDescriptor#writeTo(StreamOutput)} method
+     * and then SHA256 hashing the bytes.
+     *
+     * @param roleDescriptor the role descriptor to hash
+     * @return the base64 encoded SHA256 hash of the role descriptor
+     */
+    public static String calculateHash(final RoleDescriptor roleDescriptor) {
+        final MessageDigest hash = MessageDigests.sha256();
+        try (XContentBuilder jsonBuilder = XContentFactory.jsonBuilder()) {
+            roleDescriptor.toXContent(jsonBuilder, EMPTY_PARAMS);
+            final Map<String, Object> flattenMap = Maps.flatten(
+                XContentHelper.convertToMap(BytesReference.bytes(jsonBuilder), true, XContentType.JSON).v2(),
+                false,
+                true
+            );
+            hash.update(flattenMap.toString().getBytes(StandardCharsets.UTF_8));
+        } catch (IOException e) {
+            throw new IllegalStateException("failed to compute digest for [" + roleDescriptor.getName() + "] role", e);
+        }
+        // HEX vs Base64 encoding is a trade-off between readability and space efficiency
+        // opting for Base64 here to reduce the size of the cluster state
+        return Base64.getEncoder().encodeToString(hash.digest());
+    }
+
+    /**
+     * Determines the roles to delete by comparing the indexed roles with the roles in the built-in roles.
+     * @return the set of roles to delete
+     */
+    public static Set<String> determineRolesToDelete(final QueryableBuiltInRoles roles, final Map<String, String> indexedRolesDigests) {
+        assert roles != null;
+        if (indexedRolesDigests == null) {
+            // nothing indexed, nothing to delete
+            return Set.of();
+        }
+        final Set<String> rolesToDelete = Sets.difference(indexedRolesDigests.keySet(), roles.rolesDigest().keySet());
+        return Collections.unmodifiableSet(rolesToDelete);
+    }
+
+    /**
+     * Determines the roles to upsert by comparing the indexed roles and their digests with the current built-in roles.
+     * @return the set of roles to upsert (create or update)
+     */
+    public static Set<RoleDescriptor> determineRolesToUpsert(
+        final QueryableBuiltInRoles roles,
+        final Map<String, String> indexedRolesDigests
+    ) {
+        assert roles != null;
+        final Set<RoleDescriptor> rolesToUpsert = new HashSet<>();
+        for (RoleDescriptor role : roles.roleDescriptors()) {
+            final String roleDigest = roles.rolesDigest().get(role.getName());
+            if (indexedRolesDigests == null || indexedRolesDigests.containsKey(role.getName()) == false) {
+                rolesToUpsert.add(role);  // a new role to create
+            } else if (indexedRolesDigests.get(role.getName()).equals(roleDigest) == false) {
+                rolesToUpsert.add(role);  // an existing role that needs to be updated
+            }
+        }
+        return Collections.unmodifiableSet(rolesToUpsert);
+    }
+
+    private QueryableBuiltInRolesUtils() {
+        throw new IllegalAccessError("not allowed");
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableReservedRolesProvider.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableReservedRolesProvider.java
new file mode 100644
index 0000000000000..1786d20410990
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableReservedRolesProvider.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.support;
+
+import org.elasticsearch.common.util.CachedSupplier;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+/**
+ * A provider of the built-in reserved roles.
+ * <p>
+ * This provider fetches all reserved roles from the {@link ReservedRolesStore} and calculates their hashes lazily.
+ * The reserved roles are static and do not change during runtime, hence this provider will never notify any listeners.
+ * </p>
+ */
+public class QueryableReservedRolesProvider implements QueryableBuiltInRoles.Provider {
+
+    private final Supplier<QueryableBuiltInRoles> reservedRolesSupplier;
+
+    /**
+     * Constructs a new reserved roles provider.
+     *
+     * @param reservedRolesStore the store to fetch the reserved roles from.
+     *                           Having a store reference here is necessary to ensure that static fields are initialized.
+     */
+    public QueryableReservedRolesProvider(ReservedRolesStore reservedRolesStore) {
+        this.reservedRolesSupplier = CachedSupplier.wrap(() -> {
+            final Collection<RoleDescriptor> roleDescriptors = Collections.unmodifiableCollection(ReservedRolesStore.roleDescriptors());
+            return new QueryableBuiltInRoles(
+                roleDescriptors.stream()
+                    .collect(Collectors.toUnmodifiableMap(RoleDescriptor::getName, QueryableBuiltInRolesUtils::calculateHash)),
+                roleDescriptors
+            );
+        });
+    }
+
+    @Override
+    public QueryableBuiltInRoles getRoles() {
+        return reservedRolesSupplier.get();
+    }
+
+    @Override
+    public void addListener(QueryableBuiltInRoles.Listener listener) {
+        // no-op: reserved roles are static and do not change
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java
index f3222a74b530c..39c7a45d51dfd 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java
@@ -69,7 +69,6 @@
 import static org.elasticsearch.xpack.core.security.action.UpdateIndexMigrationVersionAction.MIGRATION_VERSION_CUSTOM_DATA_KEY;
 import static org.elasticsearch.xpack.core.security.action.UpdateIndexMigrationVersionAction.MIGRATION_VERSION_CUSTOM_KEY;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.State.UNRECOVERED_STATE;
-import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MIGRATION_FRAMEWORK;
 
 /**
  * Manages the lifecycle, mapping and data upgrades/migrations of the {@code RestrictedIndicesNames#SECURITY_MAIN_ALIAS}
@@ -539,7 +538,6 @@ public boolean isReadyForSecurityMigration(SecurityMigrations.SecurityMigration
             && state.indexAvailableForSearch
             && state.isIndexUpToDate
             && state.indexExists()
-            && state.securityFeatures.contains(SECURITY_MIGRATION_FRAMEWORK)
             && isEligibleSecurityMigration(securityMigration);
     }
 
@@ -586,7 +584,7 @@ private static int readMappingVersion(String indexName, MappingMetadata mappingM
      * Resolves a concrete index name or alias to a {@link IndexMetadata} instance.  Requires
      * that if supplied with an alias, the alias resolves to at most one concrete index.
      */
-    private static IndexMetadata resolveConcreteIndex(final String indexOrAliasName, final Metadata metadata) {
+    public static IndexMetadata resolveConcreteIndex(final String indexOrAliasName, final Metadata metadata) {
         final IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(indexOrAliasName);
         if (indexAbstraction != null) {
             final List<Index> indices = indexAbstraction.getIndices();
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java
index 203dec9e25b91..c380b9fbee93a 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java
@@ -142,7 +142,7 @@ private void updateRolesByQuery(
 
         @Override
         public Set<NodeFeature> nodeFeaturesRequired() {
-            return Set.of(SecuritySystemIndices.SECURITY_ROLES_METADATA_FLATTENED);
+            return Set.of();
         }
 
         @Override
@@ -221,7 +221,7 @@ public boolean checkPreConditions(SecurityIndexManager.State securityIndexManage
 
         @Override
         public Set<NodeFeature> nodeFeaturesRequired() {
-            return Set.of(SecuritySystemIndices.SECURITY_ROLE_MAPPING_CLEANUP);
+            return Set.of();
         }
 
         @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java
index 7b3f6a8d2ae55..806b545b85772 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.common.VersionId;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.features.FeatureService;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.indices.ExecutorNames;
 import org.elasticsearch.indices.SystemIndexDescriptor;
@@ -56,9 +55,6 @@ public class SecuritySystemIndices {
 
     public static final String INTERNAL_SECURITY_PROFILE_INDEX_8 = ".security-profile-8";
     public static final String SECURITY_PROFILE_ALIAS = ".security-profile";
-    public static final NodeFeature SECURITY_MIGRATION_FRAMEWORK = new NodeFeature("security.migration_framework");
-    public static final NodeFeature SECURITY_ROLES_METADATA_FLATTENED = new NodeFeature("security.roles_metadata_flattened");
-    public static final NodeFeature SECURITY_ROLE_MAPPING_CLEANUP = new NodeFeature("security.role_mapping_cleanup");
 
     /**
      * Security managed index mappings used to be updated based on the product version. They are now updated based on per-index mappings
diff --git a/x-pack/plugin/security/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/security/src/main/plugin-metadata/entitlement-policy.yaml
new file mode 100644
index 0000000000000..98c6b81553572
--- /dev/null
+++ b/x-pack/plugin/security/src/main/plugin-metadata/entitlement-policy.yaml
@@ -0,0 +1,2 @@
+org.elasticsearch.security:
+  - set_https_connection_properties # for CommandLineHttpClient
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java
index 702af75141093..75c2507a1dc5f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java
@@ -126,6 +126,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
@@ -147,6 +148,7 @@ public class TokenServiceTests extends ESTestCase {
     private SecurityIndexManager securityMainIndex;
     private SecurityIndexManager securityTokensIndex;
     private ClusterService clusterService;
+    private DiscoveryNode pre72OldNode;
     private DiscoveryNode pre8500040OldNode;
     private Settings tokenServiceEnabledSettings = Settings.builder()
         .put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), true)
@@ -226,12 +228,31 @@ public void setupClient() {
         licenseState = mock(MockLicenseState.class);
         when(licenseState.isAllowed(Security.TOKEN_SERVICE_FEATURE)).thenReturn(true);
 
+        if (randomBoolean()) {
+            // version 7.2 was an "inflection" point in the Token Service development (access_tokens as UUIDS, multiple concurrent
+            // refreshes,
+            // tokens docs on a separate index)
+            pre72OldNode = addAnother7071DataNode(this.clusterService);
+        }
         if (randomBoolean()) {
             // before refresh tokens used GET, i.e. TokenService#VERSION_GET_TOKEN_DOC_FOR_REFRESH
             pre8500040OldNode = addAnotherPre8500DataNode(this.clusterService);
         }
     }
 
+    private static DiscoveryNode addAnother7071DataNode(ClusterService clusterService) {
+        Version version;
+        TransportVersion transportVersion;
+        if (randomBoolean()) {
+            version = Version.V_7_0_0;
+            transportVersion = TransportVersions.V_7_0_0;
+        } else {
+            version = Version.V_7_1_0;
+            transportVersion = TransportVersions.V_7_1_0;
+        }
+        return addAnotherDataNodeWithVersion(clusterService, version, transportVersion);
+    }
+
     private static DiscoveryNode addAnotherPre8500DataNode(ClusterService clusterService) {
         Version version;
         TransportVersion transportVersion;
@@ -280,6 +301,53 @@ public static void shutdownThreadpool() {
         threadPool = null;
     }
 
+    public void testAttachAndGetToken() throws Exception {
+        TokenService tokenService = createTokenService(tokenServiceEnabledSettings, systemUTC());
+        // This test only makes sense in mixed clusters with pre v7.2.0 nodes where the Token Service Key is used (to encrypt tokens)
+        if (null == pre72OldNode) {
+            pre72OldNode = addAnother7071DataNode(this.clusterService);
+        }
+        Authentication authentication = AuthenticationTestHelper.builder()
+            .user(new User("joe", "admin"))
+            .realmRef(new RealmRef("native_realm", "native", "node1"))
+            .build(false);
+        PlainActionFuture<TokenService.CreateTokenResult> tokenFuture = new PlainActionFuture<>();
+        Tuple<byte[], byte[]> newTokenBytes = tokenService.getRandomTokenBytes(randomBoolean());
+        tokenService.createOAuth2Tokens(
+            newTokenBytes.v1(),
+            newTokenBytes.v2(),
+            authentication,
+            authentication,
+            Collections.emptyMap(),
+            tokenFuture
+        );
+        final String accessToken = tokenFuture.get().getAccessToken();
+        assertNotNull(accessToken);
+        mockGetTokenFromAccessTokenBytes(tokenService, newTokenBytes.v1(), authentication, false, null);
+
+        ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
+        requestContext.putHeader("Authorization", randomFrom("Bearer ", "BEARER ", "bearer ") + accessToken);
+
+        try (ThreadContext.StoredContext ignore = requestContext.newStoredContextPreservingResponseHeaders()) {
+            PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+            final SecureString bearerToken = Authenticator.extractBearerTokenFromHeader(requestContext);
+            tokenService.tryAuthenticateToken(bearerToken, future);
+            UserToken serialized = future.get();
+            assertAuthentication(authentication, serialized.getAuthentication());
+        }
+
+        try (ThreadContext.StoredContext ignore = requestContext.newStoredContextPreservingResponseHeaders()) {
+            // verify a second separate token service with its own salt can also verify
+            TokenService anotherService = createTokenService(tokenServiceEnabledSettings, systemUTC());
+            anotherService.refreshMetadata(tokenService.getTokenMetadata());
+            PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+            final SecureString bearerToken = Authenticator.extractBearerTokenFromHeader(requestContext);
+            anotherService.tryAuthenticateToken(bearerToken, future);
+            UserToken fromOtherService = future.get();
+            assertAuthentication(authentication, fromOtherService.getAuthentication());
+        }
+    }
+
     public void testInvalidAuthorizationHeader() throws Exception {
         TokenService tokenService = createTokenService(tokenServiceEnabledSettings, systemUTC());
         ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
@@ -296,6 +364,89 @@ public void testInvalidAuthorizationHeader() throws Exception {
         }
     }
 
+    public void testPassphraseWorks() throws Exception {
+        TokenService tokenService = createTokenService(tokenServiceEnabledSettings, systemUTC());
+        // This test only makes sense in mixed clusters with pre v7.1.0 nodes where the Key is actually used
+        if (null == pre72OldNode) {
+            pre72OldNode = addAnother7071DataNode(this.clusterService);
+        }
+        Authentication authentication = AuthenticationTestHelper.builder()
+            .user(new User("joe", "admin"))
+            .realmRef(new RealmRef("native_realm", "native", "node1"))
+            .build(false);
+        PlainActionFuture<TokenService.CreateTokenResult> tokenFuture = new PlainActionFuture<>();
+        Tuple<byte[], byte[]> newTokenBytes = tokenService.getRandomTokenBytes(randomBoolean());
+        tokenService.createOAuth2Tokens(
+            newTokenBytes.v1(),
+            newTokenBytes.v2(),
+            authentication,
+            authentication,
+            Collections.emptyMap(),
+            tokenFuture
+        );
+        final String accessToken = tokenFuture.get().getAccessToken();
+        assertNotNull(accessToken);
+        mockGetTokenFromAccessTokenBytes(tokenService, newTokenBytes.v1(), authentication, false, null);
+
+        ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
+        storeTokenHeader(requestContext, accessToken);
+
+        try (ThreadContext.StoredContext ignore = requestContext.newStoredContextPreservingResponseHeaders()) {
+            PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+            final SecureString bearerToken = Authenticator.extractBearerTokenFromHeader(requestContext);
+            tokenService.tryAuthenticateToken(bearerToken, future);
+            UserToken serialized = future.get();
+            assertAuthentication(authentication, serialized.getAuthentication());
+        }
+
+        try (ThreadContext.StoredContext ignore = requestContext.newStoredContextPreservingResponseHeaders()) {
+            // verify a second separate token service with its own passphrase cannot verify
+            TokenService anotherService = createTokenService(tokenServiceEnabledSettings, systemUTC());
+            PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+            final SecureString bearerToken = Authenticator.extractBearerTokenFromHeader(requestContext);
+            anotherService.tryAuthenticateToken(bearerToken, future);
+            assertNull(future.get());
+        }
+    }
+
+    public void testGetTokenWhenKeyCacheHasExpired() throws Exception {
+        TokenService tokenService = createTokenService(tokenServiceEnabledSettings, systemUTC());
+        // This test only makes sense in mixed clusters with pre v7.1.0 nodes where the Key is actually used
+        if (null == pre72OldNode) {
+            pre72OldNode = addAnother7071DataNode(this.clusterService);
+        }
+        Authentication authentication = AuthenticationTestHelper.builder()
+            .user(new User("joe", "admin"))
+            .realmRef(new RealmRef("native_realm", "native", "node1"))
+            .build(false);
+
+        PlainActionFuture<TokenService.CreateTokenResult> tokenFuture = new PlainActionFuture<>();
+        Tuple<byte[], byte[]> newTokenBytes = tokenService.getRandomTokenBytes(randomBoolean());
+        tokenService.createOAuth2Tokens(
+            newTokenBytes.v1(),
+            newTokenBytes.v2(),
+            authentication,
+            authentication,
+            Collections.emptyMap(),
+            tokenFuture
+        );
+        String accessToken = tokenFuture.get().getAccessToken();
+        assertThat(accessToken, notNullValue());
+
+        tokenService.clearActiveKeyCache();
+
+        tokenService.createOAuth2Tokens(
+            newTokenBytes.v1(),
+            newTokenBytes.v2(),
+            authentication,
+            authentication,
+            Collections.emptyMap(),
+            tokenFuture
+        );
+        accessToken = tokenFuture.get().getAccessToken();
+        assertThat(accessToken, notNullValue());
+    }
+
     public void testAuthnWithInvalidatedToken() throws Exception {
         when(securityMainIndex.indexExists()).thenReturn(true);
         TokenService tokenService = createTokenService(tokenServiceEnabledSettings, systemUTC());
@@ -669,6 +820,57 @@ public void testMalformedRefreshTokens() throws Exception {
         }
     }
 
+    public void testNonExistingPre72Token() throws Exception {
+        TokenService tokenService = createTokenService(tokenServiceEnabledSettings, systemUTC());
+        // mock another random token so that we don't find a token in TokenService#getUserTokenFromId
+        Authentication authentication = AuthenticationTestHelper.builder()
+            .user(new User("joe", "admin"))
+            .realmRef(new RealmRef("native_realm", "native", "node1"))
+            .build(false);
+        mockGetTokenFromAccessTokenBytes(tokenService, tokenService.getRandomTokenBytes(randomBoolean()).v1(), authentication, false, null);
+        ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
+        storeTokenHeader(
+            requestContext,
+            tokenService.prependVersionAndEncodeAccessToken(
+                TransportVersions.V_7_1_0,
+                tokenService.getRandomTokenBytes(TransportVersions.V_7_1_0, randomBoolean()).v1()
+            )
+        );
+
+        try (ThreadContext.StoredContext ignore = requestContext.newStoredContextPreservingResponseHeaders()) {
+            PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+            final SecureString bearerToken = Authenticator.extractBearerTokenFromHeader(requestContext);
+            tokenService.tryAuthenticateToken(bearerToken, future);
+            assertNull(future.get());
+        }
+    }
+
+    public void testNonExistingUUIDToken() throws Exception {
+        TokenService tokenService = createTokenService(tokenServiceEnabledSettings, systemUTC());
+        // mock another random token so that we don't find a token in TokenService#getUserTokenFromId
+        Authentication authentication = AuthenticationTestHelper.builder()
+            .user(new User("joe", "admin"))
+            .realmRef(new RealmRef("native_realm", "native", "node1"))
+            .build(false);
+        mockGetTokenFromAccessTokenBytes(tokenService, tokenService.getRandomTokenBytes(randomBoolean()).v1(), authentication, false, null);
+        ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
+        TransportVersion uuidTokenVersion = randomFrom(TransportVersions.V_7_2_0, TransportVersions.V_7_3_2);
+        storeTokenHeader(
+            requestContext,
+            tokenService.prependVersionAndEncodeAccessToken(
+                uuidTokenVersion,
+                tokenService.getRandomTokenBytes(uuidTokenVersion, randomBoolean()).v1()
+            )
+        );
+
+        try (ThreadContext.StoredContext ignore = requestContext.newStoredContextPreservingResponseHeaders()) {
+            PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+            final SecureString bearerToken = Authenticator.extractBearerTokenFromHeader(requestContext);
+            tokenService.tryAuthenticateToken(bearerToken, future);
+            assertNull(future.get());
+        }
+    }
+
     public void testNonExistingLatestTokenVersion() throws Exception {
         TokenService tokenService = createTokenService(tokenServiceEnabledSettings, systemUTC());
         // mock another random token so that we don't find a token in TokenService#getUserTokenFromId
@@ -723,11 +925,18 @@ public void testIndexNotAvailable() throws Exception {
             return Void.TYPE;
         }).when(client).get(any(GetRequest.class), anyActionListener());
 
-        final SecurityIndexManager tokensIndex = securityTokensIndex;
-        when(securityMainIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(false);
-        when(securityMainIndex.indexExists()).thenReturn(false);
-        when(securityMainIndex.defensiveCopy()).thenReturn(securityMainIndex);
-
+        final SecurityIndexManager tokensIndex;
+        if (pre72OldNode != null) {
+            tokensIndex = securityMainIndex;
+            when(securityTokensIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(false);
+            when(securityTokensIndex.indexExists()).thenReturn(false);
+            when(securityTokensIndex.defensiveCopy()).thenReturn(securityTokensIndex);
+        } else {
+            tokensIndex = securityTokensIndex;
+            when(securityMainIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(false);
+            when(securityMainIndex.indexExists()).thenReturn(false);
+            when(securityMainIndex.defensiveCopy()).thenReturn(securityMainIndex);
+        }
         try (ThreadContext.StoredContext ignore = requestContext.newStoredContextPreservingResponseHeaders()) {
             PlainActionFuture<UserToken> future = new PlainActionFuture<>();
             final SecureString bearerToken3 = Authenticator.extractBearerTokenFromHeader(requestContext);
@@ -779,6 +988,7 @@ public void testGetAuthenticationWorksWithExpiredUserToken() throws Exception {
     }
 
     public void testSupersedingTokenEncryption() throws Exception {
+        assumeTrue("Superseding tokens are only created in post 7.2 clusters", pre72OldNode == null);
         TokenService tokenService = createTokenService(tokenServiceEnabledSettings, Clock.systemUTC());
         Authentication authentication = AuthenticationTests.randomAuthentication(null, null);
         PlainActionFuture<TokenService.CreateTokenResult> tokenFuture = new PlainActionFuture<>();
@@ -813,11 +1023,13 @@ public void testSupersedingTokenEncryption() throws Exception {
             authentication,
             tokenFuture
         );
-
-        assertThat(
-            tokenService.prependVersionAndEncodeAccessToken(version, newTokenBytes.v1()),
-            equalTo(tokenFuture.get().getAccessToken())
-        );
+        if (version.onOrAfter(TokenService.VERSION_ACCESS_TOKENS_AS_UUIDS)) {
+            // previous versions serialized the access token encrypted and the cipher text was different each time (due to different IVs)
+            assertThat(
+                tokenService.prependVersionAndEncodeAccessToken(version, newTokenBytes.v1()),
+                equalTo(tokenFuture.get().getAccessToken())
+            );
+        }
         assertThat(
             TokenService.prependVersionAndEncodeRefreshToken(version, newTokenBytes.v2()),
             equalTo(tokenFuture.get().getRefreshToken())
@@ -946,8 +1158,10 @@ public static String tokenDocIdFromAccessTokenBytes(byte[] accessTokenBytes, Tra
             MessageDigest userTokenIdDigest = sha256();
             userTokenIdDigest.update(accessTokenBytes, RAW_TOKEN_BYTES_LENGTH, RAW_TOKEN_DOC_ID_BYTES_LENGTH);
             return Base64.getUrlEncoder().withoutPadding().encodeToString(userTokenIdDigest.digest());
-        } else {
+        } else if (tokenVersion.onOrAfter(TokenService.VERSION_ACCESS_TOKENS_AS_UUIDS)) {
             return TokenService.hashTokenString(Base64.getUrlEncoder().withoutPadding().encodeToString(accessTokenBytes));
+        } else {
+            return Base64.getUrlEncoder().withoutPadding().encodeToString(accessTokenBytes);
         }
     }
 
@@ -964,9 +1178,12 @@ private void mockTokenForRefreshToken(
         if (userToken.getTransportVersion().onOrAfter(VERSION_GET_TOKEN_DOC_FOR_REFRESH)) {
             storedAccessToken = Base64.getUrlEncoder().withoutPadding().encodeToString(sha256().digest(accessTokenBytes));
             storedRefreshToken = Base64.getUrlEncoder().withoutPadding().encodeToString(sha256().digest(refreshTokenBytes));
-        } else {
+        } else if (userToken.getTransportVersion().onOrAfter(TokenService.VERSION_HASHED_TOKENS)) {
             storedAccessToken = null;
             storedRefreshToken = TokenService.hashTokenString(Base64.getUrlEncoder().withoutPadding().encodeToString(refreshTokenBytes));
+        } else {
+            storedAccessToken = null;
+            storedRefreshToken = Base64.getUrlEncoder().withoutPadding().encodeToString(refreshTokenBytes);
         }
         final RealmRef realmRef = new RealmRef(
             refreshTokenStatus == null ? randomAlphaOfLength(6) : refreshTokenStatus.getAssociatedRealm(),
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactoryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactoryTests.java
index acb4359b37323..5482d7711031e 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactoryTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactoryTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.env.TestEnvironment;
@@ -45,7 +46,6 @@
 import java.util.Locale;
 
 import static org.elasticsearch.xpack.core.security.authc.RealmSettings.getFullSettingKey;
-import static org.elasticsearch.xpack.core.security.authc.ldap.PoolingSessionFactorySettings.BIND_DN;
 import static org.elasticsearch.xpack.core.security.authc.ldap.PoolingSessionFactorySettings.LEGACY_BIND_PASSWORD;
 import static org.elasticsearch.xpack.core.security.authc.ldap.PoolingSessionFactorySettings.SECURE_BIND_PASSWORD;
 import static org.hamcrest.Matchers.containsString;
@@ -199,7 +199,7 @@ public void testUserSearchBaseScopeFailsWithWrongBaseDN() throws Exception {
         assertDeprecationWarnings(config.identifier(), useAttribute, useLegacyBindPassword);
     }
 
-    public void testConstructorLogsErrorIfBindDnSetWithoutPassword() throws Exception {
+    public void testConstructorThrowsIfBindDnSetWithoutPassword() throws Exception {
         String groupSearchBase = "o=sevenSeas";
         String userSearchBase = "cn=William Bush,ou=people,o=sevenSeas";
 
@@ -216,19 +216,18 @@ public void testConstructorLogsErrorIfBindDnSetWithoutPassword() throws Exceptio
             new ThreadContext(globalSettings)
         );
 
-        try (LdapUserSearchSessionFactory ignored = getLdapUserSearchSessionFactory(config, sslService, threadPool)) {
-            assertCriticalWarnings(
-                String.format(
-                    Locale.ROOT,
-                    "[%s] is set but no bind password is specified. Without a corresponding bind password, "
-                        + "all ldap realm authentication will fail. Specify a bind password via [%s] or [%s]. "
-                        + "In the next major release, nodes with incomplete bind credentials will fail to start.",
-                    RealmSettings.getFullSettingKey(config, BIND_DN),
-                    RealmSettings.getFullSettingKey(config, SECURE_BIND_PASSWORD),
-                    RealmSettings.getFullSettingKey(config, LEGACY_BIND_PASSWORD)
-                )
-            );
-        }
+        Exception ex = expectThrows(SettingsException.class, () -> getLdapUserSearchSessionFactory(config, sslService, threadPool));
+        assertEquals(
+            String.format(
+                Locale.ROOT,
+                "[%s] is set but no bind password is specified. Without a corresponding bind password, "
+                    + "all %s realm authentication will fail. Specify a bind password via [%s].",
+                RealmSettings.getFullSettingKey(config, PoolingSessionFactorySettings.BIND_DN),
+                config.type(),
+                RealmSettings.getFullSettingKey(config, SECURE_BIND_PASSWORD)
+            ),
+            ex.getMessage()
+        );
     }
 
     public void testConstructorThrowsIfBothLegacyAndSecureBindPasswordSet() throws Exception {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
index 904fb1cff820a..a0d63d26d475f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
@@ -158,8 +158,10 @@ public void setup() {
         final String otherDataStreamName = "logs-foo";
         IndexMetadata dataStreamIndex1 = DataStreamTestHelper.createBackingIndex(dataStreamName, 1).build();
         IndexMetadata dataStreamIndex2 = DataStreamTestHelper.createBackingIndex(dataStreamName, 2).build();
+        IndexMetadata dataStreamFailureStore1 = DataStreamTestHelper.createFailureStore(dataStreamName, 1).build();
+        IndexMetadata dataStreamFailureStore2 = DataStreamTestHelper.createFailureStore(dataStreamName, 2).build();
         IndexMetadata dataStreamIndex3 = DataStreamTestHelper.createBackingIndex(otherDataStreamName, 1).build();
-        Metadata metadata = Metadata.builder()
+        Metadata.Builder metadataBuilder = Metadata.builder()
             .put(
                 indexBuilder("foo").putAlias(AliasMetadata.builder("foofoobar"))
                     .putAlias(AliasMetadata.builder("foounauthorized"))
@@ -217,15 +219,27 @@ public void setup() {
             .put(dataStreamIndex1, true)
             .put(dataStreamIndex2, true)
             .put(dataStreamIndex3, true)
-            .put(newInstance(dataStreamName, List.of(dataStreamIndex1.getIndex(), dataStreamIndex2.getIndex())))
             .put(newInstance(otherDataStreamName, List.of(dataStreamIndex3.getIndex())))
-            .put(indexBuilder(securityIndexName).settings(settings))
-            .build();
+            .put(indexBuilder(securityIndexName).settings(settings));
 
+        // Only add the failure indices if the failure store flag is enabled
+        if (DataStream.isFailureStoreFeatureFlagEnabled()) {
+            metadataBuilder.put(dataStreamFailureStore1, true).put(dataStreamFailureStore2, true);
+        }
+        metadataBuilder.put(
+            newInstance(
+                dataStreamName,
+                List.of(dataStreamIndex1.getIndex(), dataStreamIndex2.getIndex()),
+                DataStream.isFailureStoreFeatureFlagEnabled()
+                    ? List.of(dataStreamFailureStore1.getIndex(), dataStreamFailureStore2.getIndex())
+                    : List.of()
+            )
+        );
         if (withAlias) {
-            metadata = SecurityTestUtils.addAliasToMetadata(metadata, securityIndexName);
+            this.metadata = SecurityTestUtils.addAliasToMetadata(metadataBuilder.build(), securityIndexName);
+        } else {
+            this.metadata = metadataBuilder.build();
         }
-        this.metadata = metadata;
 
         user = new User("user", "role");
         userDashIndices = new User("dash", "dash");
@@ -337,7 +351,9 @@ public void setup() {
             new RoleDescriptor(
                 "backing_index_test_wildcards",
                 null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices(".ds-logs*").privileges("all").build() },
+                new IndicesPrivileges[] {
+                    IndicesPrivileges.builder().indices(".ds-logs*").privileges("all").build(),
+                    IndicesPrivileges.builder().indices(".fs-logs*").privileges("all").build() },
                 null
             )
         );
@@ -347,7 +363,8 @@ public void setup() {
                 "backing_index_test_name",
                 null,
                 new IndicesPrivileges[] {
-                    IndicesPrivileges.builder().indices(dataStreamIndex1.getIndex().getName()).privileges("all").build() },
+                    IndicesPrivileges.builder().indices(dataStreamIndex1.getIndex().getName()).privileges("all").build(),
+                    IndicesPrivileges.builder().indices(dataStreamFailureStore1.getIndex().getName()).privileges("all").build() },
                 null
             )
         );
@@ -432,6 +449,38 @@ public void testWildcardsAreNotAllowedInShardLevelRequests() {
         );
     }
 
+    public void testSelectorsDoNotImpactWildcardDetection() {
+        ShardSearchRequest request = mock(ShardSearchRequest.class);
+        when(request.indices()).thenReturn(new String[] { "index*::data" });
+        IllegalArgumentException exception = expectThrows(
+            IllegalArgumentException.class,
+            () -> defaultIndicesResolver.resolveIndicesAndAliasesWithoutWildcards(TransportSearchAction.TYPE.name() + "[s]", request)
+        );
+        assertThat(
+            exception,
+            throwableWithMessage(
+                "the action indices:data/read/search[s] does not support wildcards;"
+                    + " the provided index expression(s) [index*::data] are not allowed"
+            )
+        );
+    }
+
+    public void testWildcardSelectorsAreNotAllowedInShardLevelRequests() {
+        ShardSearchRequest request = mock(ShardSearchRequest.class);
+        when(request.indices()).thenReturn(new String[] { "index10::*" });
+        IllegalArgumentException exception = expectThrows(
+            IllegalArgumentException.class,
+            () -> defaultIndicesResolver.resolveIndicesAndAliasesWithoutWildcards(TransportSearchAction.TYPE.name() + "[s]", request)
+        );
+        assertThat(
+            exception,
+            throwableWithMessage(
+                "the action indices:data/read/search[s] does not support wildcard selectors;"
+                    + " the provided index expression(s) [index10::*] are not allowed"
+            )
+        );
+    }
+
     public void testAllIsNotAllowedInShardLevelRequests() {
         ShardSearchRequest request = mock(ShardSearchRequest.class);
         final boolean literalAll = randomBoolean();
@@ -460,6 +509,23 @@ public void testAllIsNotAllowedInShardLevelRequests() {
         );
     }
 
+    public void testSelectorsDoNotImpactAllPatternDetection() {
+        ShardSearchRequest request = mock(ShardSearchRequest.class);
+        when(request.indices()).thenReturn(new String[] { "_all::data" });
+        IllegalArgumentException exception = expectThrows(
+            IllegalArgumentException.class,
+            () -> defaultIndicesResolver.resolveIndicesAndAliasesWithoutWildcards(TransportSearchAction.TYPE.name() + "[s]", request)
+        );
+
+        assertThat(
+            exception,
+            throwableWithMessage(
+                "the action indices:data/read/search[s] does not support accessing all indices;"
+                    + " the provided index expression [_all::data] is not allowed"
+            )
+        );
+    }
+
     public void testResolveIndicesAndAliasesWithoutWildcardsWithSingleIndexNoWildcardsRequest() {
         // test 1: matching local index
         {
@@ -1162,7 +1228,7 @@ public void testResolveWildcardsIndicesAliasesRequestAddAndDeleteActions() {
     }
 
     public void testResolveGetAliasesRequestStrict() {
-        GetAliasesRequest request = new GetAliasesRequest("alias1").indices("foo", "foofoo");
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "alias1").indices("foo", "foofoo");
         request.indicesOptions(IndicesOptions.fromOptions(false, randomBoolean(), randomBoolean(), randomBoolean()));
         final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
@@ -1175,7 +1241,7 @@ public void testResolveGetAliasesRequestStrict() {
     }
 
     public void testResolveGetAliasesRequestIgnoreUnavailable() {
-        GetAliasesRequest request = new GetAliasesRequest("alias1").indices("foo", "foofoo");
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "alias1").indices("foo", "foofoo");
         request.indicesOptions(IndicesOptions.fromOptions(true, randomBoolean(), randomBoolean(), randomBoolean()));
         final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
@@ -1187,7 +1253,7 @@ public void testResolveGetAliasesRequestIgnoreUnavailable() {
     }
 
     public void testResolveGetAliasesRequestMissingIndexStrict() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.indicesOptions(IndicesOptions.fromOptions(false, randomBoolean(), true, randomBoolean()));
         request.indices("missing");
         request.aliases("alias2");
@@ -1206,7 +1272,7 @@ public void testResolveGetAliasesRequestMissingIndexStrict() {
     }
 
     public void testGetAliasesRequestMissingIndexIgnoreUnavailableDisallowNoIndices() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.indicesOptions(IndicesOptions.fromOptions(true, false, randomBoolean(), randomBoolean()));
         request.indices("missing");
         request.aliases("alias2");
@@ -1224,7 +1290,7 @@ public void testGetAliasesRequestMissingIndexIgnoreUnavailableDisallowNoIndices(
     }
 
     public void testGetAliasesRequestMissingIndexIgnoreUnavailableAllowNoIndices() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.indicesOptions(IndicesOptions.fromOptions(true, true, randomBoolean(), randomBoolean()));
         request.indices("missing");
         request.aliases("alias2");
@@ -1238,7 +1304,7 @@ public void testGetAliasesRequestMissingIndexIgnoreUnavailableAllowNoIndices() {
     }
 
     public void testGetAliasesRequestMissingIndexStrict() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.indicesOptions(IndicesOptions.fromOptions(false, randomBoolean(), randomBoolean(), randomBoolean()));
         request.indices("missing");
         request.aliases("alias2");
@@ -1252,7 +1318,7 @@ public void testGetAliasesRequestMissingIndexStrict() {
     }
 
     public void testResolveWildcardsGetAliasesRequestStrictExpand() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.indicesOptions(IndicesOptions.fromOptions(false, randomBoolean(), true, true));
         request.aliases("alias1");
         request.indices("foo*");
@@ -1268,7 +1334,7 @@ public void testResolveWildcardsGetAliasesRequestStrictExpand() {
     }
 
     public void testResolveWildcardsGetAliasesRequestStrictExpandOpen() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.indicesOptions(IndicesOptions.fromOptions(false, randomBoolean(), true, false));
         request.aliases("alias1");
         request.indices("foo*");
@@ -1284,7 +1350,7 @@ public void testResolveWildcardsGetAliasesRequestStrictExpandOpen() {
     }
 
     public void testResolveWildcardsGetAliasesRequestLenientExpandOpen() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false));
         request.aliases("alias1");
         request.indices("foo*", "bar", "missing");
@@ -1298,7 +1364,7 @@ public void testResolveWildcardsGetAliasesRequestLenientExpandOpen() {
     }
 
     public void testWildcardsGetAliasesRequestNoMatchingIndicesDisallowNoIndices() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), false, true, randomBoolean()));
         request.aliases("alias3");
         request.indices("non_matching_*");
@@ -1310,7 +1376,7 @@ public void testWildcardsGetAliasesRequestNoMatchingIndicesDisallowNoIndices() {
     }
 
     public void testWildcardsGetAliasesRequestNoMatchingIndicesAllowNoIndices() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), true, true, randomBoolean()));
         request.aliases("alias3");
         request.indices("non_matching_*");
@@ -1318,7 +1384,7 @@ public void testWildcardsGetAliasesRequestNoMatchingIndicesAllowNoIndices() {
     }
 
     public void testResolveAllGetAliasesRequest() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         // even if not set, empty means _all
         if (randomBoolean()) {
             request.indices("_all");
@@ -1361,7 +1427,7 @@ public void testResolveAllGetAliasesRequest() {
     }
 
     public void testResolveAllGetAliasesRequestExpandWildcardsOpenOnly() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         // set indices options to have wildcards resolved to open indices only (default is open and closed)
         request.indicesOptions(IndicesOptions.fromOptions(true, false, true, false));
         // even if not set, empty means _all
@@ -1382,7 +1448,7 @@ public void testResolveAllGetAliasesRequestExpandWildcardsOpenOnly() {
     }
 
     public void testAllGetAliasesRequestNoAuthorizedIndicesAllowNoIndices() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), true, true, randomBoolean()));
         request.aliases("alias1");
         request.indices("_all");
@@ -1390,7 +1456,7 @@ public void testAllGetAliasesRequestNoAuthorizedIndicesAllowNoIndices() {
     }
 
     public void testAllGetAliasesRequestNoAuthorizedIndicesDisallowNoIndices() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), false, true, randomBoolean()));
         request.aliases("alias1");
         request.indices("_all");
@@ -1402,7 +1468,7 @@ public void testAllGetAliasesRequestNoAuthorizedIndicesDisallowNoIndices() {
     }
 
     public void testWildcardsGetAliasesRequestNoAuthorizedIndicesAllowNoIndices() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.aliases("alias1");
         request.indices("foo*");
         request.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), true, true, randomBoolean()));
@@ -1410,7 +1476,7 @@ public void testWildcardsGetAliasesRequestNoAuthorizedIndicesAllowNoIndices() {
     }
 
     public void testWildcardsGetAliasesRequestNoAuthorizedIndicesDisallowNoIndices() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), false, true, randomBoolean()));
         request.aliases("alias1");
         request.indices("foo*");
@@ -1423,7 +1489,7 @@ public void testWildcardsGetAliasesRequestNoAuthorizedIndicesDisallowNoIndices()
     }
 
     public void testResolveAllAliasesGetAliasesRequest() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         if (randomBoolean()) {
             request.aliases("_all");
         }
@@ -1453,7 +1519,7 @@ public void testResolveAllAliasesGetAliasesRequest() {
     }
 
     public void testResolveAllAndExplicitAliasesGetAliasesRequest() {
-        GetAliasesRequest request = new GetAliasesRequest(new String[] { "_all", "explicit" });
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, new String[] { "_all", "explicit" });
         if (randomBoolean()) {
             request.indices("_all");
         }
@@ -1498,7 +1564,7 @@ public void testResolveAllAndExplicitAliasesGetAliasesRequest() {
     }
 
     public void testResolveAllAndWildcardsAliasesGetAliasesRequest() {
-        GetAliasesRequest request = new GetAliasesRequest(new String[] { "_all", "foo*", "non_matching_*" });
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, new String[] { "_all", "foo*", "non_matching_*" });
         if (randomBoolean()) {
             request.indices("_all");
         }
@@ -1525,7 +1591,7 @@ public void testResolveAllAndWildcardsAliasesGetAliasesRequest() {
     }
 
     public void testResolveAliasesWildcardsGetAliasesRequest() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.indices("*bar");
         request.aliases("foo*");
         final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
@@ -1541,7 +1607,7 @@ public void testResolveAliasesWildcardsGetAliasesRequest() {
     }
 
     public void testResolveAliasesWildcardsGetAliasesRequestNoAuthorizedIndices() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         // no authorized aliases match bar*, hence aliases are replaced with the no-aliases-expression
         request.aliases("bar*");
         request.indices("*bar");
@@ -1550,7 +1616,7 @@ public void testResolveAliasesWildcardsGetAliasesRequestNoAuthorizedIndices() {
     }
 
     public void testResolveAliasesExclusionWildcardsGetAliasesRequest() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         request.aliases("foo*", "-foobar*");
         final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
@@ -1594,7 +1660,7 @@ public void testResolveAliasesExclusionWildcardsGetAliasesRequest() {
     }
 
     public void testResolveAliasesAllGetAliasesRequestNoAuthorizedIndices() {
-        GetAliasesRequest request = new GetAliasesRequest();
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT);
         if (randomBoolean()) {
             request.aliases("_all");
         }
@@ -1854,7 +1920,7 @@ public void testAliasDateMathExpressionNotSupported() {
                 null
             )
         );
-        GetAliasesRequest request = new GetAliasesRequest("<datetime-{now/M}>").indices("foo", "foofoo");
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "<datetime-{now/M}>").indices("foo", "foofoo");
         List<String> indices = resolveIndices(request, buildAuthorizedIndices(user, GetAliasesAction.NAME)).getLocal();
         // the union of all indices and aliases gets returned
         String[] expectedIndices = new String[] { "<datetime-{now/M}>", "foo", "foofoo" };
@@ -2158,7 +2224,7 @@ public void testDataStreamResolution() {
 
     public void testDataStreamsAreNotVisibleWhenNotIncludedByRequestWithWildcard() {
         final User user = new User("data-stream-tester2", "data_stream_test2");
-        GetAliasesRequest request = new GetAliasesRequest("*");
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "*");
         assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
         assertThat(request.includeDataStreams(), is(true));
 
@@ -2199,7 +2265,7 @@ public void testDataStreamsAreNotVisibleWhenNotIncludedByRequestWithWildcard() {
     public void testDataStreamsAreNotVisibleWhenNotIncludedByRequestWithoutWildcard() {
         final User user = new User("data-stream-tester2", "data_stream_test2");
         String dataStreamName = "logs-foobar";
-        GetAliasesRequest request = new GetAliasesRequest(dataStreamName);
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, dataStreamName);
         assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
         assertThat(request.includeDataStreams(), is(true));
 
@@ -2305,12 +2371,13 @@ public void testDataStreamsAreVisibleWhenIncludedByRequestWithoutWildcard() {
 
     public void testBackingIndicesAreVisibleWhenIncludedByRequestWithWildcard() {
         final User user = new User("data-stream-tester3", "data_stream_test3");
-        SearchRequest request = new SearchRequest(".ds-logs*");
+        boolean failureStore = runFailureStore();
+        SearchRequest request = new SearchRequest(failureStore ? ".fs-logs*" : ".ds-logs*");
         assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
         assertThat(request.includeDataStreams(), is(true));
 
         // data streams and their backing indices should be included in the authorized list
-        List<String> expectedDataStreams = List.of("logs-foo", "logs-foobar");
+        List<String> expectedDataStreams = failureStore ? List.of("logs-foobar") : List.of("logs-foo", "logs-foobar");
         final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, TransportSearchAction.TYPE.name(), request);
         for (String dsName : expectedDataStreams) {
             DataStream dataStream = metadata.dataStreams().get(dsName);
@@ -2320,6 +2387,10 @@ public void testBackingIndicesAreVisibleWhenIncludedByRequestWithWildcard() {
                 assertThat(authorizedIndices.all().get(), hasItem(i.getName()));
                 assertThat(authorizedIndices.check(i.getName()), is(true));
             }
+            for (Index i : dataStream.getFailureIndices().getIndices()) {
+                assertThat(authorizedIndices.all().get(), hasItem(i.getName()));
+                assertThat(authorizedIndices.check(i.getName()), is(true));
+            }
         }
 
         // data streams should _not_ be included in the resolved list because they do not match the pattern but their backing indices
@@ -2333,7 +2404,7 @@ public void testBackingIndicesAreVisibleWhenIncludedByRequestWithWildcard() {
         for (String dsName : expectedDataStreams) {
             DataStream dataStream = metadata.dataStreams().get(dsName);
             assertThat(resolvedIndices.getLocal(), not(hasItem(dsName)));
-            for (Index i : dataStream.getIndices()) {
+            for (Index i : dataStream.getDataStreamIndices(failureStore).getIndices()) {
                 assertThat(resolvedIndices.getLocal(), hasItem(i.getName()));
             }
         }
@@ -2342,7 +2413,7 @@ public void testBackingIndicesAreVisibleWhenIncludedByRequestWithWildcard() {
     public void testBackingIndicesAreNotVisibleWhenNotIncludedByRequestWithoutWildcard() {
         final User user = new User("data-stream-tester2", "data_stream_test2");
         String dataStreamName = "logs-foobar";
-        GetAliasesRequest request = new GetAliasesRequest(dataStreamName);
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, dataStreamName);
         assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
         assertThat(request.includeDataStreams(), is(true));
 
@@ -2352,12 +2423,14 @@ public void testBackingIndicesAreNotVisibleWhenNotIncludedByRequestWithoutWildca
         assertThat(authorizedIndices.all().get(), hasItem(dataStreamName));
         assertThat(authorizedIndices.check(dataStreamName), is(true));
         DataStream dataStream = metadata.dataStreams().get(dataStreamName);
-        assertThat(authorizedIndices.all().get(), hasItem(dataStreamName));
-        assertThat(authorizedIndices.check(dataStreamName), is(true));
         for (Index i : dataStream.getIndices()) {
             assertThat(authorizedIndices.all().get(), hasItem(i.getName()));
             assertThat(authorizedIndices.check(i.getName()), is(true));
         }
+        for (Index i : dataStream.getFailureIndices().getIndices()) {
+            assertThat(authorizedIndices.all().get(), hasItem(i.getName()));
+            assertThat(authorizedIndices.check(i.getName()), is(true));
+        }
 
         // neither data streams nor their backing indices will be in the resolved list since the request does not support data streams
         // and the backing indices do not match the requested name
@@ -2371,11 +2444,15 @@ public void testBackingIndicesAreNotVisibleWhenNotIncludedByRequestWithoutWildca
         for (Index i : dataStream.getIndices()) {
             assertThat(resolvedIndices.getLocal(), hasItem(i.getName()));
         }
+        for (Index i : dataStream.getFailureIndices().getIndices()) {
+            assertThat(resolvedIndices.getLocal(), hasItem(i.getName()));
+        }
     }
 
     public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaWildcardAndRequestThatIncludesDataStreams() {
         final User user = new User("data-stream-tester2", "backing_index_test_wildcards");
-        String indexName = ".ds-logs-foobar-*";
+        boolean failureStore = runFailureStore();
+        String indexName = failureStore ? ".fs-logs-foobar-*" : ".ds-logs-foobar-*";
         SearchRequest request = new SearchRequest(indexName);
         assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
         assertThat(request.includeDataStreams(), is(true));
@@ -2393,6 +2470,10 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaWildcar
             assertThat(authorizedIndices.all().get(), hasItem(i.getName()));
             assertThat(authorizedIndices.check(i.getName()), is(true));
         }
+        for (Index i : dataStream.getFailureIndices().getIndices()) {
+            assertThat(authorizedIndices.all().get(), hasItem(i.getName()));
+            assertThat(authorizedIndices.check(i.getName()), is(true));
+        }
 
         // only the backing indices will be in the resolved list since the request does not support data streams
         // but the backing indices match the requested pattern
@@ -2403,14 +2484,15 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaWildcar
             authorizedIndices
         );
         assertThat(resolvedIndices.getLocal(), not(hasItem(dataStream.getName())));
-        for (Index i : dataStream.getIndices()) {
+        for (Index i : dataStream.getDataStreamIndices(failureStore).getIndices()) {
             assertThat(resolvedIndices.getLocal(), hasItem(i.getName()));
         }
     }
 
     public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaNameAndRequestThatIncludesDataStreams() {
         final User user = new User("data-stream-tester2", "backing_index_test_name");
-        String indexName = ".ds-logs-foobar-*";
+        boolean failureStore = runFailureStore();
+        String indexName = failureStore ? ".fs-logs-foobar-*" : ".ds-logs-foobar-*";
         SearchRequest request = new SearchRequest(indexName);
         assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
         assertThat(request.includeDataStreams(), is(true));
@@ -2420,8 +2502,12 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaNameAnd
         final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
         assertThat(authorizedIndices.all().get(), not(hasItem("logs-foobar")));
         assertThat(authorizedIndices.check("logs-foobar"), is(false));
-        assertThat(authorizedIndices.all().get(), contains(DataStream.getDefaultBackingIndexName("logs-foobar", 1)));
-        assertThat(authorizedIndices.check(DataStream.getDefaultBackingIndexName("logs-foobar", 1)), is(true));
+
+        String expectedIndex = failureStore
+            ? DataStream.getDefaultFailureStoreName("logs-foobar", 1, System.currentTimeMillis())
+            : DataStream.getDefaultBackingIndexName("logs-foobar", 1);
+        assertThat(authorizedIndices.all().get(), hasItem(expectedIndex));
+        assertThat(authorizedIndices.check(expectedIndex), is(true));
 
         // only the single backing index will be in the resolved list since the request does not support data streams
         // but one of the backing indices matched the requested pattern
@@ -2432,13 +2518,14 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaNameAnd
             authorizedIndices
         );
         assertThat(resolvedIndices.getLocal(), not(hasItem("logs-foobar")));
-        assertThat(resolvedIndices.getLocal(), contains(DataStream.getDefaultBackingIndexName("logs-foobar", 1)));
+        assertThat(resolvedIndices.getLocal(), contains(expectedIndex));
     }
 
     public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaWildcardAndRequestThatExcludesDataStreams() {
         final User user = new User("data-stream-tester2", "backing_index_test_wildcards");
-        String indexName = ".ds-logs-foobar-*";
-        GetAliasesRequest request = new GetAliasesRequest(indexName);
+        boolean failureStore = runFailureStore();
+        String indexName = failureStore ? ".fs-logs-foobar-*" : ".ds-logs-foobar-*";
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, indexName);
         assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
         assertThat(request.includeDataStreams(), is(true));
 
@@ -2455,6 +2542,10 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaWildcar
             assertThat(authorizedIndices.all().get(), hasItem(i.getName()));
             assertThat(authorizedIndices.check(i.getName()), is(true));
         }
+        for (Index i : dataStream.getFailureIndices().getIndices()) {
+            assertThat(authorizedIndices.all().get(), hasItem(i.getName()));
+            assertThat(authorizedIndices.check(i.getName()), is(true));
+        }
 
         // only the backing indices will be in the resolved list since the request does not support data streams
         // but the backing indices match the requested pattern
@@ -2465,25 +2556,29 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaWildcar
             authorizedIndices
         );
         assertThat(resolvedIndices.getLocal(), not(hasItem(dataStream.getName())));
-        for (Index i : dataStream.getIndices()) {
+        for (Index i : dataStream.getDataStreamIndices(failureStore).getIndices()) {
             assertThat(resolvedIndices.getLocal(), hasItem(i.getName()));
         }
     }
 
     public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaNameAndRequestThatExcludesDataStreams() {
         final User user = new User("data-stream-tester2", "backing_index_test_name");
-        String indexName = ".ds-logs-foobar-*";
-        GetAliasesRequest request = new GetAliasesRequest(indexName);
+        boolean failureStore = runFailureStore();
+        String indexName = failureStore ? ".fs-logs-foobar-*" : ".ds-logs-foobar-*";
+        GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, indexName);
         assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
         assertThat(request.includeDataStreams(), is(true));
 
         // data streams should _not_ be in the authorized list but a single backing index that matched the requested pattern
         // and the authorized name should be in the list
+        String expectedIndex = failureStore
+            ? DataStream.getDefaultFailureStoreName("logs-foobar", 1, System.currentTimeMillis())
+            : DataStream.getDefaultBackingIndexName("logs-foobar", 1);
         final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
         assertThat(authorizedIndices.all().get(), not(hasItem("logs-foobar")));
         assertThat(authorizedIndices.check("logs-foobar"), is(false));
-        assertThat(authorizedIndices.all().get(), contains(DataStream.getDefaultBackingIndexName("logs-foobar", 1)));
-        assertThat(authorizedIndices.check(DataStream.getDefaultBackingIndexName("logs-foobar", 1)), is(true));
+        assertThat(authorizedIndices.all().get(), hasItem(expectedIndex));
+        assertThat(authorizedIndices.check(expectedIndex), is(true));
 
         // only the single backing index will be in the resolved list since the request does not support data streams
         // but one of the backing indices matched the requested pattern
@@ -2494,7 +2589,7 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaNameAnd
             authorizedIndices
         );
         assertThat(resolvedIndices.getLocal(), not(hasItem("logs-foobar")));
-        assertThat(resolvedIndices.getLocal(), contains(DataStream.getDefaultBackingIndexName("logs-foobar", 1)));
+        assertThat(resolvedIndices.getLocal(), hasItem(expectedIndex));
     }
 
     public void testResolveSearchShardRequestAgainstDataStream() {
@@ -2583,4 +2678,8 @@ private void assertSameValues(List<String> indices, String[] expectedIndices) {
         assertThat(indices.stream().distinct().count(), equalTo((long) expectedIndices.length));
         assertThat(indices, hasItems(expectedIndices));
     }
+
+    private boolean runFailureStore() {
+        return DataStream.isFailureStoreFeatureFlagEnabled() && randomBoolean();
+    }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/DeprecationRoleDescriptorConsumerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/DeprecationRoleDescriptorConsumerTests.java
index ee91b8b65e540..c572b3a35d5a4 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/DeprecationRoleDescriptorConsumerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/DeprecationRoleDescriptorConsumerTests.java
@@ -342,7 +342,7 @@ public void testMultipleIndicesSameAlias() throws Exception {
 
     private void addIndex(Metadata.Builder metadataBuilder, String index, String... aliases) {
         final IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(index)
-            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion(random())))
+            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion()))
             .numberOfShards(1)
             .numberOfReplicas(1);
         for (final String alias : aliases) {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesSynchronizerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesSynchronizerTests.java
new file mode 100644
index 0000000000000..e3c6c0cdd4e5b
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesSynchronizerTests.java
@@ -0,0 +1,526 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.support;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlocks;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.node.VersionInformation;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.features.FeatureService;
+import org.elasticsearch.index.IndexVersions;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.index.IndexVersionUtils;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.security.action.role.BulkRolesResponse;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
+import org.elasticsearch.xpack.core.security.test.TestRestrictedIndices;
+import org.elasticsearch.xpack.security.authz.store.FileRolesStore;
+import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
+import org.elasticsearch.xpack.security.support.QueryableBuiltInRolesSynchronizer.MarkRolesAsSyncedTask;
+import org.elasticsearch.xpack.security.test.SecurityTestUtils;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
+import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesSynchronizer.QUERYABLE_BUILT_IN_ROLES_FEATURE;
+import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesUtilsTests.buildQueryableBuiltInRoles;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class QueryableBuiltInRolesSynchronizerTests extends ESTestCase {
+
+    private static final ClusterName CLUSTER_NAME = new ClusterName("queryable-built-in-roles-synchronizer-tests");
+    private static final ClusterState EMPTY_CLUSTER_STATE = new ClusterState.Builder(CLUSTER_NAME).build();
+
+    private QueryableBuiltInRolesSynchronizer synchronizer;
+    private NativeRolesStore nativeRolesStore;
+    private ClusterService clusterService;
+    private FeatureService featureService;
+    private ThreadPool threadPool;
+
+    private MasterServiceTaskQueue<MarkRolesAsSyncedTask> taskQueue;
+
+    private QueryableReservedRolesProvider reservedRolesProvider;
+
+    @BeforeClass
+    public static void setupReservedRolesStore() {
+        new ReservedRolesStore(); // initialize the store
+    }
+
+    @Before
+    public void setupSynchronizer() {
+        threadPool = mock(ThreadPool.class);
+        when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY));
+        when(threadPool.generic()).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE);
+
+        nativeRolesStore = mock(NativeRolesStore.class);
+        clusterService = mock(ClusterService.class);
+        taskQueue = mockTaskQueue(clusterService);
+
+        QueryableBuiltInRolesProviderFactory rolesProviderFactory = mock(QueryableBuiltInRolesProviderFactory.class);
+        reservedRolesProvider = mock(QueryableReservedRolesProvider.class);
+        when(rolesProviderFactory.createProvider(any(), any())).thenReturn(reservedRolesProvider);
+
+        featureService = mock(FeatureService.class);
+
+        synchronizer = new QueryableBuiltInRolesSynchronizer(
+            clusterService,
+            featureService,
+            rolesProviderFactory,
+            nativeRolesStore,
+            mock(ReservedRolesStore.class),
+            mock(FileRolesStore.class),
+            threadPool
+        );
+    }
+
+    private void assertInitialState() {
+        verifyNoInteractions(nativeRolesStore);
+        verifyNoInteractions(featureService);
+        verifyNoInteractions(taskQueue);
+
+        verify(reservedRolesProvider, times(1)).addListener(any());
+        verifyNoMoreInteractions(reservedRolesProvider);
+
+        verify(clusterService, times(1)).addLifecycleListener(any());
+        verify(clusterService, times(1)).createTaskQueue(eq("mark-built-in-roles-as-synced-task-queue"), eq(Priority.LOW), any());
+        verifyNoMoreInteractions(clusterService);
+
+        verify(threadPool, times(1)).generic();
+        verifyNoMoreInteractions(threadPool);
+    }
+
+    public void testSuccessfulSync() {
+        assertInitialState();
+
+        ClusterState clusterState = markShardsAvailable(createClusterStateWithOpenSecurityIndex()).nodes(localNodeMaster())
+            .blocks(emptyClusterBlocks())
+            .build();
+
+        when(clusterService.state()).thenReturn(clusterState);
+        when(featureService.clusterHasFeature(any(), eq(QUERYABLE_BUILT_IN_ROLES_FEATURE))).thenReturn(true);
+
+        final QueryableBuiltInRoles builtInRoles = buildQueryableBuiltInRoles(
+            Set.of(
+                ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR,
+                ReservedRolesStore.roleDescriptor("viewer"),
+                ReservedRolesStore.roleDescriptor("editor")
+            )
+        );
+        when(reservedRolesProvider.getRoles()).thenReturn(builtInRoles);
+        mockEnabledNativeStore(builtInRoles.roleDescriptors(), Set.of());
+        assertThat(synchronizer.isSynchronizationInProgress(), equalTo(false));
+
+        synchronizer.clusterChanged(event(clusterState));
+
+        // called once on successful sync to update the digests in the cluster state
+        verify(taskQueue, times(1)).submitTask(any(), argThat(task -> task.getNewRoleDigests().equals(builtInRoles.rolesDigest())), any());
+        verify(nativeRolesStore, times(1)).isEnabled();
+        verify(featureService, times(1)).clusterHasFeature(any(), eq(QUERYABLE_BUILT_IN_ROLES_FEATURE));
+        verify(reservedRolesProvider, times(1)).getRoles();
+        verify(nativeRolesStore, times(1)).putRoles(
+            eq(WriteRequest.RefreshPolicy.IMMEDIATE),
+            eq(builtInRoles.roleDescriptors()),
+            eq(false),
+            any()
+        );
+        verify(clusterService, times(3)).state();
+        verifyNoMoreInteractions(nativeRolesStore, featureService, taskQueue, reservedRolesProvider, threadPool, clusterService);
+        assertThat(synchronizer.isSynchronizationInProgress(), equalTo(false));
+    }
+
+    public void testNotMaster() {
+        assertInitialState();
+
+        ClusterState clusterState = spy(
+            markShardsAvailable(createClusterStateWithOpenSecurityIndex()).nodes(localNodeNotMaster()).blocks(emptyClusterBlocks()).build()
+        );
+
+        when(clusterService.state()).thenReturn(clusterState);
+        when(featureService.clusterHasFeature(any(), eq(QUERYABLE_BUILT_IN_ROLES_FEATURE))).thenReturn(true);
+
+        final QueryableBuiltInRoles builtInRoles = buildQueryableBuiltInRoles(Set.of(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR));
+        when(reservedRolesProvider.getRoles()).thenReturn(builtInRoles);
+        mockEnabledNativeStore(builtInRoles.roleDescriptors(), Set.of());
+
+        synchronizer.clusterChanged(event(clusterState));
+
+        verify(clusterState, times(0)).clusterRecovered();
+        verify(reservedRolesProvider, times(0)).getRoles();
+        verifyNoMoreInteractions(nativeRolesStore, featureService, taskQueue, reservedRolesProvider, threadPool, clusterService);
+    }
+
+    public void testClusterNotRecovered() {
+        assertInitialState();
+
+        ClusterState clusterState = spy(
+            markShardsAvailable(createClusterStateWithOpenSecurityIndex()).nodes(localNodeMaster())
+                .blocks(stateNotRecoveredClusterBlocks())
+                .build()
+        );
+
+        when(clusterService.state()).thenReturn(clusterState);
+        when(featureService.clusterHasFeature(any(), eq(QUERYABLE_BUILT_IN_ROLES_FEATURE))).thenReturn(true);
+
+        final QueryableBuiltInRoles builtInRoles = buildQueryableBuiltInRoles(Set.of(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR));
+        when(reservedRolesProvider.getRoles()).thenReturn(builtInRoles);
+        mockEnabledNativeStore(builtInRoles.roleDescriptors(), Set.of());
+
+        synchronizer.clusterChanged(event(clusterState));
+
+        verify(clusterState, times(1)).clusterRecovered();
+        verify(reservedRolesProvider, times(0)).getRoles();
+        verifyNoMoreInteractions(nativeRolesStore, featureService, taskQueue, reservedRolesProvider, threadPool, clusterService);
+    }
+
+    public void testNativeRolesDisabled() {
+        assertInitialState();
+
+        ClusterState clusterState = spy(
+            markShardsAvailable(createClusterStateWithOpenSecurityIndex()).nodes(localNodeMaster()).blocks(emptyClusterBlocks()).build()
+        );
+
+        when(clusterService.state()).thenReturn(clusterState);
+        when(featureService.clusterHasFeature(any(), eq(QUERYABLE_BUILT_IN_ROLES_FEATURE))).thenReturn(true);
+
+        when(reservedRolesProvider.getRoles()).thenReturn(buildQueryableBuiltInRoles(Set.of(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR)));
+        mockDisabledNativeStore();
+
+        synchronizer.clusterChanged(event(clusterState));
+
+        verify(clusterState, times(1)).clusterRecovered();
+        verify(nativeRolesStore, times(1)).isEnabled();
+        verify(clusterState, times(2)).nodes();
+        verifyNoMoreInteractions(nativeRolesStore, featureService, taskQueue, reservedRolesProvider, threadPool, clusterService);
+    }
+
+    public void testMixedVersionsCluster() {
+        assertInitialState();
+
+        ClusterState clusterState = spy(
+            markShardsAvailable(createClusterStateWithOpenSecurityIndex()).nodes(mixedVersionNodes()).blocks(emptyClusterBlocks()).build()
+        );
+
+        when(clusterService.state()).thenReturn(clusterState);
+        when(featureService.clusterHasFeature(any(), eq(QUERYABLE_BUILT_IN_ROLES_FEATURE))).thenReturn(true);
+
+        final QueryableBuiltInRoles builtInRoles = buildQueryableBuiltInRoles(Set.of(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR));
+        when(reservedRolesProvider.getRoles()).thenReturn(builtInRoles);
+        mockEnabledNativeStore(builtInRoles.roleDescriptors(), Set.of());
+
+        synchronizer.clusterChanged(event(clusterState));
+
+        verify(clusterState, times(1)).clusterRecovered();
+        verify(nativeRolesStore, times(1)).isEnabled();
+        verify(clusterState, times(4)).nodes();
+        verifyNoMoreInteractions(nativeRolesStore, featureService, taskQueue, reservedRolesProvider, threadPool, clusterService);
+
+    }
+
+    public void testNoDataNodes() {
+        assertInitialState();
+
+        ClusterState clusterState = spy(
+            markShardsAvailable(createClusterStateWithOpenSecurityIndex()).nodes(noDataNodes()).blocks(emptyClusterBlocks()).build()
+        );
+
+        when(clusterService.state()).thenReturn(clusterState);
+        when(featureService.clusterHasFeature(any(), eq(QUERYABLE_BUILT_IN_ROLES_FEATURE))).thenReturn(true);
+
+        final QueryableBuiltInRoles builtInRoles = buildQueryableBuiltInRoles(Set.of(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR));
+        when(reservedRolesProvider.getRoles()).thenReturn(builtInRoles);
+        mockEnabledNativeStore(builtInRoles.roleDescriptors(), Set.of());
+
+        synchronizer.clusterChanged(event(clusterState));
+
+        verify(clusterState, times(1)).clusterRecovered();
+        verify(nativeRolesStore, times(1)).isEnabled();
+        verify(clusterState, times(3)).nodes();
+        verifyNoMoreInteractions(nativeRolesStore, featureService, taskQueue, reservedRolesProvider, threadPool, clusterService);
+    }
+
+    public void testFeatureNotSupported() {
+        assertInitialState();
+
+        ClusterState clusterState = spy(
+            markShardsAvailable(createClusterStateWithOpenSecurityIndex()).nodes(localNodeMaster()).blocks(emptyClusterBlocks()).build()
+        );
+
+        when(clusterService.state()).thenReturn(clusterState);
+        when(featureService.clusterHasFeature(any(), eq(QUERYABLE_BUILT_IN_ROLES_FEATURE))).thenReturn(false);
+
+        final QueryableBuiltInRoles builtInRoles = buildQueryableBuiltInRoles(Set.of(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR));
+        when(reservedRolesProvider.getRoles()).thenReturn(builtInRoles);
+        mockEnabledNativeStore(builtInRoles.roleDescriptors(), Set.of());
+
+        synchronizer.clusterChanged(event(clusterState));
+
+        verify(clusterState, times(1)).clusterRecovered();
+        verify(nativeRolesStore, times(1)).isEnabled();
+        verify(clusterState, times(4)).nodes();
+        verify(featureService, times(1)).clusterHasFeature(eq(clusterState), eq(QUERYABLE_BUILT_IN_ROLES_FEATURE));
+        verifyNoMoreInteractions(nativeRolesStore, featureService, taskQueue, reservedRolesProvider, threadPool, clusterService);
+    }
+
+    public void testSecurityIndexDeleted() {
+        assertInitialState();
+
+        ClusterState previousClusterState = spy(
+            markShardsAvailable(createClusterStateWithOpenSecurityIndex()).nodes(localNodeMaster()).blocks(emptyClusterBlocks()).build()
+        );
+        ClusterState currentClusterState = spy(
+            ClusterState.builder(CLUSTER_NAME)
+                .nodes(localNodeMaster())
+                .blocks(emptyClusterBlocks())
+                .metadata(Metadata.builder().generateClusterUuidIfNeeded())
+                .build()
+        );
+        when(clusterService.state()).thenReturn(currentClusterState);
+
+        synchronizer.clusterChanged(event(currentClusterState, previousClusterState));
+
+        verify(previousClusterState, times(1)).metadata();
+        verify(currentClusterState, times(1)).metadata();
+        verifyNoMoreInteractions(nativeRolesStore, featureService, taskQueue, reservedRolesProvider, threadPool, clusterService);
+    }
+
+    public void testSecurityIndexCreated() {
+        assertInitialState();
+
+        ClusterState currentClusterState = spy(
+            markShardsAvailable(createClusterStateWithOpenSecurityIndex()).nodes(localNodeMaster()).blocks(emptyClusterBlocks()).build()
+        );
+
+        ClusterState previousClusterState = spy(
+            ClusterState.builder(CLUSTER_NAME)
+                .nodes(localNodeMaster())
+                .blocks(emptyClusterBlocks())
+                .metadata(Metadata.builder().generateClusterUuidIfNeeded())
+                .build()
+        );
+
+        when(clusterService.state()).thenReturn(currentClusterState);
+        when(featureService.clusterHasFeature(any(), eq(QUERYABLE_BUILT_IN_ROLES_FEATURE))).thenReturn(true);
+
+        final QueryableBuiltInRoles builtInRoles = buildQueryableBuiltInRoles(Set.of(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR));
+        when(reservedRolesProvider.getRoles()).thenReturn(builtInRoles);
+        mockEnabledNativeStore(builtInRoles.roleDescriptors(), Set.of());
+        assertThat(synchronizer.isSynchronizationInProgress(), equalTo(false));
+
+        synchronizer.clusterChanged(event(currentClusterState, previousClusterState));
+
+        verify(taskQueue, times(1)).submitTask(any(), argThat(task -> task.getNewRoleDigests().equals(builtInRoles.rolesDigest())), any());
+        verify(nativeRolesStore, times(1)).isEnabled();
+        verify(featureService, times(1)).clusterHasFeature(any(), eq(QUERYABLE_BUILT_IN_ROLES_FEATURE));
+        verify(reservedRolesProvider, times(1)).getRoles();
+        verify(nativeRolesStore, times(1)).putRoles(
+            eq(WriteRequest.RefreshPolicy.IMMEDIATE),
+            eq(builtInRoles.roleDescriptors()),
+            eq(false),
+            any()
+        );
+        verify(clusterService, times(3)).state();
+        verifyNoMoreInteractions(nativeRolesStore, featureService, taskQueue, reservedRolesProvider, threadPool, clusterService);
+        assertThat(synchronizer.isSynchronizationInProgress(), equalTo(false));
+    }
+
+    public void testSecurityIndexClosed() {
+        assertInitialState();
+
+        ClusterState clusterState = spy(
+            markShardsAvailable(createClusterStateWithClosedSecurityIndex()).nodes(localNodeMaster()).blocks(emptyClusterBlocks()).build()
+        );
+
+        when(clusterService.state()).thenReturn(clusterState);
+        when(featureService.clusterHasFeature(any(), eq(QUERYABLE_BUILT_IN_ROLES_FEATURE))).thenReturn(false);
+
+        final QueryableBuiltInRoles builtInRoles = buildQueryableBuiltInRoles(Set.of(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR));
+        when(reservedRolesProvider.getRoles()).thenReturn(builtInRoles);
+        mockEnabledNativeStore(builtInRoles.roleDescriptors(), Set.of());
+
+        synchronizer.clusterChanged(event(clusterState));
+
+        verify(clusterState, times(1)).clusterRecovered();
+        verify(nativeRolesStore, times(1)).isEnabled();
+        verify(clusterState, times(4)).nodes();
+        verify(featureService, times(1)).clusterHasFeature(eq(clusterState), eq(QUERYABLE_BUILT_IN_ROLES_FEATURE));
+        verifyNoMoreInteractions(nativeRolesStore, featureService, taskQueue, reservedRolesProvider, threadPool, clusterService);
+    }
+
+    private static ClusterState.Builder markShardsAvailable(ClusterState.Builder clusterStateBuilder) {
+        final ClusterState cs = clusterStateBuilder.build();
+        return ClusterState.builder(cs)
+            .routingTable(
+                SecurityTestUtils.buildIndexRoutingTable(
+                    cs.metadata().index(TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7).getIndex()
+                )
+            );
+    }
+
+    @SuppressWarnings("unchecked")
+    private MasterServiceTaskQueue<MarkRolesAsSyncedTask> mockTaskQueue(ClusterService clusterService) {
+        final MasterServiceTaskQueue<MarkRolesAsSyncedTask> masterServiceTaskQueue = mock(MasterServiceTaskQueue.class);
+        when(clusterService.<MarkRolesAsSyncedTask>createTaskQueue(eq("mark-built-in-roles-as-synced-task-queue"), eq(Priority.LOW), any()))
+            .thenReturn(masterServiceTaskQueue);
+        doAnswer(i -> {
+            assertThat(synchronizer.isSynchronizationInProgress(), equalTo(true));
+            MarkRolesAsSyncedTask task = i.getArgument(1);
+            var result = task.execute(clusterService.state());
+            task.success(result.v2());
+            return null;
+        }).when(masterServiceTaskQueue).submitTask(any(), any(), any());
+        return masterServiceTaskQueue;
+    }
+
+    private ClusterChangedEvent event(ClusterState clusterState) {
+        return new ClusterChangedEvent("test-event", clusterState, EMPTY_CLUSTER_STATE);
+    }
+
+    private ClusterChangedEvent event(ClusterState currentClusterState, ClusterState previousClusterState) {
+        return new ClusterChangedEvent("test-event", currentClusterState, previousClusterState);
+    }
+
+    private static DiscoveryNodes localNodeMaster() {
+        return nodes(randomIntBetween(1, 3), true).build();
+    }
+
+    private static DiscoveryNodes localNodeNotMaster() {
+        return nodes(randomIntBetween(1, 3), false).build();
+    }
+
+    private static DiscoveryNodes noDataNodes() {
+        return nodes(0, true).build();
+    }
+
+    private DiscoveryNodes mixedVersionNodes() {
+        VersionInformation oldVersion = new VersionInformation(
+            VersionUtils.randomVersionBetween(random(), null, VersionUtils.getPreviousVersion()),
+            IndexVersions.MINIMUM_COMPATIBLE,
+            IndexVersionUtils.randomCompatibleVersion(random())
+        );
+        return nodes(randomIntBetween(1, 3), true).add(
+            DiscoveryNodeUtils.builder("old-data-node")
+                .name("old-data-node")
+                .roles(Set.of(DiscoveryNodeRole.DATA_ROLE))
+                .version(oldVersion)
+                .build()
+        ).build();
+    }
+
+    private static ClusterBlocks emptyClusterBlocks() {
+        return ClusterBlocks.EMPTY_CLUSTER_BLOCK;
+    }
+
+    private static ClusterBlocks stateNotRecoveredClusterBlocks() {
+        return ClusterBlocks.builder().addGlobalBlock(STATE_NOT_RECOVERED_BLOCK).build();
+    }
+
+    private static DiscoveryNodes.Builder nodes(int dataNodes, boolean localNodeMaster) {
+        final DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder();
+
+        int totalNodes = 0;
+        List<String> dataNodeIds = new ArrayList<>();
+        for (int i = 0; i < dataNodes; i++) {
+            String dataNodeId = "data-id-" + totalNodes++;
+            dataNodeIds.add(dataNodeId);
+            nodesBuilder.add(
+                DiscoveryNodeUtils.builder(dataNodeId).name("data-node-" + i).roles(Set.of(DiscoveryNodeRole.DATA_ROLE)).build()
+            );
+        }
+
+        String masterNodeId = "master-id-" + totalNodes++;
+        nodesBuilder.add(DiscoveryNodeUtils.builder(masterNodeId).name("master-node").roles(Set.of(DiscoveryNodeRole.MASTER_ROLE)).build());
+
+        final String localNodeId;
+        if (localNodeMaster) {
+            localNodeId = masterNodeId;
+        } else {
+            localNodeId = randomFrom(dataNodeIds);
+        }
+        return nodesBuilder.localNodeId(localNodeId).masterNodeId(masterNodeId);
+    }
+
+    private void mockDisabledNativeStore() {
+        when(nativeRolesStore.isEnabled()).thenReturn(false);
+    }
+
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    private void mockEnabledNativeStore(final Collection<RoleDescriptor> rolesToUpsert, final Collection<String> rolesToDelete) {
+        when(nativeRolesStore.isEnabled()).thenReturn(true);
+        doAnswer(i -> {
+            assertThat(synchronizer.isSynchronizationInProgress(), equalTo(true));
+            ((ActionListener) i.getArgument(3)).onResponse(
+                new BulkRolesResponse(
+                    rolesToUpsert.stream()
+                        .map(role -> BulkRolesResponse.Item.success(role.getName(), DocWriteResponse.Result.CREATED))
+                        .toList()
+                )
+            );
+            return null;
+        }).when(nativeRolesStore)
+            .putRoles(eq(WriteRequest.RefreshPolicy.IMMEDIATE), eq(rolesToUpsert), eq(false), any(ActionListener.class));
+
+        doAnswer(i -> {
+            ((ActionListener) i.getArgument(3)).onResponse(
+                new BulkRolesResponse(
+                    rolesToDelete.stream().map(role -> BulkRolesResponse.Item.success(role, DocWriteResponse.Result.DELETED)).toList()
+                )
+            );
+            return null;
+        }).when(nativeRolesStore)
+            .deleteRoles(eq(rolesToDelete), eq(WriteRequest.RefreshPolicy.IMMEDIATE), eq(false), any(ActionListener.class));
+    }
+
+    private static ClusterState.Builder createClusterStateWithOpenSecurityIndex() {
+        return SecurityIndexManagerTests.createClusterState(
+            TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7,
+            SecuritySystemIndices.SECURITY_MAIN_ALIAS,
+            IndexMetadata.State.OPEN
+        );
+    }
+
+    private static ClusterState.Builder createClusterStateWithClosedSecurityIndex() {
+        return SecurityIndexManagerTests.createClusterState(
+            TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7,
+            SecuritySystemIndices.SECURITY_MAIN_ALIAS,
+            IndexMetadata.State.CLOSE
+        );
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesUtilsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesUtilsTests.java
new file mode 100644
index 0000000000000..933512d3426c4
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesUtilsTests.java
@@ -0,0 +1,296 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.support;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper;
+import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissionGroup;
+import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
+import org.junit.BeforeClass;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.xpack.core.security.support.MetadataUtils.RESERVED_METADATA_KEY;
+import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesUtils.determineRolesToDelete;
+import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesUtils.determineRolesToUpsert;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+
+public class QueryableBuiltInRolesUtilsTests extends ESTestCase {
+
+    @BeforeClass
+    public static void setupReservedRolesStore() {
+        new ReservedRolesStore(); // initialize the store
+    }
+
+    public void testCalculateHash() {
+        assertThat(
+            QueryableBuiltInRolesUtils.calculateHash(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR),
+            equalTo("bWEFdFo4WX229wdhdecfiz5QHMYEssh3ex8hizRgg+Q=")
+        );
+    }
+
+    public void testEmptyOrNullRolesToUpsertOrDelete() {
+        // test empty roles and index digests
+        final QueryableBuiltInRoles emptyRoles = new QueryableBuiltInRoles(Map.of(), Set.of());
+        assertThat(determineRolesToDelete(emptyRoles, Map.of()), is(empty()));
+        assertThat(determineRolesToUpsert(emptyRoles, Map.of()), is(empty()));
+
+        // test empty roles and null indexed digests
+        assertThat(determineRolesToDelete(emptyRoles, null), is(empty()));
+        assertThat(determineRolesToUpsert(emptyRoles, null), is(empty()));
+    }
+
+    public void testNoRolesToUpsertOrDelete() {
+        {
+            QueryableBuiltInRoles currentBuiltInRoles = buildQueryableBuiltInRoles(
+                Set.of(
+                    ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR,
+                    ReservedRolesStore.roleDescriptor("viewer"),
+                    ReservedRolesStore.roleDescriptor("editor")
+                )
+            );
+
+            // no roles to delete or upsert since the built-in roles are the same as the indexed roles
+            assertThat(determineRolesToDelete(currentBuiltInRoles, currentBuiltInRoles.rolesDigest()), is(empty()));
+            assertThat(determineRolesToUpsert(currentBuiltInRoles, currentBuiltInRoles.rolesDigest()), is(empty()));
+        }
+        {
+            QueryableBuiltInRoles currentBuiltInRoles = buildQueryableBuiltInRoles(
+                Set.of(
+                    ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR,
+                    ReservedRolesStore.roleDescriptor("viewer"),
+                    ReservedRolesStore.roleDescriptor("editor"),
+                    supermanRole("monitor", "read")
+                )
+            );
+
+            Map<String, String> digests = buildDigests(
+                Set.of(
+                    ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR,
+                    ReservedRolesStore.roleDescriptor("viewer"),
+                    ReservedRolesStore.roleDescriptor("editor"),
+                    supermanRole("monitor", "read")
+                )
+            );
+
+            // no roles to delete or upsert since the built-in roles are the same as the indexed roles
+            assertThat(determineRolesToDelete(currentBuiltInRoles, digests), is(empty()));
+            assertThat(determineRolesToUpsert(currentBuiltInRoles, digests), is(empty()));
+        }
+        {
+            final RoleDescriptor randomRole = RoleDescriptorTestHelper.randomRoleDescriptor();
+            final QueryableBuiltInRoles currentBuiltInRoles = buildQueryableBuiltInRoles(Set.of(randomRole));
+            final Map<String, String> digests = buildDigests(
+                Set.of(
+                    new RoleDescriptor(
+                        randomRole.getName(),
+                        randomRole.getClusterPrivileges(),
+                        randomRole.getIndicesPrivileges(),
+                        randomRole.getApplicationPrivileges(),
+                        randomRole.getConditionalClusterPrivileges(),
+                        randomRole.getRunAs(),
+                        randomRole.getMetadata(),
+                        randomRole.getTransientMetadata(),
+                        randomRole.getRemoteIndicesPrivileges(),
+                        randomRole.getRemoteClusterPermissions(),
+                        randomRole.getRestriction(),
+                        randomRole.getDescription()
+                    )
+                )
+            );
+
+            assertThat(determineRolesToDelete(currentBuiltInRoles, digests), is(empty()));
+            assertThat(determineRolesToUpsert(currentBuiltInRoles, digests), is(empty()));
+        }
+    }
+
+    public void testRolesToDeleteOnly() {
+        Map<String, String> indexedDigests = buildDigests(
+            Set.of(
+                ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR,
+                ReservedRolesStore.roleDescriptor("viewer"),
+                ReservedRolesStore.roleDescriptor("editor"),
+                supermanRole("monitor", "read", "view_index_metadata", "read_cross_cluster")
+            )
+        );
+
+        QueryableBuiltInRoles currentBuiltInRoles = buildQueryableBuiltInRoles(
+            Set.of(
+                ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR,
+                ReservedRolesStore.roleDescriptor("viewer"),
+                ReservedRolesStore.roleDescriptor("editor")
+            )
+        );
+
+        // superman is the only role that needs to be deleted since it is not in a current built-in role
+        assertThat(determineRolesToDelete(currentBuiltInRoles, indexedDigests), containsInAnyOrder("superman"));
+        assertThat(determineRolesToUpsert(currentBuiltInRoles, indexedDigests), is(empty()));
+
+        // passing empty built-in roles should result in all indexed roles needing to be deleted
+        QueryableBuiltInRoles emptyBuiltInRoles = new QueryableBuiltInRoles(Map.of(), Set.of());
+        assertThat(
+            determineRolesToDelete(emptyBuiltInRoles, indexedDigests),
+            containsInAnyOrder("superman", "viewer", "editor", "superuser")
+        );
+        assertThat(determineRolesToUpsert(emptyBuiltInRoles, indexedDigests), is(empty()));
+    }
+
+    public void testRolesToUpdateOnly() {
+        Map<String, String> indexedDigests = buildDigests(
+            Set.of(
+                ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR,
+                ReservedRolesStore.roleDescriptor("viewer"),
+                ReservedRolesStore.roleDescriptor("editor"),
+                supermanRole("monitor", "read", "write")
+            )
+        );
+
+        RoleDescriptor updatedSupermanRole = supermanRole("monitor", "read", "view_index_metadata", "read_cross_cluster");
+        QueryableBuiltInRoles currentBuiltInRoles = buildQueryableBuiltInRoles(
+            Set.of(
+                ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR,
+                ReservedRolesStore.roleDescriptor("viewer"),
+                ReservedRolesStore.roleDescriptor("editor"),
+                updatedSupermanRole
+            )
+        );
+
+        // superman is the only role that needs to be updated since its definition has changed
+        assertThat(determineRolesToDelete(currentBuiltInRoles, indexedDigests), is(empty()));
+        assertThat(determineRolesToUpsert(currentBuiltInRoles, indexedDigests), containsInAnyOrder(updatedSupermanRole));
+        assertThat(currentBuiltInRoles.rolesDigest().get("superman"), is(not(equalTo(indexedDigests.get("superman")))));
+    }
+
+    public void testRolesToCreateOnly() {
+        Map<String, String> indexedDigests = buildDigests(
+            Set.of(
+                ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR,
+                ReservedRolesStore.roleDescriptor("viewer"),
+                ReservedRolesStore.roleDescriptor("editor")
+            )
+        );
+
+        RoleDescriptor newSupermanRole = supermanRole("monitor", "read", "view_index_metadata", "read_cross_cluster");
+        QueryableBuiltInRoles currentBuiltInRoles = buildQueryableBuiltInRoles(
+            Set.of(
+                ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR,
+                ReservedRolesStore.roleDescriptor("viewer"),
+                ReservedRolesStore.roleDescriptor("editor"),
+                newSupermanRole
+            )
+        );
+
+        // superman is the only role that needs to be created since it is not in the indexed roles
+        assertThat(determineRolesToDelete(currentBuiltInRoles, indexedDigests), is(empty()));
+        assertThat(determineRolesToUpsert(currentBuiltInRoles, indexedDigests), containsInAnyOrder(newSupermanRole));
+
+        // passing empty indexed roles should result in all roles needing to be created
+        assertThat(determineRolesToDelete(currentBuiltInRoles, Map.of()), is(empty()));
+        assertThat(
+            determineRolesToUpsert(currentBuiltInRoles, Map.of()),
+            containsInAnyOrder(currentBuiltInRoles.roleDescriptors().toArray(new RoleDescriptor[0]))
+        );
+    }
+
+    public void testRolesToUpsertAndDelete() {
+        Map<String, String> indexedDigests = buildDigests(
+            Set.of(
+                ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR,
+                ReservedRolesStore.roleDescriptor("viewer"),
+                ReservedRolesStore.roleDescriptor("editor")
+            )
+        );
+
+        RoleDescriptor newSupermanRole = supermanRole("monitor");
+        QueryableBuiltInRoles currentBuiltInRoles = buildQueryableBuiltInRoles(
+            Set.of(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR, newSupermanRole)
+        );
+
+        // superman is the only role that needs to be updated since its definition has changed
+        assertThat(determineRolesToDelete(currentBuiltInRoles, indexedDigests), containsInAnyOrder("viewer", "editor"));
+        assertThat(determineRolesToUpsert(currentBuiltInRoles, indexedDigests), containsInAnyOrder(newSupermanRole));
+    }
+
+    private static RoleDescriptor supermanRole(String... indicesPrivileges) {
+        return new RoleDescriptor(
+            "superman",
+            new String[] { "all" },
+            new RoleDescriptor.IndicesPrivileges[] {
+                RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("all").allowRestrictedIndices(false).build(),
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices("*")
+                    .privileges(indicesPrivileges)
+                    .allowRestrictedIndices(true)
+                    .build() },
+            new RoleDescriptor.ApplicationResourcePrivileges[] {
+                RoleDescriptor.ApplicationResourcePrivileges.builder().application("*").privileges("*").resources("*").build() },
+            null,
+            new String[] { "*" },
+            randomlyOrderedSupermanMetadata(),
+            Collections.emptyMap(),
+            new RoleDescriptor.RemoteIndicesPrivileges[] {
+                new RoleDescriptor.RemoteIndicesPrivileges(
+                    RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("all").allowRestrictedIndices(false).build(),
+                    "*"
+                ),
+                new RoleDescriptor.RemoteIndicesPrivileges(
+                    RoleDescriptor.IndicesPrivileges.builder()
+                        .indices("*")
+                        .privileges(indicesPrivileges)
+                        .allowRestrictedIndices(true)
+                        .build(),
+                    "*"
+                ) },
+            new RemoteClusterPermissions().addGroup(
+                new RemoteClusterPermissionGroup(
+                    RemoteClusterPermissions.getSupportedRemoteClusterPermissions().toArray(new String[0]),
+                    new String[] { "*" }
+                )
+            ),
+            null,
+            "Grants full access to cluster management and data indices."
+        );
+    }
+
+    private static Map<String, Object> randomlyOrderedSupermanMetadata() {
+        final LinkedHashMap<String, Object> metadata = new LinkedHashMap<>();
+        if (randomBoolean()) {
+            metadata.put("foo", "bar");
+            metadata.put("baz", "qux");
+            metadata.put(RESERVED_METADATA_KEY, true);
+        } else {
+            metadata.put(RESERVED_METADATA_KEY, true);
+            metadata.put("foo", "bar");
+            metadata.put("baz", "qux");
+        }
+        return metadata;
+    }
+
+    public static QueryableBuiltInRoles buildQueryableBuiltInRoles(Set<RoleDescriptor> roles) {
+        final Map<String, String> digests = buildDigests(roles);
+        return new QueryableBuiltInRoles(digests, roles);
+    }
+
+    public static Map<String, String> buildDigests(Set<RoleDescriptor> roles) {
+        final Map<String, String> digests = new HashMap<>();
+        for (RoleDescriptor role : roles) {
+            digests.put(role.getName(), QueryableBuiltInRolesUtils.calculateHash(role));
+        }
+        return digests;
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/QueryableReservedRolesProviderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/QueryableReservedRolesProviderTests.java
new file mode 100644
index 0000000000000..7beb078795b29
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/QueryableReservedRolesProviderTests.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.support;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
+
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class QueryableReservedRolesProviderTests extends ESTestCase {
+
+    public void testReservedRoleProvider() {
+        QueryableReservedRolesProvider provider = new QueryableReservedRolesProvider(new ReservedRolesStore());
+        assertNotNull(provider.getRoles());
+        assertThat(provider.getRoles(), equalTo(provider.getRoles()));
+        assertThat(provider.getRoles().rolesDigest().size(), equalTo(ReservedRolesStore.roleDescriptors().size()));
+        assertThat(
+            provider.getRoles().rolesDigest().keySet(),
+            equalTo(ReservedRolesStore.roleDescriptors().stream().map(RoleDescriptor::getName).collect(Collectors.toSet()))
+        );
+    }
+
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageCertificateVerificationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageCertificateVerificationTests.java
index ad8e15db6f032..ccb73c9ead210 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageCertificateVerificationTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageCertificateVerificationTests.java
@@ -39,7 +39,6 @@
 import java.util.regex.Pattern;
 
 import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.SSLHandshakeException;
 import javax.net.ssl.SSLParameters;
 import javax.net.ssl.SSLSocket;
 import javax.net.ssl.SSLSocketFactory;
@@ -104,7 +103,6 @@ public void testMessageForRestClientHostnameVerificationFailure() throws IOExcep
     }
 
     public void testDiagnosticTrustManagerForHostnameVerificationFailure() throws Exception {
-        assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm());
         final Settings settings = getPemSSLSettings(
             HTTP_SERVER_SSL,
             "not-this-host.crt",
@@ -133,7 +131,7 @@ public void testDiagnosticTrustManagerForHostnameVerificationFailure() throws Ex
                     DiagnosticTrustManager.class.getName(),
                     Level.WARN,
                     "failed to establish trust with server at \\["
-                        + Pattern.quote(webServer.getHostName())
+                        + (inFipsJvm() ? "<unknown host>" : Pattern.quote(webServer.getHostName()))
                         + "\\];"
                         + " the server provided a certificate with subject name \\[CN=not-this-host\\],"
                         + " fingerprint \\[[0-9a-f]{40}\\], no keyUsage and no extendedKeyUsage;"
@@ -154,13 +152,12 @@ public void testDiagnosticTrustManagerForHostnameVerificationFailure() throws Ex
             enableHttpsHostnameChecking(clientSocket);
             connect(clientSocket, webServer);
             assertThat(clientSocket.isConnected(), is(true));
-            final SSLHandshakeException handshakeException = expectThrows(
-                SSLHandshakeException.class,
-                () -> clientSocket.getInputStream().read()
-            );
-            assertThat(handshakeException, throwableWithMessage(containsStringIgnoringCase("subject alternative names")));
-            assertThat(handshakeException, throwableWithMessage(containsString(webServer.getHostName())));
-
+            final Exception handshakeException = expectThrows(Exception.class, () -> clientSocket.getInputStream().read());
+            // Bouncy Castle throws a different exception message
+            if (inFipsJvm() == false) {
+                assertThat(handshakeException, throwableWithMessage(containsStringIgnoringCase("subject alternative names")));
+                assertThat(handshakeException, throwableWithMessage(containsString(webServer.getHostName())));
+            }
             // Logging message failures are tricky to debug because you just get a "didn't find match" assertion failure.
             // You should be able to check the log output for the text that was logged and compare to the regex above.
             mockLog.assertAllExpectationsMatched();
diff --git a/x-pack/plugin/shutdown/build.gradle b/x-pack/plugin/shutdown/build.gradle
index 8eabb51b3e131..df1a7575cdd05 100644
--- a/x-pack/plugin/shutdown/build.gradle
+++ b/x-pack/plugin/shutdown/build.gradle
@@ -4,9 +4,9 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-    name 'x-pack-shutdown'
-    description 'Elasticsearch Expanded Pack Plugin - Shutdown'
-    classname 'org.elasticsearch.xpack.shutdown.ShutdownPlugin'
+    name = 'x-pack-shutdown'
+    description = 'Elasticsearch Expanded Pack Plugin - Shutdown'
+    classname ='org.elasticsearch.xpack.shutdown.ShutdownPlugin'
     extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java
index 95fd97cd5931f..6a9116daf2ca7 100644
--- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java
+++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java
@@ -12,10 +12,11 @@
 import org.elasticsearch.cluster.metadata.ShutdownShardMigrationStatus;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
@@ -24,6 +25,10 @@
 import java.util.Iterator;
 import java.util.Objects;
 
+import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.chunk;
+import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.endObject;
+import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.startObject;
+
 public class SingleNodeShutdownStatus implements Writeable, ChunkedToXContentObject {
 
     private final SingleNodeShutdownMetadata metadata;
@@ -111,40 +116,37 @@ public String toString() {
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return ChunkedToXContent.builder(params).object(b -> {
-            b.append((builder, p) -> {
-                builder.field(SingleNodeShutdownMetadata.NODE_ID_FIELD.getPreferredName(), metadata.getNodeId());
-                builder.field(SingleNodeShutdownMetadata.TYPE_FIELD.getPreferredName(), metadata.getType());
-                builder.field(SingleNodeShutdownMetadata.REASON_FIELD.getPreferredName(), metadata.getReason());
-                if (metadata.getAllocationDelay() != null) {
-                    builder.field(
-                        SingleNodeShutdownMetadata.ALLOCATION_DELAY_FIELD.getPreferredName(),
-                        metadata.getAllocationDelay().getStringRep()
-                    );
-                }
-                builder.timestampFieldsFromUnixEpochMillis(
-                    SingleNodeShutdownMetadata.STARTED_AT_MILLIS_FIELD.getPreferredName(),
-                    SingleNodeShutdownMetadata.STARTED_AT_READABLE_FIELD,
-                    metadata.getStartedAtMillis()
+        return Iterators.concat(startObject(), chunk((builder, p) -> {
+            builder.field(SingleNodeShutdownMetadata.NODE_ID_FIELD.getPreferredName(), metadata.getNodeId());
+            builder.field(SingleNodeShutdownMetadata.NODE_EPHEMERAL_ID_FIELD.getPreferredName(), metadata.getNodeEphemeralId());
+            builder.field(SingleNodeShutdownMetadata.TYPE_FIELD.getPreferredName(), metadata.getType());
+            builder.field(SingleNodeShutdownMetadata.REASON_FIELD.getPreferredName(), metadata.getReason());
+            if (metadata.getAllocationDelay() != null) {
+                builder.field(
+                    SingleNodeShutdownMetadata.ALLOCATION_DELAY_FIELD.getPreferredName(),
+                    metadata.getAllocationDelay().getStringRep()
+                );
+            }
+            builder.timestampFieldsFromUnixEpochMillis(
+                SingleNodeShutdownMetadata.STARTED_AT_MILLIS_FIELD.getPreferredName(),
+                SingleNodeShutdownMetadata.STARTED_AT_READABLE_FIELD,
+                metadata.getStartedAtMillis()
+            );
+            builder.field(STATUS.getPreferredName(), overallStatus());
+            return builder;
+        }), ChunkedToXContentHelper.field(SHARD_MIGRATION_FIELD.getPreferredName(), shardMigrationStatus, params), chunk((builder, p) -> {
+            builder.field(PERSISTENT_TASKS_FIELD.getPreferredName(), persistentTasksStatus);
+            builder.field(PLUGINS_STATUS.getPreferredName(), pluginsStatus);
+            if (metadata.getTargetNodeName() != null) {
+                builder.field(TARGET_NODE_NAME_FIELD.getPreferredName(), metadata.getTargetNodeName());
+            }
+            if (metadata.getGracePeriod() != null) {
+                builder.timestampField(
+                    SingleNodeShutdownMetadata.GRACE_PERIOD_FIELD.getPreferredName(),
+                    metadata.getGracePeriod().getStringRep()
                 );
-                builder.field(STATUS.getPreferredName(), overallStatus());
-                return builder;
-            });
-            b.field(SHARD_MIGRATION_FIELD.getPreferredName(), shardMigrationStatus);
-            b.append((builder, p) -> {
-                builder.field(PERSISTENT_TASKS_FIELD.getPreferredName(), persistentTasksStatus);
-                builder.field(PLUGINS_STATUS.getPreferredName(), pluginsStatus);
-                if (metadata.getTargetNodeName() != null) {
-                    builder.field(TARGET_NODE_NAME_FIELD.getPreferredName(), metadata.getTargetNodeName());
-                }
-                if (metadata.getGracePeriod() != null) {
-                    builder.timestampField(
-                        SingleNodeShutdownMetadata.GRACE_PERIOD_FIELD.getPreferredName(),
-                        metadata.getGracePeriod().getStringRep()
-                    );
-                }
-                return builder;
-            });
-        });
+            }
+            return builder;
+        }), endObject());
     }
 }
diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java
index 4c163bc1a9361..f0197b3c998d4 100644
--- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java
+++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterService;
@@ -49,7 +50,8 @@ public class TransportPutShutdownNodeAction extends AcknowledgedTransportMasterN
     private static boolean putShutdownNodeState(
         Map<String, SingleNodeShutdownMetadata> shutdownMetadata,
         Predicate<String> nodeExists,
-        Request request
+        Request request,
+        String nodeEphemeralId
     ) {
         if (isNoop(shutdownMetadata, request)) {
             return false;
@@ -58,6 +60,7 @@ private static boolean putShutdownNodeState(
         final boolean nodeSeen = nodeExists.test(request.getNodeId());
         SingleNodeShutdownMetadata newNodeMetadata = SingleNodeShutdownMetadata.builder()
             .setNodeId(request.getNodeId())
+            .setNodeEphemeralId(nodeEphemeralId)
             .setType(request.getType())
             .setReason(request.getReason())
             .setStartedAtMillis(System.currentTimeMillis())
@@ -103,8 +106,13 @@ public ClusterState execute(BatchExecutionContext<PutShutdownNodeTask> batchExec
             boolean needsReroute = false;
             for (final var taskContext : batchExecutionContext.taskContexts()) {
                 var request = taskContext.getTask().request();
+                String nodeEphemeralId = null;
+                DiscoveryNode discoveryNode = initialState.nodes().getNodes().get(request.getNodeId());
+                if (discoveryNode != null) {
+                    nodeEphemeralId = discoveryNode.getEphemeralId();
+                }
                 try (var ignored = taskContext.captureResponseHeaders()) {
-                    changed |= putShutdownNodeState(shutdownMetadata, nodeExistsPredicate, request);
+                    changed |= putShutdownNodeState(shutdownMetadata, nodeExistsPredicate, request, nodeEphemeralId);
                 } catch (Exception e) {
                     taskContext.onFailure(e);
                     continue;
diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/GetShutdownStatusResponseTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/GetShutdownStatusResponseTests.java
index 2c68e8d7f39be..e3a8fb4482c3d 100644
--- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/GetShutdownStatusResponseTests.java
+++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/GetShutdownStatusResponseTests.java
@@ -55,6 +55,7 @@ public static SingleNodeShutdownMetadata randomNodeShutdownMetadata() {
         final TimeValue gracefulShutdown = type == SIGTERM ? randomPositiveTimeValue() : null;
         return SingleNodeShutdownMetadata.builder()
             .setNodeId(randomAlphaOfLength(5))
+            .setNodeEphemeralId(randomAlphaOfLength(5))
             .setType(type)
             .setReason(randomAlphaOfLength(5))
             .setStartedAtMillis(randomNonNegativeLong())
diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java
index 9a1dda99674c9..c8f1d8b58a490 100644
--- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java
+++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java
@@ -603,6 +603,7 @@ public void testNodeNotInCluster() {
                                     .setStartedAtMillis(randomNonNegativeLong())
                                     .setReason(this.getTestName())
                                     .setNodeId(bogusNodeId)
+                                    .setNodeEphemeralId(bogusNodeId)
                                     .build()
                             )
                         )
@@ -866,6 +867,7 @@ private ClusterState createTestClusterState(
                                     .setStartedAtMillis(randomNonNegativeLong())
                                     .setReason(this.getTestName())
                                     .setNodeId(SHUTTING_DOWN_NODE_ID)
+                                    .setNodeEphemeralId(SHUTTING_DOWN_NODE_ID)
                                     .build()
                             )
                         )
diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java
index de5c5d393f39d..ed603d30d0593 100644
--- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java
+++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java
@@ -15,6 +15,8 @@
 import org.elasticsearch.cluster.ClusterStateTaskExecutor.TaskContext;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata.Type;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
@@ -32,6 +34,8 @@
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 import static org.elasticsearch.core.Strings.format;
 import static org.hamcrest.Matchers.containsString;
@@ -96,14 +100,16 @@ public void testNoop() throws Exception {
             targetNodeName,
             null
         );
-        action.masterOperation(null, request, ClusterState.EMPTY_STATE, ActionListener.noop());
+        var dummyNode = new DiscoveryNode(targetNodeName, "node1", "eph-node1", "abc", "abc", null, Map.of(), Set.of(), null);
+        var state = ClusterState.builder(ClusterState.EMPTY_STATE).nodes(DiscoveryNodes.builder().add(dummyNode).build()).build();
+        action.masterOperation(null, request, state, ActionListener.noop());
         var updateTask = ArgumentCaptor.forClass(PutShutdownNodeTask.class);
         var taskExecutor = ArgumentCaptor.forClass(PutShutdownNodeExecutor.class);
         verify(clusterService).createTaskQueue(any(), any(), taskExecutor.capture());
         verify(taskQueue).submitTask(any(), updateTask.capture(), any());
         when(taskContext.getTask()).thenReturn(updateTask.getValue());
         ClusterState stableState = taskExecutor.getValue()
-            .execute(new ClusterStateTaskExecutor.BatchExecutionContext<>(ClusterState.EMPTY_STATE, List.of(taskContext), () -> null));
+            .execute(new ClusterStateTaskExecutor.BatchExecutionContext<>(state, List.of(taskContext), () -> null));
 
         // run the request again, there should be no call to submit an update task
         clearTaskQueueInvocations();
diff --git a/x-pack/plugin/slm/build.gradle b/x-pack/plugin/slm/build.gradle
index b54e31315f709..a827891e4b167 100644
--- a/x-pack/plugin/slm/build.gradle
+++ b/x-pack/plugin/slm/build.gradle
@@ -9,12 +9,12 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  name 'x-pack-slm'
-  description 'Elasticsearch Expanded Pack Plugin - Snapshot Lifecycle Management'
-  classname 'org.elasticsearch.xpack.slm.SnapshotLifecycle'
+  name = 'x-pack-slm'
+  description = 'Elasticsearch Expanded Pack Plugin - Snapshot Lifecycle Management'
+  classname ='org.elasticsearch.xpack.slm.SnapshotLifecycle'
   extendedPlugins = ['x-pack-core']
-  hasNativeController false
-  requiresKeystore true
+  hasNativeController =false
+  requiresKeystore =true
 }
 base {
   archivesName = 'x-pack-slm'
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycle.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycle.java
index cc01d5b101106..3c7e9310744cb 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycle.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycle.java
@@ -237,7 +237,7 @@ public List<RestHandler> getRestHandlers(
     }
 
     List<ReservedClusterStateHandler<?>> reservedClusterStateHandlers() {
-        return List.of(new ReservedSnapshotAction(featureService.get()));
+        return List.of(new ReservedSnapshotAction());
     }
 
     @Override
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleFeatures.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleFeatures.java
index 274dec75865a8..ca4a632f7a76b 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleFeatures.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleFeatures.java
@@ -15,6 +15,6 @@
 public class SnapshotLifecycleFeatures implements FeatureSpecification {
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(SnapshotLifecycleService.INTERVAL_SCHEDULE);
+        return Set.of();
     }
 }
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleService.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleService.java
index b93f90de73f05..2c14e0804d1f5 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleService.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleService.java
@@ -20,8 +20,6 @@
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.features.FeatureService;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 import org.elasticsearch.xpack.core.ilm.OperationMode;
 import org.elasticsearch.xpack.core.ilm.OperationModeUpdateTask;
@@ -46,7 +44,6 @@
  * task according to the policy's schedule.
  */
 public class SnapshotLifecycleService implements Closeable, ClusterStateListener {
-    public static final NodeFeature INTERVAL_SCHEDULE = new NodeFeature("slm.interval_schedule");
     private static final Logger logger = LogManager.getLogger(SnapshotLifecycleService.class);
     private static final String JOB_PATTERN_SUFFIX = "-\\d+$";
 
@@ -261,18 +258,6 @@ public static void validateMinimumInterval(final SnapshotLifecyclePolicy lifecyc
         }
     }
 
-    /**
-     * Validate that interval schedule feature is not supported by all nodes
-     * @throws IllegalArgumentException if is interval expression but interval schedule not supported
-     */
-    public static void validateIntervalScheduleSupport(String schedule, FeatureService featureService, ClusterState state) {
-        if (SnapshotLifecyclePolicy.isIntervalSchedule(schedule) && featureService.clusterHasFeature(state, INTERVAL_SCHEDULE) == false) {
-            throw new IllegalArgumentException(
-                "Unable to use slm interval schedules in mixed-clusters with nodes that do not support feature " + INTERVAL_SCHEDULE.id()
-            );
-        }
-    }
-
     @Override
     public void close() {
         if (this.running.compareAndSet(true, false)) {
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotAction.java
index a98e110ed88de..b915f26310c14 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotAction.java
@@ -9,7 +9,6 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.reservedstate.TransformState;
 import org.elasticsearch.xcontent.XContentParser;
@@ -41,12 +40,6 @@ public class ReservedSnapshotAction implements ReservedClusterStateHandler<List<
 
     public static final String NAME = "slm";
 
-    private final FeatureService featureService;
-
-    public ReservedSnapshotAction(FeatureService featureService) {
-        this.featureService = featureService;
-    }
-
     @Override
     public String name() {
         return NAME;
@@ -66,7 +59,6 @@ private Collection<PutSnapshotLifecycleAction.Request> prepare(List<SnapshotLife
             );
             try {
                 validate(request);
-                SnapshotLifecycleService.validateIntervalScheduleSupport(request.getLifecycle().getSchedule(), featureService, state);
                 SnapshotLifecycleService.validateRepositoryExists(request.getLifecycle().getRepository(), state);
                 SnapshotLifecycleService.validateMinimumInterval(request.getLifecycle(), state);
                 result.add(request);
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java
index f14afefe6f47a..3410d92ab4277 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.tasks.Task;
@@ -50,7 +49,6 @@ public class TransportPutSnapshotLifecycleAction extends TransportMasterNodeActi
     AcknowledgedResponse> {
 
     private static final Logger logger = LogManager.getLogger(TransportPutSnapshotLifecycleAction.class);
-    private final FeatureService featureService;
 
     @Inject
     public TransportPutSnapshotLifecycleAction(
@@ -58,8 +56,7 @@ public TransportPutSnapshotLifecycleAction(
         ClusterService clusterService,
         ThreadPool threadPool,
         ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver,
-        FeatureService featureService
+        IndexNameExpressionResolver indexNameExpressionResolver
     ) {
         super(
             PutSnapshotLifecycleAction.NAME,
@@ -72,7 +69,6 @@ public TransportPutSnapshotLifecycleAction(
             AcknowledgedResponse::readFrom,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
-        this.featureService = featureService;
     }
 
     @Override
@@ -82,7 +78,6 @@ protected void masterOperation(
         final ClusterState state,
         final ActionListener<AcknowledgedResponse> listener
     ) {
-        SnapshotLifecycleService.validateIntervalScheduleSupport(request.getLifecycle().getSchedule(), featureService, state);
         SnapshotLifecycleService.validateRepositoryExists(request.getLifecycle().getRepository(), state);
         SnapshotLifecycleService.validateMinimumInterval(request.getLifecycle(), state);
 
diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java
index 36887681f5575..b48a3be0728e8 100644
--- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java
+++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -50,7 +49,6 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -525,41 +523,6 @@ public void submitUnbatchedStateUpdateTask(String source, ClusterStateUpdateTask
         }
     }
 
-    public void testValidateIntervalScheduleSupport() {
-        var featureService = new FeatureService(List.of(new SnapshotLifecycleFeatures()));
-        {
-            ClusterState state = ClusterState.builder(new ClusterName("cluster"))
-                .nodeFeatures(Map.of("a", Set.of(), "b", Set.of(SnapshotLifecycleService.INTERVAL_SCHEDULE.id())))
-                .build();
-
-            IllegalArgumentException e = expectThrows(
-                IllegalArgumentException.class,
-                () -> SnapshotLifecycleService.validateIntervalScheduleSupport("30d", featureService, state)
-            );
-            assertThat(e.getMessage(), containsString("Unable to use slm interval schedules"));
-        }
-        {
-            ClusterState state = ClusterState.builder(new ClusterName("cluster"))
-                .nodeFeatures(Map.of("a", Set.of(SnapshotLifecycleService.INTERVAL_SCHEDULE.id())))
-                .build();
-            try {
-                SnapshotLifecycleService.validateIntervalScheduleSupport("30d", featureService, state);
-            } catch (Exception e) {
-                fail("interval schedule is supported by version and should not fail");
-            }
-        }
-        {
-            ClusterState state = ClusterState.builder(new ClusterName("cluster"))
-                .nodeFeatures(Map.of("a", Set.of(), "b", Set.of(SnapshotLifecycleService.INTERVAL_SCHEDULE.id())))
-                .build();
-            try {
-                SnapshotLifecycleService.validateIntervalScheduleSupport("*/1 * * * * ?", featureService, state);
-            } catch (Exception e) {
-                fail("cron schedule does not need feature check and should not fail");
-            }
-        }
-    }
-
     class FakeSnapshotTask extends SnapshotLifecycleTask {
         private final Consumer<SchedulerEngine.Event> onTriggered;
 
diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java
index b993633e3d17d..e61e73d656ca7 100644
--- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java
+++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Releasable;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.reservedstate.TransformState;
 import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction;
@@ -81,7 +80,7 @@ private TransformState processJSON(ReservedSnapshotAction action, TransformState
     }
 
     public void testDependencies() {
-        var action = new ReservedSnapshotAction(mock(FeatureService.class));
+        var action = new ReservedSnapshotAction();
         assertThat(action.optionalDependencies(), contains(ReservedRepositoryAction.NAME));
     }
 
@@ -91,7 +90,7 @@ public void testValidationFailsNeitherScheduleOrInterval() {
         final ClusterName clusterName = new ClusterName("elasticsearch");
 
         ClusterState state = ClusterState.builder(clusterName).build();
-        ReservedSnapshotAction action = new ReservedSnapshotAction(mock(FeatureService.class));
+        ReservedSnapshotAction action = new ReservedSnapshotAction();
         TransformState prevState = new TransformState(state, Set.of());
 
         String badPolicyJSON = """
@@ -119,56 +118,6 @@ public void testValidationFailsNeitherScheduleOrInterval() {
         );
     }
 
-    public void testIntervalScheduleSupportValidation() {
-        Client client = mock(Client.class);
-        when(client.settings()).thenReturn(Settings.EMPTY);
-        final ClusterName clusterName = new ClusterName("elasticsearch");
-        List<RepositoryMetadata> repositoriesMetadata = List.of(new RepositoryMetadata("repo", "fs", Settings.EMPTY));
-        Metadata.Builder mdBuilder = Metadata.builder();
-        mdBuilder.putCustom(RepositoriesMetadata.TYPE, new RepositoriesMetadata(repositoriesMetadata));
-        ClusterState state = ClusterState.builder(clusterName).metadata(mdBuilder).build();
-        TransformState prevState = new TransformState(state, Set.of());
-        String goodPolicyJSON = """
-            {
-               "daily-snapshots": {
-                 "schedule": "30d",
-                 "name": "<production-snap-{now/d}>",
-                 "repository": "repo",
-                 "config": {
-                   "indices": ["foo-*", "important"],
-                   "ignore_unavailable": true,
-                   "include_global_state": false
-                 },
-                 "retention": {
-                   "expire_after": "30d",
-                   "min_count": 1,
-                   "max_count": 50
-                 }
-               }
-            }
-            """;
-
-        {
-            FeatureService featureService = mock(FeatureService.class);
-            when(featureService.clusterHasFeature(any(), any())).thenReturn(false);
-            ReservedSnapshotAction action = new ReservedSnapshotAction(featureService);
-            assertThat(
-                expectThrows(IllegalArgumentException.class, () -> processJSON(action, prevState, goodPolicyJSON)).getMessage(),
-                is("Error on validating SLM requests")
-            );
-        }
-        {
-            FeatureService featureService = mock(FeatureService.class);
-            when(featureService.clusterHasFeature(any(), any())).thenReturn(true);
-            ReservedSnapshotAction action = new ReservedSnapshotAction(featureService);
-            try {
-                processJSON(action, prevState, goodPolicyJSON);
-            } catch (Exception e) {
-                fail("interval schedule with interval feature should pass validation");
-            }
-        }
-    }
-
     public void testActionAddRemove() throws Exception {
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
@@ -180,7 +129,7 @@ public void testActionAddRemove() throws Exception {
         mdBuilder.putCustom(RepositoriesMetadata.TYPE, new RepositoriesMetadata(repositoriesMetadata));
         ClusterState state = ClusterState.builder(clusterName).metadata(mdBuilder).build();
 
-        ReservedSnapshotAction action = new ReservedSnapshotAction(mock(FeatureService.class));
+        ReservedSnapshotAction action = new ReservedSnapshotAction();
 
         String emptyJSON = "";
 
@@ -414,7 +363,7 @@ public void testOperatorControllerFromJSONContent() throws IOException {
             null,
             List.of(
                 new ReservedClusterSettingsAction(clusterSettings),
-                new ReservedSnapshotAction(mock(FeatureService.class)),
+                new ReservedSnapshotAction(),
                 new ReservedRepositoryAction(repositoriesService)
             )
         );
@@ -448,8 +397,7 @@ public void testPutSLMReservedStateHandler() throws Exception {
             mock(ClusterService.class),
             threadPool,
             mock(ActionFilters.class),
-            mock(IndexNameExpressionResolver.class),
-            mock(FeatureService.class)
+            mock(IndexNameExpressionResolver.class)
         );
         assertThat(putAction.reservedStateHandlerName().get(), equalTo(ReservedSnapshotAction.NAME));
 
diff --git a/x-pack/plugin/snapshot-based-recoveries/build.gradle b/x-pack/plugin/snapshot-based-recoveries/build.gradle
index f1bed57838c46..d11f78a13f95d 100644
--- a/x-pack/plugin/snapshot-based-recoveries/build.gradle
+++ b/x-pack/plugin/snapshot-based-recoveries/build.gradle
@@ -3,9 +3,9 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-test-artifact'
 
 esplugin {
-  name 'snapshot-based-recoveries'
-  description 'A plugin that enables snapshot based recoveries'
-  classname 'org.elasticsearch.xpack.snapshotbasedrecoveries.SnapshotBasedRecoveriesPlugin'
+  name = 'snapshot-based-recoveries'
+  description = 'A plugin that enables snapshot based recoveries'
+  classname ='org.elasticsearch.xpack.snapshotbasedrecoveries.SnapshotBasedRecoveriesPlugin'
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/snapshot-repo-test-kit/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/build.gradle
index 45839a049f457..58afbeb237e46 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/build.gradle
+++ b/x-pack/plugin/snapshot-repo-test-kit/build.gradle
@@ -3,9 +3,9 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-test-artifact'
 
 esplugin {
-  name 'snapshot-repo-test-kit'
-  description 'A plugin for a test kit for snapshot repositories'
-  classname 'org.elasticsearch.repositories.blobstore.testkit.SnapshotRepositoryTestKit'
+  name = 'snapshot-repo-test-kit'
+  description = 'A plugin for a test kit for snapshot repositories'
+  classname = 'org.elasticsearch.repositories.blobstore.testkit.SnapshotRepositoryTestKit'
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/GCSRepositoryAnalysisRestIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/GCSRepositoryAnalysisRestIT.java
index 7f7540d138825..01ca881f9a119 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/GCSRepositoryAnalysisRestIT.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/GCSRepositoryAnalysisRestIT.java
@@ -46,12 +46,6 @@ public class GCSRepositoryAnalysisRestIT extends AbstractRepositoryAnalysisRestT
                 );
             }
         })
-        .apply(c -> {
-            if (USE_FIXTURE) {
-                // test fixture does not support CAS yet; TODO fix this
-                c.systemProperty("test.repository_test_kit.skip_cas", "true");
-            }
-        })
         .build();
 
     @ClassRule
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/build.gradle
index 81eb82a522389..d4615260952d1 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/build.gradle
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/build.gradle
@@ -10,7 +10,8 @@ apply plugin: 'elasticsearch.rest-resources'
 
 dependencies {
   javaRestTestImplementation testArtifact(project(xpackModule('snapshot-repo-test-kit')))
-  javaRestTestImplementation project(path: ':test:fixtures:hdfs-fixture', configuration:"shadow")
+  javaRestTestCompileOnly project(path: ':test:fixtures:hdfs-fixture')
+  javaRestTestRuntimeOnly project(path: ':test:fixtures:hdfs-fixture', configuration:"shadow")
   javaRestTestImplementation project(':test:fixtures:krb5kdc-fixture')
   javaRestTestImplementation "org.slf4j:slf4j-api:${versions.slf4j}"
   javaRestTestImplementation "org.slf4j:slf4j-simple:${versions.slf4j}"
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/20_verify_integrity.yml b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/20_verify_integrity.yml
index be6929a15ff44..18707ea2d1d83 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/20_verify_integrity.yml
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/20_verify_integrity.yml
@@ -1,9 +1,5 @@
 ---
 setup:
-  - requires:
-      cluster_features: "snapshot.repository_verify_integrity"
-      reason: "required feature"
-
   - do:
       snapshot.create_repository:
         repository: test_repo
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/module-info.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/module-info.java
index 70385cdc4cf04..e2c3df252ae2e 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/module-info.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/module-info.java
@@ -17,8 +17,4 @@
 
     exports org.elasticsearch.repositories.blobstore.testkit.analyze;
     exports org.elasticsearch.repositories.blobstore.testkit.integrity;
-
-    provides org.elasticsearch.features.FeatureSpecification
-        with
-            org.elasticsearch.repositories.blobstore.testkit.SnapshotRepositoryTestKitFeatures;
 }
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityResponseStream.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityResponseStream.java
index 7ea9bfe6f2b23..0eb7182ef99e7 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityResponseStream.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityResponseStream.java
@@ -61,7 +61,7 @@ void startResponse(Releasable releasable) throws IOException {
         assert streamingXContentResponse == null;
         streamingXContentResponse = new StreamingXContentResponse(restChannel, restChannel.request(), () -> {});
         streamingXContentResponse.writeFragment(
-            p0 -> ChunkedToXContentHelper.singleChunk((b, p) -> b.startObject().startArray("log")),
+            p0 -> ChunkedToXContentHelper.chunk((b, p) -> b.startObject().startArray("log")),
             releasable
         );
     }
@@ -74,7 +74,7 @@ void writeChunk(RepositoryVerifyIntegrityResponseChunk chunk, Releasable releasa
             anomalyCount.incrementAndGet();
         }
         streamingXContentResponse.writeFragment(
-            p0 -> ChunkedToXContentHelper.singleChunk((b, p) -> b.startObject().value(chunk, p).endObject()),
+            p0 -> ChunkedToXContentHelper.chunk((b, p) -> b.startObject().value(chunk, p).endObject()),
             releasable
         );
     }
@@ -89,7 +89,7 @@ public void onResponse(RepositoryVerifyIntegrityResponse repositoryVerifyIntegri
                     // success - finish the response with the final results
                     assert streamingXContentResponse != null;
                     streamingXContentResponse.writeFragment(
-                        p0 -> ChunkedToXContentHelper.singleChunk(
+                        p0 -> ChunkedToXContentHelper.chunk(
                             (b, p) -> b.endArray()
                                 .startObject("results")
                                 .field("status", repositoryVerifyIntegrityResponse.finalTaskStatus())
@@ -116,7 +116,7 @@ public void onFailure(Exception e) {
                     if (streamingXContentResponse != null) {
                         // failure after starting the response - finish the response with a rendering of the final exception
                         streamingXContentResponse.writeFragment(
-                            p0 -> ChunkedToXContentHelper.singleChunk(
+                            p0 -> ChunkedToXContentHelper.chunk(
                                 (b, p) -> b.endArray()
                                     .startObject("exception")
                                     .value((bb, pp) -> ElasticsearchException.generateFailureXContent(bb, pp, e, true))
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityTask.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityTask.java
index eaae913fe9c6f..dd3193cd1d047 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityTask.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityTask.java
@@ -59,7 +59,7 @@ public record Status(
         long throttledNanos
     ) implements org.elasticsearch.tasks.Task.Status {
 
-        public static String NAME = "verify_repository_integrity_status";
+        public static final String NAME = "verify_repository_integrity_status";
 
         public Status(StreamInput in) throws IOException {
             this(
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RestRepositoryVerifyIntegrityAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RestRepositoryVerifyIntegrityAction.java
index 16cdb9140411c..53db8efde9ead 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RestRepositoryVerifyIntegrityAction.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RestRepositoryVerifyIntegrityAction.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.repositories.blobstore.testkit.integrity;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -22,8 +21,6 @@
 @ServerlessScope(Scope.INTERNAL)
 public class RestRepositoryVerifyIntegrityAction extends BaseRestHandler {
 
-    public static final NodeFeature REPOSITORY_VERIFY_INTEGRITY_FEATURE = new NodeFeature("snapshot.repository_verify_integrity");
-
     @Override
     public List<Route> routes() {
         return List.of(new Route(POST, "/_snapshot/{repository}/_verify_integrity"));
diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle
index 6299908f0dc14..49af925c0a422 100644
--- a/x-pack/plugin/spatial/build.gradle
+++ b/x-pack/plugin/spatial/build.gradle
@@ -9,9 +9,9 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  name 'spatial'
-  description 'A plugin for Basic Spatial features'
-  classname 'org.elasticsearch.xpack.spatial.SpatialPlugin'
+  name = 'spatial'
+  description = 'A plugin for Basic Spatial features'
+  classname ='org.elasticsearch.xpack.spatial.SpatialPlugin'
   extendedPlugins = ['x-pack-core', 'legacy-geo', 'lang-painless']
 }
 
diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/CartesianShapeIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/CartesianShapeIT.java
index 83fbd7262461d..eb4515a897118 100644
--- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/CartesianShapeIT.java
+++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/CartesianShapeIT.java
@@ -36,7 +36,7 @@ protected void getGeoShapeMapping(XContentBuilder b) throws IOException {
 
     @Override
     protected IndexVersion randomSupportedVersion() {
-        return IndexVersionUtils.randomCompatibleVersion(random());
+        return IndexVersionUtils.randomCompatibleWriteVersion(random());
     }
 
     @Override
diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryGeoShapeWithDocValuesIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryGeoShapeWithDocValuesIT.java
index 3d91fb443aabd..4a6fa5d545bef 100644
--- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryGeoShapeWithDocValuesIT.java
+++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryGeoShapeWithDocValuesIT.java
@@ -40,6 +40,6 @@ public XContentBuilder getMapping() throws IOException {
 
     @Override
     public IndexVersion randomSupportedVersion() {
-        return IndexVersionUtils.randomCompatibleVersion(random());
+        return IndexVersionUtils.randomCompatibleWriteVersion(random());
     }
 }
diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoShapeWithDocValuesIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoShapeWithDocValuesIT.java
index b4d7a472591bd..0857b078be579 100644
--- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoShapeWithDocValuesIT.java
+++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoShapeWithDocValuesIT.java
@@ -60,7 +60,7 @@ protected void getGeoShapeMapping(XContentBuilder b) throws IOException {
 
     @Override
     protected IndexVersion randomSupportedVersion() {
-        return IndexVersionUtils.randomCompatibleVersion(random());
+        return IndexVersionUtils.randomCompatibleWriteVersion(random());
     }
 
     @Override
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/CartesianShapeValues.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/CartesianShapeValues.java
index f9481bb8169ea..4621929e8c62c 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/CartesianShapeValues.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/CartesianShapeValues.java
@@ -27,7 +27,7 @@
  */
 public abstract class CartesianShapeValues extends ShapeValues<CartesianShapeValues.CartesianShapeValue> {
 
-    public static CartesianShapeValues EMPTY = new CartesianShapeValues() {
+    public static final CartesianShapeValues EMPTY = new CartesianShapeValues() {
         private final CartesianShapeValuesSourceType DEFAULT_VALUES_SOURCE_TYPE = CartesianShapeValuesSourceType.instance();
 
         @Override
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeoShapeValues.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeoShapeValues.java
index c5193d5412ec2..d7099c904085f 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeoShapeValues.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeoShapeValues.java
@@ -28,7 +28,7 @@
  */
 public abstract class GeoShapeValues extends ShapeValues<GeoShapeValues.GeoShapeValue> {
 
-    public static GeoShapeValues EMPTY = new GeoShapeValues() {
+    public static final GeoShapeValues EMPTY = new GeoShapeValues() {
         private final GeoShapeValuesSourceType DEFAULT_VALUES_SOURCE_TYPE = GeoShapeValuesSourceType.instance();
 
         @Override
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java
index 23505eda493af..67d25556a2aa7 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java
@@ -298,6 +298,17 @@ public List<Object> parseStoredValues(List<Object> storedValues) {
         protected Function<List<Geometry>, List<Object>> getFormatter(String format) {
             return geoFormatterFactory.getFormatter(format, Function.identity());
         }
+
+        @Override
+        protected boolean isBoundsExtractionSupported() {
+            // Extracting bounds for geo shapes is not implemented yet.
+            return false;
+        }
+
+        @Override
+        protected CoordinateEncoder coordinateEncoder() {
+            return CoordinateEncoder.GEO;
+        }
     }
 
     public static class TypeParser implements Mapper.TypeParser {
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java
index 50221d838e05a..44e68c2ebea74 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java
@@ -121,7 +121,7 @@ public FieldMapper build(MapperBuilderContext context) {
 
     }
 
-    public static TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, IGNORE_MALFORMED_SETTING.get(c.getSettings())));
+    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, IGNORE_MALFORMED_SETTING.get(c.getSettings())));
 
     private final Builder builder;
 
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java
index e5d5354327f5a..f1140093f2368 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java
@@ -126,7 +126,7 @@ public ShapeFieldMapper build(MapperBuilderContext context) {
         }
     }
 
-    public static TypeParser PARSER = new TypeParser(
+    public static final TypeParser PARSER = new TypeParser(
         (n, c) -> new Builder(
             n,
             c.indexVersionCreated(),
@@ -184,6 +184,16 @@ public String typeName() {
         protected Function<List<Geometry>, List<Object>> getFormatter(String format) {
             return GeometryFormatterFactory.getFormatter(format, Function.identity());
         }
+
+        @Override
+        protected boolean isBoundsExtractionSupported() {
+            return true;
+        }
+
+        @Override
+        protected CoordinateEncoder coordinateEncoder() {
+            return CoordinateEncoder.CARTESIAN;
+        }
     }
 
     private final Builder builder;
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java
index 4b13a7bf1f829..5999a3ff1e151 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java
@@ -10,7 +10,6 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.geo.Orientation;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.utils.GeometryValidator;
 import org.elasticsearch.geometry.utils.WellKnownBinary;
@@ -92,6 +91,16 @@ public void testDefaultConfiguration() throws IOException {
         assertTrue(fieldType.hasDocValues());
     }
 
+    public void testDefaultDocValueConfigurationOnPre7_8() throws IOException {
+        IndexVersion oldVersion = IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_7_0);
+        DocumentMapper defaultMapper = createDocumentMapper(oldVersion, fieldMapping(this::minimalMapping));
+        Mapper fieldMapper = defaultMapper.mappers().getMapper(FIELD_NAME);
+        assertThat(fieldMapper, instanceOf(fieldMapperClass()));
+
+        GeoShapeWithDocValuesFieldMapper geoShapeFieldMapper = (GeoShapeWithDocValuesFieldMapper) fieldMapper;
+        assertFalse(geoShapeFieldMapper.fieldType().hasDocValues());
+    }
+
     /**
      * Test that orientation parameter correctly parses
      */
@@ -280,8 +289,6 @@ public void testInvalidCurrentVersion() {
         );
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_ANALYTICS)
-    @AwaitsFix(bugUrl = "this is testing legacy functionality so can likely be removed in 9.0")
     public void testGeoShapeLegacyMerge() throws Exception {
         IndexVersion version = IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.V_8_0_0);
         MapperService m = createMapperService(version, fieldMapping(b -> b.field("type", getFieldName())));
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java
index d030a2bbf81ad..5d2624735bebe 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java
@@ -113,8 +113,11 @@ public void testDefaultConfiguration() throws IOException {
     }
 
     public void testDefaultDocValueConfigurationOnPre8_4() throws IOException {
-        // TODO verify which version this test is actually valid for (when PR is actually merged)
-        IndexVersion oldVersion = IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.V_8_3_0);
+        IndexVersion oldVersion = IndexVersionUtils.randomVersionBetween(
+            random(),
+            IndexVersions.MINIMUM_READONLY_COMPATIBLE,
+            IndexVersions.V_8_3_0
+        );
         DocumentMapper defaultMapper = createDocumentMapper(oldVersion, fieldMapping(this::minimalMapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper(FIELD_NAME);
         assertThat(fieldMapper, instanceOf(fieldMapperClass()));
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoShapeQueryBuilderGeoShapeTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoShapeQueryBuilderGeoShapeTests.java
index 405ef5c480687..e5212a804e057 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoShapeQueryBuilderGeoShapeTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoShapeQueryBuilderGeoShapeTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.geo.GeometryTestUtils;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.ShapeType;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.GeoShapeQueryBuilder;
 import org.elasticsearch.index.query.SearchExecutionContext;
@@ -87,15 +88,27 @@ protected GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) {
         }
         if (ESTestCase.randomBoolean()) {
             SearchExecutionContext context = AbstractBuilderTestCase.createSearchExecutionContext();
-            if (shapeType == ShapeType.LINESTRING || shapeType == ShapeType.MULTILINESTRING) {
-                builder.relation(ESTestCase.randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.CONTAINS));
+            if (context.indexVersionCreated().onOrAfter(IndexVersions.V_7_5_0)) { // CONTAINS is only supported from version 7.5
+                if (shapeType == ShapeType.LINESTRING || shapeType == ShapeType.MULTILINESTRING) {
+                    builder.relation(ESTestCase.randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.CONTAINS));
+                } else {
+                    builder.relation(
+                        ESTestCase.randomFrom(
+                            ShapeRelation.DISJOINT,
+                            ShapeRelation.INTERSECTS,
+                            ShapeRelation.WITHIN,
+                            ShapeRelation.CONTAINS
+                        )
+                    );
+                }
             } else {
-                builder.relation(
-                    ESTestCase.randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.WITHIN, ShapeRelation.CONTAINS)
-                );
+                if (shapeType == ShapeType.LINESTRING || shapeType == ShapeType.MULTILINESTRING) {
+                    builder.relation(ESTestCase.randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS));
+                } else {
+                    builder.relation(ESTestCase.randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.WITHIN));
+                }
             }
         }
-
         if (ESTestCase.randomBoolean()) {
             builder.ignoreUnmapped(ESTestCase.randomBoolean());
         }
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilderOverShapeTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilderOverShapeTests.java
index 72073a6eff550..aa5ae72df2b9e 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilderOverShapeTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilderOverShapeTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.geo.ShapeTestUtils;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.ShapeType;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.SearchExecutionContext;
 
@@ -32,10 +33,18 @@ protected void initializeAdditionalMappings(MapperService mapperService) throws
     @Override
     protected ShapeRelation getShapeRelation(ShapeType type) {
         SearchExecutionContext context = createSearchExecutionContext();
-        if (type == ShapeType.LINESTRING || type == ShapeType.MULTILINESTRING) {
-            return randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.CONTAINS);
+        if (context.indexVersionCreated().onOrAfter(IndexVersions.V_7_5_0)) { // CONTAINS is only supported from version 7.5
+            if (type == ShapeType.LINESTRING || type == ShapeType.MULTILINESTRING) {
+                return randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.CONTAINS);
+            } else {
+                return randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.WITHIN, ShapeRelation.CONTAINS);
+            }
         } else {
-            return randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.WITHIN, ShapeRelation.CONTAINS);
+            if (type == ShapeType.LINESTRING || type == ShapeType.MULTILINESTRING) {
+                return randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS);
+            } else {
+                return randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.WITHIN);
+            }
         }
     }
 
diff --git a/x-pack/plugin/sql/build.gradle b/x-pack/plugin/sql/build.gradle
index 69468bf574956..74936d119492f 100644
--- a/x-pack/plugin/sql/build.gradle
+++ b/x-pack/plugin/sql/build.gradle
@@ -10,7 +10,7 @@ apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
   name = 'x-pack-sql'
-  description 'The Elasticsearch plugin that powers SQL for Elasticsearch'
+  description = 'The Elasticsearch plugin that powers SQL for Elasticsearch'
   classname = 'org.elasticsearch.xpack.sql.plugin.SqlPlugin'
   extendedPlugins = ['x-pack-ql', 'lang-painless']
 }
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java
index e12661122f3ac..298ee54b89e9a 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java
@@ -40,7 +40,7 @@
 public class JdbcConfiguration extends ConnectionConfiguration {
     static final String URL_PREFIX = "jdbc:es://";
     static final String URL_FULL_PREFIX = "jdbc:elasticsearch://";
-    public static URI DEFAULT_URI = URI.create("http://localhost:9200/");
+    public static final URI DEFAULT_URI = URI.create("http://localhost:9200/");
 
     static final String DEBUG = "debug";
     static final String DEBUG_DEFAULT = "false";
diff --git a/x-pack/plugin/sql/qa/jdbc/security/build.gradle b/x-pack/plugin/sql/qa/jdbc/security/build.gradle
index b40a8aaaa5580..4248423d4ff43 100644
--- a/x-pack/plugin/sql/qa/jdbc/security/build.gradle
+++ b/x-pack/plugin/sql/qa/jdbc/security/build.gradle
@@ -20,7 +20,7 @@ subprojects {
   def clusterPath = getPath()
 
   // Use tests from the root security qa project in subprojects
-  configurations.create('testArtifacts').transitive(false)
+  configurations.create('testArtifacts').transitive = false
 
   dependencies {
     javaRestTestImplementation project(":x-pack:plugin:core")
diff --git a/x-pack/plugin/sql/qa/server/security/build.gradle b/x-pack/plugin/sql/qa/server/security/build.gradle
index ef81ebda5cb89..37ae3edaf51db 100644
--- a/x-pack/plugin/sql/qa/server/security/build.gradle
+++ b/x-pack/plugin/sql/qa/server/security/build.gradle
@@ -20,7 +20,7 @@ Project mainProject = project
 
 subprojects {
   // Use tests from the root security qa project in subprojects
-  configurations.create('testArtifacts').transitive(false)
+  configurations.create('testArtifacts').transitive = false
 
   def clusterPath = getPath()
 
diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec
index 60e81be43cc96..2fa82c05cc1aa 100644
--- a/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec
+++ b/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec
@@ -3353,7 +3353,7 @@ Alejandro
 Amabile
 Anoosh
 Basil
-Bojan
+Brendon
 // end::filterToday
 ;
 
diff --git a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/ConnectionBuilder.java b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/ConnectionBuilder.java
index 3d6324bc44557..9886d3b2637d9 100644
--- a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/ConnectionBuilder.java
+++ b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/ConnectionBuilder.java
@@ -24,8 +24,8 @@
  * Connection Builder. Can interactively ask users for the password if it is not provided
  */
 public class ConnectionBuilder {
-    public static String DEFAULT_CONNECTION_STRING = "http://localhost:9200/";
-    public static URI DEFAULT_URI = URI.create(DEFAULT_CONNECTION_STRING);
+    public static final String DEFAULT_CONNECTION_STRING = "http://localhost:9200/";
+    public static final URI DEFAULT_URI = URI.create(DEFAULT_CONNECTION_STRING);
 
     private CliTerminal cliTerminal;
 
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/Histogram.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/Histogram.java
index 5f9cd8443d50b..092fa78f8a3e5 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/Histogram.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/Histogram.java
@@ -31,9 +31,9 @@ public class Histogram extends GroupingFunction {
 
     private final Literal interval;
     private final ZoneId zoneId;
-    public static String YEAR_INTERVAL = DateHistogramInterval.YEAR.toString();
-    public static String MONTH_INTERVAL = DateHistogramInterval.MONTH.toString();
-    public static String DAY_INTERVAL = DateHistogramInterval.DAY.toString();
+    public static final String YEAR_INTERVAL = DateHistogramInterval.YEAR.toString();
+    public static final String MONTH_INTERVAL = DateHistogramInterval.MONTH.toString();
+    public static final String DAY_INTERVAL = DateHistogramInterval.DAY.toString();
 
     public Histogram(Source source, Expression field, Expression interval, ZoneId zoneId) {
         super(source, field, Collections.singletonList(interval));
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/100_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/100_synthetic_source.yml
index 2576a51e8b80e..3723f6cd4d450 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/100_synthetic_source.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/100_synthetic_source.yml
@@ -61,10 +61,6 @@ aggregate_metric_double:
 
 ---
 aggregate_metric_double with ignore_malformed:
-  - requires:
-      cluster_features: ["mapper.track_ignored_source"]
-      reason: requires tracking ignored source
-
   - do:
       indices.create:
         index: test
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/enrich/20_standard_index.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/enrich/20_standard_index.yml
index 971e276aab32a..a4c8d7cdf593d 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/enrich/20_standard_index.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/enrich/20_standard_index.yml
@@ -145,6 +145,7 @@ enrich documents over _bulk via an alias:
           - '{"baz": "lazy", "c": 2}'
           - '{"index": {"_id": "3"}}'
           - '{"baz": "slow", "c": 3}'
+  - match: { errors: false }
 
   - do:
       get:
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml
index cffc161b11539..a77a5a775442a 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml
@@ -257,9 +257,6 @@
 
 ---
 "mv_dedupe from index #104745":
-  - requires:
-      cluster_features: ["esql.mv_ordering_sorted_ascending"]
-      reason: "fixed by introducing a sorted, non-deduplicated MvOrdering"
   - do:
       indices.create:
         index: idx_with_multivalues
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml
index 663c0dc78acb3..118783b412d48 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml
@@ -170,7 +170,7 @@ setup:
   - match: { error.reason: "Found 1 problem\nline 1:36: Unknown column [content], did you mean [count(*)]?" }
 
 ---
-"match with functions":
+"match with disjunctions":
   - do:
       catch: bad_request
       allowed_warnings_regex:
@@ -181,7 +181,20 @@ setup:
 
   - match: { status: 400 }
   - match: { error.type: verification_exception }
-  - match: { error.reason: "Found 1 problem\nline 1:19: Invalid condition [content:\"fox\" OR to_upper(content) == \"FOX\"]. [:] operator can't be used as part of an or condition" }
+  - match: { error.reason: "/.+Invalid\\ condition\\ \\[content\\:\"fox\"\\ OR\\ to_upper\\(content\\)\\ ==\\ \"FOX\"\\]\\./" }
+
+  - do:
+      catch: bad_request
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test | WHERE content:"fox" OR to_upper(content) == "FOX"'
+
+  - match: { status: 400 }
+  - match: { error.type: verification_exception }
+  - match: { error.reason: "/.+Invalid\\ condition\\ \\[content\\:\"fox\"\\ OR\\ to_upper\\(content\\)\\ ==\\ \"FOX\"\\]\\./" }
+
 
 ---
 "match within eval":
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/190_lookup_join.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/190_lookup_join.yml
new file mode 100644
index 0000000000000..1567b6b556bdd
--- /dev/null
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/190_lookup_join.yml
@@ -0,0 +1,126 @@
+---
+setup:
+  - requires:
+      test_runner_features: [capabilities, contains]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [join_lookup_v10]
+      reason: "uses LOOKUP JOIN"
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            properties:
+              key:
+                type: long
+              color:
+                type: keyword
+  - do:
+      indices.create:
+        index: test-lookup
+        body:
+          settings:
+            index:
+              mode: lookup
+            number_of_shards: 1
+          mappings:
+            properties:
+              key:
+                type: long
+              color:
+                type: keyword
+  - do:
+      indices.update_aliases:
+        body:
+          actions:
+            - add:
+                index: test-lookup
+                alias: test-lookup-alias
+  - do:
+      bulk:
+        index: "test"
+        refresh: true
+        body:
+          - { "index": { } }
+          - { "key": 1, "color": "red" }
+          - { "index": { } }
+          - { "key": 2, "color": "blue" }
+  - do:
+      bulk:
+        index: "test-lookup"
+        refresh: true
+        body:
+          - { "index": { } }
+          - { "key": 1, "color": "cyan" }
+          - { "index": { } }
+          - { "key": 2, "color": "yellow" }
+
+---
+basic:
+  - do:
+      esql.query:
+        body:
+          query: 'FROM test | SORT key | LOOKUP JOIN `test-lookup` ON key | LIMIT 3'
+
+  - match: {columns.0.name: "key"}
+  - match: {columns.0.type: "long"}
+  - match: {columns.1.name: "color"}
+  - match: {columns.1.type: "keyword"}
+  - match: {values.0: [1, "cyan"]}
+  - match: {values.1: [2, "yellow"]}
+
+---
+non-lookup index:
+  - do:
+      esql.query:
+        body:
+          query: 'FROM test-lookup | SORT key | LOOKUP JOIN `test` ON key | LIMIT 3'
+      catch: "bad_request"
+
+  - match: { error.type: "verification_exception" }
+  - contains: { error.reason: "Found 1 problem\nline 1:43: invalid [test] resolution in lookup mode to an index in [standard] mode" }
+
+---
+alias:
+  - do:
+      esql.query:
+        body:
+          query: 'FROM test | SORT key | LOOKUP JOIN `test-lookup-alias` ON key | LIMIT 3'
+
+  - match: {columns.0.name: "key"}
+  - match: {columns.0.type: "long"}
+  - match: {columns.1.name: "color"}
+  - match: {columns.1.type: "keyword"}
+  - match: {values.0: [1, "cyan"]}
+  - match: {values.1: [2, "yellow"]}
+
+---
+alias-repeated-alias:
+  - do:
+      esql.query:
+        body:
+          query: 'FROM test-lookup-alias | SORT key | LOOKUP JOIN `test-lookup-alias` ON key | LIMIT 3'
+
+  - match: {columns.0.name: "key"}
+  - match: {columns.0.type: "long"}
+  - match: {columns.1.name: "color"}
+  - match: {columns.1.type: "keyword"}
+  - match: {values.0: [1, "cyan"]}
+  - match: {values.1: [2, "yellow"]}
+
+---
+alias-repeated-index:
+  - do:
+      esql.query:
+        body:
+          query: 'FROM test-lookup | SORT key | LOOKUP JOIN `test-lookup-alias` ON key | LIMIT 3'
+
+  - match: {columns.0.name: "key"}
+  - match: {columns.0.type: "long"}
+  - match: {columns.1.name: "color"}
+  - match: {columns.1.type: "keyword"}
+  - match: {values.0: [1, "cyan"]}
+  - match: {values.1: [2, "yellow"]}
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml
index ebf464ba667db..b9415bce62ea9 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml
@@ -1,7 +1,5 @@
 setup:
   - requires:
-      cluster_features: ["esql.metrics_counter_fields"]
-      reason: "require metrics counter fields"
       test_runner_features: allowed_warnings_regex
   - do:
       indices.create:
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml
index 049895bc9f31a..e100f30717aef 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml
@@ -504,52 +504,3 @@ double nested declared in mapping:
 
   # The `nested` field is not visible, nor are any of it's subfields.
   - match: { columns: [{name: name, type: keyword}] }
-
----
-semantic_text declared in mapping:
-  - requires:
-      test_runner_features: [ capabilities ]
-      capabilities:
-        - method: POST
-          path: /_query
-          parameters: [ ]
-          capabilities: [ semantic_text_type ]
-      reason: "support for semantic_text type"
-  - do:
-      indices.create:
-        index: test_semantic_text
-        body:
-          settings:
-            number_of_shards: 5
-          mappings:
-            properties:
-              semantic_text_field:
-                type: semantic_text
-                inference_id: my_inference_id
-  - do:
-      bulk:
-        index: test_semantic_text
-        refresh: true
-        body:
-          - { "index": { } }
-          - {
-              "semantic_text_field": {
-                "text": "be excellent to each other",
-                "inference": {
-                  "inference_id": "my_inference_id",
-                  "model_settings": {
-                    "task_type": "sparse_embedding"
-                  },
-                  "chunks": [{ "text": "be excellent to each other", "embeddings": { "a": 1,"b": 2 } }]
-              }
-            }
-          }
-  - do:
-      allowed_warnings_regex:
-        - "No limit defined, adding default limit of \\[.*\\]"
-      esql.query:
-        body:
-          query: 'FROM test_semantic_text'
-  - match: { columns: [{name: semantic_text_field, type: semantic_text}] }
-  - length: { values: 1 }
-  - match: { values.0: ["be excellent to each other"] }
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml
index 2a4cde9a680e9..0d9c66012dbfc 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml
@@ -92,7 +92,7 @@ setup:
   - gt: {esql.functions.to_long: $functions_to_long}
   - match: {esql.functions.coalesce: $functions_coalesce}
   # Testing for the entire function set isn't feasbile, so we just check that we return the correct count as an approximation.
-  - length: {esql.functions: 129} # check the "sister" test below for a likely update to the same esql.functions length check
+  - length: {esql.functions: 133} # check the "sister" test below for a likely update to the same esql.functions length check
 
 ---
 "Basic ESQL usage output (telemetry) non-snapshot version":
@@ -163,4 +163,4 @@ setup:
   - match: {esql.functions.cos: $functions_cos}
   - gt: {esql.functions.to_long: $functions_to_long}
   - match: {esql.functions.coalesce: $functions_coalesce}
-  - length: {esql.functions: 125} # check the "sister" test above for a likely update to the same esql.functions length check
+  - length: {esql.functions: 130} # check the "sister" test above for a likely update to the same esql.functions length check
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml
index 3a989d2c87bf3..bfb49bdc7e5c5 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml
@@ -574,10 +574,6 @@ setup:
 
 ---
 "values function":
-  - requires:
-      cluster_features: esql.agg_values
-      reason: "values is available in 8.14+"
-
   - do:
       esql.query:
         body:
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_usage.yml
index 731082378fe17..b7fdd16111db8 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_usage.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_usage.yml
@@ -1,8 +1,5 @@
 ---
 logsdb usage:
-  - requires:
-      cluster_features: ["logsdb_telemetry_stats"]
-      reason: "requires stats"
   - do:
       indices.create:
         index: test1
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/20_reindex_status.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/20_reindex_status.yml
index c94ce8dd211ae..80ca95d631491 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/20_reindex_status.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/20_reindex_status.yml
@@ -61,10 +61,10 @@ setup:
       migrate.get_reindex_status:
         index: "my-data-stream"
   - match: { complete: true }
-  - match: { total_indices: 1 }
+  - match: { total_indices_in_data_stream: 1 }
   - match: { total_indices_requiring_upgrade: 0 }
   - match: { successes: 0 }
-  - match: { in_progress: 0 }
+  - match: { in_progress: [] }
   - match: { pending: 0 }
   - match: { errors: [] }
 
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/30_create_from.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/30_create_from.yml
new file mode 100644
index 0000000000000..dfffea9a049a1
--- /dev/null
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/30_create_from.yml
@@ -0,0 +1,121 @@
+---
+setup:
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+
+---
+teardown:
+  - do:
+      indices.delete:
+        index: source-index-1
+        ignore_unavailable: true
+  - do:
+      indices.delete:
+        index: dest-index-1
+        ignore_unavailable: true
+
+---
+"Test create from with nonexistent source index":
+  - requires:
+      reason: "migration reindex is behind a feature flag"
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_migration/reindex
+          capabilities: [migration_reindex]
+  - do:
+      catch: /index_not_found_exception/
+      migrate.create_from:
+        source: "does_not_exist"
+        dest: "dest1"
+
+---
+"Test create_from with existing source index":
+  - requires:
+      reason: "migration reindex is behind a feature flag"
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_migration/reindex
+          capabilities: [migration_reindex]
+  - do:
+      indices.create:
+        index: source-index-1
+        body:
+          settings:
+            index:
+              number_of_shards: 3
+              number_of_replicas: 5
+          mappings:
+            dynamic: strict
+            properties:
+              bar:
+                type: text
+  - do:
+      migrate.create_from:
+        source: "source-index-1"
+        dest: "dest-index-1"
+  - do:
+      indices.get_settings:
+        index: dest-index-1
+  - match: { dest-index-1.settings.index.number_of_shards: "3" }
+  - match: { dest-index-1.settings.index.number_of_replicas: "5" }
+  - do:
+      indices.get_mapping:
+        index: dest-index-1
+  - match: {dest-index-1.mappings.properties.bar.type: text}
+
+---
+"Test create_from with existing source index and overrides":
+  - requires:
+      reason: "migration reindex is behind a feature flag"
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_migration/reindex
+          capabilities: [migration_reindex]
+  - do:
+      indices.create:
+        index: source-index-1
+        body:
+          settings:
+            index:
+              number_of_shards: 3
+              number_of_replicas: 5
+          mappings:
+            dynamic: strict
+            properties:
+              bar:
+                type: text
+              foo:
+                type: boolean
+  - do:
+      migrate.create_from:
+        source: "source-index-1"
+        dest: "dest-index-1"
+        body:
+          settings_override:
+            index:
+              number_of_shards: 1
+              default_pipeline: "pipeline-1"
+          mappings_override:
+            dynamic: strict
+            properties:
+              bar:
+                type: keyword
+              baz:
+                type: integer
+  - do:
+      indices.get_settings:
+        index: dest-index-1
+  - match: { dest-index-1.settings.index.number_of_shards: "1" }
+  - match: { dest-index-1.settings.index.number_of_replicas: "5" }
+  - match: { dest-index-1.settings.index.default_pipeline: "pipeline-1" }
+  - do:
+      indices.get_mapping:
+        index: dest-index-1
+  - match: {dest-index-1.mappings.properties.bar.type: keyword}
+  - match: {dest-index-1.mappings.properties.foo.type: boolean}
+  - match: {dest-index-1.mappings.properties.baz.type: integer}
+
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/sparse_vector_search.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/sparse_vector_search.yml
index 332981a580802..3481773b0bab3 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/sparse_vector_search.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/sparse_vector_search.yml
@@ -268,7 +268,7 @@ setup:
   - match: { hits.hits.0._score: 0.25 }
 
 ---
-"Test sparse_vector requires one of inference_id or query_vector":
+"Test sparse_vector requires one of query or query_vector":
   - do:
       catch: /\[sparse_vector\] requires one of \[query_vector\] or \[inference_id\]/
       search:
@@ -281,7 +281,41 @@ setup:
   - match: { status: 400 }
 
 ---
-"Test sparse_vector only allows one of inference_id or query_vector":
+"Test sparse_vector returns an error if inference ID not specified with query":
+  - do:
+      catch: bad_request # This is for BWC, the actual error message is tested in a subsequent test
+      search:
+        index: index-with-sparse-vector
+        body:
+          query:
+            sparse_vector:
+              field: text
+              query: "octopus comforter smells"
+
+  - match: { status: 400 }
+
+---
+"Test sparse_vector requires an inference ID to be specified on sparse_vector fields":
+  - requires:
+      cluster_features: [ "search.semantic_sparse_vector_query_rewrite_interception_supported" ]
+      reason: "Error message changed in 8.18"
+  - do:
+      catch: /inference_id required to perform vector search on query string/
+      search:
+        index: index-with-sparse-vector
+        body:
+          query:
+            sparse_vector:
+              field: text
+              query: "octopus comforter smells"
+
+  - match: { status: 400 }
+
+---
+"Test sparse_vector only allows one of query or query_vector (note the error message is misleading)":
+  - requires:
+      cluster_features: [ "search.semantic_sparse_vector_query_rewrite_interception_supported" ]
+      reason: "sparse vector inference checks updated in 8.18 to support sparse_vector on semantic_text fields"
   - do:
       catch: /\[sparse_vector\] requires one of \[query_vector\] or \[inference_id\]/
       search:
@@ -290,7 +324,7 @@ setup:
           query:
             sparse_vector:
               field: text
-              inference_id: text_expansion_model
+              query: "octopus comforter smells"
               query_vector:
                 the: 1.0
                 comforter: 1.0
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/10_basic.yml
index 04c7c0011e6b2..44fc75c57a16c 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/10_basic.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/10_basic.yml
@@ -213,8 +213,6 @@
   - do:
       indices.close:
         index: .monitoring-beats-*
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       catch: /export_exception/
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_rank_vectors.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml
similarity index 92%
rename from rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_rank_vectors.yml
rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml
index ecf34f46c3383..791712ee925a5 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_rank_vectors.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml
@@ -1,11 +1,7 @@
 setup:
   - requires:
-      capabilities:
-        - method: POST
-          path: /_search
-          capabilities: [ rank_vectors_field_mapper ]
-      test_runner_features: capabilities
-      reason: "Support for rank vectors field mapper capability required"
+      cluster_features: [ "rank_vectors" ]
+      reason: "requires rank_vectors feature"
 ---
 "Test create multi-vector field":
   - do:
diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_rank_vectors_dv_fields_api.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_dv_fields_api.yml
similarity index 71%
rename from modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_rank_vectors_dv_fields_api.yml
rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_dv_fields_api.yml
index f37e554fca7bf..bcfb9bcd79b90 100644
--- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_rank_vectors_dv_fields_api.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_dv_fields_api.yml
@@ -1,11 +1,7 @@
 setup:
   - requires:
-      capabilities:
-        - method: POST
-          path: /_search
-          capabilities: [ rank_vectors_script_access ]
-      test_runner_features: capabilities
-      reason: "Support for rank vector field script access capability required"
+      cluster_features: [ "rank_vectors" ]
+      reason: "requires rank_vectors feature"
   - skip:
       features: headers
 
@@ -17,6 +13,8 @@ setup:
             number_of_shards: 1
           mappings:
             properties:
+              id:
+                type: keyword
               vector:
                  type: rank_vectors
                  dims: 5
@@ -33,6 +31,7 @@ setup:
         index: test-index
         id: "1"
         body:
+          id: "1"
           vector: [[230.0, 300.33, -34.8988, 15.555, -200.0],  [-0.5, 100.0, -13, 14.8, -156.0]]
           byte_vector: [[8, 5, -15, 1, -7], [-1, 115, -3, 4, -128]]
           bit_vector: [[8, 5, -15, 1, -7], [-1, 115, -3, 4, -128]]
@@ -42,6 +41,7 @@ setup:
         index: test-index
         id: "3"
         body:
+          id: "3"
           vector: [[0.5, 111.3, -13.0, 14.8, -156.0]]
           byte_vector: [[2, 18, -5, 0, -124]]
           bit_vector: [[2, 18, -5, 0, -124]]
@@ -176,3 +176,35 @@ setup:
 
   - match: {hits.hits.1._id: "3"}
   - close_to: {hits.hits.1._score: {value: 2, error: 0.01}}
+---
+"Test doc value vector access":
+    - skip:
+        features: close_to
+
+    - do:
+        search:
+          _source: false
+          index: test-index
+          body:
+            docvalue_fields: [name, bit_vector, byte_vector, vector]
+            sort: [{id: "asc"}]
+
+    - match: {hits.hits.0._id: "1"}
+    # vector: [[230.0, 300.33, -34.8988, 15.555, -200.0],  [-0.5, 100.0, -13, 14.8, -156.0]]
+    - close_to: {hits.hits.0.fields.vector.0.0.0: {value: 230, error: 0.0001}}
+    - close_to: {hits.hits.0.fields.vector.0.0.1: {value: 300.33, error: 0.0001}}
+    - close_to: {hits.hits.0.fields.vector.0.0.2: {value: -34.8988, error: 0.0001}}
+    - close_to: {hits.hits.0.fields.vector.0.0.3: {value: 15.555, error: 0.0001}}
+    - close_to: {hits.hits.0.fields.vector.0.0.4: {value: -200, error: 0.0001}}
+    - close_to: {hits.hits.0.fields.vector.0.1.0: {value: -0.5, error: 0.0001}}
+    - close_to: {hits.hits.0.fields.vector.0.1.1: {value: 100, error: 0.0001}}
+    - close_to: {hits.hits.0.fields.vector.0.1.2: {value: -13, error: 0.0001}}
+    - close_to: {hits.hits.0.fields.vector.0.1.3: {value: 14.8, error: 0.0001}}
+    - close_to: {hits.hits.0.fields.vector.0.1.4: {value: -156, error: 0.0001}}
+
+    # byte_vector: [[8, 5, -15, 1, -7], [-1, 115, -3, 4, -128]]
+    - match: {hits.hits.0.fields.byte_vector.0.0: [8, 5, -15, 1, -7]}
+    - match: {hits.hits.0.fields.bit_vector.0.0: [8, 5, -15, 1, -7]}
+    - match: {hits.hits.0.fields.byte_vector.0.1: [-1, 115, -3, 4, -128]}
+    - match: {hits.hits.0.fields.bit_vector.0.1: [-1, 115, -3, 4, -128]}
+
diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_rank_vectors_max_sim.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_max_sim.yml
similarity index 95%
rename from modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_rank_vectors_max_sim.yml
rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_max_sim.yml
index 7c46fbc9a26a5..acaf1b99b626e 100644
--- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_rank_vectors_max_sim.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_max_sim.yml
@@ -1,11 +1,7 @@
 setup:
   - requires:
-      capabilities:
-        - method: POST
-          path: /_search
-          capabilities: [ rank_vectors_script_max_sim_with_bugfix ]
-      test_runner_features: capabilities
-      reason: "Support for rank vectors max-sim functions capability required"
+      cluster_features: [ "rank_vectors" ]
+      reason: "requires rank_vectors feature"
   - skip:
       features: headers
 
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/14_cat_indices.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/14_cat_indices.yml
index 553201680f8c0..2e03907b73713 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/14_cat_indices.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/14_cat_indices.yml
@@ -55,8 +55,6 @@ setup:
   - do:
       indices.close:
         index:  index3
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
 ---
 teardown:
@@ -155,8 +153,6 @@ teardown:
   - do:
       indices.close:
         index:  index_to_monitor
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       headers: { Authorization: "Basic Y2F0X3VzZXI6Y2F0X3NlY3JldF9wYXNzd29yZA==" } # cat_user
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml
index 9ba0b5e4088af..ec73081cd424b 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml
@@ -54,8 +54,6 @@ setup:
   - do:
       indices.close:
         index : test_index
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       snapshot.restore:
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml
index 5e9faa84ee088..10556a49bcb4e 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml
@@ -5,10 +5,6 @@ setup:
 
 ---
 "geo_shape":
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_fallback"]
-      reason: introduced in 8.15.0
-
   - do:
       indices.create:
         index: test
@@ -72,10 +68,6 @@ setup:
 
 ---
 "geo_shape with ignore_malformed":
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_fallback"]
-      reason: introduced in 8.15.0
-
   - do:
       indices.create:
         index: test
@@ -156,10 +148,6 @@ setup:
 
 ---
 "shape":
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_fallback"]
-      reason: introduced in 8.15.0
-
   - do:
       indices.create:
         index: test
@@ -223,10 +211,6 @@ setup:
 
 ---
 "shape with ignore_malformed":
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_fallback"]
-      reason: introduced in 8.15.0
-
   - do:
       indices.create:
         index: test
@@ -424,8 +408,6 @@ setup:
 ---
 "geo_point with ignore_malformed":
   - requires:
-      cluster_features: ["mapper.track_ignored_source"]
-      reason: introduced in 8.15.0
       test_runner_features: close_to
 
   - do:
@@ -504,10 +486,6 @@ setup:
 
 ---
 "point":
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_fallback"]
-      reason: introduced in 8.15.0
-
   - do:
       indices.create:
         index: test
@@ -601,10 +579,6 @@ setup:
 
 ---
 "point with ignore_malformed":
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_fallback"]
-      reason: introduced in 8.15.0
-
   - do:
       indices.create:
         index: test
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/20_ignore_above_stored_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/20_ignore_above_stored_source.yml
index 252bafbdbe15a..12c2a9858a16b 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/20_ignore_above_stored_source.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/20_ignore_above_stored_source.yml
@@ -1,8 +1,5 @@
 ---
 wildcard field type ignore_above:
-  - requires:
-      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
-      reason: introduce ignore_above index level setting
   - do:
       indices.create:
         index:  test
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/30_ignore_above_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/30_ignore_above_synthetic_source.yml
index 26beb3aa19075..5076e7556d911 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/30_ignore_above_synthetic_source.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/30_ignore_above_synthetic_source.yml
@@ -5,9 +5,6 @@ setup:
 
 ---
 wildcard field type ignore_above:
-  - requires:
-      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
-      reason: introduce ignore_above index level setting
   - do:
       indices.create:
         index:  test
diff --git a/x-pack/plugin/stack/build.gradle b/x-pack/plugin/stack/build.gradle
index c4b950ad9cb59..1796ba334cdaf 100644
--- a/x-pack/plugin/stack/build.gradle
+++ b/x-pack/plugin/stack/build.gradle
@@ -2,12 +2,12 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-java-rest-test'
 
 esplugin {
-  name 'x-pack-stack'
-  description 'Elasticsearch Expanded Pack Plugin - Stack'
-  classname 'org.elasticsearch.xpack.stack.StackPlugin'
+  name = 'x-pack-stack'
+  description = 'Elasticsearch Expanded Pack Plugin - Stack'
+  classname ='org.elasticsearch.xpack.stack.StackPlugin'
   extendedPlugins = ['x-pack-core']
-  hasNativeController false
-  requiresKeystore true
+  hasNativeController =false
+  requiresKeystore =true
 }
 
 base {
diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java
index aeb9bf2bfa5cb..55d10c38f9644 100644
--- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java
+++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java
@@ -90,6 +90,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
     //////////////////////////////////////////////////////////
     public static final String TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME = "traces@mappings";
     public static final String TRACES_SETTINGS_COMPONENT_TEMPLATE_NAME = "traces@settings";
+    public static final String TRACES_ILM_POLICY_NAME = "traces@lifecycle";
 
     //////////////////////////////////////////////////////////
     // Synthetics components (for matching synthetics-*-* indices)
@@ -99,6 +100,13 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
     public static final String SYNTHETICS_ILM_POLICY_NAME = "synthetics@lifecycle";
     public static final String SYNTHETICS_INDEX_TEMPLATE_NAME = "synthetics";
 
+    //////////////////////////////////////////////////////////
+    // Agentless components (for matching agentless-*-* indices)
+    //////////////////////////////////////////////////////////
+    public static final String AGENTLESS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "agentless@mappings";
+    public static final String AGENTLESS_SETTINGS_COMPONENT_TEMPLATE_NAME = "agentless@settings";
+    public static final String AGENTLESS_INDEX_TEMPLATE_NAME = "agentless";
+
     ///////////////////////////////////
     // Kibana reporting template
     ///////////////////////////////////
@@ -198,6 +206,20 @@ private Map<String, ComponentTemplate> loadComponentTemplateConfigs() {
                 TEMPLATE_VERSION_VARIABLE,
                 ADDITIONAL_TEMPLATE_VARIABLES
             ),
+            new IndexTemplateConfig(
+                AGENTLESS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
+                "/agentless@mappings.json",
+                REGISTRY_VERSION,
+                TEMPLATE_VERSION_VARIABLE,
+                ADDITIONAL_TEMPLATE_VARIABLES
+            ),
+            new IndexTemplateConfig(
+                AGENTLESS_SETTINGS_COMPONENT_TEMPLATE_NAME,
+                "/agentless@settings.json",
+                REGISTRY_VERSION,
+                TEMPLATE_VERSION_VARIABLE,
+                ADDITIONAL_TEMPLATE_VARIABLES
+            ),
             new IndexTemplateConfig(
                 KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME,
                 "/kibana-reporting@settings.json",
@@ -241,6 +263,7 @@ private void updateEnabledSetting(boolean newValue) {
         new LifecyclePolicyConfig(LOGS_ILM_POLICY_NAME, "/logs@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES),
         new LifecyclePolicyConfig(METRICS_ILM_POLICY_NAME, "/metrics@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES),
         new LifecyclePolicyConfig(SYNTHETICS_ILM_POLICY_NAME, "/synthetics@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES),
+        new LifecyclePolicyConfig(TRACES_ILM_POLICY_NAME, "/traces@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES),
         new LifecyclePolicyConfig(ILM_7_DAYS_POLICY_NAME, "/7-days@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES),
         new LifecyclePolicyConfig(ILM_30_DAYS_POLICY_NAME, "/30-days@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES),
         new LifecyclePolicyConfig(ILM_90_DAYS_POLICY_NAME, "/90-days@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES),
@@ -285,6 +308,13 @@ protected Map<String, ComponentTemplate> getComponentTemplateConfigs() {
             TEMPLATE_VERSION_VARIABLE,
             ADDITIONAL_TEMPLATE_VARIABLES
         ),
+        new IndexTemplateConfig(
+            AGENTLESS_INDEX_TEMPLATE_NAME,
+            "/agentless@template.json",
+            REGISTRY_VERSION,
+            TEMPLATE_VERSION_VARIABLE,
+            ADDITIONAL_TEMPLATE_VARIABLES
+        ),
         new IndexTemplateConfig(
             KIBANA_REPORTING_INDEX_TEMPLATE_NAME,
             "/kibana-reporting@template.json",
diff --git a/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java
index a8043f3d5e4e5..600c1ccba8ca4 100644
--- a/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java
+++ b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java
@@ -174,6 +174,7 @@ public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception {
                         equalTo(StackTemplateRegistry.LOGS_ILM_POLICY_NAME),
                         equalTo(StackTemplateRegistry.METRICS_ILM_POLICY_NAME),
                         equalTo(StackTemplateRegistry.SYNTHETICS_ILM_POLICY_NAME),
+                        equalTo(StackTemplateRegistry.TRACES_ILM_POLICY_NAME),
                         equalTo(StackTemplateRegistry.ILM_7_DAYS_POLICY_NAME),
                         equalTo(StackTemplateRegistry.ILM_30_DAYS_POLICY_NAME),
                         equalTo(StackTemplateRegistry.ILM_90_DAYS_POLICY_NAME),
@@ -197,7 +198,7 @@ public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception {
 
         ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes);
         registry.clusterChanged(event);
-        assertBusy(() -> assertThat(calledTimes.get(), equalTo(8)));
+        assertBusy(() -> assertThat(calledTimes.get(), equalTo(9)));
     }
 
     public void testPolicyAlreadyExists() {
@@ -206,7 +207,7 @@ public void testPolicyAlreadyExists() {
 
         Map<String, LifecyclePolicy> policyMap = new HashMap<>();
         List<LifecyclePolicy> policies = registry.getLifecyclePolicies();
-        assertThat(policies, hasSize(8));
+        assertThat(policies, hasSize(9));
         policies.forEach(p -> policyMap.put(p.getName(), p));
 
         client.setVerifier((action, request, listener) -> {
@@ -277,7 +278,7 @@ public void testPolicyAlreadyExistsButDiffers() throws IOException {
         Map<String, LifecyclePolicy> policyMap = new HashMap<>();
         String policyStr = "{\"phases\":{\"delete\":{\"min_age\":\"1m\",\"actions\":{\"delete\":{}}}}}";
         List<LifecyclePolicy> policies = registry.getLifecyclePolicies();
-        assertThat(policies, hasSize(8));
+        assertThat(policies, hasSize(9));
         policies.forEach(p -> policyMap.put(p.getName(), p));
 
         client.setVerifier((action, request, listener) -> {
@@ -413,6 +414,8 @@ public void testSameOrHigherVersionTemplateNotUpgraded() {
         versions.put(StackTemplateRegistry.METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
         versions.put(StackTemplateRegistry.SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
         versions.put(StackTemplateRegistry.SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
+        versions.put(StackTemplateRegistry.AGENTLESS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
+        versions.put(StackTemplateRegistry.AGENTLESS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
         versions.put(StackTemplateRegistry.KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
         versions.put(StackTemplateRegistry.TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
         versions.put(StackTemplateRegistry.TRACES_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
@@ -471,6 +474,14 @@ public void testSameOrHigherVersionTemplateNotUpgraded() {
             StackTemplateRegistry.SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
             StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
         );
+        versions.put(
+            StackTemplateRegistry.AGENTLESS_SETTINGS_COMPONENT_TEMPLATE_NAME,
+            StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
+        );
+        versions.put(
+            StackTemplateRegistry.AGENTLESS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
+            StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
+        );
         versions.put(
             StackTemplateRegistry.KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME,
             StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
diff --git a/x-pack/plugin/text-structure/build.gradle b/x-pack/plugin/text-structure/build.gradle
index 5bb6d8ef50274..4ebe380e7c724 100644
--- a/x-pack/plugin/text-structure/build.gradle
+++ b/x-pack/plugin/text-structure/build.gradle
@@ -1,8 +1,8 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 esplugin {
-  name 'x-pack-text-structure'
-  description 'Elasticsearch Expanded Pack Plugin - Text Structure'
-  classname 'org.elasticsearch.xpack.textstructure.TextStructurePlugin'
+  name = 'x-pack-text-structure'
+  description = 'Elasticsearch Expanded Pack Plugin - Text Structure'
+  classname ='org.elasticsearch.xpack.textstructure.TextStructurePlugin'
   extendedPlugins = ['x-pack-core']
 }
 base {
diff --git a/x-pack/plugin/transform/build.gradle b/x-pack/plugin/transform/build.gradle
index 9e50aea3a8fca..02c63b06ac083 100644
--- a/x-pack/plugin/transform/build.gradle
+++ b/x-pack/plugin/transform/build.gradle
@@ -1,9 +1,9 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 esplugin {
-  name 'transform'
-  description 'A plugin to transform data'
-  classname 'org.elasticsearch.xpack.transform.Transform'
+  name = 'transform'
+  description = 'A plugin to transform data'
+  classname ='org.elasticsearch.xpack.transform.Transform'
   extendedPlugins = ['x-pack-core']
 }
 
diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java
index ab478dc16f224..84519166eddb6 100644
--- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java
+++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig;
 import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
+import org.elasticsearch.xpack.core.transform.transforms.TransformStats;
 import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfig;
 import org.elasticsearch.xpack.core.transform.transforms.pivot.SingleGroupSource;
 import org.elasticsearch.xpack.core.transform.transforms.pivot.TermsGroupSource;
@@ -152,7 +153,7 @@ public void testBasicTransformStats() throws Exception {
     public void testContinuousTransformCrud() throws Exception {
         var transformId = "transform-continuous-crud";
         var indexName = "continuous-crud-reviews";
-        createContinuousTransform(indexName, transformId);
+        createContinuousTransform(indexName, transformId, "reviews-by-user-business-day");
         var transformStats = getBasicTransformStats(transformId);
         assertThat(transformStats.get("state"), equalTo("started"));
 
@@ -181,7 +182,7 @@ public void testContinuousTransformCrud() throws Exception {
         deleteTransform(transformId);
     }
 
-    private void createContinuousTransform(String indexName, String transformId) throws Exception {
+    private void createContinuousTransform(String indexName, String transformId, String destinationIndex) throws Exception {
         createReviewsIndex(indexName, 100, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow);
 
         var groups = Map.of(
@@ -197,8 +198,9 @@ private void createContinuousTransform(String indexName, String transformId) thr
             .addAggregator(AggregationBuilders.avg("review_score").field("stars"))
             .addAggregator(AggregationBuilders.max("timestamp").field("timestamp"));
 
-        var config = createTransformConfigBuilder(transformId, "reviews-by-user-business-day", QueryConfig.matchAll(), indexName)
-            .setPivotConfig(createPivotConfig(groups, aggs))
+        var config = createTransformConfigBuilder(transformId, destinationIndex, QueryConfig.matchAll(), indexName).setPivotConfig(
+            createPivotConfig(groups, aggs)
+        )
             .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)))
             .setSettings(new SettingsConfig.Builder().setAlignCheckpoints(false).build())
             .build();
@@ -216,7 +218,7 @@ private void createContinuousTransform(String indexName, String transformId) thr
     @SuppressWarnings("unchecked")
     public void testBasicContinuousTransformStats() throws Exception {
         var transformId = "transform-continuous-basic-stats";
-        createContinuousTransform("continuous-basic-stats-reviews", transformId);
+        createContinuousTransform("continuous-basic-stats-reviews", transformId, "reviews-by-user-business-day");
         var transformStats = getBasicTransformStats(transformId);
 
         assertEquals("started", XContentMapValues.extractValue("state", transformStats));
@@ -230,6 +232,40 @@ public void testBasicContinuousTransformStats() throws Exception {
         deleteTransform(transformId);
     }
 
+    public void testDestinationIndexBlocked() throws Exception {
+        var transformId = "transform-continuous-blocked-destination";
+        var sourceIndexName = "source-reviews";
+        var destIndexName = "destination-reviews";
+
+        // create transform & indices, wait until 1st checkpoint is finished
+        createContinuousTransform(sourceIndexName, transformId, destIndexName);
+
+        // block destination index
+        Request request = new Request("PUT", destIndexName + "/_block/write");
+        assertAcknowledged(adminClient().performRequest(request));
+
+        // index more docs so the checkpoint tries to run, wait until transform stops
+        assertBusy(() -> {
+            indexDoc(42, sourceIndexName);
+            assertEquals(TransformStats.State.WAITING.value(), getTransformState(transformId));
+        }, 30, TimeUnit.SECONDS);
+
+        // unblock index
+        request = new Request("PUT", destIndexName + "/_settings");
+        request.setJsonEntity("""
+                { "blocks.write": false }
+            """);
+        assertAcknowledged(adminClient().performRequest(request));
+
+        assertBusy(() -> {
+            indexDoc(42, sourceIndexName);
+            assertEquals(TransformStats.State.STARTED.value(), getTransformState(transformId));
+        }, 30, TimeUnit.SECONDS);
+
+        stopTransform(transformId);
+        deleteTransform(transformId);
+    }
+
     public void testTransformLifecycleInALoop() throws Exception {
         String transformId = "lifecycle-in-a-loop";
         String indexName = transformId + "-src";
@@ -652,4 +688,17 @@ private void indexMoreDocs(long timestamp, long userId, String index) throws Exc
         bulkBuilder.append("\r\n");
         doBulk(bulkBuilder.toString(), true);
     }
+
+    private void indexDoc(long userId, String index) throws Exception {
+        StringBuilder bulkBuilder = new StringBuilder();
+        bulkBuilder.append(format("""
+            {"create":{"_index":"%s"}}
+            """, index));
+        String source = format("""
+            {"user_id":"user_%s","count":%s,"business_id":"business_%s","stars":%s,"timestamp":%s}
+            """, userId, 1, 2, 5, Instant.now().toEpochMilli());
+        bulkBuilder.append(source);
+        bulkBuilder.append("\r\n");
+        doBulk(bulkBuilder.toString(), true);
+    }
 }
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformGetAndGetStatsIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformGetAndGetStatsIT.java
index d73dc2a4e3aac..570feaf28cc58 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformGetAndGetStatsIT.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformGetAndGetStatsIT.java
@@ -37,11 +37,6 @@
 import static org.hamcrest.Matchers.oneOf;
 
 public class TransformGetAndGetStatsIT extends TransformRestTestCase {
-
-    private static final String TEST_USER_NAME = "transform_user";
-    private static final String BASIC_AUTH_VALUE_TRANSFORM_USER = basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING);
-    private static final String TEST_ADMIN_USER_NAME = "transform_admin";
-    private static final String BASIC_AUTH_VALUE_TRANSFORM_ADMIN = basicAuthHeaderValue(TEST_ADMIN_USER_NAME, TEST_PASSWORD_SECURE_STRING);
     private static final String DANGLING_TASK_ERROR_MESSAGE =
         "Found task for transform [pivot_continuous], but no configuration for it. To delete this transform use DELETE with force=true.";
 
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java
index 50b078730063d..322ac63a819fc 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java
@@ -7,14 +7,18 @@
 
 package org.elasticsearch.xpack.transform.integration;
 
+import org.apache.http.HttpHost;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.client.WarningsHandler;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.junit.Before;
@@ -51,6 +55,14 @@ public class TransformPivotRestIT extends TransformRestTestCase {
 
     private static boolean indicesCreated = false;
 
+    @Override
+    protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException {
+        RestClientBuilder builder = RestClient.builder(hosts);
+        configureClient(builder, settings);
+        builder.setStrictDeprecationMode(false);
+        return builder.build();
+    }
+
     // preserve indices in order to reuse source indices in several test cases
     @Override
     protected boolean preserveIndicesUponCompletion() {
@@ -2707,6 +2719,54 @@ public void testExportAndImport() throws Exception {
         assertThat(storedConfig, equalTo(importConfig));
     }
 
+    public void testPivotWithDeprecatedMaxPageSearchSize() throws Exception {
+        String transformId = "deprecated_settings_pivot";
+        String transformIndex = "deprecated_settings_pivot_index";
+        setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex);
+
+        final Request createTransformRequest = createRequestWithAuth(
+            "PUT",
+            getTransformEndpoint() + transformId,
+            BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
+        );
+
+        String config = Strings.format("""
+            {
+              "source": {
+                "index": "%s"
+              },
+              "dest": {
+                "index": "%s"
+              },
+              "pivot": {
+                "group_by": {
+                  "every_2": {
+                    "histogram": {
+                      "interval": 2,
+                      "field": "stars"
+                    }
+                  }
+                },
+                "aggregations": {
+                  "avg_rating": {
+                    "avg": {
+                      "field": "stars"
+                    }
+                  }
+                },
+                "max_page_search_size": 1234
+              }
+            }""", REVIEWS_INDEX_NAME, transformIndex);
+
+        createTransformRequest.setJsonEntity(config);
+        Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
+        assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+
+        Map<String, Object> transform = getTransformConfig(transformId, BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS);
+        assertNull(XContentMapValues.extractValue("pivot.max_page_search_size", transform));
+        assertThat(XContentMapValues.extractValue("settings.max_page_search_size", transform), equalTo(1234));
+    }
+
     private void createDateNanoIndex(String indexName, int numDocs) throws IOException {
         // create mapping
         try (XContentBuilder builder = jsonBuilder()) {
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java
index 09fbea29d4b15..537f50a30b5dd 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java
@@ -39,9 +39,16 @@
 import static org.hamcrest.Matchers.is;
 
 public abstract class TransformRestTestCase extends TransformCommonRestTestCase {
-
     protected static final String TEST_PASSWORD = "x-pack-test-password";
     protected static final SecureString TEST_PASSWORD_SECURE_STRING = new SecureString(TEST_PASSWORD.toCharArray());
+
+    protected static final String TEST_USER_NAME = "transform_user";
+    protected static final String BASIC_AUTH_VALUE_TRANSFORM_USER = basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING);
+    protected static final String TEST_ADMIN_USER_NAME = "transform_admin";
+    protected static final String BASIC_AUTH_VALUE_TRANSFORM_ADMIN = basicAuthHeaderValue(
+        TEST_ADMIN_USER_NAME,
+        TEST_PASSWORD_SECURE_STRING
+    );
     private static final String BASIC_AUTH_VALUE_SUPER_USER = basicAuthHeaderValue("x_pack_rest_user", TEST_PASSWORD_SECURE_STRING);
 
     protected static final String REVIEWS_INDEX_NAME = "reviews";
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUpdateIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUpdateIT.java
index 939dbbbb762f8..6fd5df3a60c74 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUpdateIT.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUpdateIT.java
@@ -40,7 +40,6 @@
 public class TransformUpdateIT extends TransformRestTestCase {
 
     private static final String TEST_USER_NAME = "transform_user";
-    private static final String BASIC_AUTH_VALUE_TRANSFORM_USER = basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING);
     private static final String TEST_ADMIN_USER_NAME_1 = "transform_admin_1";
     private static final String BASIC_AUTH_VALUE_TRANSFORM_ADMIN_1 = basicAuthHeaderValue(
         TEST_ADMIN_USER_NAME_1,
@@ -101,69 +100,6 @@ public void shutdownThreadPool() {
         }
     }
 
-    @SuppressWarnings("unchecked")
-    public void testUpdateDeprecatedSettings() throws Exception {
-        String transformId = "old_transform";
-        String transformDest = transformId + "_idx";
-        setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest);
-
-        final Request createTransformRequest = createRequestWithAuth(
-            "PUT",
-            getTransformEndpoint() + transformId,
-            BASIC_AUTH_VALUE_TRANSFORM_ADMIN_1
-        );
-        String config = Strings.format("""
-            {
-              "dest": {
-                "index": "%s"
-              },
-              "source": {
-                "index": "%s"
-              },
-              "pivot": {
-                "group_by": {
-                  "reviewer": {
-                    "terms": {
-                      "field": "user_id"
-                    }
-                  }
-                },
-                "aggregations": {
-                  "avg_rating": {
-                    "avg": {
-                      "field": "stars"
-                    }
-                  }
-                },
-                "max_page_search_size": 555
-              }
-            }""", transformDest, REVIEWS_INDEX_NAME);
-
-        createTransformRequest.setJsonEntity(config);
-        Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
-        assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
-
-        Map<String, Object> transform = getTransformConfig(transformId, BASIC_AUTH_VALUE_TRANSFORM_USER);
-        assertThat(XContentMapValues.extractValue("pivot.max_page_search_size", transform), equalTo(555));
-
-        final Request updateRequest = createRequestWithAuth(
-            "POST",
-            getTransformEndpoint() + transformId + "/_update",
-            BASIC_AUTH_VALUE_TRANSFORM_ADMIN_1
-        );
-        updateRequest.setJsonEntity("{}");
-
-        Map<String, Object> updateResponse = entityAsMap(client().performRequest(updateRequest));
-
-        assertNull(XContentMapValues.extractValue("pivot.max_page_search_size", updateResponse));
-        assertThat(XContentMapValues.extractValue("settings.max_page_search_size", updateResponse), equalTo(555));
-
-        transform = getTransformConfig(transformId, BASIC_AUTH_VALUE_TRANSFORM_USER);
-
-        assertNull(XContentMapValues.extractValue("pivot.max_page_search_size", transform));
-        assertThat(XContentMapValues.extractValue("settings.max_page_search_size", transform), equalTo(555));
-    }
-
     public void testUpdateTransferRights() throws Exception {
         updateTransferRightsTester(false);
     }
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUpgradeModeIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUpgradeModeIT.java
new file mode 100644
index 0000000000000..437d7aa5e26d9
--- /dev/null
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUpgradeModeIT.java
@@ -0,0 +1,197 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.transform.integration;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.CheckedRunnable;
+import org.elasticsearch.xpack.core.transform.TransformField;
+import org.elasticsearch.xpack.core.transform.transforms.TransformStats;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+import static java.util.Collections.singletonList;
+import static org.hamcrest.Matchers.anEmptyMap;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+
+public class TransformUpgradeModeIT extends TransformRestTestCase {
+    private static boolean indicesCreated = false;
+
+    // preserve indices in order to reuse source indices in several test cases
+    @Override
+    protected boolean preserveIndicesUponCompletion() {
+        return true;
+    }
+
+    @Before
+    public void createIndexes() throws IOException {
+        setupUser(TEST_USER_NAME, singletonList("transform_user"));
+        setupUser(TEST_ADMIN_USER_NAME, singletonList("transform_admin"));
+        // it's not possible to run it as @BeforeClass as clients aren't initialized then, so we need this little hack
+        if (indicesCreated) {
+            return;
+        }
+
+        createReviewsIndex();
+        indicesCreated = true;
+    }
+
+    @After
+    public void clearOutTransforms() throws Exception {
+        adminClient().performRequest(new Request("POST", "/_features/_reset"));
+    }
+
+    public void testUpgradeMode() throws Exception {
+        var transformId = startTransform();
+
+        assertAcknowledged(setUpgradeMode(BASIC_AUTH_VALUE_TRANSFORM_ADMIN, true));
+
+        var statsAndState = getTransformStateAndStats(transformId);
+        assertThat(readString(statsAndState, "state"), equalTo(TransformStats.State.WAITING.value()));
+        assertThat(readString(statsAndState, "reason"), equalTo("Transform task will not be assigned while upgrade mode is enabled."));
+
+        assertAcknowledged(setUpgradeMode(BASIC_AUTH_VALUE_TRANSFORM_ADMIN, false));
+
+        // upgrade mode only waits for the assignment block to be removed, not for the transform to restart
+        // so we need to wait until the transform restarts
+        assertBusy(() -> {
+            var nextStatsAndState = getTransformStateAndStats(transformId);
+            assertThat(readString(nextStatsAndState, "state"), equalTo(TransformStats.State.STARTED.value()));
+            assertThat(readString(nextStatsAndState, "reason"), nullValue());
+        }, 30, TimeUnit.SECONDS);
+    }
+
+    private String startTransform() throws Exception {
+        var transformId = "pivot_continuous";
+        createContinuousPivotReviewsTransform(transformId, "pivot_reviews_continuous", null);
+        startAndWaitForContinuousTransform(transformId, "pivot_reviews_continuous", null);
+        return transformId;
+    }
+
+    private Response setUpgradeMode(String auth, boolean enabled) throws Exception {
+        var path = enabled ? "set_upgrade_mode?enabled" : "set_upgrade_mode";
+        var setUpgradeMode = createRequestWithAuth("POST", getTransformEndpoint() + path, auth);
+        return client().performRequest(setUpgradeMode);
+    }
+
+    private String readString(Map<?, ?> statsAndState, String key) {
+        return statsAndState == null ? null : (String) XContentMapValues.extractValue(key, statsAndState);
+    }
+
+    public void testUpgradeModeFailsWithNonAdminUser() throws Exception {
+        assertRestException(
+            () -> setUpgradeMode(TEST_USER_NAME, true),
+            e -> assertThat(
+                "Expected security error using non-admin transform user",
+                e.getResponse().getStatusLine().getStatusCode(),
+                is(401)
+            )
+        );
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testUpgradeModeBlocksRestApi() throws Exception {
+        var stoppedTransform = "stopped-transform";
+        createContinuousPivotReviewsTransform(stoppedTransform, "pivot_reviews_continuous", null);
+
+        var startedTransform = startTransform();
+
+        var uncreatedTransform = "some-new-transform";
+
+        assertAcknowledged(setUpgradeMode(BASIC_AUTH_VALUE_TRANSFORM_ADMIN, true));
+
+        // start an existing transform
+        assertConflict(() -> startTransform(stoppedTransform), "Cannot start any Transform while the Transform feature is upgrading.");
+
+        // stop an existing transform
+        assertConflict(() -> stopTransform(startedTransform, false), "Cannot stop any Transform while the Transform feature is upgrading.");
+
+        // create a new transform
+        assertConflict(
+            () -> createContinuousPivotReviewsTransform(uncreatedTransform, "pivot_reviews_continuous", null),
+            "Cannot create new Transform while the Transform feature is upgrading."
+        );
+
+        // update an existing transform
+        assertConflict(
+            () -> updateTransform(stoppedTransform, "{ \"settings\": { \"max_page_search_size\": 123 } }", false),
+            "Cannot update any Transform while the Transform feature is upgrading."
+        );
+
+        // upgrade all transforms
+        assertConflict(this::upgrade, "Cannot upgrade Transforms while the Transform feature is upgrading.");
+
+        // schedule transform
+        assertConflict(
+            () -> scheduleNowTransform(startedTransform),
+            "Cannot schedule any Transform while the Transform feature is upgrading."
+        );
+
+        // reset transform
+        assertConflict(
+            () -> resetTransform(startedTransform, false),
+            "Cannot reset any Transform while the Transform feature is upgrading."
+        );
+
+        assertAcknowledged(setUpgradeMode(BASIC_AUTH_VALUE_TRANSFORM_ADMIN, false));
+
+        // started transform should go back into started
+        assertBusy(() -> {
+            var statsAndState = getTransformStateAndStats(startedTransform);
+            assertThat(readString(statsAndState, "state"), equalTo(TransformStats.State.STARTED.value()));
+            assertThat(readString(statsAndState, "reason"), nullValue());
+        }, 30, TimeUnit.SECONDS);
+
+        // stopped transform should have never started
+        var statsAndState = getTransformStateAndStats(stoppedTransform);
+        assertThat(readString(statsAndState, "state"), equalTo(TransformStats.State.STOPPED.value()));
+        assertThat(readString(statsAndState, "reason"), nullValue());
+
+        // new transform shouldn't exist
+        assertRestException(() -> getTransformConfig(uncreatedTransform, BASIC_AUTH_VALUE_TRANSFORM_ADMIN), e -> {
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), is(404));
+            assertThat(e.getMessage(), containsString("Transform with id [" + uncreatedTransform + "] could not be found"));
+        });
+
+        // stopped transform should not have been updated
+        var stoppedConfig = getTransformConfig(stoppedTransform, BASIC_AUTH_VALUE_TRANSFORM_ADMIN);
+        assertThat((Map<String, Object>) stoppedConfig.get("settings"), is(anEmptyMap()));
+    }
+
+    private void assertConflict(CheckedRunnable<Exception> runnable, String message) throws Exception {
+        assertRestException(runnable, e -> {
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), is(409));
+            assertThat(e.getMessage(), containsString(message));
+        });
+    }
+
+    private void assertRestException(CheckedRunnable<Exception> runnable, Consumer<ResponseException> assertThat) throws Exception {
+        try {
+            runnable.run();
+            fail("Expected code to throw ResponseException.");
+        } catch (ResponseException e) {
+            assertThat.accept(e);
+        }
+    }
+
+    private void upgrade() throws IOException {
+        client().performRequest(
+            createRequestWithAuth("POST", TransformField.REST_BASE_PATH_TRANSFORMS + "_upgrade", BASIC_AUTH_VALUE_TRANSFORM_ADMIN)
+        );
+    }
+}
diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformOldTransformsIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformOldTransformsIT.java
index 346765c515d31..cb8580054f8cc 100644
--- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformOldTransformsIT.java
+++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformOldTransformsIT.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
@@ -56,16 +57,7 @@ public void testStopThrowsForDeprecatedTransformConfig() throws Exception {
 
         // The mapping does not need to actually be the "OLD" mapping, we are testing that the old doc gets deleted, and the new one
         // created.
-        try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
-            builder.startObject();
-            builder.field(TransformInternalIndex.DYNAMIC, "false");
-            builder.startObject("properties");
-            builder.startObject(TransformField.INDEX_DOC_TYPE.getPreferredName()).field("type", "keyword").endObject();
-            TransformInternalIndex.addTransformsConfigMappings(builder);
-            builder.endObject();
-            builder.endObject();
-            indicesAdmin().create(new CreateIndexRequest(OLD_INDEX).mapping(builder).origin(ClientHelper.TRANSFORM_ORIGIN)).actionGet();
-        }
+        createTransformIndex();
         String transformIndex = "transform-index";
         createSourceIndex(transformIndex);
         String transformId = "transform-throws-for-old-config";
@@ -106,11 +98,7 @@ public void testStopThrowsForDeprecatedTransformConfig() throws Exception {
               "frequency": "1s",
               "version": "%s"
             }""", transformIndex, transformId, transformVersion);
-        IndexRequest indexRequest = new IndexRequest(OLD_INDEX).id(TransformConfig.documentId(transformId))
-            .source(config, XContentType.JSON)
-            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
-        DocWriteResponse indexResponse = client().index(indexRequest).actionGet();
-        assertThat(indexResponse.getResult(), is(DocWriteResponse.Result.CREATED));
+        putTransform(transformId, config);
 
         GetTransformAction.Request getTransformRequest = new GetTransformAction.Request(transformId);
         GetTransformAction.Response getTransformResponse = client().execute(GetTransformAction.INSTANCE, getTransformRequest).actionGet();
@@ -155,7 +143,114 @@ public void testStopThrowsForDeprecatedTransformConfig() throws Exception {
         assertTrue(stopTransformActionResponse.isAcknowledged());
     }
 
+    private void createTransformIndex() throws Exception {
+        try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
+            builder.startObject();
+            builder.field(TransformInternalIndex.DYNAMIC, "false");
+            builder.startObject("properties");
+            builder.startObject(TransformField.INDEX_DOC_TYPE.getPreferredName()).field("type", "keyword").endObject();
+            TransformInternalIndex.addTransformsConfigMappings(builder);
+            builder.endObject();
+            builder.endObject();
+            indicesAdmin().create(new CreateIndexRequest(OLD_INDEX).mapping(builder).origin(ClientHelper.TRANSFORM_ORIGIN)).actionGet();
+        }
+    }
+
     private void createSourceIndex(String index) {
         indicesAdmin().create(new CreateIndexRequest(index)).actionGet();
     }
+
+    private void putTransform(String transformId, String config) {
+        IndexRequest indexRequest = new IndexRequest(OLD_INDEX).id(TransformConfig.documentId(transformId))
+            .source(config, XContentType.JSON)
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+        DocWriteResponse indexResponse = client().index(indexRequest).actionGet();
+        assertThat(indexResponse.getResult(), is(DocWriteResponse.Result.CREATED));
+    }
+
+    public void testUpdateReplacesDeprecatedTransformSettings() throws Exception {
+        var expectedMaxPageSearchSize = 555;
+        var transformId = createTransformWithDeprecatedMaxPageSearchSize(expectedMaxPageSearchSize);
+
+        assertMaxPageSearchSizeInPivotConfig(transformId, expectedMaxPageSearchSize);
+
+        var updateRequest = new UpdateTransformAction.Request(TransformConfigUpdate.EMPTY, transformId, false, TimeValue.THIRTY_SECONDS);
+        client().execute(UpdateTransformAction.INSTANCE, updateRequest).actionGet();
+
+        assertMaxPageSearchSizeInSettings(transformId, expectedMaxPageSearchSize);
+    }
+
+    private String createTransformWithDeprecatedMaxPageSearchSize(int maxPageSearchSize) throws Exception {
+        createTransformIndex();
+        String transformIndex = "transform-index";
+        createSourceIndex(transformIndex);
+        String transformId = "transform-update-fixes-deprecated-settings";
+        String config = Strings.format("""
+            {
+              "dest": {
+                "index": "bar"
+              },
+              "source": {
+                "index": "%s",
+                "query": {
+                  "match_all": {}
+                }
+              },
+              "id": "%s",
+              "doc_type": "data_frame_transform_config",
+              "pivot": {
+                "group_by": {
+                  "reviewer": {
+                    "terms": {
+                      "field": "user_id"
+                    }
+                  }
+                },
+                "aggregations": {
+                  "avg_rating": {
+                    "avg": {
+                      "field": "stars"
+                    }
+                  }
+                },
+                "max_page_search_size": %d
+              },
+              "frequency": "1s",
+              "version": "%s"
+            }""", transformIndex, transformId, maxPageSearchSize, TransformConfigVersion.CURRENT);
+        putTransform(transformId, config);
+        return transformId;
+    }
+
+    private void assertMaxPageSearchSizeInPivotConfig(String transformId, int expectedMaxPageSearchSize) {
+        var getTransformRequest = new GetTransformAction.Request(transformId);
+        var getTransformResponse = client().execute(GetTransformAction.INSTANCE, getTransformRequest).actionGet();
+        var transformConfig = getTransformResponse.getTransformConfigurations().get(0);
+        assertThat(transformConfig.getId(), equalTo(transformId));
+        assertThat(transformConfig.getPivotConfig().getMaxPageSearchSize(), equalTo(expectedMaxPageSearchSize));
+        assertThat(transformConfig.getSettings().getMaxPageSearchSize(), equalTo(null));
+    }
+
+    private void assertMaxPageSearchSizeInSettings(String transformId, int expectedMaxPageSearchSize) {
+        var getTransformRequest = new GetTransformAction.Request(transformId);
+        var getTransformResponse = client().execute(GetTransformAction.INSTANCE, getTransformRequest).actionGet();
+        var transformConfig = getTransformResponse.getTransformConfigurations().get(0);
+        assertThat(transformConfig.getId(), equalTo(transformId));
+        assertThat(transformConfig.getPivotConfig().getMaxPageSearchSize(), equalTo(null));
+        assertThat(transformConfig.getSettings().getMaxPageSearchSize(), equalTo(expectedMaxPageSearchSize));
+    }
+
+    public void testStartReplacesDeprecatedTransformSettings() throws Exception {
+        var expectedMaxPageSearchSize = 1234;
+        var transformId = createTransformWithDeprecatedMaxPageSearchSize(expectedMaxPageSearchSize);
+
+        assertMaxPageSearchSizeInPivotConfig(transformId, expectedMaxPageSearchSize);
+
+        var startTransformRequest = new StartTransformAction.Request(transformId, null, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT);
+        var startTransformActionResponse = client().execute(StartTransformAction.INSTANCE, startTransformRequest).actionGet();
+        assertTrue(startTransformActionResponse.isAcknowledged());
+
+        assertMaxPageSearchSizeInSettings(transformId, expectedMaxPageSearchSize);
+    }
+
 }
diff --git a/x-pack/plugin/transform/src/main/java/module-info.java b/x-pack/plugin/transform/src/main/java/module-info.java
index 33a722725d5da..246e207f1c36d 100644
--- a/x-pack/plugin/transform/src/main/java/module-info.java
+++ b/x-pack/plugin/transform/src/main/java/module-info.java
@@ -22,6 +22,7 @@
     requires org.apache.logging.log4j;
     requires org.apache.lucene.core;
     requires org.apache.lucene.join;
+    requires org.elasticsearch.plugin;
 
     exports org.elasticsearch.xpack.transform;
     exports org.elasticsearch.xpack.transform.action;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
index 732d64059d734..a6681104b636f 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
@@ -48,11 +48,13 @@
 import org.elasticsearch.xcontent.NamedXContentRegistry.Entry;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.action.SetResetModeActionRequest;
+import org.elasticsearch.xpack.core.action.SetUpgradeModeActionRequest;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.transform.TransformConfigVersion;
 import org.elasticsearch.xpack.core.transform.TransformField;
 import org.elasticsearch.xpack.core.transform.TransformMessages;
+import org.elasticsearch.xpack.core.transform.TransformMetadata;
 import org.elasticsearch.xpack.core.transform.TransformNamedXContentProvider;
 import org.elasticsearch.xpack.core.transform.action.DeleteTransformAction;
 import org.elasticsearch.xpack.core.transform.action.GetCheckpointAction;
@@ -65,6 +67,7 @@
 import org.elasticsearch.xpack.core.transform.action.ResetTransformAction;
 import org.elasticsearch.xpack.core.transform.action.ScheduleNowTransformAction;
 import org.elasticsearch.xpack.core.transform.action.SetResetModeAction;
+import org.elasticsearch.xpack.core.transform.action.SetTransformUpgradeModeAction;
 import org.elasticsearch.xpack.core.transform.action.StartTransformAction;
 import org.elasticsearch.xpack.core.transform.action.StopTransformAction;
 import org.elasticsearch.xpack.core.transform.action.UpdateTransformAction;
@@ -82,6 +85,7 @@
 import org.elasticsearch.xpack.transform.action.TransportResetTransformAction;
 import org.elasticsearch.xpack.transform.action.TransportScheduleNowTransformAction;
 import org.elasticsearch.xpack.transform.action.TransportSetTransformResetModeAction;
+import org.elasticsearch.xpack.transform.action.TransportSetTransformUpgradeModeAction;
 import org.elasticsearch.xpack.transform.action.TransportStartTransformAction;
 import org.elasticsearch.xpack.transform.action.TransportStopTransformAction;
 import org.elasticsearch.xpack.transform.action.TransportUpdateTransformAction;
@@ -101,10 +105,12 @@
 import org.elasticsearch.xpack.transform.rest.action.RestPutTransformAction;
 import org.elasticsearch.xpack.transform.rest.action.RestResetTransformAction;
 import org.elasticsearch.xpack.transform.rest.action.RestScheduleNowTransformAction;
+import org.elasticsearch.xpack.transform.rest.action.RestSetTransformUpgradeModeAction;
 import org.elasticsearch.xpack.transform.rest.action.RestStartTransformAction;
 import org.elasticsearch.xpack.transform.rest.action.RestStopTransformAction;
 import org.elasticsearch.xpack.transform.rest.action.RestUpdateTransformAction;
 import org.elasticsearch.xpack.transform.rest.action.RestUpgradeTransformsAction;
+import org.elasticsearch.xpack.transform.telemetry.TransformMeterRegistry;
 import org.elasticsearch.xpack.transform.transforms.TransformPersistentTasksExecutor;
 import org.elasticsearch.xpack.transform.transforms.scheduling.TransformScheduler;
 
@@ -133,6 +139,7 @@ public class Transform extends Plugin implements SystemIndexPlugin, PersistentTa
 
     private final Settings settings;
     private final SetOnce<TransformServices> transformServices = new SetOnce<>();
+    private final SetOnce<TransformConfigAutoMigration> transformConfigAutoMigration = new SetOnce<>();
     private final TransformExtension transformExtension = new DefaultTransformExtension();
 
     public static final Integer DEFAULT_INITIAL_MAX_PAGE_SEARCH_SIZE = Integer.valueOf(500);
@@ -177,6 +184,45 @@ protected XPackLicenseState getLicenseState() {
         return XPackPlugin.getSharedLicenseState();
     }
 
+    @Override
+    public void prepareForIndicesMigration(ClusterService clusterService, Client client, ActionListener<Map<String, Object>> listener) {
+        if (TransformMetadata.upgradeMode(clusterService.state())) {
+            // Transform is already in upgrade mode, so nothing will write to the Transform system indices during their upgrade
+            listener.onResponse(Map.of("already_in_upgrade_mode", true));
+            return;
+        }
+
+        // Enable Transform upgrade mode before upgrading the system indices to ensure nothing writes to them during the upgrade
+        var originClient = new OriginSettingClient(client, TRANSFORM_ORIGIN);
+        originClient.execute(
+            SetTransformUpgradeModeAction.INSTANCE,
+            new SetUpgradeModeActionRequest(true),
+            listener.delegateFailureAndWrap((l, r) -> l.onResponse(Map.of("already_in_upgrade_mode", false)))
+        );
+    }
+
+    @Override
+    public void indicesMigrationComplete(
+        Map<String, Object> preUpgradeMetadata,
+        ClusterService clusterService,
+        Client client,
+        ActionListener<Boolean> listener
+    ) {
+        var wasAlreadyInUpgradeMode = (boolean) preUpgradeMetadata.getOrDefault("already_in_upgrade_mode", false);
+        if (wasAlreadyInUpgradeMode) {
+            // Transform was already in upgrade mode before system indices upgrade started - we shouldn't disable it
+            listener.onResponse(true);
+            return;
+        }
+
+        var originClient = new OriginSettingClient(client, TRANSFORM_ORIGIN);
+        originClient.execute(
+            SetTransformUpgradeModeAction.INSTANCE,
+            new SetUpgradeModeActionRequest(false),
+            listener.delegateFailureAndWrap((l, r) -> l.onResponse(r.isAcknowledged()))
+        );
+    }
+
     @Override
     public List<RestHandler> getRestHandlers(
         final Settings unused,
@@ -203,7 +249,8 @@ public List<RestHandler> getRestHandlers(
             new RestUpgradeTransformsAction(),
             new RestResetTransformAction(),
             new RestScheduleNowTransformAction(),
-            new RestGetTransformNodeStatsAction()
+            new RestGetTransformNodeStatsAction(),
+            new RestSetTransformUpgradeModeAction()
         );
     }
 
@@ -224,6 +271,7 @@ public List<RestHandler> getRestHandlers(
             new ActionHandler<>(ResetTransformAction.INSTANCE, TransportResetTransformAction.class),
             new ActionHandler<>(ScheduleNowTransformAction.INSTANCE, TransportScheduleNowTransformAction.class),
             new ActionHandler<>(GetTransformNodeStatsAction.INSTANCE, TransportGetTransformNodeStatsAction.class),
+            new ActionHandler<>(SetTransformUpgradeModeAction.INSTANCE, TransportSetTransformUpgradeModeAction.class),
 
             // internal, no rest endpoint
             new ActionHandler<>(ValidateTransformAction.INSTANCE, TransportValidateTransformAction.class),
@@ -273,7 +321,15 @@ public Collection<?> createComponents(PluginServices services) {
 
         transformServices.set(new TransformServices(configManager, checkpointService, auditor, scheduler, transformNode));
 
-        return List.of(transformServices.get(), clusterStateListener, new TransformExtensionHolder(getTransformExtension()));
+        var transformMeterRegistry = TransformMeterRegistry.create(services.telemetryProvider().getMeterRegistry());
+        transformConfigAutoMigration.set(new TransformConfigAutoMigration(configManager, auditor, transformMeterRegistry));
+
+        return List.of(
+            transformServices.get(),
+            clusterStateListener,
+            new TransformExtensionHolder(getTransformExtension()),
+            transformConfigAutoMigration.get()
+        );
     }
 
     @Override
@@ -286,6 +342,7 @@ public List<PersistentTasksExecutor<?>> getPersistentTasksExecutor(
     ) {
         // the transform services should have been created
         assert transformServices.get() != null;
+        assert transformConfigAutoMigration.get() != null;
 
         return List.of(
             new TransformPersistentTasksExecutor(
@@ -295,7 +352,8 @@ public List<PersistentTasksExecutor<?>> getPersistentTasksExecutor(
                 clusterService,
                 settingsModule.getSettings(),
                 getTransformExtension(),
-                expressionResolver
+                expressionResolver,
+                transformConfigAutoMigration.get()
             )
         );
     }
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformConfigAutoMigration.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformConfigAutoMigration.java
new file mode 100644
index 0000000000000..fa36f646b1a6a
--- /dev/null
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformConfigAutoMigration.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.transform;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.logging.DeprecationCategory;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.xpack.core.transform.TransformField;
+import org.elasticsearch.xpack.core.transform.TransformMessages;
+import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
+import org.elasticsearch.xpack.transform.notifications.TransformAuditor;
+import org.elasticsearch.xpack.transform.persistence.TransformConfigManager;
+import org.elasticsearch.xpack.transform.telemetry.TransformMeterRegistry;
+
+/**
+ * Intended to automatically apply configuration changes as part of `PUT _transform` or during the persistent task's start
+ * in {@link org.elasticsearch.xpack.transform.transforms.TransformPersistentTasksExecutor}.
+ *
+ * This is intentionally separate from {@link org.elasticsearch.xpack.transform.action.TransformUpdater}, which is designed to be called
+ * from `_update` and `_upgrade` and apply potentially breaking changes and a broader set of changes.
+ * This class is more designed for serverless rolling updates to apply a smaller non-breaking subset of changes.
+ */
+public class TransformConfigAutoMigration {
+
+    private static final Logger logger = LogManager.getLogger(TransformConfigAutoMigration.class);
+    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(TransformConfigAutoMigration.class);
+    private final TransformConfigManager transformConfigManager;
+    private final TransformAuditor auditor;
+    private final TransformMeterRegistry transformMeterRegistry;
+
+    TransformConfigAutoMigration(
+        TransformConfigManager transformConfigManager,
+        TransformAuditor auditor,
+        TransformMeterRegistry transformMeterRegistry
+    ) {
+        this.transformConfigManager = transformConfigManager;
+        this.auditor = auditor;
+        this.transformMeterRegistry = transformMeterRegistry;
+    }
+
+    public TransformConfig migrate(TransformConfig currentConfig) {
+        // most transforms shouldn't need to migrate, so let's exit early
+        if (currentConfig.shouldAutoMigrateMaxPageSearchSize() == false) {
+            return currentConfig;
+        }
+
+        var updatedConfig = TransformConfig.migrateMaxPageSearchSize(new TransformConfig.Builder(currentConfig)).build();
+
+        var validationException = updatedConfig.validate(null);
+        if (validationException == null) {
+            auditor.info(updatedConfig.getId(), TransformMessages.MAX_PAGE_SEARCH_SIZE_MIGRATION);
+            transformMeterRegistry.autoMigrationCount().increment();
+            deprecationLogger.warn(
+                DeprecationCategory.API,
+                TransformField.MAX_PAGE_SEARCH_SIZE.getPreferredName(),
+                TransformMessages.MAX_PAGE_SEARCH_SIZE_MIGRATION
+            );
+            return updatedConfig;
+        } else {
+            logger.atDebug()
+                .withThrowable(validationException)
+                .log(
+                    "Failed to validate auto-migrated Config. Please use the _update API to correct and update "
+                        + "the Transform configuration. Continuing with old config."
+                );
+            return currentConfig;
+        }
+    }
+
+    public void migrateAndSave(TransformConfig currentConfig, ActionListener<TransformConfig> listener) {
+        var updatedConfig = migrate(currentConfig);
+        if (currentConfig == updatedConfig) {
+            listener.onResponse(currentConfig);
+        } else {
+            ActionListener<Boolean> putConfigListener = ActionListener.wrap(r -> listener.onResponse(updatedConfig), e -> {
+                var errorMessage = "Failed to persist auto-migrated Config. Please see Elasticsearch logs. Continuing with old config.";
+                logger.atWarn().withThrowable(e).log(errorMessage);
+                auditor.warning(currentConfig.getId(), errorMessage);
+                listener.onResponse(currentConfig);
+            });
+
+            transformConfigManager.putTransformConfiguration(updatedConfig, putConfigListener);
+        }
+    }
+}
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java
index aad3cc4dc6ac0..13aaf10793e0c 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.transform.TransformMetadata;
 import org.elasticsearch.xpack.core.transform.action.DeleteTransformAction;
 import org.elasticsearch.xpack.core.transform.action.DeleteTransformAction.Request;
 import org.elasticsearch.xpack.core.transform.action.StopTransformAction;
@@ -80,6 +81,15 @@ public TransportDeleteTransformAction(
 
     @Override
     protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<AcknowledgedResponse> listener) {
+        if (TransformMetadata.upgradeMode(state)) {
+            listener.onFailure(
+                new ElasticsearchStatusException(
+                    "Cannot delete any Transform while the Transform feature is upgrading.",
+                    RestStatus.CONFLICT
+                )
+            );
+            return;
+        }
         final TaskId parentTaskId = new TaskId(clusterService.localNode().getId(), task.getId());
         final boolean transformIsRunning = TransformTask.getTransformTask(request.getId(), state) != null;
         if (transformIsRunning && request.isForce() == false) {
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java
index ae0eb000e70e5..7de774224541c 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java
@@ -270,6 +270,9 @@ static TransformStats deriveStats(TransformTask transformTask, @Nullable Transfo
             && derivedState.equals(TransformStats.State.FAILED) == false) {
             derivedState = TransformStats.State.STOPPING;
             reason = Strings.isNullOrEmpty(reason) ? "transform is set to stop at the next checkpoint" : reason;
+        } else if (derivedState.equals(TransformStats.State.STARTED) && transformTask.getContext().isWaitingForIndexToUnblock()) {
+            derivedState = TransformStats.State.WAITING;
+            reason = Strings.isNullOrEmpty(reason) ? "transform is paused while destination index is blocked" : reason;
         }
         return new TransformStats(
             transformTask.getTransformId(),
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPutTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPutTransformAction.java
index 35c97f50b6824..a160269fb5056 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPutTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPutTransformAction.java
@@ -9,6 +9,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
@@ -25,6 +26,7 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -35,11 +37,13 @@
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.transform.TransformConfigVersion;
 import org.elasticsearch.xpack.core.transform.TransformMessages;
+import org.elasticsearch.xpack.core.transform.TransformMetadata;
 import org.elasticsearch.xpack.core.transform.action.PutTransformAction;
 import org.elasticsearch.xpack.core.transform.action.PutTransformAction.Request;
 import org.elasticsearch.xpack.core.transform.action.ValidateTransformAction;
 import org.elasticsearch.xpack.core.transform.transforms.AuthorizationState;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
+import org.elasticsearch.xpack.transform.TransformConfigAutoMigration;
 import org.elasticsearch.xpack.transform.TransformServices;
 import org.elasticsearch.xpack.transform.notifications.TransformAuditor;
 import org.elasticsearch.xpack.transform.persistence.AuthorizationStatePersistenceUtils;
@@ -59,6 +63,7 @@ public class TransportPutTransformAction extends AcknowledgedTransportMasterNode
     private final TransformConfigManager transformConfigManager;
     private final SecurityContext securityContext;
     private final TransformAuditor auditor;
+    private final TransformConfigAutoMigration transformConfigAutoMigration;
 
     @Inject
     public TransportPutTransformAction(
@@ -69,7 +74,8 @@ public TransportPutTransformAction(
         IndexNameExpressionResolver indexNameExpressionResolver,
         ClusterService clusterService,
         TransformServices transformServices,
-        Client client
+        Client client,
+        TransformConfigAutoMigration transformConfigAutoMigration
     ) {
         super(
             PutTransformAction.NAME,
@@ -88,12 +94,23 @@ public TransportPutTransformAction(
             ? new SecurityContext(settings, threadPool.getThreadContext())
             : null;
         this.auditor = transformServices.auditor();
+        this.transformConfigAutoMigration = transformConfigAutoMigration;
     }
 
     @Override
     protected void masterOperation(Task task, Request request, ClusterState clusterState, ActionListener<AcknowledgedResponse> listener) {
         XPackPlugin.checkReadyForXPackCustomMetadata(clusterState);
 
+        if (TransformMetadata.upgradeMode(clusterState)) {
+            listener.onFailure(
+                new ElasticsearchStatusException(
+                    "Cannot create new Transform while the Transform feature is upgrading.",
+                    RestStatus.CONFLICT
+                )
+            );
+            return;
+        }
+
         TransformConfig config = request.getConfig().setCreateTime(Instant.now()).setVersion(TransformConfigVersion.CURRENT);
         config.setHeaders(getSecurityHeadersPreferringSecondary(threadPool, securityContext, clusterState));
 
@@ -169,7 +186,7 @@ protected ClusterBlockException checkBlock(PutTransformAction.Request request, C
     }
 
     private void putTransform(Request request, ActionListener<AcknowledgedResponse> listener) {
-        var config = request.getConfig();
+        var config = transformConfigAutoMigration.migrate(request.getConfig());
         transformConfigManager.putTransformConfiguration(config, listener.delegateFailureAndWrap((l, unused) -> {
             var transformId = config.getId();
             logger.info("[{}] created transform", transformId);
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportResetTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportResetTransformAction.java
index e817463385a13..6dd126628d72a 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportResetTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportResetTransformAction.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.transform.TransformMetadata;
 import org.elasticsearch.xpack.core.transform.action.ResetTransformAction;
 import org.elasticsearch.xpack.core.transform.action.ResetTransformAction.Request;
 import org.elasticsearch.xpack.core.transform.action.StopTransformAction;
@@ -96,6 +97,16 @@ public TransportResetTransformAction(
 
     @Override
     protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<AcknowledgedResponse> listener) {
+        if (TransformMetadata.upgradeMode(state)) {
+            listener.onFailure(
+                new ElasticsearchStatusException(
+                    "Cannot reset any Transform while the Transform feature is upgrading.",
+                    RestStatus.CONFLICT
+                )
+            );
+            return;
+        }
+
         final boolean transformIsRunning = TransformTask.getTransformTask(request.getId(), state) != null;
         if (transformIsRunning && request.isForce() == false) {
             listener.onFailure(
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java
index ee82c28913143..fc25d7fbba878 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java
@@ -10,6 +10,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.FailedNodeException;
@@ -21,11 +22,13 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.ActionNotFoundTransportException;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.core.transform.TransformMetadata;
 import org.elasticsearch.xpack.core.transform.action.ScheduleNowTransformAction;
 import org.elasticsearch.xpack.core.transform.action.ScheduleNowTransformAction.Request;
 import org.elasticsearch.xpack.core.transform.action.ScheduleNowTransformAction.Response;
@@ -72,6 +75,15 @@ public TransportScheduleNowTransformAction(
     protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
         final ClusterState clusterState = clusterService.state();
         XPackPlugin.checkReadyForXPackCustomMetadata(clusterState);
+        if (TransformMetadata.upgradeMode(clusterState)) {
+            listener.onFailure(
+                new ElasticsearchStatusException(
+                    "Cannot schedule any Transform while the Transform feature is upgrading.",
+                    RestStatus.CONFLICT
+                )
+            );
+            return;
+        }
 
         ActionListener<TransformConfig> getTransformListener = ActionListener.wrap(unusedConfig -> {
             PersistentTasksCustomMetadata.PersistentTask<?> transformTask = TransformTask.getTransformTask(request.getId(), clusterState);
@@ -116,6 +128,10 @@ protected void taskOperation(
         TransformTask transformTask,
         ActionListener<Response> listener
     ) {
+        if (transformTask.getContext().isWaitingForIndexToUnblock()) {
+            logger.debug("[{}] Destination index is blocked. User requested a retry.", transformTask.getTransformId());
+            transformTask.getContext().setIsWaitingForIndexToUnblock(false);
+        }
         transformScheduler.scheduleNow(request.getId());
         listener.onResponse(Response.TRUE);
     }
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportSetTransformResetModeAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportSetTransformResetModeAction.java
index b5b50ec6d3997..460cad9bd6b28 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportSetTransformResetModeAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportSetTransformResetModeAction.java
@@ -34,7 +34,7 @@ public TransportSetTransformResetModeAction(
 
     @Override
     protected boolean isResetMode(ClusterState clusterState) {
-        return TransformMetadata.getTransformMetadata(clusterState).isResetMode();
+        return TransformMetadata.getTransformMetadata(clusterState).resetMode();
     }
 
     @Override
@@ -50,7 +50,7 @@ protected ClusterState setState(ClusterState oldState, SetResetModeActionRequest
             newState.metadata(Metadata.builder(oldState.getMetadata()).removeCustom(TransformMetadata.TYPE).build());
         } else {
             TransformMetadata.Builder builder = TransformMetadata.Builder.from(oldState.metadata().custom(TransformMetadata.TYPE))
-                .isResetMode(request.isEnabled());
+                .resetMode(request.isEnabled());
             newState.metadata(Metadata.builder(oldState.getMetadata()).putCustom(TransformMetadata.TYPE, builder.build()).build());
         }
         return newState.build();
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportSetTransformUpgradeModeAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportSetTransformUpgradeModeAction.java
new file mode 100644
index 0000000000000..0d41d1a9a55d7
--- /dev/null
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportSetTransformUpgradeModeAction.java
@@ -0,0 +1,199 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.transform.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.SubscribableListener;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.client.internal.OriginSettingClient;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.persistent.PersistentTasksClusterService;
+import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
+import org.elasticsearch.persistent.PersistentTasksService;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.action.AbstractTransportSetUpgradeModeAction;
+import org.elasticsearch.xpack.core.action.SetUpgradeModeActionRequest;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.transform.TransformField;
+import org.elasticsearch.xpack.core.transform.TransformMetadata;
+import org.elasticsearch.xpack.core.transform.action.SetTransformUpgradeModeAction;
+
+import java.util.Comparator;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.ExceptionsHelper.rethrowAndSuppress;
+import static org.elasticsearch.xpack.core.ClientHelper.TRANSFORM_ORIGIN;
+import static org.elasticsearch.xpack.core.transform.TransformField.AWAITING_UPGRADE;
+
+public class TransportSetTransformUpgradeModeAction extends AbstractTransportSetUpgradeModeAction {
+    private static final Logger logger = LogManager.getLogger(TransportSetTransformUpgradeModeAction.class);
+    private final PersistentTasksClusterService persistentTasksClusterService;
+    private final PersistentTasksService persistentTasksService;
+    private final OriginSettingClient client;
+
+    @Inject
+    public TransportSetTransformUpgradeModeAction(
+        TransportService transportService,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ActionFilters actionFilters,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        PersistentTasksClusterService persistentTasksClusterService,
+        PersistentTasksService persistentTasksService,
+        Client client
+    ) {
+        super(
+            SetTransformUpgradeModeAction.NAME,
+            "transform",
+            transportService,
+            clusterService,
+            threadPool,
+            actionFilters,
+            indexNameExpressionResolver
+        );
+        this.persistentTasksClusterService = persistentTasksClusterService;
+        this.persistentTasksService = persistentTasksService;
+        this.client = new OriginSettingClient(client, TRANSFORM_ORIGIN);
+    }
+
+    @Override
+    protected String featureName() {
+        return "transform-set-upgrade-mode";
+    }
+
+    @Override
+    protected boolean upgradeMode(ClusterState state) {
+        return TransformMetadata.upgradeMode(state);
+    }
+
+    @Override
+    protected ClusterState createUpdatedState(SetUpgradeModeActionRequest request, ClusterState state) {
+        var updatedTransformMetadata = TransformMetadata.getTransformMetadata(state).builder().upgradeMode(request.enabled()).build();
+        var updatedClusterMetadata = state.metadata().copyAndUpdate(b -> b.putCustom(TransformMetadata.TYPE, updatedTransformMetadata));
+        return state.copyAndUpdate(b -> b.metadata(updatedClusterMetadata));
+    }
+
+    @Override
+    protected void upgradeModeSuccessfullyChanged(
+        Task task,
+        SetUpgradeModeActionRequest request,
+        ClusterState state,
+        ActionListener<AcknowledgedResponse> listener
+    ) {
+        PersistentTasksCustomMetadata tasksCustomMetadata = state.metadata().custom(PersistentTasksCustomMetadata.TYPE);
+        if (tasksCustomMetadata == null
+            || tasksCustomMetadata.tasks().isEmpty()
+            || tasksCustomMetadata.tasks().stream().noneMatch(this::isTransformTask)) {
+            logger.info("No tasks to worry about after state update");
+            listener.onResponse(AcknowledgedResponse.TRUE);
+            return;
+        }
+
+        if (request.enabled()) {
+            stopTransforms(request, state, listener);
+        } else {
+            waitForTransformsToRestart(request, listener);
+        }
+    }
+
+    private void stopTransforms(SetUpgradeModeActionRequest request, ClusterState state, ActionListener<AcknowledgedResponse> listener) {
+        unassignTransforms(state, listener.delegateFailureAndWrap((l, r) -> waitForTransformsToStop(request, l)));
+    }
+
+    private void unassignTransforms(ClusterState state, ActionListener<Void> listener) {
+        PersistentTasksCustomMetadata tasksCustomMetadata = state.metadata().custom(PersistentTasksCustomMetadata.TYPE);
+        var transformTasks = tasksCustomMetadata.tasks()
+            .stream()
+            .filter(this::isTransformTask)
+            // We want to always have the same ordering of which tasks we un-allocate first.
+            // However, the order in which the distributed tasks handle the un-allocation event is not guaranteed.
+            .sorted(Comparator.comparing(PersistentTasksCustomMetadata.PersistentTask::getTaskName))
+            .toList();
+
+        logger.info(
+            "Un-assigning persistent tasks : {}",
+            () -> transformTasks.stream()
+                .map(PersistentTasksCustomMetadata.PersistentTask::getId)
+                .collect(Collectors.joining(", ", "[ ", " ]"))
+        );
+
+        // chain each call one at a time
+        // because that is what we are doing for ML, and that is all that is supported in the persistentTasksClusterService (for now)
+        SubscribableListener<PersistentTasksCustomMetadata.PersistentTask<?>> chainListener = SubscribableListener.newSucceeded(null);
+        for (var task : transformTasks) {
+            chainListener = chainListener.andThen(executor, threadPool.getThreadContext(), (l, unused) -> {
+                persistentTasksClusterService.unassignPersistentTask(
+                    task.getId(),
+                    task.getAllocationId(),
+                    AWAITING_UPGRADE.getExplanation(),
+                    l.delegateResponse((ll, ex) -> {
+                        // ignore ResourceNotFoundException since a delete task is already unassigned
+                        if (ExceptionsHelper.unwrapCause(ex) instanceof ResourceNotFoundException) {
+                            ll.onResponse(null);
+                        } else {
+                            ll.onFailure(ex);
+                        }
+                    })
+                );
+            });
+        }
+        chainListener.addListener(listener.delegateFailure((l, ignored) -> l.onResponse(null)), executor, threadPool.getThreadContext());
+    }
+
+    private void waitForTransformsToStop(SetUpgradeModeActionRequest request, ActionListener<AcknowledgedResponse> listener) {
+        client.admin()
+            .cluster()
+            .prepareListTasks()
+            .setActions(TransformField.TASK_NAME + "[c]")
+            // There is a chance that we failed un-allocating a task due to allocation_id being changed
+            // This call will timeout in that case and return an error
+            .setWaitForCompletion(true)
+            .setTimeout(request.ackTimeout())
+            .execute(listener.delegateFailureAndWrap((l, r) -> {
+                try {
+                    // Handle potential node timeouts,
+                    // these should be considered failures as tasks as still potentially executing
+                    logger.info("Waited for tasks to be unassigned");
+                    if (r.getNodeFailures().isEmpty() == false) {
+                        logger.info("There were node failures waiting for tasks", r.getNodeFailures().get(0));
+                    }
+                    rethrowAndSuppress(r.getNodeFailures());
+                    l.onResponse(AcknowledgedResponse.TRUE);
+                } catch (ElasticsearchException ex) {
+                    logger.info("Caught node failures waiting for tasks to be unassigned", ex);
+                    l.onFailure(ex);
+                }
+            }));
+    }
+
+    private boolean isTransformTask(PersistentTasksCustomMetadata.PersistentTask<?> task) {
+        return TransformField.TASK_NAME.equals(task.getTaskName());
+    }
+
+    private void waitForTransformsToRestart(SetUpgradeModeActionRequest request, ActionListener<AcknowledgedResponse> listener) {
+        logger.info("Disabling upgrade mode for Transforms, must wait for tasks to not have AWAITING_UPGRADE assignment");
+        persistentTasksService.waitForPersistentTasksCondition(
+            persistentTasksCustomMetadata -> persistentTasksCustomMetadata.tasks()
+                .stream()
+                .noneMatch(t -> isTransformTask(t) && t.getAssignment().equals(AWAITING_UPGRADE)),
+            request.ackTimeout(),
+            listener.delegateFailureAndWrap((d, r) -> d.onResponse(AcknowledgedResponse.TRUE))
+        );
+    }
+}
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java
index cd9b79d921548..69afce887222a 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.transform.TransformMessages;
+import org.elasticsearch.xpack.core.transform.TransformMetadata;
 import org.elasticsearch.xpack.core.transform.action.StartTransformAction;
 import org.elasticsearch.xpack.core.transform.action.ValidateTransformAction;
 import org.elasticsearch.xpack.core.transform.transforms.AuthorizationState;
@@ -135,6 +136,15 @@ protected void masterOperation(
         ActionListener<StartTransformAction.Response> listener
     ) {
         TransformNodes.warnIfNoTransformNodes(state);
+        if (TransformMetadata.upgradeMode(state)) {
+            listener.onFailure(
+                new ElasticsearchStatusException(
+                    "Cannot start any Transform while the Transform feature is upgrading.",
+                    RestStatus.CONFLICT
+                )
+            );
+            return;
+        }
 
         var transformTaskParamsHolder = new SetOnce<TransformTaskParams>();
         var transformConfigHolder = new SetOnce<TransformConfig>();
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java
index fd24fd7c0faba..37bd800477be6 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java
@@ -40,6 +40,7 @@
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.action.util.PageParams;
 import org.elasticsearch.xpack.core.transform.TransformMessages;
+import org.elasticsearch.xpack.core.transform.TransformMetadata;
 import org.elasticsearch.xpack.core.transform.action.StopTransformAction;
 import org.elasticsearch.xpack.core.transform.action.StopTransformAction.Request;
 import org.elasticsearch.xpack.core.transform.action.StopTransformAction.Response;
@@ -125,6 +126,13 @@ static void validateTaskState(ClusterState state, List<String> transformIds, boo
     @Override
     protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
         final ClusterState state = clusterService.state();
+        if (TransformMetadata.upgradeMode(state)) {
+            listener.onFailure(
+                new ElasticsearchStatusException("Cannot stop any Transform while the Transform feature is upgrading.", RestStatus.CONFLICT)
+            );
+            return;
+        }
+
         final DiscoveryNodes nodes = state.nodes();
 
         if (nodes.isLocalNodeElectedMaster() == false) {
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java
index 6251dab36b2ca..ecd93ae1ae721 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java
@@ -10,6 +10,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
 import org.elasticsearch.action.FailedNodeException;
@@ -26,6 +27,7 @@
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -34,6 +36,7 @@
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.transform.TransformMetadata;
 import org.elasticsearch.xpack.core.transform.action.UpdateTransformAction;
 import org.elasticsearch.xpack.core.transform.action.UpdateTransformAction.Request;
 import org.elasticsearch.xpack.core.transform.action.UpdateTransformAction.Response;
@@ -106,6 +109,15 @@ public TransportUpdateTransformAction(
     protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
         final ClusterState clusterState = clusterService.state();
         XPackPlugin.checkReadyForXPackCustomMetadata(clusterState);
+        if (TransformMetadata.upgradeMode(clusterState)) {
+            listener.onFailure(
+                new ElasticsearchStatusException(
+                    "Cannot update any Transform while the Transform feature is upgrading.",
+                    RestStatus.CONFLICT
+                )
+            );
+            return;
+        }
 
         final DiscoveryNodes nodes = clusterState.nodes();
 
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java
index 5e2fa7f7a590e..80a84c8123091 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.ml.utils.TransportVersionUtils;
 import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.transform.TransformMetadata;
 import org.elasticsearch.xpack.core.transform.action.UpgradeTransformsAction;
 import org.elasticsearch.xpack.core.transform.action.UpgradeTransformsAction.Request;
 import org.elasticsearch.xpack.core.transform.action.UpgradeTransformsAction.Response;
@@ -99,6 +100,12 @@ public TransportUpgradeTransformsAction(
     protected void masterOperation(Task ignoredTask, Request request, ClusterState state, ActionListener<Response> listener)
         throws Exception {
         TransformNodes.warnIfNoTransformNodes(state);
+        if (TransformMetadata.upgradeMode(state)) {
+            listener.onFailure(
+                new ElasticsearchStatusException("Cannot upgrade Transforms while the Transform feature is upgrading.", RestStatus.CONFLICT)
+            );
+            return;
+        }
 
         // do not allow in mixed clusters
         if (TransportVersionUtils.isMinTransportVersionSameAsCurrent(state) == false) {
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/notifications/TransformAuditor.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/notifications/TransformAuditor.java
index 6da7b6190bd45..7a1f874da66a7 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/notifications/TransformAuditor.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/notifications/TransformAuditor.java
@@ -64,7 +64,7 @@ public TransformAuditor(Client client, String nodeName, ClusterService clusterSe
         clusterService.addListener(event -> {
             if (event.metadataChanged()) {
                 boolean oldIsResetMode = isResetMode;
-                boolean newIsResetMode = TransformMetadata.getTransformMetadata(event.state()).isResetMode();
+                boolean newIsResetMode = TransformMetadata.getTransformMetadata(event.state()).resetMode();
                 if (oldIsResetMode != newIsResetMode) {
                     logger.debug("TransformAuditor has noticed change of isResetMode bit from {} to {}", oldIsResetMode, newIsResetMode);
                 }
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java
index 9d5175922c892..f78868112e8cf 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java
@@ -68,6 +68,7 @@
 import org.elasticsearch.xpack.core.action.util.PageParams;
 import org.elasticsearch.xpack.core.transform.TransformField;
 import org.elasticsearch.xpack.core.transform.TransformMessages;
+import org.elasticsearch.xpack.core.transform.TransformMetadata;
 import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
 import org.elasticsearch.xpack.core.transform.transforms.TransformStoredDoc;
@@ -129,6 +130,10 @@ public IndexBasedTransformConfigManager(
 
     @Override
     public void putTransformCheckpoint(TransformCheckpoint checkpoint, ActionListener<Boolean> listener) {
+        if (isUpgrading()) {
+            listener.onFailure(conflictStatusException("Cannot store Transform while the Transform feature is upgrading."));
+            return;
+        }
         try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
             XContentBuilder source = checkpoint.toXContent(builder, new ToXContent.MapParams(TO_XCONTENT_PARAMS));
 
@@ -148,6 +153,10 @@ public void putTransformCheckpoint(TransformCheckpoint checkpoint, ActionListene
 
     @Override
     public void putTransformConfiguration(TransformConfig transformConfig, ActionListener<Boolean> listener) {
+        if (isUpgrading()) {
+            listener.onFailure(conflictStatusException("Cannot store Transform while the Transform feature is upgrading."));
+            return;
+        }
         putTransformConfiguration(transformConfig, DocWriteRequest.OpType.CREATE, null, listener);
     }
 
@@ -157,6 +166,10 @@ public void updateTransformConfiguration(
         SeqNoPrimaryTermAndIndex seqNoPrimaryTermAndIndex,
         ActionListener<Boolean> listener
     ) {
+        if (isUpgrading()) {
+            listener.onFailure(conflictStatusException("Cannot update Transform while the Transform feature is upgrading."));
+            return;
+        }
         if (seqNoPrimaryTermAndIndex.getIndex().equals(TransformInternalIndexConstants.LATEST_INDEX_NAME)) {
             // update the config in the same, current index using optimistic concurrency control
             putTransformConfiguration(transformConfig, DocWriteRequest.OpType.INDEX, seqNoPrimaryTermAndIndex, listener);
@@ -169,6 +182,10 @@ public void updateTransformConfiguration(
 
     @Override
     public void deleteOldTransformConfigurations(String transformId, ActionListener<Boolean> listener) {
+        if (isUpgrading()) {
+            listener.onFailure(conflictStatusException("Cannot delete Transform while the Transform feature is upgrading."));
+            return;
+        }
         DeleteByQueryRequest deleteByQueryRequest = createDeleteByQueryRequest();
         deleteByQueryRequest.indices(
             TransformInternalIndexConstants.INDEX_NAME_PATTERN,
@@ -196,6 +213,10 @@ public void deleteOldTransformConfigurations(String transformId, ActionListener<
 
     @Override
     public void deleteOldTransformStoredDocuments(String transformId, ActionListener<Long> listener) {
+        if (isUpgrading()) {
+            listener.onFailure(conflictStatusException("Cannot delete Transform while the Transform feature is upgrading."));
+            return;
+        }
         DeleteByQueryRequest deleteByQueryRequest = createDeleteByQueryRequest();
         deleteByQueryRequest.indices(
             TransformInternalIndexConstants.INDEX_NAME_PATTERN,
@@ -213,6 +234,10 @@ public void deleteOldTransformStoredDocuments(String transformId, ActionListener
 
     @Override
     public void deleteOldCheckpoints(String transformId, long deleteCheckpointsBelow, long deleteOlderThan, ActionListener<Long> listener) {
+        if (isUpgrading()) {
+            listener.onFailure(conflictStatusException("Cannot delete checkpoints while the Transform feature is upgrading."));
+            return;
+        }
         DeleteByQueryRequest deleteByQueryRequest = createDeleteByQueryRequest();
         deleteByQueryRequest.indices(
             TransformInternalIndexConstants.INDEX_NAME_PATTERN,
@@ -246,6 +271,10 @@ private void deleteByQuery(ActionListener<Long> listener, DeleteByQueryRequest d
 
     @Override
     public void deleteOldIndices(ActionListener<Boolean> listener) {
+        if (isUpgrading()) {
+            listener.onFailure(conflictStatusException("Cannot delete internal indices while the Transform feature is upgrading."));
+            return;
+        }
         ClusterState state = clusterService.state();
         Set<String> indicesToDelete = new HashSet<>();
 
@@ -568,6 +597,10 @@ public void getAllOutdatedTransformIds(TimeValue timeout, ActionListener<Tuple<L
 
     @Override
     public void resetTransform(String transformId, ActionListener<Boolean> listener) {
+        if (isUpgrading()) {
+            listener.onFailure(conflictStatusException("Cannot reset Transform while the Transform feature is upgrading."));
+            return;
+        }
         ActionListener<BulkByScrollResponse> deleteListener = ActionListener.wrap(dbqResponse -> listener.onResponse(true), e -> {
             if (e.getClass() == IndexNotFoundException.class) {
                 listener.onFailure(
@@ -613,6 +646,10 @@ public void resetTransform(String transformId, ActionListener<Boolean> listener)
 
     @Override
     public void deleteTransform(String transformId, ActionListener<Boolean> listener) {
+        if (isUpgrading()) {
+            listener.onFailure(conflictStatusException("Cannot delete Transform while the Transform feature is upgrading."));
+            return;
+        }
         DeleteByQueryRequest request = createDeleteByQueryRequest();
 
         request.indices(TransformInternalIndexConstants.INDEX_NAME_PATTERN, TransformInternalIndexConstants.INDEX_NAME_PATTERN_DEPRECATED);
@@ -645,6 +682,10 @@ public void putOrUpdateTransformStoredDoc(
         SeqNoPrimaryTermAndIndex seqNoPrimaryTermAndIndex,
         ActionListener<SeqNoPrimaryTermAndIndex> listener
     ) {
+        if (isUpgrading()) {
+            listener.onFailure(conflictStatusException("Cannot store Transform while the Transform feature is upgrading."));
+            return;
+        }
         try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
             XContentBuilder source = storedDoc.toXContent(builder, new ToXContent.MapParams(TO_XCONTENT_PARAMS));
 
@@ -940,6 +981,14 @@ private void recursiveExpandAllTransformIds(
         }), client::search);
     }
 
+    private boolean isUpgrading() {
+        return TransformMetadata.upgradeMode(clusterService.state());
+    }
+
+    private Exception conflictStatusException(String message) {
+        return new ElasticsearchStatusException(message, RestStatus.CONFLICT);
+    }
+
     private static Tuple<RestStatus, Throwable> getStatusAndReason(final BulkByScrollResponse response) {
         RestStatus status = RestStatus.OK;
         Throwable reason = new Exception("Unknown error");
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestSetTransformUpgradeModeAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestSetTransformUpgradeModeAction.java
new file mode 100644
index 0000000000000..f4e7606c1c1d4
--- /dev/null
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestSetTransformUpgradeModeAction.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.transform.rest.action;
+
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.Scope;
+import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.action.SetUpgradeModeActionRequest;
+import org.elasticsearch.xpack.core.transform.action.SetTransformUpgradeModeAction;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.rest.RestUtils.getAckTimeout;
+import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
+import static org.elasticsearch.xpack.core.transform.TransformField.REST_BASE_PATH_TRANSFORMS;
+
+@ServerlessScope(Scope.INTERNAL)
+public class RestSetTransformUpgradeModeAction extends BaseRestHandler {
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(POST, REST_BASE_PATH_TRANSFORMS + "set_upgrade_mode"));
+    }
+
+    @Override
+    public String getName() {
+        return "transform_set_upgrade_mode_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        SetUpgradeModeActionRequest request = new SetUpgradeModeActionRequest(restRequest.paramAsBoolean("enabled", false));
+        request.ackTimeout(getAckTimeout(restRequest));
+        request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
+        return channel -> client.execute(SetTransformUpgradeModeAction.INSTANCE, request, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/telemetry/TransformMeterRegistry.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/telemetry/TransformMeterRegistry.java
new file mode 100644
index 0000000000000..c217355027fb1
--- /dev/null
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/telemetry/TransformMeterRegistry.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.transform.telemetry;
+
+import org.elasticsearch.telemetry.metric.LongCounter;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
+
+import java.util.Objects;
+
+public record TransformMeterRegistry(LongCounter autoMigrationCount) {
+    public TransformMeterRegistry {
+        Objects.requireNonNull(autoMigrationCount);
+    }
+
+    public static TransformMeterRegistry create(MeterRegistry meterRegistry) {
+        return new TransformMeterRegistry(
+            meterRegistry.registerLongCounter(
+                "es.transform.automigration.count.total",
+                "Count of when a Transform is automatically migrated from a deprecated setting or feature",
+                "count"
+            )
+        );
+    }
+
+    public static TransformMeterRegistry noOp() {
+        return new TransformMeterRegistry(LongCounter.NOOP);
+    }
+}
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java
index df8c3f62034e5..c7fd6df4467dd 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java
@@ -25,8 +25,10 @@
 import org.elasticsearch.action.search.TransportClosePointInTimeAction;
 import org.elasticsearch.action.search.TransportOpenPointInTimeAction;
 import org.elasticsearch.action.search.TransportSearchAction;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.client.internal.ParentTaskAssigningClient;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -47,6 +49,7 @@
 import org.elasticsearch.transport.ActionNotFoundTransportException;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.indexing.IndexerState;
+import org.elasticsearch.xpack.core.transform.TransformMetadata;
 import org.elasticsearch.xpack.core.transform.action.ValidateTransformAction;
 import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig;
 import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint;
@@ -436,6 +439,42 @@ protected void afterFinishOrFailure() {
         super.afterFinishOrFailure();
     }
 
+    @Override
+    public boolean maybeTriggerAsyncJob(long now) {
+        if (TransformMetadata.upgradeMode(clusterService.state())) {
+            logger.debug("[{}] schedule was triggered but the Transform is upgrading. Ignoring trigger.", getJobId());
+            return false;
+        }
+        if (context.isWaitingForIndexToUnblock()) {
+            if (destinationIndexHasWriteBlock()) {
+                logger.debug("[{}] schedule was triggered but the destination index has a write block. Ignoring trigger.", getJobId());
+                return false;
+            }
+            logger.debug("[{}] destination index is no longer blocked.", getJobId());
+            context.setIsWaitingForIndexToUnblock(false);
+        }
+
+        return super.maybeTriggerAsyncJob(now);
+    }
+
+    private boolean destinationIndexHasWriteBlock() {
+        var clusterState = clusterService.state();
+        if (clusterState == null) {
+            // if we can't determine if the index is blocked, we assume it isn't, even though the bulk request may fail again
+            return false;
+        }
+
+        var destinationIndexName = transformConfig.getDestination().getIndex();
+        var destinationIndex = indexNameExpressionResolver.concreteWriteIndex(
+            clusterState,
+            IndicesOptions.lenientExpandOpen(),
+            destinationIndexName,
+            true,
+            false
+        );
+        return destinationIndex != null && clusterState.blocks().indexBlocked(ClusterBlockLevel.WRITE, destinationIndex.getName());
+    }
+
     @Override
     protected void onStop() {
         closePointInTime();
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java
index 53c3ff4091a99..73872fac097c1 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java
@@ -49,6 +49,14 @@ public interface Listener {
     private volatile AuthorizationState authState;
     private volatile int pageSize = 0;
 
+    /**
+     * If the destination index is blocked (e.g. during a reindex), the Transform will fail to write to it.
+     * {@link TransformFailureHandler} will silence the error so the Transform automatically retries.
+     * Every time the Transform runs, it will check if the index is unblocked and reset this to false.
+     * Users can override this via the `_schedule_now` API.
+     */
+    private volatile boolean isWaitingForIndexToUnblock = false;
+
     // the checkpoint of this transform, storing the checkpoint until data indexing from source to dest is _complete_
     // Note: Each indexer run creates a new future checkpoint which becomes the current checkpoint only after the indexer run finished
     private final AtomicLong currentCheckpoint;
@@ -183,6 +191,14 @@ public void setShouldRecreateDestinationIndex(boolean shouldRecreateDestinationI
         this.shouldRecreateDestinationIndex = shouldRecreateDestinationIndex;
     }
 
+    public boolean isWaitingForIndexToUnblock() {
+        return isWaitingForIndexToUnblock;
+    }
+
+    public void setIsWaitingForIndexToUnblock(boolean isWaitingForIndexToUnblock) {
+        this.isWaitingForIndexToUnblock = isWaitingForIndexToUnblock;
+    }
+
     public AuthorizationState getAuthState() {
         return authState;
     }
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandler.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandler.java
index 24586e5f36337..eeeabff9efd77 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandler.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandler.java
@@ -170,6 +170,7 @@ private void handleScriptException(ScriptException scriptException, boolean unat
      */
     private void handleBulkIndexingException(BulkIndexingException bulkIndexingException, boolean unattended, int numFailureRetries) {
         if (bulkIndexingException.getCause() instanceof ClusterBlockException) {
+            context.setIsWaitingForIndexToUnblock(true);
             retryWithoutIncrementingFailureCount(
                 bulkIndexingException,
                 bulkIndexingException.getDetailedMessage(),
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformNodes.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformNodes.java
index 56e5fd5900cfb..6aea39b34752b 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformNodes.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformNodes.java
@@ -136,7 +136,7 @@ public static boolean hasAnyTransformNode(DiscoveryNodes nodes) {
      * @param clusterState state
      */
     public static void warnIfNoTransformNodes(ClusterState clusterState) {
-        if (TransformMetadata.getTransformMetadata(clusterState).isResetMode() == false) {
+        if (TransformMetadata.getTransformMetadata(clusterState).resetMode() == false) {
             if (hasAnyTransformNode(clusterState.getNodes()) == false) {
                 HeaderWarning.addWarning(TransformMessages.REST_WARN_NO_TRANSFORM_NODES);
             }
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java
index 3774760114657..9cd0f3abcb07d 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java
@@ -49,6 +49,7 @@
 import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState;
 import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
 import org.elasticsearch.xpack.transform.Transform;
+import org.elasticsearch.xpack.transform.TransformConfigAutoMigration;
 import org.elasticsearch.xpack.transform.TransformExtension;
 import org.elasticsearch.xpack.transform.TransformServices;
 import org.elasticsearch.xpack.transform.notifications.TransformAuditor;
@@ -68,6 +69,8 @@
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.xpack.core.common.notifications.Level.ERROR;
 import static org.elasticsearch.xpack.core.common.notifications.Level.INFO;
+import static org.elasticsearch.xpack.core.transform.TransformField.AWAITING_UPGRADE;
+import static org.elasticsearch.xpack.core.transform.TransformField.RESET_IN_PROGRESS;
 import static org.elasticsearch.xpack.transform.transforms.TransformNodes.nodeCanRunThisTransform;
 
 public class TransformPersistentTasksExecutor extends PersistentTasksExecutor<TransformTaskParams> {
@@ -83,6 +86,7 @@ public class TransformPersistentTasksExecutor extends PersistentTasksExecutor<Tr
     private final IndexNameExpressionResolver resolver;
     private final TransformAuditor auditor;
     private final TransformExtension transformExtension;
+    private final TransformConfigAutoMigration transformConfigAutoMigration;
     private volatile int numFailureRetries;
 
     public TransformPersistentTasksExecutor(
@@ -92,7 +96,8 @@ public TransformPersistentTasksExecutor(
         ClusterService clusterService,
         Settings settings,
         TransformExtension transformExtension,
-        IndexNameExpressionResolver resolver
+        IndexNameExpressionResolver resolver,
+        TransformConfigAutoMigration transformConfigAutoMigration
     ) {
         super(TransformField.TASK_NAME, threadPool.generic());
         this.client = client;
@@ -104,6 +109,7 @@ public TransformPersistentTasksExecutor(
         this.numFailureRetries = Transform.NUM_FAILURE_RETRIES_SETTING.get(settings);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(Transform.NUM_FAILURE_RETRIES_SETTING, this::setNumFailureRetries);
         this.transformExtension = transformExtension;
+        this.transformConfigAutoMigration = transformConfigAutoMigration;
     }
 
     @Override
@@ -119,11 +125,12 @@ public PersistentTasksCustomMetadata.Assignment getAssignment(
          *
          * Operations on the transform node happen in {@link #nodeOperation()}
          */
-        if (TransformMetadata.getTransformMetadata(clusterState).isResetMode()) {
-            return new PersistentTasksCustomMetadata.Assignment(
-                null,
-                "Transform task will not be assigned as a feature reset is in progress."
-            );
+        var transformMetadata = TransformMetadata.getTransformMetadata(clusterState);
+        if (transformMetadata.upgradeMode()) {
+            return AWAITING_UPGRADE;
+        }
+        if (transformMetadata.resetMode()) {
+            return RESET_IN_PROGRESS;
         }
         List<String> unavailableIndices = verifyIndicesPrimaryShardsAreActive(clusterState, resolver);
         if (unavailableIndices.size() != 0) {
@@ -217,7 +224,7 @@ protected void nodeOperation(AllocatedPersistentTask task, @Nullable TransformTa
 
         final SetOnce<TransformState> stateHolder = new SetOnce<>();
 
-        // <7> log the start result
+        // <8> log the start result
         ActionListener<StartTransformAction.Response> startTaskListener = ActionListener.wrap(
             response -> logger.info("[{}] successfully completed and scheduled task in node operation", transformId),
             failure -> {
@@ -235,7 +242,7 @@ protected void nodeOperation(AllocatedPersistentTask task, @Nullable TransformTa
             }
         );
 
-        // <6> load next checkpoint
+        // <7> load next checkpoint
         ActionListener<TransformCheckpoint> getTransformNextCheckpointListener = ActionListener.wrap(nextCheckpoint -> {
             // threadpool: system_read
 
@@ -260,7 +267,7 @@ protected void nodeOperation(AllocatedPersistentTask task, @Nullable TransformTa
             markAsFailed(buildTask, error, msg);
         });
 
-        // <5> load last checkpoint
+        // <6> load last checkpoint
         ActionListener<TransformCheckpoint> getTransformLastCheckpointListener = ActionListener.wrap(lastCheckpoint -> {
             // threadpool: system_read
 
@@ -274,7 +281,7 @@ protected void nodeOperation(AllocatedPersistentTask task, @Nullable TransformTa
             markAsFailed(buildTask, error, msg);
         });
 
-        // <4> Set the previous stats (if they exist), initialize the indexer, start the task (If it is STOPPED)
+        // <5> Set the previous stats (if they exist), initialize the indexer, start the task (If it is STOPPED)
         // Since we don't create the task until `_start` is called, if we see that the task state is stopped, attempt to start
         // Schedule execution regardless
         ActionListener<Tuple<TransformStoredDoc, SeqNoPrimaryTermAndIndex>> transformStatsActionListener = ActionListener.wrap(
@@ -324,8 +331,8 @@ protected void nodeOperation(AllocatedPersistentTask task, @Nullable TransformTa
             }
         );
 
-        // <3> Validate the transform, assigning it to the indexer, and get the previous stats (if they exist)
-        ActionListener<TransformConfig> getTransformConfigListener = ActionListener.wrap(config -> {
+        // <4> Validate the transform, assigning it to the indexer, and get the previous stats (if they exist)
+        ActionListener<TransformConfig> getTransformConfigListener = transformStatsActionListener.delegateFailureAndWrap((l, config) -> {
             // threadpool: system_read
 
             // fail if a transform is too old, this can only happen on a rolling upgrade
@@ -344,7 +351,7 @@ protected void nodeOperation(AllocatedPersistentTask task, @Nullable TransformTa
             ValidationException validationException = config.validate(null);
             if (validationException == null) {
                 indexerBuilder.setTransformConfig(config);
-                transformServices.configManager().getTransformStoredDoc(transformId, false, transformStatsActionListener);
+                transformServices.configManager().getTransformStoredDoc(transformId, false, l);
             } else {
                 auditor.error(transformId, validationException.getMessage());
                 markAsFailed(
@@ -357,13 +364,18 @@ protected void nodeOperation(AllocatedPersistentTask task, @Nullable TransformTa
                     )
                 );
             }
-        }, error -> {
-            String msg = TransformMessages.getMessage(TransformMessages.FAILED_TO_LOAD_TRANSFORM_CONFIGURATION, transformId);
-            markAsFailed(buildTask, error, msg);
         });
 
+        // <3> Automatically migrate the Transform off of deprecated features
+        ActionListener<TransformConfig> autoMigrateListener = getTransformConfigListener.delegateFailureAndWrap(
+            (l, currentConfig) -> transformConfigAutoMigration.migrateAndSave(currentConfig, l)
+        );
+
         // <2> Get the transform config
-        var templateCheckListener = getTransformConfig(buildTask, params, getTransformConfigListener);
+        var templateCheckListener = getTransformConfig(buildTask, params, autoMigrateListener.delegateResponse((l, error) -> {
+            String msg = TransformMessages.getMessage(TransformMessages.FAILED_TO_LOAD_TRANSFORM_CONFIGURATION, transformId);
+            markAsFailed(buildTask, error, msg);
+        }));
 
         // <1> Check the latest internal index (IMPORTANT: according to _this_ node, which might be newer than master) is installed
         TransformInternalIndex.createLatestVersionedIndexIfRequired(
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/latest/Latest.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/latest/Latest.java
index 6c37df72769d6..8a66ceeed0394 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/latest/Latest.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/latest/Latest.java
@@ -10,6 +10,8 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
@@ -90,7 +92,12 @@ private static Map<String, Object> convertBucketToDocument(
                 topHits.getHits().getHits().length
             );
         }
-        Map<String, Object> document = topHits.getHits().getHits()[0].getSourceAsMap();
+
+        // We don't use #getSourceAsMap here because we don't want to cache the object as we
+        // only need it here. More over we are modifying the map of maps so we will be holding
+        // the wrong map.
+        BytesReference bytes = topHits.getHits().getHits()[0].getSourceRef();
+        Map<String, Object> document = XContentHelper.convertToMap(bytes, true).v2();
 
         // generator to create unique but deterministic document ids, so we
         // - do not create duplicates if we re-run after failure
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformConfigAutoMigrationTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformConfigAutoMigrationTests.java
new file mode 100644
index 0000000000000..1bc47f27e28e8
--- /dev/null
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformConfigAutoMigrationTests.java
@@ -0,0 +1,180 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.transform;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.transform.TransformConfigVersion;
+import org.elasticsearch.xpack.core.transform.TransformMessages;
+import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig;
+import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
+import org.elasticsearch.xpack.core.transform.transforms.pivot.AggregationConfigTests;
+import org.elasticsearch.xpack.core.transform.transforms.pivot.GroupConfigTests;
+import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfig;
+import org.elasticsearch.xpack.transform.notifications.TransformAuditor;
+import org.elasticsearch.xpack.transform.persistence.TransformConfigManager;
+import org.elasticsearch.xpack.transform.telemetry.TransformMeterRegistry;
+import org.junit.Before;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.action.support.ActionTestUtils.assertNoFailureListener;
+import static org.elasticsearch.xpack.core.transform.transforms.TransformConfigTests.randomTransformConfigWithSettings;
+import static org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfigTests.randomPivotConfig;
+import static org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfigTests.randomPivotConfigWithDeprecatedFields;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.only;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+
+public class TransformConfigAutoMigrationTests extends ESTestCase {
+    private TransformConfigManager transformConfigManager;
+    private TransformAuditor auditor;
+    private TransformConfigAutoMigration autoMigration;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        transformConfigManager = mock();
+        auditor = mock();
+
+        autoMigration = new TransformConfigAutoMigration(transformConfigManager, auditor, TransformMeterRegistry.noOp());
+    }
+
+    protected boolean enableWarningsCheck() {
+        return false;
+    }
+
+    public void testMigrateWithNoDeprecatedSettings() {
+        var config = randomTransformConfigWithNoDeprecatedSettings();
+
+        var updatedConfig = autoMigration.migrate(config);
+
+        assertThat(updatedConfig, sameInstance(config));
+        verifyNoInteractions(auditor);
+    }
+
+    private static TransformConfig randomTransformConfigWithNoDeprecatedSettings() {
+        return randomTransformConfigWithSettings(SettingsConfig.EMPTY, randomPivotConfig(), null);
+    }
+
+    public void testMigrateWithMaxPageSearchSize() {
+        var config = randomTransformConfigWithDeprecatedSettings();
+
+        var updatedConfig = autoMigration.migrate(config);
+
+        assertThatMaxPageSearchSizeMigrated(updatedConfig, config);
+        verify(auditor, only()).info(eq(updatedConfig.getId()), eq(TransformMessages.MAX_PAGE_SEARCH_SIZE_MIGRATION));
+    }
+
+    private static TransformConfig randomTransformConfigWithDeprecatedSettings() {
+        return randomTransformConfigWithSettings(SettingsConfig.EMPTY, randomPivotConfigWithDeprecatedFields(), null);
+    }
+
+    private void assertThatMaxPageSearchSizeMigrated(TransformConfig updatedConfig, TransformConfig originalConfig) {
+        assertThat(updatedConfig, not(sameInstance(originalConfig)));
+        assertThat(updatedConfig.getPivotConfig().getMaxPageSearchSize(), nullValue());
+        assertThat(updatedConfig.getSettings().getMaxPageSearchSize(), equalTo(originalConfig.getPivotConfig().getMaxPageSearchSize()));
+    }
+
+    public void testMigrateWithInvalidMaxPageSearchSize() {
+        var config = randomTransformConfigWithSettings(
+            SettingsConfig.EMPTY,
+            new PivotConfig(
+                GroupConfigTests.randomGroupConfig(TransformConfigVersion.CURRENT),
+                AggregationConfigTests.randomAggregationConfig(),
+                1
+            ),
+            null
+        );
+
+        var updatedConfig = autoMigration.migrate(config);
+
+        assertThat(updatedConfig, sameInstance(config));
+        verifyNoInteractions(auditor);
+    }
+
+    public void testMigrateAndSaveWithNoChanges() throws InterruptedException {
+        var originalConfig = randomTransformConfigWithNoDeprecatedSettings();
+
+        testMigration(originalConfig, updatedConfig -> {
+            assertThat(updatedConfig, sameInstance(originalConfig));
+            verifyNoInteractions(transformConfigManager);
+            verifyNoInteractions(auditor);
+        });
+    }
+
+    private void testMigration(TransformConfig config, CheckedConsumer<TransformConfig, Exception> updatedConfigListener)
+        throws InterruptedException {
+        CountDownLatch latch = new CountDownLatch(1);
+        autoMigration.migrateAndSave(config, ActionListener.runAfter(assertNoFailureListener(updatedConfigListener), latch::countDown));
+        assertTrue(latch.await(1, TimeUnit.SECONDS));
+    }
+
+    public void testMigrateAndSaveSuccess() throws InterruptedException {
+        doAnswer(ans -> {
+            ActionListener<Boolean> listener = ans.getArgument(1);
+            listener.onResponse(true);
+            return null;
+        }).when(transformConfigManager).putTransformConfiguration(any(), any());
+        var originalConfig = randomTransformConfigWithDeprecatedSettings();
+
+        testMigration(originalConfig, updatedConfig -> {
+            assertThatMaxPageSearchSizeMigrated(updatedConfig, originalConfig);
+            verify(auditor, only()).info(eq(updatedConfig.getId()), eq(TransformMessages.MAX_PAGE_SEARCH_SIZE_MIGRATION));
+        });
+    }
+
+    public void testMigrateAndSaveWithErrors() throws InterruptedException {
+        doAnswer(ans -> {
+            ActionListener<Boolean> listener = ans.getArgument(1);
+            listener.onFailure(new IllegalStateException("This is a failure"));
+            return null;
+        }).when(transformConfigManager).putTransformConfiguration(any(), any());
+        var originalConfig = randomTransformConfigWithDeprecatedSettings();
+
+        testMigration(originalConfig, updatedConfig -> {
+            assertThat(updatedConfig, sameInstance(originalConfig));
+            verify(auditor).info(eq(updatedConfig.getId()), eq(TransformMessages.MAX_PAGE_SEARCH_SIZE_MIGRATION));
+            verify(auditor).warning(
+                eq(originalConfig.getId()),
+                eq("Failed to persist auto-migrated Config. Please see Elasticsearch logs. Continuing with old config.")
+            );
+        });
+    }
+
+    public void testMigrateWithMaxPageSearchSizeInBothLocations() {
+        var settingsMaxPageSearchSize = 555;
+        var settingsConfig = new SettingsConfig.Builder(SettingsConfig.EMPTY).setMaxPageSearchSize(settingsMaxPageSearchSize).build();
+
+        var pivotConfigMaxPageSearchSize = 1234;
+        var pivotConfig = new PivotConfig(
+            GroupConfigTests.randomGroupConfig(TransformConfigVersion.CURRENT),
+            AggregationConfigTests.randomAggregationConfig(),
+            pivotConfigMaxPageSearchSize
+        );
+
+        var originalConfig = randomTransformConfigWithSettings(settingsConfig, pivotConfig, null);
+
+        var updatedConfig = autoMigration.migrate(originalConfig);
+
+        assertThat(updatedConfig, not(sameInstance(originalConfig)));
+        assertThat(updatedConfig.getPivotConfig().getMaxPageSearchSize(), nullValue());
+        assertThat(updatedConfig.getSettings().getMaxPageSearchSize(), equalTo(settingsMaxPageSearchSize));
+        verify(auditor, only()).info(eq(updatedConfig.getId()), eq(TransformMessages.MAX_PAGE_SEARCH_SIZE_MIGRATION));
+    }
+}
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformMetadataTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformMetadataTests.java
index 39ea10ef570f6..108bbab85935e 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformMetadataTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformMetadataTests.java
@@ -16,7 +16,7 @@ public class TransformMetadataTests extends AbstractChunkedSerializingTestCase<T
 
     @Override
     protected TransformMetadata createTestInstance() {
-        return new TransformMetadata.Builder().isResetMode(randomBoolean()).build();
+        return new TransformMetadata.Builder().resetMode(randomBoolean()).upgradeMode(randomBoolean()).build();
     }
 
     @Override
@@ -31,6 +31,8 @@ protected TransformMetadata doParseInstance(XContentParser parser) {
 
     @Override
     protected TransformMetadata mutateInstance(TransformMetadata instance) {
-        return new TransformMetadata.Builder().isResetMode(instance.isResetMode() == false).build();
+        return new TransformMetadata.Builder().resetMode(instance.resetMode() == false)
+            .upgradeMode(instance.upgradeMode() == false)
+            .build();
     }
 }
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformNodeTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformNodeTests.java
index 3df566fd70047..8c7a6abe3f055 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformNodeTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformNodeTests.java
@@ -88,6 +88,7 @@ private Supplier<Optional<ClusterState>> clusterState(String nodeId) {
                 SHUTTING_DOWN_ID,
                 SingleNodeShutdownMetadata.builder()
                     .setNodeId(SHUTTING_DOWN_ID)
+                    .setNodeEphemeralId(SHUTTING_DOWN_ID)
                     .setReason("shutdown for a unit test")
                     .setType(SingleNodeShutdownMetadata.Type.RESTART)
                     .setStartedAtMillis(randomNonNegativeLong())
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformTests.java
new file mode 100644
index 0000000000000..1e8bcc5b96882
--- /dev/null
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformTests.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.transform;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionTestUtils;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.xpack.core.action.SetUpgradeModeActionRequest;
+import org.elasticsearch.xpack.core.transform.TransformMetadata;
+import org.elasticsearch.xpack.core.transform.action.SetTransformUpgradeModeAction;
+
+import java.util.Collections;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.same;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class TransformTests extends ESTestCase {
+
+    public void testSetTransformUpgradeMode() {
+        var threadPool = new TestThreadPool("testSetTransformUpgradeMode");
+        ClusterService clusterService = mock();
+        when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE);
+        Client client = mock();
+        when(client.threadPool()).thenReturn(threadPool);
+        doAnswer(invocationOnMock -> {
+            ActionListener<AcknowledgedResponse> listener = invocationOnMock.getArgument(2);
+            listener.onResponse(AcknowledgedResponse.TRUE);
+            return null;
+        }).when(client).execute(same(SetTransformUpgradeModeAction.INSTANCE), any(), any());
+
+        try (var transformPlugin = new Transform(Settings.EMPTY)) {
+            SetOnce<Map<String, Object>> response = new SetOnce<>();
+            transformPlugin.prepareForIndicesMigration(clusterService, client, ActionTestUtils.assertNoFailureListener(response::set));
+
+            assertThat(response.get(), equalTo(Collections.singletonMap("already_in_upgrade_mode", false)));
+            verify(client).execute(same(SetTransformUpgradeModeAction.INSTANCE), eq(new SetUpgradeModeActionRequest(true)), any());
+
+            transformPlugin.indicesMigrationComplete(
+                response.get(),
+                clusterService,
+                client,
+                ActionTestUtils.assertNoFailureListener(ESTestCase::assertTrue)
+            );
+
+            verify(client).execute(same(SetTransformUpgradeModeAction.INSTANCE), eq(new SetUpgradeModeActionRequest(false)), any());
+        } finally {
+            terminate(threadPool);
+        }
+    }
+
+    public void testIgnoreSetTransformUpgradeMode() {
+        ClusterService clusterService = mock();
+        when(clusterService.state()).thenReturn(
+            ClusterState.builder(ClusterName.DEFAULT)
+                .metadata(Metadata.builder().putCustom(TransformMetadata.TYPE, new TransformMetadata.Builder().upgradeMode(true).build()))
+                .build()
+        );
+        Client client = mock();
+
+        try (var transformPlugin = new Transform(Settings.EMPTY)) {
+            SetOnce<Map<String, Object>> response = new SetOnce<>();
+            transformPlugin.prepareForIndicesMigration(clusterService, client, ActionTestUtils.assertNoFailureListener(response::set));
+
+            assertThat(response.get(), equalTo(Collections.singletonMap("already_in_upgrade_mode", true)));
+            verifyNoMoreInteractions(client);
+
+            transformPlugin.indicesMigrationComplete(
+                response.get(),
+                clusterService,
+                client,
+                ActionTestUtils.assertNoFailureListener(ESTestCase::assertTrue)
+            );
+
+            verifyNoMoreInteractions(client);
+        }
+    }
+}
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsActionTests.java
index a3f56efa9e138..52ea738b03313 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsActionTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsActionTests.java
@@ -343,6 +343,46 @@ public void testDeriveStats() {
         );
     }
 
+    public void testDeriveStatsWithIndexBlock() {
+        String transformId = "transform-with-stats";
+        String reason = "transform is paused while destination index is blocked";
+        TransformIndexerStats stats = TransformIndexerStatsTests.randomStats();
+        TransformState runningState = new TransformState(
+            TransformTaskState.STARTED,
+            IndexerState.STARTED,
+            null,
+            0,
+            null,
+            null,
+            null,
+            false,
+            null
+        );
+
+        var context = new TransformContext(TransformTaskState.STARTED, "", 0, mock());
+        context.setIsWaitingForIndexToUnblock(true);
+        var task = mock(TransformTask.class);
+        when(task.getContext()).thenReturn(context);
+        when(task.getTransformId()).thenReturn(transformId);
+        when(task.getState()).thenReturn(runningState);
+        when(task.getStats()).thenReturn(stats);
+
+        assertThat(
+            TransportGetTransformStatsAction.deriveStats(task, null),
+            equalTo(
+                new TransformStats(
+                    transformId,
+                    TransformStats.State.WAITING,
+                    reason,
+                    null,
+                    stats,
+                    TransformCheckpointingInfo.EMPTY,
+                    TransformHealth.GREEN
+                )
+            )
+        );
+    }
+
     private void withIdStateAndStats(String transformId, TransformState state, TransformIndexerStats stats) {
         when(task.getTransformId()).thenReturn(transformId);
         when(task.getState()).thenReturn(state);
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportSetTransformUpgradeModeActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportSetTransformUpgradeModeActionTests.java
new file mode 100644
index 0000000000000..97ef367dca8e5
--- /dev/null
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportSetTransformUpgradeModeActionTests.java
@@ -0,0 +1,202 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.transform.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.internal.AdminClient;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.persistent.PersistentTasksClusterService;
+import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
+import org.elasticsearch.persistent.PersistentTasksService;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.action.SetUpgradeModeActionRequest;
+import org.elasticsearch.xpack.core.transform.TransformField;
+import org.elasticsearch.xpack.core.transform.TransformMetadata;
+import org.junit.Before;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.action.support.ActionTestUtils.assertNoFailureListener;
+import static org.elasticsearch.xpack.core.action.AbstractTransportSetUpgradeModeActionTests.clusterService;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+public class TransportSetTransformUpgradeModeActionTests extends ESTestCase {
+    private ClusterService clusterService;
+    private PersistentTasksService persistentTasksService;
+    private Client client;
+    private TransportSetTransformUpgradeModeAction action;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        clusterService = clusterService();
+        doAnswer(ans -> {
+            ClusterStateUpdateTask task = ans.getArgument(1);
+            task.clusterStateProcessed(ClusterState.EMPTY_STATE, ClusterState.EMPTY_STATE);
+            return null;
+        }).when(clusterService).submitUnbatchedStateUpdateTask(any(), any(ClusterStateUpdateTask.class));
+        when(clusterService.getClusterSettings()).thenReturn(ClusterSettings.createBuiltInClusterSettings());
+        PersistentTasksClusterService persistentTasksClusterService = new PersistentTasksClusterService(
+            Settings.EMPTY,
+            mock(),
+            clusterService,
+            mock()
+        );
+        persistentTasksService = mock();
+        client = mock();
+        ThreadPool threadPool = mock();
+        when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY));
+        when(client.threadPool()).thenReturn(threadPool);
+        action = new TransportSetTransformUpgradeModeAction(
+            mock(),
+            clusterService,
+            threadPool,
+            mock(),
+            mock(),
+            persistentTasksClusterService,
+            persistentTasksService,
+            client
+        );
+    }
+
+    public void testUpgradeMode() {
+        assertTrue(action.upgradeMode(state(true)));
+        assertFalse(action.upgradeMode(state(false)));
+    }
+
+    private ClusterState state(boolean upgradeMode) {
+        return ClusterState.EMPTY_STATE.copyAndUpdate(
+            u -> u.metadata(
+                ClusterState.EMPTY_STATE.metadata()
+                    .copyAndUpdate(
+                        b -> b.putCustom(
+                            TransformMetadata.TYPE,
+                            TransformMetadata.getTransformMetadata(ClusterState.EMPTY_STATE).builder().upgradeMode(upgradeMode).build()
+                        )
+                    )
+            )
+        );
+    }
+
+    public void testCreateUpdatedState() {
+        var updatedState = action.createUpdatedState(new SetUpgradeModeActionRequest(true), state(false));
+        assertTrue(TransformMetadata.upgradeMode(updatedState));
+
+        updatedState = action.createUpdatedState(new SetUpgradeModeActionRequest(false), state(true));
+        assertFalse(TransformMetadata.upgradeMode(updatedState));
+    }
+
+    public void testUpgradeModeWithNoMetadata() throws InterruptedException {
+        upgradeModeSuccessfullyChanged(ClusterState.EMPTY_STATE, assertNoFailureListener(r -> {
+            assertThat(r, is(AcknowledgedResponse.TRUE));
+            verifyNoInteractions(persistentTasksService);
+            verify(client, never()).admin();
+        }));
+    }
+
+    public void testUpgradeModeWithNoTasks() throws InterruptedException {
+        upgradeModeSuccessfullyChanged(
+            ClusterState.EMPTY_STATE.copyAndUpdateMetadata(
+                m -> m.putCustom(PersistentTasksCustomMetadata.TYPE, new PersistentTasksCustomMetadata(1, Map.of()))
+            ),
+            assertNoFailureListener(r -> {
+                assertThat(r, is(AcknowledgedResponse.TRUE));
+                verifyNoInteractions(persistentTasksService);
+                verify(client, never()).admin();
+            })
+        );
+    }
+
+    public void testUpgradeModeWithNoTransformTasks() throws InterruptedException {
+        upgradeModeSuccessfullyChanged(
+            ClusterState.EMPTY_STATE.copyAndUpdateMetadata(
+                m -> m.putCustom(
+                    PersistentTasksCustomMetadata.TYPE,
+                    new PersistentTasksCustomMetadata(1, Map.of("not a transform", mock()))
+                )
+            ),
+            assertNoFailureListener(r -> {
+                assertThat(r, is(AcknowledgedResponse.TRUE));
+                verifyNoInteractions(persistentTasksService);
+                verify(client, never()).admin();
+            })
+        );
+    }
+
+    private void upgradeModeSuccessfullyChanged(ClusterState state, ActionListener<AcknowledgedResponse> listener)
+        throws InterruptedException {
+        upgradeModeSuccessfullyChanged(new SetUpgradeModeActionRequest(true), state, listener);
+    }
+
+    private void upgradeModeSuccessfullyChanged(
+        SetUpgradeModeActionRequest request,
+        ClusterState state,
+        ActionListener<AcknowledgedResponse> listener
+    ) throws InterruptedException {
+        CountDownLatch latch = new CountDownLatch(1);
+        action.upgradeModeSuccessfullyChanged(mock(), request, state, ActionListener.runAfter(listener, latch::countDown));
+        assertTrue("Failed to finish test after 10s", latch.await(10, TimeUnit.SECONDS));
+    }
+
+    public void testEnableUpgradeMode() throws InterruptedException {
+        doAnswer(ans -> {
+            ActionListener<ListTasksResponse> listener = ans.getArgument(2);
+            ListTasksResponse response = mock();
+            when(response.getNodeFailures()).thenReturn(List.of());
+            listener.onResponse(response);
+            return null;
+        }).when(client).execute(any(), any(), any());
+
+        when(client.admin()).thenReturn(new AdminClient(client));
+
+        upgradeModeSuccessfullyChanged(stateWithTransformTask(), assertNoFailureListener(r -> {
+            assertThat(r, is(AcknowledgedResponse.TRUE));
+            verify(clusterService).submitUnbatchedStateUpdateTask(eq("unassign persistent task from any node"), any());
+        }));
+    }
+
+    private ClusterState stateWithTransformTask() {
+        PersistentTasksCustomMetadata.PersistentTask<?> task = mock();
+        when(task.getTaskName()).thenReturn(TransformField.TASK_NAME);
+        return ClusterState.EMPTY_STATE.copyAndUpdateMetadata(
+            m -> m.putCustom(PersistentTasksCustomMetadata.TYPE, new PersistentTasksCustomMetadata(1, Map.of("a transform", task)))
+        );
+    }
+
+    public void testDisableUpgradeMode() throws InterruptedException {
+        doAnswer(ans -> {
+            ActionListener<Boolean> listener = ans.getArgument(2);
+            listener.onResponse(true);
+            return null;
+        }).when(persistentTasksService).waitForPersistentTasksCondition(any(), any(), any());
+        upgradeModeSuccessfullyChanged(new SetUpgradeModeActionRequest(false), stateWithTransformTask(), assertNoFailureListener(r -> {
+            assertThat(r, is(AcknowledgedResponse.TRUE));
+            verify(clusterService, never()).submitUnbatchedStateUpdateTask(eq("unassign persistent task from any node"), any());
+        }));
+    }
+}
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerTests.java
index c8677c2816fc9..1fbe5c53caccb 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerTests.java
@@ -22,13 +22,18 @@
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.action.support.ActionTestUtils;
 import org.elasticsearch.client.internal.ParentTaskAssigningClient;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.CompositeBytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.search.SearchContextMissingException;
 import org.elasticsearch.search.SearchHit;
@@ -44,6 +49,7 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ActionNotFoundTransportException;
 import org.elasticsearch.xpack.core.indexing.IndexerState;
+import org.elasticsearch.xpack.core.transform.TransformMetadata;
 import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig;
 import org.elasticsearch.xpack.core.transform.transforms.SourceConfig;
 import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint;
@@ -53,6 +59,7 @@
 import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition;
 import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
 import org.elasticsearch.xpack.core.transform.transforms.TransformProgress;
+import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState;
 import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
 import org.elasticsearch.xpack.transform.TransformExtension;
 import org.elasticsearch.xpack.transform.TransformNode;
@@ -77,6 +84,10 @@
 import java.util.function.Consumer;
 import java.util.stream.IntStream;
 
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -467,6 +478,53 @@ public void testHandlePitIndexNotFound() throws InterruptedException {
         }
     }
 
+    public void testIndexBlocked() {
+        var service = serviceWithBlockCheck(true);
+        var context = new TransformContext(TransformTaskState.STARTED, "", 0, mock());
+
+        var indexer = createTestIndexer(mock(), service, resolver(), context);
+        context.setIsWaitingForIndexToUnblock(true);
+
+        assertFalse(indexer.maybeTriggerAsyncJob(Instant.now().toEpochMilli()));
+        assertTrue(context.isWaitingForIndexToUnblock());
+    }
+
+    public void testIndexUnblocked() {
+        var service = serviceWithBlockCheck(false);
+        // set state to failed so that TransformIndexer returns false
+        var context = new TransformContext(TransformTaskState.FAILED, "", 0, mock());
+
+        var indexer = createTestIndexer(mock(), service, resolver(), context);
+        context.setIsWaitingForIndexToUnblock(true);
+
+        assertFalse(indexer.maybeTriggerAsyncJob(Instant.now().toEpochMilli()));
+        // ClientTransformIndexer's maybeTriggerAsyncJob should reset isWaitingForIndexToUnblock to false
+        assertFalse(context.isWaitingForIndexToUnblock());
+    }
+
+    private ClusterService serviceWithBlockCheck(boolean checkResponse) {
+        var clusterBlocks = mock(ClusterBlocks.class);
+        when(clusterBlocks.indexBlocked(eq(ClusterBlockLevel.WRITE), anyString())).thenReturn(checkResponse);
+        var metadata = mock(Metadata.class);
+        when(metadata.custom(eq(TransformMetadata.TYPE))).thenReturn(TransformMetadata.EMPTY_METADATA);
+        var clusterState = mock(ClusterState.class);
+        when(clusterState.blocks()).thenReturn(clusterBlocks);
+        when(clusterState.getMetadata()).thenReturn(metadata);
+        var clusterService = mock(ClusterService.class);
+        when(clusterService.state()).thenReturn(clusterState);
+        return clusterService;
+    }
+
+    private IndexNameExpressionResolver resolver() {
+        var resolver = mock(IndexNameExpressionResolver.class);
+        when(resolver.concreteWriteIndex(any(), any(), any(), anyBoolean(), anyBoolean())).thenAnswer(ans -> {
+            Index destIndex = mock();
+            when(destIndex.getName()).thenReturn(ans.getArgument(2));
+            return destIndex;
+        });
+        return resolver;
+    }
+
     private static class MockClientTransformIndexer extends ClientTransformIndexer {
 
         MockClientTransformIndexer(
@@ -627,13 +685,22 @@ private ClientTransformIndexer createTestIndexer() {
     }
 
     private ClientTransformIndexer createTestIndexer(ParentTaskAssigningClient client) {
+        return createTestIndexer(client, mock(), mock(), mock(TransformContext.class));
+    }
+
+    private ClientTransformIndexer createTestIndexer(
+        ParentTaskAssigningClient client,
+        ClusterService service,
+        IndexNameExpressionResolver resolver,
+        TransformContext context
+    ) {
         ThreadPool threadPool = mock(ThreadPool.class);
         when(threadPool.executor("generic")).thenReturn(mock(ExecutorService.class));
 
         return new ClientTransformIndexer(
             mock(ThreadPool.class),
-            mock(ClusterService.class),
-            mock(IndexNameExpressionResolver.class),
+            service,
+            resolver,
             mock(TransformExtension.class),
             new TransformServices(
                 mock(IndexBasedTransformConfigManager.class),
@@ -652,7 +719,7 @@ private ClientTransformIndexer createTestIndexer(ParentTaskAssigningClient clien
             new TransformCheckpoint("transform", Instant.now().toEpochMilli(), 0L, Collections.emptyMap(), Instant.now().toEpochMilli()),
             new TransformCheckpoint("transform", Instant.now().toEpochMilli(), 2L, Collections.emptyMap(), Instant.now().toEpochMilli()),
             new SeqNoPrimaryTermAndIndex(1, 1, TransformInternalIndexConstants.LATEST_INDEX_NAME),
-            mock(TransformContext.class),
+            context,
             false
         );
     }
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandlerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandlerTests.java
index 3894ff3043ccd..9f16e0d078b3c 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandlerTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandlerTests.java
@@ -93,7 +93,7 @@ public void testHandleIndexerFailure_BulkIndexExceptionWrappingClusterBlockExcep
             randomBoolean()
         );
 
-        List.of(true, false).forEach((unattended) -> { assertRetryFailureCountNotIncremented(bulkIndexingException, unattended); });
+        List.of(true, false).forEach((unattended) -> { assertClusterBlockHandled(bulkIndexingException, unattended); });
     }
 
     public void testHandleIndexerFailure_IrrecoverableBulkIndexException() {
@@ -197,7 +197,7 @@ private void assertRetry(Exception e, boolean unattended) {
         }
     }
 
-    private void assertRetryFailureCountNotIncremented(Exception e, boolean unattended) {
+    private void assertClusterBlockHandled(Exception e, boolean unattended) {
         String transformId = randomAlphaOfLength(10);
         SettingsConfig settings = new SettingsConfig.Builder().setNumFailureRetries(2).setUnattended(unattended).build();
 
@@ -211,6 +211,7 @@ private void assertRetryFailureCountNotIncremented(Exception e, boolean unattend
         assertNoFailure(handler, e, contextListener, settings, false);
         assertNoFailure(handler, e, contextListener, settings, false);
         assertNoFailure(handler, e, contextListener, settings, false);
+        assertTrue(context.isWaitingForIndexToUnblock());
     }
 
     private void assertFailure(Exception e) {
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java
index 70ded62b4c7d4..fa509143f9ba9 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java
@@ -48,6 +48,7 @@
 import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
 import org.elasticsearch.xpack.transform.DefaultTransformExtension;
 import org.elasticsearch.xpack.transform.Transform;
+import org.elasticsearch.xpack.transform.TransformConfigAutoMigration;
 import org.elasticsearch.xpack.transform.TransformNode;
 import org.elasticsearch.xpack.transform.TransformServices;
 import org.elasticsearch.xpack.transform.checkpoint.TransformCheckpointService;
@@ -57,6 +58,7 @@
 import org.elasticsearch.xpack.transform.persistence.TransformInternalIndexTests;
 import org.elasticsearch.xpack.transform.transforms.scheduling.TransformScheduler;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.time.Clock;
@@ -80,6 +82,7 @@
 
 public class TransformPersistentTasksExecutorTests extends ESTestCase {
     private static ThreadPool threadPool;
+    private TransformConfigAutoMigration autoMigration;
 
     @BeforeClass
     public static void setUpThreadPool() {
@@ -102,6 +105,16 @@ public static void tearDownThreadPool() {
         terminate(threadPool);
     }
 
+    @Before
+    public void initMocks() {
+        autoMigration = mock();
+        doAnswer(ans -> {
+            ActionListener<?> listener = ans.getArgument(1);
+            listener.onResponse(ans.getArgument(0));
+            return null;
+        }).when(autoMigration).migrateAndSave(any(), any());
+    }
+
     public void testNodeVersionAssignment() {
         DiscoveryNodes.Builder nodes = buildNodes(false, true, true, true, true);
         ClusterState cs = buildClusterState(nodes);
@@ -609,7 +622,8 @@ private TransformPersistentTasksExecutor buildTaskExecutor(TransformServices tra
             clusterService(),
             Settings.EMPTY,
             new DefaultTransformExtension(),
-            TestIndexNameExpressionResolver.newInstance()
+            TestIndexNameExpressionResolver.newInstance(),
+            autoMigration
         );
     }
 
diff --git a/x-pack/plugin/vector-tile/build.gradle b/x-pack/plugin/vector-tile/build.gradle
index 5855757b417fe..4ff3240d239e8 100644
--- a/x-pack/plugin/vector-tile/build.gradle
+++ b/x-pack/plugin/vector-tile/build.gradle
@@ -16,9 +16,9 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 esplugin {
-  name 'vector-tile'
-  description 'A plugin for mapbox vector tile features'
-  classname 'org.elasticsearch.xpack.vectortile.VectorTilePlugin'
+  name = 'vector-tile'
+  description = 'A plugin for mapbox vector tile features'
+  classname ='org.elasticsearch.xpack.vectortile.VectorTilePlugin'
   extendedPlugins = ['spatial']
 }
 
diff --git a/x-pack/plugin/voting-only-node/build.gradle b/x-pack/plugin/voting-only-node/build.gradle
index 60dd79ef7f6bf..1202f90977da7 100644
--- a/x-pack/plugin/voting-only-node/build.gradle
+++ b/x-pack/plugin/voting-only-node/build.gradle
@@ -1,9 +1,9 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 esplugin {
-  name 'x-pack-voting-only-node'
-  description 'Elasticsearch Expanded Pack Plugin - Voting-only node'
-  classname 'org.elasticsearch.cluster.coordination.votingonly.VotingOnlyNodePlugin'
+  name = 'x-pack-voting-only-node'
+  description = 'Elasticsearch Expanded Pack Plugin - Voting-only node'
+  classname ='org.elasticsearch.cluster.coordination.votingonly.VotingOnlyNodePlugin'
   extendedPlugins = ['x-pack-core']
 }
 
diff --git a/x-pack/plugin/watcher/build.gradle b/x-pack/plugin/watcher/build.gradle
index edf767fad675c..ce7c8e5d6311a 100644
--- a/x-pack/plugin/watcher/build.gradle
+++ b/x-pack/plugin/watcher/build.gradle
@@ -1,11 +1,11 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 esplugin {
-  name 'x-pack-watcher'
-  description 'Elasticsearch Expanded Pack Plugin - Watcher'
-  classname 'org.elasticsearch.xpack.watcher.Watcher'
-  hasNativeController false
-  requiresKeystore false
+  name = 'x-pack-watcher'
+  description = 'Elasticsearch Expanded Pack Plugin - Watcher'
+  classname ='org.elasticsearch.xpack.watcher.Watcher'
+  hasNativeController =false
+  requiresKeystore =false
   extendedPlugins = ['x-pack-core', 'lang-painless']
 }
 
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java
index ffac36846414e..ea9d76fb643fb 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java
@@ -172,7 +172,8 @@ public void testExceptionMapping() throws Exception {
 
         // ensure that enabled is set to false
         List<Boolean> indexed = new ArrayList<>();
-        GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(HistoryStoreField.INDEX_PREFIX + "*").get();
+        GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, HistoryStoreField.INDEX_PREFIX + "*")
+            .get();
         for (MappingMetadata mapping : mappingsResponse.getMappings().values()) {
             Map<String, Object> docMapping = mapping.getSourceAsMap();
             if (abortAtInput) {
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java
index c78315383df2f..34d38dfde064d 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java
@@ -46,7 +46,7 @@ public void testTimeFields() throws Exception {
 
         assertWatchWithMinimumActionsCount("_id", ExecutionState.EXECUTED, 1);
         assertBusy(() -> {
-            GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings()
+            GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT)
                 .setIndicesOptions(IndicesOptions.strictExpandHidden())
                 .get();
             assertThat(mappingsResponse, notNullValue());
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java
index 5eaf27e7b2670..626e1e1e9e9d5 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java
@@ -280,9 +280,12 @@ public void replaceWatcherIndexWithRandomlyNamedIndex(String originalIndexOrAlia
         ensureGreen(to);
 
         AtomicReference<String> originalIndex = new AtomicReference<>(originalIndexOrAlias);
-        boolean watchesIsAlias = indicesAdmin().prepareGetAliases(originalIndexOrAlias).get().getAliases().isEmpty() == false;
+        boolean watchesIsAlias = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, originalIndexOrAlias)
+            .get()
+            .getAliases()
+            .isEmpty() == false;
         if (watchesIsAlias) {
-            GetAliasesResponse aliasesResponse = indicesAdmin().prepareGetAliases(originalIndexOrAlias).get();
+            GetAliasesResponse aliasesResponse = indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, originalIndexOrAlias).get();
             assertEquals(1, aliasesResponse.getAliases().size());
             aliasesResponse.getAliases().entrySet().forEach((aliasRecord) -> {
                 assertEquals(1, aliasRecord.getValue().size());
@@ -510,7 +513,7 @@ private void ensureWatcherTemplatesAdded() throws Exception {
         assertBusy(() -> {
             GetComposableIndexTemplateAction.Response response = client().execute(
                 GetComposableIndexTemplateAction.INSTANCE,
-                new GetComposableIndexTemplateAction.Request(HISTORY_TEMPLATE_NAME)
+                new GetComposableIndexTemplateAction.Request(TEST_REQUEST_TIMEOUT, HISTORY_TEMPLATE_NAME)
             ).get();
             assertThat("[" + HISTORY_TEMPLATE_NAME + "] is missing", response.indexTemplates().size(), equalTo(1));
         });
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java
index 1bcdd060994ce..732b0800b4316 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java
@@ -120,7 +120,7 @@ public void testFailedInputResultWithDotsInFieldNameGetsStored() throws Exceptio
         assertBusy(() -> { assertHitCount(getWatchHistory(), 1); });
 
         // as fields with dots are allowed in 5.0 again, the mapping must be checked in addition
-        GetMappingsResponse response = indicesAdmin().prepareGetMappings(".watcher-history*").get();
+        GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, ".watcher-history*").get();
         XContentSource source = new XContentSource(
             response.getMappings().values().iterator().next().source().uncompressed(),
             XContentType.JSON
@@ -162,7 +162,7 @@ public void testPayloadInputWithDotsInFieldNameWorks() throws Exception {
         assertBusy(() -> { assertHitCount(getWatchHistory(), 1); });
 
         // as fields with dots are allowed in 5.0 again, the mapping must be checked in addition
-        GetMappingsResponse response = indicesAdmin().prepareGetMappings(".watcher-history*").get();
+        GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, ".watcher-history*").get();
         XContentSource source = new XContentSource(
             response.getMappings().values().iterator().next().source().uncompressed(),
             XContentType.JSON
@@ -225,7 +225,7 @@ public void testThatHistoryContainsStatus() throws Exception {
 
         assertBusy(() -> {
             // also ensure that the status field is disabled in the watch history
-            GetMappingsResponse response = indicesAdmin().prepareGetMappings(".watcher-history*").get();
+            GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, ".watcher-history*").get();
             XContentSource mappingSource = new XContentSource(
                 response.getMappings().values().iterator().next().source().uncompressed(),
                 XContentType.JSON
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/condition/WatcherConditionScript.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/condition/WatcherConditionScript.java
index 724ba58c7e996..d1a2e1c846586 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/condition/WatcherConditionScript.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/condition/WatcherConditionScript.java
@@ -43,7 +43,7 @@ public interface Factory {
         WatcherConditionScript newInstance(Map<String, Object> params, WatchExecutionContext watcherContext);
     }
 
-    public static ScriptContext<Factory> CONTEXT = new ScriptContext<>(
+    public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>(
         "watcher_condition",
         Factory.class,
         200,
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/DataAttachment.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/DataAttachment.java
index d22a608e131f9..7ce1b9ca8b60a 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/DataAttachment.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/DataAttachment.java
@@ -56,7 +56,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
     };
 
-    public static DataAttachment DEFAULT = YAML;
+    public static final DataAttachment DEFAULT = YAML;
 
     public abstract String contentType();
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/script/WatcherTransformScript.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/script/WatcherTransformScript.java
index 93e3eefb4aa26..d8c079e1cb00b 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/script/WatcherTransformScript.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/script/WatcherTransformScript.java
@@ -44,7 +44,7 @@ public interface Factory {
         WatcherTransformScript newInstance(Map<String, Object> params, WatchExecutionContext watcherContext, Payload payload);
     }
 
-    public static ScriptContext<Factory> CONTEXT = new ScriptContext<>(
+    public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>(
         "watcher_transform",
         Factory.class,
         200,
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailSslTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailSslTests.java
index b00c54bf500fd..50101d08b711e 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailSslTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailSslTests.java
@@ -38,6 +38,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import javax.mail.MessagingException;
 import javax.mail.internet.MimeMessage;
@@ -74,7 +75,6 @@ public void stopSmtpServer() {
     }
 
     public void testFailureSendingMessageToSmtpServerWithUntrustedCertificateAuthority() throws Exception {
-        assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm());
         final Settings.Builder settings = Settings.builder();
         final MockSecureSettings secureSettings = new MockSecureSettings();
         final ExecutableEmailAction emailAction = buildEmailAction(settings, secureSettings);
@@ -84,7 +84,14 @@ public void testFailureSendingMessageToSmtpServerWithUntrustedCertificateAuthori
             () -> emailAction.execute("my_action_id", ctx, Payload.EMPTY)
         );
         final List<Throwable> allCauses = getAllCauses(exception);
-        assertThat(allCauses, Matchers.hasItem(Matchers.instanceOf(SSLException.class)));
+        if (inFipsJvm()) {
+            assertThat(
+                allCauses.stream().map(c -> c.getClass().getCanonicalName()).collect(Collectors.toSet()),
+                Matchers.hasItem("org.bouncycastle.tls.TlsFatalAlert")
+            );
+        } else {
+            assertThat(allCauses, Matchers.hasItem(Matchers.instanceOf(SSLException.class)));
+        }
     }
 
     public void testCanSendMessageToSmtpServerUsingTrustStore() throws Exception {
@@ -149,7 +156,6 @@ public void testCanSendMessageToSmtpServerUsingSmtpSslTrust() throws Exception {
      * over the account level "smtp.ssl.trust" setting) but smtp.ssl.trust was ignored for a period of time (see #52153)
      * so this is the least breaking way to resolve that.
      */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/49094")
     public void testNotificationSslSettingsOverrideSmtpSslTrust() throws Exception {
         List<MimeMessage> messages = new ArrayList<>();
         server.addListener(messages::add);
@@ -165,9 +171,16 @@ public void testNotificationSslSettingsOverrideSmtpSslTrust() throws Exception {
                 MessagingException.class,
                 () -> emailAction.execute("my_action_id", ctx, Payload.EMPTY)
             );
-
             final List<Throwable> allCauses = getAllCauses(exception);
-            assertThat(allCauses, Matchers.hasItem(Matchers.instanceOf(SSLException.class)));
+            if (inFipsJvm()) {
+                assertThat(
+                    allCauses.stream().map(c -> c.getClass().getCanonicalName()).collect(Collectors.toSet()),
+                    Matchers.hasItem("org.bouncycastle.tls.TlsFatalAlert")
+                );
+            } else {
+                assertThat(allCauses, Matchers.hasItem(Matchers.instanceOf(SSLException.class)));
+            }
+
         } finally {
             server.clearListeners();
         }
diff --git a/x-pack/plugin/wildcard/build.gradle b/x-pack/plugin/wildcard/build.gradle
index 1a4f133402582..d87a461eaf858 100644
--- a/x-pack/plugin/wildcard/build.gradle
+++ b/x-pack/plugin/wildcard/build.gradle
@@ -9,9 +9,9 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
 esplugin {
-  name 'wildcard'
-  description 'A plugin for a keyword field type with efficient wildcard search'
-  classname 'org.elasticsearch.xpack.wildcard.Wildcard'
+  name = 'wildcard'
+  description = 'A plugin for a keyword field type with efficient wildcard search'
+  classname = 'org.elasticsearch.xpack.wildcard.Wildcard'
   extendedPlugins = ['x-pack-core', 'lang-painless']
 }
 
diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
index f3b01bb898126..f87c73273a7ac 100644
--- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
+++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
@@ -94,7 +94,7 @@
 public class WildcardFieldMapper extends FieldMapper {
 
     public static final String CONTENT_TYPE = "wildcard";
-    public static short MAX_CLAUSES_IN_APPROXIMATION_QUERY = 10;
+    public static final short MAX_CLAUSES_IN_APPROXIMATION_QUERY = 10;
     public static final int NGRAM_SIZE = 3;
 
     static final NamedAnalyzer WILDCARD_ANALYZER_7_10 = new NamedAnalyzer("_wildcard_7_10", AnalyzerScope.GLOBAL, new Analyzer() {
@@ -252,7 +252,7 @@ public WildcardFieldMapper build(MapperBuilderContext context) {
         }
     }
 
-    public static TypeParser PARSER = new TypeParser(
+    public static final TypeParser PARSER = new TypeParser(
         (n, c) -> new Builder(n, IGNORE_ABOVE_SETTING.get(c.getSettings()), c.indexVersionCreated())
     );
 
diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
index a1a01ebdcc590..0b31e96ece84a 100644
--- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
+++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java
@@ -51,6 +51,7 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -106,6 +107,7 @@ static SearchExecutionContext createMockSearchExecutionContext(boolean allowExpe
 
     static final int MAX_FIELD_LENGTH = 30;
     static WildcardFieldMapper wildcardFieldType;
+    static WildcardFieldMapper wildcardFieldType79;
     static KeywordFieldMapper keywordFieldType;
     private DirectoryReader rewriteReader;
     private BaseDirectoryWrapper rewriteDir;
@@ -127,6 +129,9 @@ public void setUp() throws Exception {
         builder.ignoreAbove(MAX_FIELD_LENGTH);
         wildcardFieldType = builder.build(MapperBuilderContext.root(false, false));
 
+        Builder builder79 = new WildcardFieldMapper.Builder(WILDCARD_FIELD_NAME, IndexVersions.V_7_9_0);
+        wildcardFieldType79 = builder79.build(MapperBuilderContext.root(false, false));
+
         org.elasticsearch.index.mapper.KeywordFieldMapper.Builder kwBuilder = new KeywordFieldMapper.Builder(
             KEYWORD_FIELD_NAME,
             IndexVersion.current()
@@ -207,6 +212,37 @@ public void testIgnoreAbove() throws IOException {
         assertTrue(fields.stream().anyMatch(field -> "field".equals(field.stringValue())));
     }
 
+    public void testBWCIndexVersion() throws IOException {
+        // Create old format index using wildcard ngram analyzer used in 7.9 launch
+        Directory dir = newDirectory();
+        IndexWriterConfig iwc = newIndexWriterConfig(WildcardFieldMapper.WILDCARD_ANALYZER_7_9);
+        iwc.setMergePolicy(newTieredMergePolicy(random()));
+        RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+
+        Document doc = new Document();
+        LuceneDocument parseDoc = new LuceneDocument();
+        addFields(parseDoc, doc, "a b");
+        indexDoc(parseDoc, doc, iw);
+
+        iw.forceMerge(1);
+        DirectoryReader reader = iw.getReader();
+        IndexSearcher searcher = newSearcher(reader);
+        iw.close();
+
+        // Unnatural circumstance - testing we fail if we were to use the new analyzer on old index
+        Query oldWildcardFieldQuery = wildcardFieldType.fieldType().wildcardQuery("a b", null, null);
+        TopDocs oldWildcardFieldTopDocs = searcher.search(oldWildcardFieldQuery, 10, Sort.INDEXORDER);
+        assertThat(oldWildcardFieldTopDocs.totalHits.value(), equalTo(0L));
+
+        // Natural circumstance test we revert to the old analyzer for old indices
+        Query wildcardFieldQuery = wildcardFieldType79.fieldType().wildcardQuery("a b", null, null);
+        TopDocs wildcardFieldTopDocs = searcher.search(wildcardFieldQuery, 10, Sort.INDEXORDER);
+        assertThat(wildcardFieldTopDocs.totalHits.value(), equalTo(1L));
+
+        reader.close();
+        dir.close();
+    }
+
     // Test long query strings don't cause exceptions
     public void testTooBigQueryField() throws IOException {
         Directory dir = newDirectory();
diff --git a/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/30_synthetic_source.yml b/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/30_synthetic_source.yml
index 14e4a6f5aaef8..4eaf89d0bf67a 100644
--- a/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/30_synthetic_source.yml
+++ b/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/30_synthetic_source.yml
@@ -5,10 +5,6 @@ setup:
 
 ---
 synthetic source:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"]
-      reason: requires synthetic source support
-
   - do:
       indices.create:
         index: synthetic_source_test
@@ -47,10 +43,6 @@ synthetic source:
 
 ---
 synthetic source with copy_to:
-  - requires:
-      cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"]
-      reason: requires copy_to support in synthetic source
-
   - do:
       indices.create:
         index: synthetic_source_test
diff --git a/x-pack/plugin/write-load-forecaster/build.gradle b/x-pack/plugin/write-load-forecaster/build.gradle
index 8a3db5830e83f..3eecec6363a3f 100644
--- a/x-pack/plugin/write-load-forecaster/build.gradle
+++ b/x-pack/plugin/write-load-forecaster/build.gradle
@@ -3,7 +3,7 @@ apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
   name = 'x-pack-write-load-forecaster'
-  description 'x'
+  description = 'x'
   classname = 'org.elasticsearch.xpack.writeloadforecaster.WriteLoadForecasterPlugin'
   extendedPlugins = ['x-pack-core']
 }
diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
index fe4c1c20c69c4..516dd4759861f 100644
--- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
@@ -209,7 +209,6 @@ public void testWatcher() throws Exception {
     }
 
     @SuppressWarnings("unchecked")
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/84700")
     public void testWatcherWithApiKey() throws Exception {
         final Request getWatchStatusRequest = new Request("GET", "/_watcher/watch/watch_with_api_key");
 
diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/fulfilling_cluster/10_basic.yml b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/fulfilling_cluster/10_basic.yml
index f3861264d6056..45faa23becbb9 100644
--- a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/fulfilling_cluster/10_basic.yml
+++ b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/fulfilling_cluster/10_basic.yml
@@ -188,8 +188,6 @@ setup:
   - do:
       indices.close:
         index: closed_index
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
         indices.create:
diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml
index 1b435c551fbe9..41bc0a96e737e 100644
--- a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml
+++ b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml
@@ -92,8 +92,6 @@ teardown:
 
   - do:
       catch: bad_request
-      allowed_warnings:
-        - "Line 1:21: Square brackets '[]' need to be removed in FROM METADATA declaration"
       headers: { Authorization: "Basic am9lOnMza3JpdC1wYXNzd29yZA==" }
       esql.query:
         body:
@@ -104,12 +102,10 @@ teardown:
 
   - do:
       catch: bad_request
-      allowed_warnings:
-        - "Line 1:21: Square brackets '[]' need to be removed in FROM METADATA declaration"
       headers: { Authorization: "Basic am9lOnMza3JpdC1wYXNzd29yZA==" }
       esql.query:
         body:
-          query: 'FROM *:esql*,esql_* [METADATA _index] | sort cost | KEEP _index, tag, cost | LIMIT 10'
+          query: 'FROM *:esql*,esql_* METADATA _index | sort cost | KEEP _index, tag, cost | LIMIT 10'
           filter:
             range:
               since:
diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/src/test/resources/rest-api-spec/test/fulfilling_cluster/10_basic_with_dls_fls.yml b/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/src/test/resources/rest-api-spec/test/fulfilling_cluster/10_basic_with_dls_fls.yml
index 6a54396fb46b2..5489ac3217521 100644
--- a/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/src/test/resources/rest-api-spec/test/fulfilling_cluster/10_basic_with_dls_fls.yml
+++ b/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/src/test/resources/rest-api-spec/test/fulfilling_cluster/10_basic_with_dls_fls.yml
@@ -167,8 +167,6 @@ setup:
   - do:
       indices.close:
         index: closed_index
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       indices.create:
diff --git a/x-pack/qa/repository-old-versions-compatibility/build.gradle b/x-pack/qa/repository-old-versions-compatibility/build.gradle
new file mode 100644
index 0000000000000..37e5eea85a08b
--- /dev/null
+++ b/x-pack/qa/repository-old-versions-compatibility/build.gradle
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.internal-test-artifact'
+apply plugin: 'elasticsearch.bwc-test'
+
+buildParams.bwcVersions.withLatestReadOnlyIndexCompatible { bwcVersion ->
+  tasks.named("javaRestTest").configure {
+    systemProperty("tests.minimum.index.compatible", bwcVersion)
+    usesBwcDistribution(bwcVersion)
+    enabled = true
+  }
+}
+
+tasks.withType(Test).configureEach {
+  // CI doesn't like it when there's multiple clusters running at once
+  maxParallelForks = 1
+}
+
diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java
new file mode 100644
index 0000000000000..be9f98406bc96
--- /dev/null
+++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java
@@ -0,0 +1,231 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.oldrepos;
+
+import com.carrotsearch.randomizedtesting.TestMethodAndParams;
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
+
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.repositories.fs.FsRepository;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.util.Version;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestRule;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import static org.elasticsearch.test.cluster.util.Version.CURRENT;
+import static org.elasticsearch.test.cluster.util.Version.fromString;
+import static org.elasticsearch.test.rest.ObjectPath.createFromResponse;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+@TestCaseOrdering(AbstractUpgradeCompatibilityTestCase.TestCaseOrdering.class)
+public abstract class AbstractUpgradeCompatibilityTestCase extends ESRestTestCase {
+
+    protected static final Version VERSION_MINUS_2 = fromString(System.getProperty("tests.minimum.index.compatible"));
+    protected static final Version VERSION_MINUS_1 = fromString(System.getProperty("tests.minimum.wire.compatible"));
+    protected static final Version VERSION_CURRENT = CURRENT;
+
+    protected static TemporaryFolder REPOSITORY_PATH = new TemporaryFolder();
+
+    protected static LocalClusterConfigProvider clusterConfig = c -> {};
+    private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .version(VERSION_MINUS_1)
+        .nodes(2)
+        .setting("xpack.security.enabled", "false")
+        .setting("xpack.ml.enabled", "false")
+        .setting("path.repo", () -> REPOSITORY_PATH.getRoot().getPath())
+        .apply(() -> clusterConfig)
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(REPOSITORY_PATH).around(cluster);
+
+    private static boolean upgradeFailed = false;
+
+    private final Version clusterVersion;
+    private final String indexCreatedVersion;
+    private final Consumer<List<String>> warningsConsumer;
+
+    public AbstractUpgradeCompatibilityTestCase(
+        @Name("cluster") Version clusterVersion,
+        String indexCreatedVersion,
+        Consumer<List<String>> warningsConsumer
+    ) {
+        this.clusterVersion = clusterVersion;
+        this.indexCreatedVersion = indexCreatedVersion;
+        this.warningsConsumer = warningsConsumer;
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        return Stream.of(VERSION_MINUS_1, CURRENT).map(v -> new Object[] { v }).toList();
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected boolean preserveClusterUponCompletion() {
+        return true;
+    }
+
+    /**
+     * This method verifies the currentVersion against the clusterVersion and performs a "full cluster restart" upgrade if the current
+     * is before clusterVersion. The cluster version is fetched externally and is controlled by the gradle setup.
+     *
+     * @throws Exception
+     */
+    @Before
+    public void maybeUpgrade() throws Exception {
+        // We want to use this test suite for the V9 upgrade, but we are not fully committed to necessarily having N-2 support
+        // in V10, so we add a check here to ensure we'll revisit this decision once V10 exists.
+        assertThat("Explicit check that N-2 version is Elasticsearch 7", VERSION_MINUS_2.getMajor(), equalTo(7));
+
+        var currentVersion = clusterVersion();
+        if (currentVersion.before(clusterVersion)) {
+            try {
+                cluster.upgradeToVersion(clusterVersion);
+                closeClients();
+                initClient();
+            } catch (Exception e) {
+                upgradeFailed = true;
+                throw e;
+            }
+        }
+
+        // Skip remaining tests if upgrade failed
+        assumeFalse("Cluster upgrade failed", upgradeFailed);
+    }
+
+    protected static Version clusterVersion() throws Exception {
+        var response = assertOK(client().performRequest(new Request("GET", "/")));
+        var responseBody = createFromResponse(response);
+        var version = Version.fromString(responseBody.evaluate("version.number").toString());
+        assertThat("Failed to retrieve cluster version", version, notNullValue());
+        return version;
+    }
+
+    /**
+     * Execute the test suite with the parameters provided by the {@link #parameters()} in version order.
+     */
+    public static class TestCaseOrdering implements Comparator<TestMethodAndParams> {
+        @Override
+        public int compare(TestMethodAndParams o1, TestMethodAndParams o2) {
+            var version1 = (Version) o1.getInstanceArguments().get(0);
+            var version2 = (Version) o2.getInstanceArguments().get(0);
+            return version1.compareTo(version2);
+        }
+    }
+
+    protected final void verifyCompatibility(String version, Consumer<List<String>> warningsConsumer) throws Exception {
+        final String repository = "repository";
+        final String snapshot = "snapshot";
+        final String index = "index";
+        final int numDocs = 5;
+
+        String repositoryPath = REPOSITORY_PATH.getRoot().getPath();
+
+        if (VERSION_MINUS_1.equals(clusterVersion())) {
+            assertEquals(VERSION_MINUS_1, clusterVersion());
+            assertTrue(getIndices(client()).isEmpty());
+
+            // Copy a snapshot of an index with 5 documents
+            copySnapshotFromResources(repositoryPath, version);
+            registerRepository(client(), repository, FsRepository.TYPE, true, Settings.builder().put("location", repositoryPath).build());
+            recover(client(), repository, snapshot, index, warningsConsumer);
+
+            assertTrue(getIndices(client()).contains(index));
+            assertDocCount(client(), index, numDocs);
+
+            return;
+        }
+
+        if (VERSION_CURRENT.equals(clusterVersion())) {
+            assertEquals(VERSION_CURRENT, clusterVersion());
+            assertTrue(getIndices(client()).contains(index));
+            assertDocCount(client(), index, numDocs);
+        }
+    }
+
+    protected abstract void recover(
+        RestClient restClient,
+        String repository,
+        String snapshot,
+        String index,
+        Consumer<List<String>> warningsConsumer
+    ) throws Exception;
+
+    private static String getIndices(RestClient client) throws IOException {
+        final Request request = new Request("GET", "_cat/indices");
+        Response response = client.performRequest(request);
+        return EntityUtils.toString(response.getEntity());
+    }
+
+    private static void copySnapshotFromResources(String repositoryPath, String version) throws IOException, URISyntaxException {
+        Path zipFilePath = Paths.get(
+            Objects.requireNonNull(AbstractUpgradeCompatibilityTestCase.class.getClassLoader().getResource("snapshot_v" + version + ".zip"))
+                .toURI()
+        );
+        unzip(zipFilePath, Paths.get(repositoryPath));
+    }
+
+    private static void unzip(Path zipFilePath, Path outputDir) throws IOException {
+        try (ZipInputStream zipIn = new ZipInputStream(Files.newInputStream(zipFilePath))) {
+            ZipEntry entry;
+            while ((entry = zipIn.getNextEntry()) != null) {
+                Path outputPath = outputDir.resolve(entry.getName());
+                if (entry.isDirectory()) {
+                    Files.createDirectories(outputPath);
+                } else {
+                    Files.createDirectories(outputPath.getParent());
+                    try (OutputStream out = Files.newOutputStream(outputPath)) {
+                        byte[] buffer = new byte[1024];
+                        int len;
+                        while ((len = zipIn.read(buffer)) > 0) {
+                            out.write(buffer, 0, len);
+                        }
+                    }
+                }
+                zipIn.closeEntry();
+            }
+        }
+    }
+
+    public final void testArchiveIndex() throws Exception {
+        verifyCompatibility(indexCreatedVersion, warningsConsumer);
+    }
+}
diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/TestSnapshotCases.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/TestSnapshotCases.java
new file mode 100644
index 0000000000000..da1984c482670
--- /dev/null
+++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/TestSnapshotCases.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.oldrepos;
+
+/**
+* Snapshot zip file names for archive and searchable snapshot tests
+*/
+public final class TestSnapshotCases {
+    // Index created in vES_5 - Basic mapping
+    public static final String ES_VERSION_5 = "5";
+
+    // Index created in vES_5 - Custom-Analyzer - standard token filter
+    public static final String ES_VERSION_5_STANDARD_TOKEN_FILTER = "5_custom_analyzer";
+
+    // Index created in vES_5 - Basic mapping
+    public static final String ES_VERSION_6 = "5";
+
+    // Index created in vES_5 - Custom-Analyzer - standard token filter
+    public static final String ES_VERSION_6_STANDARD_TOKEN_FILTER = "5_custom_analyzer";
+
+}
diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/ArchiveIndexTestCase.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/ArchiveIndexTestCase.java
new file mode 100644
index 0000000000000..d3704937fd912
--- /dev/null
+++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/ArchiveIndexTestCase.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.oldrepos.archiveindex;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.WarningsHandler;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.oldrepos.AbstractUpgradeCompatibilityTestCase;
+import org.elasticsearch.test.cluster.util.Version;
+
+import java.util.List;
+import java.util.function.Consumer;
+
+/**
+ * Test suite for Archive indices backward compatibility with N-2 versions.
+ * The test suite creates a cluster in the N-1 version, where N is the current version.
+ * Restores snapshots from old-clusters (version 5/6) and upgrades it to the current version.
+ * Test methods are executed after each upgrade.
+ *
+ * For example the test suite creates a cluster of version 8, then restores a snapshot of an index created
+ * when deployed ES version 5/6. The cluster then upgrades to version 9, verifying that the archive index
+ * is successfully restored.
+ */
+abstract class ArchiveIndexTestCase extends AbstractUpgradeCompatibilityTestCase {
+
+    static {
+        clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial");
+    }
+
+    protected ArchiveIndexTestCase(Version version, String indexCreatedVersion) {
+        this(version, indexCreatedVersion, o -> {});
+    }
+
+    protected ArchiveIndexTestCase(Version version, String indexCreatedVersion, Consumer<List<String>> consumer) {
+        super(version, indexCreatedVersion, consumer);
+    }
+
+    /**
+     * Overrides the snapshot-restore operation for archive-indices scenario.
+     */
+    @Override
+    public void recover(RestClient client, String repository, String snapshot, String index, Consumer<List<String>> warningsConsumer)
+        throws Exception {
+        var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_restore");
+        request.addParameter("wait_for_completion", "true");
+        request.setJsonEntity(Strings.format("""
+            {
+              "indices": "%s",
+              "include_global_state": false,
+              "rename_pattern": "(.+)",
+              "include_aliases": false
+            }""", index));
+        request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE));
+        Response response = client.performRequest(request);
+        assertOK(response);
+        warningsConsumer.accept(response.getWarnings());
+    }
+}
diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5CustomAnalyzerIT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5CustomAnalyzerIT.java
new file mode 100644
index 0000000000000..df5826fc6b725
--- /dev/null
+++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5CustomAnalyzerIT.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.oldrepos.archiveindex;
+
+import org.elasticsearch.oldrepos.TestSnapshotCases;
+import org.elasticsearch.test.cluster.util.Version;
+
+/**
+ * Test case restoring index created in ES_v5 - Custom-Analyzer - standard token filter
+ *
+ * PUT /index
+ * {
+ *   "settings": {
+ *     "analysis": {
+ *       "analyzer": {
+ *         "custom_analyzer": {
+ *           "type": "custom",
+ *           "tokenizer": "standard",
+ *           "filter": [
+ *             "standard",
+ *             "lowercase"
+ *           ]
+ *         }
+ *       }
+ *     }
+ *   },
+ *   "mappings": {
+ *     "my_type": {
+ *       "properties": {
+ *         "content": {
+ *           "type": "text",
+ *           "analyzer": "custom_analyzer"
+ *         }
+ *       }
+ *     }
+ *   }
+ * }
+ */
+public class RestoreFromVersion5CustomAnalyzerIT extends ArchiveIndexTestCase {
+
+    public RestoreFromVersion5CustomAnalyzerIT(Version version) {
+        super(version, TestSnapshotCases.ES_VERSION_5_STANDARD_TOKEN_FILTER, warnings -> {
+            assertEquals(1, warnings.size());
+            assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst());
+        });
+    }
+}
diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5IT.java
new file mode 100644
index 0000000000000..ca482e54db6bb
--- /dev/null
+++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5IT.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.oldrepos.archiveindex;
+
+import org.elasticsearch.oldrepos.TestSnapshotCases;
+import org.elasticsearch.test.cluster.util.Version;
+
+/**
+ * Test case restoring snapshot created in ES_v5 - Basic mapping
+ *
+ * PUT /index
+ * {
+ *   "settings": {
+ *     "number_of_shards": 1,
+ *     "number_of_replicas": 1
+ *   },
+ *   "mappings": {
+ *     "my_type": {
+ *       "properties": {
+ *         "title": {
+ *           "type": "text"
+ *         },
+ *         "created_at": {
+ *           "type": "date"
+ *         },
+ *         "views": {
+ *           "type": "integer"
+ *         }
+ *       }
+ *     }
+ *   }
+ * }
+ */
+public class RestoreFromVersion5IT extends ArchiveIndexTestCase {
+
+    public RestoreFromVersion5IT(Version version) {
+        super(version, TestSnapshotCases.ES_VERSION_5);
+    }
+}
diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6CustomAnalyzerIT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6CustomAnalyzerIT.java
new file mode 100644
index 0000000000000..63ab202798d2f
--- /dev/null
+++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6CustomAnalyzerIT.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.oldrepos.archiveindex;
+
+import org.elasticsearch.oldrepos.TestSnapshotCases;
+import org.elasticsearch.test.cluster.util.Version;
+
+/**
+ * Test case restoring index created in ES_v6 - Custom-Analyzer - standard token filter
+ *
+ * PUT /index
+ * {
+ *   "settings": {
+ *     "analysis": {
+ *       "analyzer": {
+ *         "custom_analyzer": {
+ *           "type": "custom",
+ *           "tokenizer": "standard",
+ *           "filter": [
+ *             "standard",
+ *             "lowercase"
+ *           ]
+ *         }
+ *       }
+ *     }
+ *   },
+ *   "mappings": {
+ *     "_doc": {
+ *       "properties": {
+ *         "content": {
+ *           "type": "text",
+ *           "analyzer": "custom_analyzer"
+ *         }
+ *       }
+ *     }
+ *   }
+ * }
+ */
+public class RestoreFromVersion6CustomAnalyzerIT extends ArchiveIndexTestCase {
+
+    public RestoreFromVersion6CustomAnalyzerIT(Version version) {
+        super(version, TestSnapshotCases.ES_VERSION_6_STANDARD_TOKEN_FILTER, warnings -> {
+            assertEquals(1, warnings.size());
+            assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst());
+        });
+    }
+}
diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6IT.java
new file mode 100644
index 0000000000000..843f4dd96297e
--- /dev/null
+++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6IT.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.oldrepos.archiveindex;
+
+import org.elasticsearch.oldrepos.TestSnapshotCases;
+import org.elasticsearch.test.cluster.util.Version;
+
+/**
+ * Test case restoring snapshot created in ES_v6 - Basic mapping
+ *
+ * PUT /index
+ * {
+ *   "settings": {
+ *     "number_of_shards": 1,
+ *     "number_of_replicas": 1
+ *   },
+ *   "mappings": {
+ *     "_doc": {
+ *       "properties": {
+ *         "title": {
+ *           "type": "text"
+ *         },
+ *         "content": {
+ *           "type": "text"
+ *         },
+ *         "created_at": {
+ *           "type": "date"
+ *         }
+ *       }
+ *     }
+ *   }
+ * }
+ */
+public class RestoreFromVersion6IT extends ArchiveIndexTestCase {
+
+    public RestoreFromVersion6IT(Version version) {
+        super(version, TestSnapshotCases.ES_VERSION_6);
+    }
+}
diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5CustomAnalyzerIT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5CustomAnalyzerIT.java
new file mode 100644
index 0000000000000..4440629729ef9
--- /dev/null
+++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5CustomAnalyzerIT.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.oldrepos.searchablesnapshot;
+
+import org.elasticsearch.oldrepos.TestSnapshotCases;
+import org.elasticsearch.test.cluster.util.Version;
+
+/**
+ * Test case mounting index created in ES_v5 - Custom-Analyzer - standard token filter
+ *
+ * PUT /index
+ * {
+ *   "settings": {
+ *     "analysis": {
+ *       "analyzer": {
+ *         "custom_analyzer": {
+ *           "type": "custom",
+ *           "tokenizer": "standard",
+ *           "filter": [
+ *             "standard",
+ *             "lowercase"
+ *           ]
+ *         }
+ *       }
+ *     }
+ *   },
+ *   "mappings": {
+ *     "my_type": {
+ *       "properties": {
+ *         "content": {
+ *           "type": "text",
+ *           "analyzer": "custom_analyzer"
+ *         }
+ *       }
+ *     }
+ *   }
+ * }
+ */
+public class MountFromVersion5CustomAnalyzerIT extends SearchableSnapshotTestCase {
+
+    public MountFromVersion5CustomAnalyzerIT(Version version) {
+        super(version, TestSnapshotCases.ES_VERSION_5_STANDARD_TOKEN_FILTER, warnings -> {
+            assertEquals(1, warnings.size());
+            assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst());
+        });
+    }
+}
diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5IT.java
new file mode 100644
index 0000000000000..d5231be121282
--- /dev/null
+++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5IT.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.oldrepos.searchablesnapshot;
+
+import org.elasticsearch.oldrepos.TestSnapshotCases;
+import org.elasticsearch.test.cluster.util.Version;
+
+/**
+ * Test case mounting snapshot created in ES_v5 - Basic mapping
+ *
+ * PUT /index
+ * {
+ *   "settings": {
+ *     "number_of_shards": 1,
+ *     "number_of_replicas": 1
+ *   },
+ *   "mappings": {
+ *     "my_type": {
+ *       "properties": {
+ *         "title": {
+ *           "type": "text"
+ *         },
+ *         "created_at": {
+ *           "type": "date"
+ *         },
+ *         "views": {
+ *           "type": "integer"
+ *         }
+ *       }
+ *     }
+ *   }
+ * }
+ */
+public class MountFromVersion5IT extends SearchableSnapshotTestCase {
+
+    public MountFromVersion5IT(Version version) {
+        super(version, TestSnapshotCases.ES_VERSION_5);
+    }
+}
diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6CustomAnalyzerIT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6CustomAnalyzerIT.java
new file mode 100644
index 0000000000000..de529c95bd72d
--- /dev/null
+++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6CustomAnalyzerIT.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.oldrepos.searchablesnapshot;
+
+import org.elasticsearch.oldrepos.TestSnapshotCases;
+import org.elasticsearch.test.cluster.util.Version;
+
+/**
+ * Test case mounting index created in ES_v6 - Custom-Analyzer - standard token filter
+ *
+ * PUT /index
+ * {
+ *   "settings": {
+ *     "analysis": {
+ *       "analyzer": {
+ *         "custom_analyzer": {
+ *           "type": "custom",
+ *           "tokenizer": "standard",
+ *           "filter": [
+ *             "standard",
+ *             "lowercase"
+ *           ]
+ *         }
+ *       }
+ *     }
+ *   },
+ *   "mappings": {
+ *     "_doc": {
+ *       "properties": {
+ *         "content": {
+ *           "type": "text",
+ *           "analyzer": "custom_analyzer"
+ *         }
+ *       }
+ *     }
+ *   }
+ * }
+ */
+public class MountFromVersion6CustomAnalyzerIT extends SearchableSnapshotTestCase {
+
+    public MountFromVersion6CustomAnalyzerIT(Version version) {
+        super(version, TestSnapshotCases.ES_VERSION_6_STANDARD_TOKEN_FILTER, warnings -> {
+            assertEquals(1, warnings.size());
+            assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst());
+        });
+    }
+}
diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6IT.java
new file mode 100644
index 0000000000000..9632fc0e279b8
--- /dev/null
+++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6IT.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.oldrepos.searchablesnapshot;
+
+import org.elasticsearch.oldrepos.TestSnapshotCases;
+import org.elasticsearch.test.cluster.util.Version;
+
+/**
+ * Test case mounting snapshot created in ES_v6 - Basic mapping
+ *
+ * PUT /index
+ * {
+ *   "settings": {
+ *     "number_of_shards": 1,
+ *     "number_of_replicas": 1
+ *   },
+ *   "mappings": {
+ *     "_doc": {
+ *       "properties": {
+ *         "title": {
+ *           "type": "text"
+ *         },
+ *         "content": {
+ *           "type": "text"
+ *         },
+ *         "created_at": {
+ *           "type": "date"
+ *         }
+ *       }
+ *     }
+ *   }
+ * }
+ */
+public class MountFromVersion6IT extends SearchableSnapshotTestCase {
+
+    public MountFromVersion6IT(Version version) {
+        super(version, TestSnapshotCases.ES_VERSION_6);
+    }
+}
diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/SearchableSnapshotTestCase.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/SearchableSnapshotTestCase.java
new file mode 100644
index 0000000000000..88d0f3901ca1c
--- /dev/null
+++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/SearchableSnapshotTestCase.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.oldrepos.searchablesnapshot;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.WarningsHandler;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.oldrepos.AbstractUpgradeCompatibilityTestCase;
+import org.elasticsearch.test.cluster.util.Version;
+
+import java.util.List;
+import java.util.function.Consumer;
+
+/**
+ * Test suite for Searchable indices backward compatibility with N-2 versions.
+ * The test suite creates a cluster in the N-1 version, where N is the current version.
+ * Restores snapshots from old-clusters (version 5/6) and upgrades it to the current version.
+ * Test methods are executed after each upgrade.
+ */
+abstract class SearchableSnapshotTestCase extends AbstractUpgradeCompatibilityTestCase {
+
+    static {
+        clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial");
+    }
+
+    protected SearchableSnapshotTestCase(Version version, String indexCreatedVersion) {
+        this(version, indexCreatedVersion, o -> {});
+    }
+
+    protected SearchableSnapshotTestCase(Version version, String indexCreatedVersion, Consumer<List<String>> warningsConsumer) {
+        super(version, indexCreatedVersion, warningsConsumer);
+    }
+
+    /**
+     * Overrides the snapshot-restore operation for archive-indices scenario.
+     */
+    @Override
+    public void recover(RestClient client, String repository, String snapshot, String index, Consumer<List<String>> warningsConsumer)
+        throws Exception {
+        var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_mount");
+        request.addParameter("wait_for_completion", "true");
+        request.addParameter("storage", "full_copy");
+        request.setJsonEntity(Strings.format("""
+             {
+              "index": "%s",
+              "renamed_index": "%s"
+            }""", index, index));
+        request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE));
+        Response response = client.performRequest(request);
+        assertOK(response);
+        warningsConsumer.accept(response.getWarnings());
+    }
+}
diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/README.md b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/README.md
new file mode 100644
index 0000000000000..5b3ff1f296414
--- /dev/null
+++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/README.md
@@ -0,0 +1,225 @@
+
+### Create data structure and config file
+```
+mkdir /tmp/sharedESData
+mkdir /tmp/sharedESData/config
+mkdir /tmp/sharedESData/data
+mkdir /tmp/sharedESData/snapshots
+```
+
+```
+touch /tmp/sharedESData/config/elasticsearch.yml
+
+cat <<EOF >> /tmp/sharedESData/config/elasticsearch.yml
+cluster.name: "archive-indides-test"
+node.name: "node-1"
+path.repo: ["/usr/share/elasticsearch/snapshots"]
+network.host: 0.0.0.0
+http.port: 9200
+
+discovery.type: single-node
+xpack.security.enabled: false
+EOF
+```
+
+### Define path
+```
+SHARED_FOLDER=/tmp/sharedESData
+```
+
+### Deploy container
+```
+docker run -d --name es \
+-p 9200:9200 -p 9300:9300 \
+-v ${SHARED_FOLDER}/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml \
+-v ${SHARED_FOLDER}/data:/usr/share/elasticsearch/data \
+-v ${SHARED_FOLDER}/snapshots:/usr/share/elasticsearch/snapshots \
+--env "discovery.type=single-node" \
+docker.elastic.co/elasticsearch/elasticsearch:5.6.16
+
+// Version 6
+docker.elastic.co/elasticsearch/elasticsearch:6.8.23
+```
+
+### Create Index Version 5
+```
+PUT /index
+{
+  "settings": {
+    "number_of_shards": 1,
+    "number_of_replicas": 1
+  },
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "title": {
+          "type": "text"
+        },
+        "created_at": {
+          "type": "date"
+        },
+        "views": {
+          "type": "integer"
+        }
+      }
+    }
+  }
+}
+```
+
+### Create Index Version 5 - Custom-Analyzer
+```
+PUT /index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "custom_analyzer": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": [
+            "standard",
+            "lowercase"
+          ]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "content": {
+          "type": "text",
+          "analyzer": "custom_analyzer"
+        }
+      }
+    }
+  }
+}
+```
+
+### Create Index Version 6
+```
+PUT /index
+{
+  "settings": {
+    "number_of_shards": 1,
+    "number_of_replicas": 1
+  },
+  "mappings": {
+    "_doc": {
+      "properties": {
+        "title": {
+          "type": "text"
+        },
+        "content": {
+          "type": "text"
+        },
+        "created_at": {
+          "type": "date"
+        }
+      }
+    }
+  }
+}
+```
+
+### Create Index Version 5 - Custom-Analyzer
+```
+PUT /index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "custom_analyzer": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": [
+            "standard",
+            "lowercase"
+          ]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "doc": {
+      "properties": {
+        "content": {
+          "type": "text",
+          "analyzer": "custom_analyzer"
+        }
+      }
+    }
+  }
+}
+```
+
+### Add documents Version 5
+```
+POST /index/my_type
+{
+  "title": "Title 5",
+  "content": "Elasticsearch is a powerful search engine.",
+  "created_at": "2024-12-16"
+}
+```
+
+### Add documents Version 5 - Custom-Analyzer
+```
+POST /index/my_type
+{
+  "content": "Doc 1"
+}
+```
+
+### Add documents Version 6
+```
+POST /index/_doc
+{
+  "title": "Title 5",
+  "content": "Elasticsearch is a powerful search engine.",
+  "created_at": "2024-12-16"
+}
+```
+
+### Add documents Version 5 - Custom-Analyzer
+```
+POST /index/_doc
+{
+  "content": "Doc 1"
+}
+```
+
+### Register repository
+```
+PUT /_snapshot/repository
+{
+  "type": "fs",
+  "settings": {
+    "location": "/usr/share/elasticsearch/snapshots",
+    "compress": true
+  }
+}
+```
+
+### Create a snapshot
+```
+PUT /_snapshot/repository/snapshot
+{
+    "indices": "index",
+    "ignore_unavailable": "true",
+    "include_global_state": false
+}
+```
+
+### Create zip file
+```
+zip -r snapshot.zip /tmp/sharedESData/snapshots/*
+```
+
+### Cleanup
+```
+docker rm -f es
+rm -rf /tmp/sharedESData/
+```
diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5.zip
new file mode 100644
index 0000000000000..54dcf4f6182cc
Binary files /dev/null and b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5.zip differ
diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5_custom_analyzer.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5_custom_analyzer.zip
new file mode 100644
index 0000000000000..cd8a28a825b89
Binary files /dev/null and b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5_custom_analyzer.zip differ
diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6.zip
new file mode 100644
index 0000000000000..d83152fb71c62
Binary files /dev/null and b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6.zip differ
diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6_custom_analyzer.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6_custom_analyzer.zip
new file mode 100644
index 0000000000000..012de2cd5d5dd
Binary files /dev/null and b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6_custom_analyzer.zip differ
diff --git a/x-pack/qa/repository-old-versions/build.gradle b/x-pack/qa/repository-old-versions/build.gradle
index ecd02ac9d209f..f21aad9cdf75a 100644
--- a/x-pack/qa/repository-old-versions/build.gradle
+++ b/x-pack/qa/repository-old-versions/build.gradle
@@ -50,6 +50,11 @@ if (OS.current() == OS.MAC && Architecture.current() == Architecture.AARCH64) {
   jdks.legacy.distributionVersion = '8.56.0.23'
 }
 
+interface Injected {
+  @Inject
+  FileSystemOperations getFs()
+}
+
 if (OS.current() == OS.WINDOWS) {
   logger.warn("Disabling repository-old-versions tests because we can't get the pid file on windows")
 } else {
@@ -95,10 +100,12 @@ if (OS.current() == OS.WINDOWS) {
       setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB'
     }
 
+    def oldesFixtureConfiguration = project.configurations.oldesFixture
+
     TaskProvider<AntFixture> fixture = tasks.register("oldES${versionNoDots}Fixture", AntFixture) {
       dependsOn project.configurations.oldesFixture, jdks.legacy, config
       executable = "${buildParams.runtimeJavaHome.get()}/bin/java"
-      env 'CLASSPATH', "${-> project.configurations.oldesFixture.asPath}"
+      env 'CLASSPATH', "${-> oldesFixtureConfiguration.asPath}"
       // old versions of Elasticsearch need JAVA_HOME
       env 'JAVA_HOME', jdks.legacy.javaHomePath
       // If we are running on certain arm systems we need to explicitly set the stack size to overcome JDK page size bug
@@ -116,11 +123,15 @@ if (OS.current() == OS.WINDOWS) {
         // We need to explicitly disable ML when running old ES versions on ARM or on systems with newer GLIBC
         args 'xpack.ml.enabled: false'
       }
+      def injected = project.objects.newInstance(Injected)
+
       doFirst {
-        delete(dataPath)
-        mkdir(dataPath)
+        injected.getFs().delete { d ->
+          d.delete(dataPath)
+        }
+        new File(dataPath).mkdirs()
       }
-      maxWaitInSeconds 60
+      maxWaitInSeconds = 60
       waitCondition = { fixture, ant ->
         // the fixture writes the ports file when Elasticsearch's HTTP service
         // is ready, so we can just wait for the file to exist
@@ -131,9 +142,13 @@ if (OS.current() == OS.WINDOWS) {
     tasks.register("javaRestTestBeforeRestart#${versionNoDots}", StandaloneRestIntegTestTask) {
       useCluster testClusterProvider
       dependsOn fixture
+      def injected = project.objects.newInstance(Injected)
+
       doFirst {
-        delete(repoLocation)
-        mkdir(repoLocation)
+        injected.getFs().delete { d ->
+          d.delete(repoLocation)
+        }
+        new File(repoLocation).mkdirs()
       }
       systemProperty 'tests.after_restart', 'false'
     }
diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java
index 968262448c87e..8cf58f0c2c082 100644
--- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java
+++ b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java
@@ -30,7 +30,7 @@
 import java.io.IOException;
 
 /**
- * Tests doc-value-based searches against indices imported from clusters older than N-1.
+ * Tests doc-value-based searches against archive indices, imported from clusters older than N-2.
  * We reuse the YAML tests in search/390_doc_values_search.yml but have to do the setup
  * manually here as the setup is done on the old cluster for which we have to use the
  * low-level REST client instead of the YAML set up that only knows how to talk to
diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle
index 2049ccb5d9cc8..f4c9f8f7ea2b0 100644
--- a/x-pack/qa/rolling-upgrade/build.gradle
+++ b/x-pack/qa/rolling-upgrade/build.gradle
@@ -47,7 +47,7 @@ buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
     testDistribution = "DEFAULT"
     versions = [oldVersion, project.version]
     numberOfNodes = 3
-
+    systemProperty 'es.queryable_built_in_roles_enabled', 'true'
     systemProperty 'ingest.geoip.downloader.enabled.default', 'true'
     //we don't want to hit real service from each test
     systemProperty 'ingest.geoip.downloader.endpoint.default', 'http://invalid.endpoint'
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java
index 40ad5bba29baa..094ca9304a695 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java
@@ -7,19 +7,29 @@
 package org.elasticsearch.upgrades;
 
 import org.apache.http.util.EntityUtils;
+import org.elasticsearch.Build;
+import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
+import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.common.time.FormatNames;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.Strings;
+import org.elasticsearch.xcontent.json.JsonXContent;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.time.Instant;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.upgrades.IndexingIT.assertCount;
+import static org.hamcrest.Matchers.equalTo;
 
 public class DataStreamsUpgradeIT extends AbstractUpgradeTestCase {
 
@@ -164,4 +174,188 @@ public void testDataStreamValidationDoesNotBreakUpgrade() throws Exception {
         }
     }
 
+    public void testUpgradeDataStream() throws Exception {
+        String dataStreamName = "reindex_test_data_stream";
+        int numRollovers = randomIntBetween(0, 5);
+        if (CLUSTER_TYPE == ClusterType.OLD) {
+            createAndRolloverDataStream(dataStreamName, numRollovers);
+        } else if (CLUSTER_TYPE == ClusterType.UPGRADED) {
+            upgradeDataStream(dataStreamName, numRollovers);
+        }
+    }
+
+    private static void createAndRolloverDataStream(String dataStreamName, int numRollovers) throws IOException {
+        // We want to create a data stream and roll it over several times so that we have several indices to upgrade
+        final String template = """
+            {
+                "settings":{
+                    "index": {
+                        "mode": "time_series"
+                    }
+                },
+                "mappings":{
+                    "dynamic_templates": [
+                        {
+                            "labels": {
+                                "path_match": "pod.labels.*",
+                                "mapping": {
+                                    "type": "keyword",
+                                    "time_series_dimension": true
+                                }
+                            }
+                        }
+                    ],
+                    "properties": {
+                        "@timestamp" : {
+                            "type": "date"
+                        },
+                        "metricset": {
+                            "type": "keyword",
+                            "time_series_dimension": true
+                        },
+                        "k8s": {
+                            "properties": {
+                                "pod": {
+                                    "properties": {
+                                        "name": {
+                                            "type": "keyword"
+                                        },
+                                        "network": {
+                                            "properties": {
+                                                "tx": {
+                                                    "type": "long"
+                                                },
+                                                "rx": {
+                                                    "type": "long"
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            """;
+        final String indexTemplate = """
+            {
+                "index_patterns": ["$PATTERN"],
+                "template": $TEMPLATE,
+                "data_stream": {
+                }
+            }""";
+        var putIndexTemplateRequest = new Request("POST", "/_index_template/reindex_test_data_stream_template");
+        putIndexTemplateRequest.setJsonEntity(indexTemplate.replace("$TEMPLATE", template).replace("$PATTERN", dataStreamName));
+        assertOK(client().performRequest(putIndexTemplateRequest));
+        bulkLoadData(dataStreamName);
+        for (int i = 0; i < numRollovers; i++) {
+            rollover(dataStreamName);
+            bulkLoadData(dataStreamName);
+        }
+    }
+
+    private void upgradeDataStream(String dataStreamName, int numRolloversOnOldCluster) throws Exception {
+        final int explicitRolloverOnNewClusterCount = randomIntBetween(0, 2);
+        for (int i = 0; i < explicitRolloverOnNewClusterCount; i++) {
+            rollover(dataStreamName);
+        }
+        Request reindexRequest = new Request("POST", "/_migration/reindex");
+        reindexRequest.setJsonEntity(Strings.format("""
+            {
+              "mode": "upgrade",
+              "source": {
+                "index": "%s"
+              }
+            }""", dataStreamName));
+        Response reindexResponse = client().performRequest(reindexRequest);
+        assertOK(reindexResponse);
+        assertBusy(() -> {
+            Request statusRequest = new Request("GET", "_migration/reindex/" + dataStreamName + "/_status");
+            Response statusResponse = client().performRequest(statusRequest);
+            Map<String, Object> statusResponseMap = XContentHelper.convertToMap(
+                JsonXContent.jsonXContent,
+                statusResponse.getEntity().getContent(),
+                false
+            );
+            assertOK(statusResponse);
+            assertThat(statusResponseMap.get("complete"), equalTo(true));
+            final int originalWriteIndex = 1;
+            if (isOriginalClusterSameMajorVersionAsCurrent()) {
+                assertThat(
+                    statusResponseMap.get("total_indices_in_data_stream"),
+                    equalTo(originalWriteIndex + numRolloversOnOldCluster + explicitRolloverOnNewClusterCount)
+                );
+                // If the original cluster was the same as this one, we don't want any indices reindexed:
+                assertThat(statusResponseMap.get("total_indices_requiring_upgrade"), equalTo(0));
+                assertThat(statusResponseMap.get("successes"), equalTo(0));
+            } else {
+                // The number of rollovers that will have happened when we call reindex:
+                final int rolloversPerformedByReindex = explicitRolloverOnNewClusterCount == 0 ? 1 : 0;
+                assertThat(
+                    statusResponseMap.get("total_indices_in_data_stream"),
+                    equalTo(originalWriteIndex + numRolloversOnOldCluster + explicitRolloverOnNewClusterCount + rolloversPerformedByReindex)
+                );
+                /*
+                 * total_indices_requiring_upgrade is made up of: (the original write index) + numRolloversOnOldCluster. The number of
+                 * rollovers on the upgraded cluster is irrelevant since those will not be reindexed.
+                 */
+                assertThat(
+                    statusResponseMap.get("total_indices_requiring_upgrade"),
+                    equalTo(originalWriteIndex + numRolloversOnOldCluster)
+                );
+                assertThat(statusResponseMap.get("successes"), equalTo(numRolloversOnOldCluster + 1));
+            }
+        }, 60, TimeUnit.SECONDS);
+        Request cancelRequest = new Request("POST", "_migration/reindex/" + dataStreamName + "/_cancel");
+        Response cancelResponse = client().performRequest(cancelRequest);
+        assertOK(cancelResponse);
+    }
+
+    /*
+     * Similar to isOriginalClusterCurrent, but returns true if the major versions of the clusters are the same. So true
+     * for 8.6 and 8.17, but false for 7.17 and 8.18.
+     */
+    private boolean isOriginalClusterSameMajorVersionAsCurrent() {
+        /*
+         * Since data stream reindex is specifically about upgrading a data stream from one major version to the next, it's ok to use the
+         * deprecated Version.fromString here
+         */
+        return Version.fromString(UPGRADE_FROM_VERSION).major == Version.fromString(Build.current().version()).major;
+    }
+
+    private static void bulkLoadData(String dataStreamName) throws IOException {
+        final String bulk = """
+            {"create": {}}
+            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cat", "network": {"tx": 2001818691, "rx": 802133794}}}}
+            {"create": {}}
+            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "hamster", "network": {"tx": 2005177954, "rx": 801479970}}}}
+            {"create": {}}
+            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cow", "network": {"tx": 2006223737, "rx": 802337279}}}}
+            {"create": {}}
+            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "rat", "network": {"tx": 2012916202, "rx": 803685721}}}}
+            {"create": {}}
+            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "dog", "network": {"tx": 1434521831, "rx": 530575198}}}}
+            {"create": {}}
+            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "tiger", "network": {"tx": 1434577921, "rx": 530600088}}}}
+            {"create": {}}
+            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "lion", "network": {"tx": 1434587694, "rx": 530604797}}}}
+            {"create": {}}
+            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "elephant", "network": {"tx": 1434595272, "rx": 530605511}}}}
+            """;
+        var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk");
+        bulkRequest.setJsonEntity(bulk.replace("$now", formatInstant(Instant.now())));
+        var response = client().performRequest(bulkRequest);
+        assertOK(response);
+    }
+
+    static String formatInstant(Instant instant) {
+        return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant);
+    }
+
+    private static void rollover(String dataStreamName) throws IOException {
+        Request rolloverRequest = new Request("POST", "/" + dataStreamName + "/_rollover");
+        Response rolloverResponse = client().performRequest(rolloverRequest);
+        assertOK(rolloverResponse);
+    }
 }
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/QueryableBuiltInRolesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/QueryableBuiltInRolesUpgradeIT.java
new file mode 100644
index 0000000000000..f9fce670bcd8c
--- /dev/null
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/QueryableBuiltInRolesUpgradeIT.java
@@ -0,0 +1,174 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.upgrades;
+
+import org.apache.http.client.methods.HttpGet;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.rest.ObjectPath;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.toSet;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
+import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class QueryableBuiltInRolesUpgradeIT extends AbstractUpgradeTestCase {
+
+    private static final String QUERYABLE_BUILT_IN_ROLES_NODE_FEATURE = "security.queryable_built_in_roles";
+    public static final String INDEX_METADATA_QUERYABLE_BUILT_IN_ROLES_DIGEST = "queryable_built_in_roles_digest";
+
+    @Before
+    public void initializeReservedRolesStore() {
+        new ReservedRolesStore();
+    }
+
+    /**
+     * Test upgrades from an older cluster versions that do not support queryable built-in roles feature.
+     */
+    public void testBuiltInRolesSyncedOnClusterUpgrade() throws Exception {
+        final int numberOfNodes = 3; // defined in build.gradle
+        waitForNodes(numberOfNodes);
+
+        final Set<TestNodeInfo> nodes = collectNodeInfos(adminClient());
+        assertThat("cluster should have " + numberOfNodes + " nodes", nodes.size(), equalTo(numberOfNodes));
+
+        final Set<TestNodeInfo> newVersionNodes = nodes.stream().filter(TestNodeInfo::isUpgradedVersionCluster).collect(toSet());
+        final Set<TestNodeInfo> oldVersionNodes = nodes.stream().filter(TestNodeInfo::isOriginalVersionCluster).collect(toSet());
+
+        assumeTrue(
+            "Old version nodes must not support queryable feature",
+            oldVersionNodes.stream().noneMatch(TestNodeInfo::supportsQueryableBuiltInRolesFeature)
+        );
+        assumeTrue(
+            "New version nodes must support queryable feature",
+            newVersionNodes.stream().allMatch(TestNodeInfo::supportsQueryableBuiltInRolesFeature)
+        );
+
+        switch (CLUSTER_TYPE) {
+            case OLD, MIXED -> {
+                // none of the old version nodes should support the queryable feature,
+                // hence the built-in roles should not exist in the security index
+                // in the mixed version cluster we do not attempt to sync the built-in roles
+                assertBuiltInRolesNotIndexed();
+            }
+            case UPGRADED -> {
+                // the built-in roles should be synced after the upgrade
+                assertBusy(() -> assertBuiltInRolesIndexed(ReservedRolesStore.names()), 45, TimeUnit.SECONDS);
+            }
+        }
+    }
+
+    record TestNodeInfo(String nodeId, String version, Set<String> features) {
+
+        public boolean isOriginalVersionCluster() {
+            return AbstractUpgradeTestCase.isOriginalCluster(this.version());
+        }
+
+        public boolean isUpgradedVersionCluster() {
+            return false == isOriginalVersionCluster();
+        }
+
+        public boolean supportsQueryableBuiltInRolesFeature() {
+            return features().contains(QUERYABLE_BUILT_IN_ROLES_NODE_FEATURE);
+        }
+
+    }
+
+    private static Set<TestNodeInfo> collectNodeInfos(RestClient adminClient) throws IOException {
+        final Request request = new Request("GET", "_cluster/state");
+        request.addParameter("filter_path", "nodes_features");
+
+        final Response response = adminClient.performRequest(request);
+
+        Map<String, Set<String>> nodeFeatures = null;
+        var responseData = responseAsMap(response);
+        if (responseData.get("nodes_features") instanceof List<?> nodesFeatures) {
+            nodeFeatures = nodesFeatures.stream()
+                .map(Map.class::cast)
+                .collect(Collectors.toUnmodifiableMap(nodeFeatureMap -> nodeFeatureMap.get("node_id").toString(), nodeFeatureMap -> {
+                    @SuppressWarnings("unchecked")
+                    var features = (List<String>) nodeFeatureMap.get("features");
+                    return new HashSet<>(features);
+                }));
+        }
+
+        Map<String, String> nodeVersions = nodesVersions();
+        assertThat(nodeVersions, is(notNullValue()));
+        // old cluster may not support node features, so we can treat it as if no features are supported
+        if (nodeFeatures == null) {
+            Set<TestNodeInfo> nodes = new HashSet<>(nodeVersions.size());
+            for (String nodeId : nodeVersions.keySet()) {
+                nodes.add(new TestNodeInfo(nodeId, nodeVersions.get(nodeId), Set.of()));
+            }
+            return nodes;
+        } else {
+            assertThat(nodeVersions.keySet(), containsInAnyOrder(nodeFeatures.keySet().toArray()));
+            Set<TestNodeInfo> nodes = new HashSet<>(nodeVersions.size());
+            for (String nodeId : nodeVersions.keySet()) {
+                nodes.add(new TestNodeInfo(nodeId, nodeVersions.get(nodeId), nodeFeatures.get(nodeId)));
+            }
+            return nodes;
+        }
+    }
+
+    private static void waitForNodes(int numberOfNodes) throws IOException {
+        final Request request = new Request(HttpGet.METHOD_NAME, "/_cluster/health");
+        request.addParameter("wait_for_nodes", String.valueOf(numberOfNodes));
+        final Response response = client().performRequest(request);
+        assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Map<String, String> nodesVersions() throws IOException {
+        final Response response = client().performRequest(new Request(HttpGet.METHOD_NAME, "_nodes/_all"));
+        assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
+        final Map<String, Object> nodes = (Map<String, Object>) extractValue(responseAsMap(response), "nodes");
+        assertNotNull("Nodes info is null", nodes);
+        final Map<String, String> nodesVersions = Maps.newMapWithExpectedSize(nodes.size());
+        for (Map.Entry<String, Object> node : nodes.entrySet()) {
+            nodesVersions.put(node.getKey(), (String) extractValue((Map<?, ?>) node.getValue(), "version"));
+        }
+        return nodesVersions;
+    }
+
+    private void assertBuiltInRolesIndexed(Set<String> expectedBuiltInRoles) throws IOException {
+        final Map<String, String> builtInRoles = readSecurityIndexBuiltInRolesMetadata();
+        assertThat(builtInRoles, is(notNullValue()));
+        assertThat(builtInRoles.keySet(), containsInAnyOrder(expectedBuiltInRoles.toArray()));
+    }
+
+    private void assertBuiltInRolesNotIndexed() throws IOException {
+        final Map<String, String> builtInRoles = readSecurityIndexBuiltInRolesMetadata();
+        assertThat(builtInRoles, is(nullValue()));
+    }
+
+    private Map<String, String> readSecurityIndexBuiltInRolesMetadata() throws IOException {
+        final Request request = new Request("GET", "_cluster/state/metadata/" + INTERNAL_SECURITY_MAIN_INDEX_7);
+        final Response response = adminClient().performRequest(request);
+        assertOK(response);
+        return ObjectPath.createFromResponse(response)
+            .evaluate("metadata.indices.\\.security-7." + INDEX_METADATA_QUERYABLE_BUILT_IN_ROLES_DIGEST);
+    }
+}
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRoleMappingCleanupIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRoleMappingCleanupIT.java
deleted file mode 100644
index 915122c97d3f1..0000000000000
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRoleMappingCleanupIT.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.upgrades;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
-import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-import static org.elasticsearch.TransportVersions.V_8_15_0;
-import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.Matchers.containsInAnyOrder;
-
-public class SecurityIndexRoleMappingCleanupIT extends AbstractUpgradeTestCase {
-    private static final Version UPGRADE_FROM_VERSION = Version.fromString(System.getProperty("tests.upgrade_from_version"));
-
-    public void testCleanupDuplicateMappings() throws Exception {
-        // see build.gradle where we set operator/settings.json for more details on this skip
-        assumeTrue(
-            "Cluster requires version higher than since operator/settings.json is only set then: " + Version.V_8_7_0,
-            UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_7_0)
-        );
-        if (CLUSTER_TYPE == ClusterType.OLD) {
-            // If we're in a state where the same operator-defined role mappings can exist both in cluster state and the native store
-            // (V_8_15_0 transport added to security.role_mapping_cleanup feature added), create a state
-            // where the native store will need to be cleaned up
-            assumeTrue(
-                "Cleanup only needed before security.role_mapping_cleanup feature available in cluster",
-                clusterHasFeature("security.role_mapping_cleanup") == false
-            );
-            assumeTrue(
-                "If role mappings are in cluster state but cleanup has not been performed yet, create duplicated role mappings",
-                minimumTransportVersion().onOrAfter(V_8_15_0)
-            );
-            // Since the old cluster has role mappings in cluster state, but doesn't check duplicates, create duplicates
-            createNativeRoleMapping("operator_role_mapping_1", Map.of("meta", "test"), true);
-            createNativeRoleMapping("operator_role_mapping_2", Map.of("meta", "test"), true);
-        } else if (CLUSTER_TYPE == ClusterType.MIXED) {
-            // Create a native role mapping that doesn't conflict with anything before the migration run
-            createNativeRoleMapping("no_name_conflict", Map.of("meta", "test"));
-        } else if (CLUSTER_TYPE == ClusterType.UPGRADED) {
-            waitForSecurityMigrationCompletion(adminClient(), 2);
-            assertAllRoleMappings(
-                client(),
-                "operator_role_mapping_1" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX,
-                "operator_role_mapping_2" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX,
-                "no_name_conflict"
-            );
-            // In the old cluster we might have created these (depending on the node features), so make sure they were removed
-            assertFalse(roleMappingExistsInSecurityIndex("operator_role_mapping_1"));
-            assertFalse(roleMappingExistsInSecurityIndex("operator_role_mapping_2"));
-            assertTrue(roleMappingExistsInSecurityIndex("no_name_conflict"));
-            // Make sure we can create and delete a conflicting role mapping again
-            createNativeRoleMapping("operator_role_mapping_1", Map.of("meta", "test"), true);
-            deleteNativeRoleMapping("operator_role_mapping_1", true);
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    private boolean roleMappingExistsInSecurityIndex(String mappingName) throws IOException {
-        final Request request = new Request("POST", "/.security/_search");
-        request.setJsonEntity(String.format(Locale.ROOT, """
-            {"query":{"bool":{"must":[{"term":{"_id":"%s_%s"}}]}}}""", "role-mapping", mappingName));
-
-        request.setOptions(
-            expectWarnings(
-                "this request accesses system indices: [.security-7],"
-                    + " but in a future major version, direct access to system indices will be prevented by default"
-            )
-        );
-
-        Response response = adminClient().performRequest(request);
-        assertOK(response);
-        final Map<String, Object> responseMap = responseAsMap(response);
-
-        Map<String, Object> hits = ((Map<String, Object>) responseMap.get("hits"));
-        return ((List<Object>) hits.get("hits")).isEmpty() == false;
-    }
-
-    private void createNativeRoleMapping(String roleMappingName, Map<String, Object> metadata) throws IOException {
-        createNativeRoleMapping(roleMappingName, metadata, false);
-    }
-
-    private void createNativeRoleMapping(String roleMappingName, Map<String, Object> metadata, boolean expectWarning) throws IOException {
-        final Request request = new Request("POST", "/_security/role_mapping/" + roleMappingName);
-        if (expectWarning) {
-            request.setOptions(
-                expectWarnings(
-                    "A read-only role mapping with the same name ["
-                        + roleMappingName
-                        + "] has been previously defined in a configuration file. "
-                        + "Both role mappings will be used to determine role assignments."
-                )
-            );
-        }
-
-        BytesReference source = BytesReference.bytes(
-            jsonBuilder().map(
-                Map.of(
-                    ExpressionRoleMapping.Fields.ROLES.getPreferredName(),
-                    List.of("superuser"),
-                    ExpressionRoleMapping.Fields.ENABLED.getPreferredName(),
-                    true,
-                    ExpressionRoleMapping.Fields.RULES.getPreferredName(),
-                    Map.of("field", Map.of("username", "role-mapping-test-user")),
-                    RoleDescriptor.Fields.METADATA.getPreferredName(),
-                    metadata
-                )
-            )
-        );
-        request.setJsonEntity(source.utf8ToString());
-        assertOK(client().performRequest(request));
-    }
-
-    private void deleteNativeRoleMapping(String roleMappingName, boolean expectWarning) throws IOException {
-        final Request request = new Request("DELETE", "/_security/role_mapping/" + roleMappingName);
-        if (expectWarning) {
-            request.setOptions(
-                expectWarnings(
-                    "A read-only role mapping with the same name ["
-                        + roleMappingName
-                        + "] has previously been defined in a configuration file. "
-                        + "The native role mapping was deleted, but the read-only mapping will remain active "
-                        + "and will be used to determine role assignments."
-                )
-            );
-        }
-        assertOK(client().performRequest(request));
-    }
-
-    private void assertAllRoleMappings(RestClient client, String... roleNames) throws IOException {
-        Request request = new Request("GET", "/_security/role_mapping");
-        Response response = client.performRequest(request);
-        assertOK(response);
-        Map<String, Object> responseMap = responseAsMap(response);
-
-        assertThat(responseMap.keySet(), containsInAnyOrder(roleNames));
-        assertThat(responseMap.size(), is(roleNames.length));
-    }
-}
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java
index 6c34e68297aa0..df8290327ee5a 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java
@@ -39,10 +39,6 @@ public void testRoleMigration() throws Exception {
         if (CLUSTER_TYPE == ClusterType.OLD) {
             createRoleWithMetadata(oldTestRole, Map.of("meta", "test"));
             assertDocInSecurityIndex(oldTestRole);
-            if (canRolesBeMigrated() == false) {
-                assertNoMigration(adminClient());
-                assertCannotQueryRolesByMetadata(client());
-            }
         } else if (CLUSTER_TYPE == ClusterType.MIXED) {
             if (FIRST_MIXED_ROUND) {
                 createRoleWithMetadata(mixed1TestRole, Map.of("meta", "test"));
@@ -51,13 +47,8 @@ public void testRoleMigration() throws Exception {
                 createRoleWithMetadata(mixed2TestRole, Map.of("meta", "test"));
                 assertDocInSecurityIndex(mixed2TestRole);
             }
-            if (canRolesBeMigrated() == false) {
-                assertNoMigration(adminClient());
-                assertCannotQueryRolesByMetadata(client());
-            }
         } else if (CLUSTER_TYPE == ClusterType.UPGRADED) {
             createRoleWithMetadata(upgradedTestRole, Map.of("meta", "test"));
-            assertTrue(canRolesBeMigrated());
             waitForSecurityMigrationCompletion(adminClient(), 1);
             assertMigratedDocInSecurityIndex(oldTestRole, "meta", "test");
             assertMigratedDocInSecurityIndex(mixed1TestRole, "meta", "test");
@@ -196,9 +187,4 @@ private void assertAllRoles(RestClient client, String... roleNames) throws IOExc
             assertThat(roles.get(i).get("name"), equalTo(roleNames[i]));
         }
     }
-
-    private boolean canRolesBeMigrated() {
-        return clusterHasFeature("security.migration_framework") != false
-            && clusterHasFeature("security.roles_metadata_flattened") != false;
-    }
 }
diff --git a/x-pack/qa/security-example-spi-extension/build.gradle b/x-pack/qa/security-example-spi-extension/build.gradle
index 13e1325848f32..66912af4a837a 100644
--- a/x-pack/qa/security-example-spi-extension/build.gradle
+++ b/x-pack/qa/security-example-spi-extension/build.gradle
@@ -2,9 +2,9 @@ apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.base-internal-es-plugin'
 
 esplugin {
-  name 'spi-extension'
-  description 'An example spi extension plugin for security'
-  classname 'org.elasticsearch.example.SpiExtensionPlugin'
+  name = 'spi-extension'
+  description = 'An example spi extension plugin for security'
+  classname = 'org.elasticsearch.example.SpiExtensionPlugin'
   extendedPlugins = ['x-pack-security']
 }
 
diff --git a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java
index 8400e7df54f4d..5d8684bf32f89 100644
--- a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java
+++ b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java
@@ -36,7 +36,8 @@
 public class ExampleSecurityExtension implements SecurityExtension {
 
     static {
-        if (RuntimeVersionFeature.isSecurityManagerAvailable()) {
+        final boolean useEntitlements = Boolean.parseBoolean(System.getProperty("es.entitlements.enabled"));
+        if (useEntitlements == false && RuntimeVersionFeature.isSecurityManagerAvailable()) {
             // check that the extension's policy works.
             AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
                 System.getSecurityManager().checkPropertyAccess("myproperty");
diff --git a/x-pack/qa/smoke-test-plugins-ssl/build.gradle b/x-pack/qa/smoke-test-plugins-ssl/build.gradle
index 461ebc4beb443..cbd837fc2ccf6 100644
--- a/x-pack/qa/smoke-test-plugins-ssl/build.gradle
+++ b/x-pack/qa/smoke-test-plugins-ssl/build.gradle
@@ -54,6 +54,14 @@ project(':plugins').getChildProjects().each { pluginName, pluginProject ->
 testClusters.matching { it.name == "yamlRestTest" }.configureEach {
   testDistribution = 'DEFAULT'
   setting 'xpack.monitoring.collection.interval', '1s'
+  setting 'xpack.monitoring.collection.enabled', 'false' //will enable in test
+  setting 'xpack.monitoring.exporters._http.enabled', 'false'//will enable in test
+  setting 'xpack.monitoring.exporters._http.type', "http"
+  setting 'xpack.monitoring.exporters._http.host', "https://example.com" //will be replaced in test
+  setting 'xpack.monitoring.exporters._http.auth.username', 'monitoring_agent'
+  setting 'xpack.monitoring.exporters._http.ssl.verification_mode', 'full'
+  setting 'xpack.monitoring.exporters._http.ssl.certificate_authorities', 'testnode.crt'
+  keystore 'xpack.monitoring.exporters._http.auth.secure_password', 'x-pack-test-password'
 
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'xpack.security.enabled', 'true'
diff --git a/x-pack/qa/smoke-test-plugins-ssl/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java b/x-pack/qa/smoke-test-plugins-ssl/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java
index f5247cad12d70..ac394b81ed5e7 100644
--- a/x-pack/qa/smoke-test-plugins-ssl/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java
+++ b/x-pack/qa/smoke-test-plugins-ssl/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -53,26 +52,25 @@ public class SmokeTestMonitoringWithSecurityIT extends ESRestTestCase {
 
     private static final String USER = "test_user";
     private static final SecureString PASS = new SecureString("x-pack-test-password".toCharArray());
-    private static final String KEYSTORE_PASS = "testnode";
     private static final String MONITORING_PATTERN = ".monitoring-*";
 
-    static Path keyStore;
+    static Path trustStore;
 
     @BeforeClass
     public static void getKeyStore() {
         try {
-            keyStore = PathUtils.get(SmokeTestMonitoringWithSecurityIT.class.getResource("/testnode.jks").toURI());
+            trustStore = PathUtils.get(SmokeTestMonitoringWithSecurityIT.class.getResource("/testnode.crt").toURI());
         } catch (URISyntaxException e) {
-            throw new ElasticsearchException("exception while reading the store", e);
+            throw new ElasticsearchException("exception while reading the truststore", e);
         }
-        if (Files.exists(keyStore) == false) {
-            throw new IllegalStateException("Keystore file [" + keyStore + "] does not exist.");
+        if (Files.exists(trustStore) == false) {
+            throw new IllegalStateException("Truststore file [" + trustStore + "] does not exist.");
         }
     }
 
     @AfterClass
     public static void clearKeyStore() {
-        keyStore = null;
+        trustStore = null;
     }
 
     @Override
@@ -85,24 +83,17 @@ protected Settings restClientSettings() {
         String token = basicAuthHeaderValue(USER, PASS);
         return Settings.builder()
             .put(ThreadContext.PREFIX + ".Authorization", token)
-            .put(ESRestTestCase.TRUSTSTORE_PATH, keyStore)
-            .put(ESRestTestCase.TRUSTSTORE_PASSWORD, KEYSTORE_PASS)
+            .put(ESRestTestCase.CERTIFICATE_AUTHORITIES, trustStore)
             .build();
     }
 
     @Before
     public void enableExporter() throws Exception {
-        MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.monitoring.exporters._http.auth.secure_password", "x-pack-test-password");
         Settings exporterSettings = Settings.builder()
             .put("xpack.monitoring.collection.enabled", true)
             .put("xpack.monitoring.exporters._http.enabled", true)
             .put("xpack.monitoring.exporters._http.type", "http")
             .put("xpack.monitoring.exporters._http.host", "https://" + randomNodeHttpAddress())
-            .put("xpack.monitoring.exporters._http.auth.username", "monitoring_agent")
-            .put("xpack.monitoring.exporters._http.ssl.verification_mode", "full")
-            .put("xpack.monitoring.exporters._http.ssl.certificate_authorities", "testnode.crt")
-            .setSecureSettings(secureSettings)
             .build();
         updateClusterSettingsIgnoringWarnings(client(), exporterSettings);
     }
@@ -140,7 +131,6 @@ private boolean getMonitoringUsageExportersDefined() throws Exception {
         return exporters != null && exporters.isEmpty() == false;
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/49094")
     public void testHTTPExporterWithSSL() throws Exception {
         // Ensures that the exporter is actually on
         assertBusy(() -> assertThat("[_http] exporter is not defined", getMonitoringUsageExportersDefined(), is(true)));
@@ -193,12 +183,11 @@ public void testHTTPExporterWithSSL() throws Exception {
         });
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/49094")
     public void testSettingsFilter() throws IOException {
         final Request request = new Request("GET", "/_cluster/settings");
         final Response response = client().performRequest(request);
         final ObjectPath path = ObjectPath.createFromResponse(response);
-        final Map<String, Object> settings = path.evaluate("transient.xpack.monitoring.exporters._http");
+        final Map<String, Object> settings = path.evaluate("persistent.xpack.monitoring.exporters._http");
         assertThat(settings, hasKey("type"));
         assertThat(settings, not(hasKey("auth")));
         assertThat(settings, not(hasKey("ssl")));